1use khive_storage::{StorageCapability, StorageError};
8use serde_json::{json, Value};
9
10use crate::{DomainDisposition, RuntimeError};
11
12pub fn runtime_error_value(error: RuntimeError, disposition: DomainDisposition) -> Value {
19 let disposition = if disposition == DomainDisposition::Committed {
20 disposition
21 } else {
22 error.batch_write_disposition().unwrap_or(disposition)
23 };
24 runtime_error_value_inner(error, disposition)
25}
26
27pub enum FailureEntrySource {
29 Runtime {
31 error: RuntimeError,
32 boundary: DomainDisposition,
33 },
34 Value {
36 error: Value,
37 disposition: DomainDisposition,
38 },
39}
40
41pub struct FailureEntryParts {
43 error: Value,
44 disposition: DomainDisposition,
45}
46
47impl FailureEntryParts {
48 pub fn into_error(self) -> Value {
50 self.error
51 }
52
53 pub fn into_entry(self, tool: impl Into<String>) -> Value {
55 let mut entry = serde_json::Map::new();
56 entry.insert("ok".into(), Value::Bool(false));
57 entry.insert("tool".into(), Value::String(tool.into()));
58 entry.insert(
59 "domain_disposition".into(),
60 json!(self.disposition.as_str()),
61 );
62 entry.insert("error".into(), self.error);
63 Value::Object(entry)
64 }
65}
66
67pub fn failure_error_disposition(error: &Value) -> DomainDisposition {
71 match error.get("domain_disposition").and_then(Value::as_str) {
72 Some("committed") => DomainDisposition::Committed,
73 Some("not_committed") => DomainDisposition::NotCommitted,
74 _ => DomainDisposition::Unknown,
75 }
76}
77
78pub fn failure_entry_parts(source: FailureEntrySource) -> FailureEntryParts {
87 use crate::daemon::{drop_error_detail_iteratively, error_detail_value_within_limit};
88
89 let (error, boundary, disposition) = match source {
90 FailureEntrySource::Runtime { error, boundary } => {
91 let projected = runtime_error_value(error, boundary);
92 let disposition = failure_error_disposition(&projected);
93 (projected, boundary, disposition)
94 }
95 FailureEntrySource::Value { error, disposition } => (error, disposition, disposition),
96 };
97 let mut error = match error {
98 Value::Object(map) => map,
99 Value::String(message) => serde_json::Map::from_iter([
100 ("kind".into(), json!("runtime_error")),
101 ("message".into(), Value::String(message)),
102 ]),
103 other => {
104 drop_error_detail_iteratively(other);
105 serde_json::Map::from_iter([
106 ("kind".into(), json!("runtime_error")),
107 ("message".into(), json!("operation failed")),
108 ])
109 }
110 };
111 let committed = |value: DomainDisposition| value == DomainDisposition::Committed;
112 if let Some(result) = error.remove("domain_result") {
113 if !committed(boundary) {
114 drop_error_detail_iteratively(result);
115 } else if !error_detail_value_within_limit(&result) {
116 drop_error_detail_iteratively(result);
117 error.insert("code".into(), json!("result_too_deep"));
118 error.insert(
119 "message".into(),
120 json!("committed domain result omitted because it exceeds the nesting depth limit"),
121 );
122 } else if !committed(disposition) {
123 drop_error_detail_iteratively(result);
124 } else {
125 error.insert("domain_result".into(), result);
126 }
127 }
128 error.insert("domain_disposition".into(), json!(disposition.as_str()));
129 FailureEntryParts {
130 error: Value::Object(error),
131 disposition,
132 }
133}
134
135fn runtime_error_value_inner(error: RuntimeError, disposition: DomainDisposition) -> Value {
138 let sqlite_write_failure = match error.refusal_source() {
141 RuntimeError::Storage(error) => error.sqlite_write_failure(),
142 _ => None,
143 };
144 let outcome_has_precedence = error.writer_task_failure_context().is_some_and(|context| {
145 context.task_terminated
146 || context.request_state == khive_storage::WriterTaskRequestState::SideEffectsUnknown
147 });
148 let receipt_projection = crate::visibility_receipts::receipt_error_projection(&error);
149 let named_disposition = match &error {
150 RuntimeError::Storage(error) if error.is_write_outcome_unknown() => Some("unknown"),
151 RuntimeError::Storage(error)
152 if matches!(
153 error.without_sqlite_write_stage(),
154 StorageError::WriterOwnerUnavailable { .. }
155 | StorageError::WriterRouteViolation { .. }
156 ) =>
157 {
158 Some("not_committed")
159 }
160 _ if receipt_projection.is_some_and(|(_, replay)| replay) => Some("not_committed"),
161 error if error.is_stream_policy_refusal() => Some("not_committed"),
165 RuntimeError::RefusedWithReceipt(_) => Some("not_committed"),
168 RuntimeError::Khive(k) => match (k.kind(), k.details().and_then(|d| d.get("reason"))) {
169 (
170 khive_types::ErrorKind::Internal,
171 Some("post_commit_degraded" | "embedding_input_truncated"),
172 ) => Some("committed"),
173 (
175 khive_types::ErrorKind::Conflict,
176 Some("key_conflict" | "fence_conflict" | "idempotency_key_conflict"),
177 ) => Some("not_committed"),
178 (khive_types::ErrorKind::Unavailable, Some("key_holder_unresolved")) => Some("unknown"),
179 (khive_types::ErrorKind::Conflict, Some("seq_conflict")) => Some("not_committed"),
187 (khive_types::ErrorKind::Conflict, Some("unknown_op")) => Some("not_committed"),
188 (khive_types::ErrorKind::Conflict, Some("version_conflict" | "identity_conflict")) => {
189 Some("not_committed")
190 }
191 (khive_types::ErrorKind::Conflict, Some("expired" | "live_until_unreadable")) => {
192 Some("not_committed")
193 }
194 (khive_types::ErrorKind::NotFound, Some("stream_write_not_found")) => {
195 Some("not_committed")
196 }
197 (khive_types::ErrorKind::NotFound, Some("keyed_replay_holder_missing")) => {
200 Some("not_committed")
201 }
202 (
203 khive_types::ErrorKind::InvalidInput,
204 Some("member_unavailable" | "sequence_exhausted"),
205 ) => Some("not_committed"),
206 _ => None,
207 },
208 _ => None,
209 };
210 let denial_message =
213 matches!(error, RuntimeError::PermissionDenied { .. }).then(|| error.to_string());
214 let payload = match error {
215 RuntimeError::SearchRead { context } => {
216 let message = context.to_string();
217 let backend = context.backend().as_str().to_owned();
218 let substrate = context.substrate().as_str();
219 let cause_class = context.source().audit_error_class().as_str();
220 let retryable = context.source().retryable_failure_context().is_some();
221 let source = runtime_error_value_inner(context.into_source(), disposition);
224 let kind = source["kind"]
225 .as_str()
226 .expect("canonical error kind is a string");
227 assert!(
228 matches!(
229 kind,
230 "runtime_error"
231 | "obligation"
232 | "internal"
233 | "remote_fetch_error"
234 | "not_found"
235 | "storage"
236 | "unavailable"
237 | "invalid_input"
238 | "unauthorized"
239 | "conflict"
240 ),
241 "canonical error kind must have an explicit privacy classification"
242 );
243 let mut value = json!({
244 "kind": kind,
245 "code": "search_read_failed",
246 "message": message,
247 "cause_class": cause_class,
248 "domain_disposition": failure_error_disposition(&source).as_str(),
249 "search_context": {"backend": backend, "substrate": substrate},
250 });
251 if retryable {
252 value["retryable"] = json!(true);
253 }
254 return value;
255 }
256 RuntimeError::CanonicalUnsupportedOperation {
257 op_index,
258 operation,
259 } => {
260 return json!({
261 "kind": "runtime_error",
262 "code": "invalid_input",
263 "reason": "KnownUnimplemented",
264 "operation": operation,
265 "op_index": op_index,
266 "message": format!("canonical operation {op_index}: {operation} is KnownUnimplemented for canonical apply"),
267 "domain_disposition": "not_committed",
268 "retryable": false,
269 });
270 }
271 RuntimeError::CanonicalAdmission { op_index, source } => {
272 let mut value = runtime_error_value_inner(*source, DomainDisposition::NotCommitted);
276 value["op_index"] = json!(op_index);
277 value["domain_disposition"] = json!("not_committed");
278 value["retryable"] = json!(false);
279 return value;
280 }
281 RuntimeError::LintFix { source, detail } => {
282 let mut value = runtime_error_value_inner(*source, DomainDisposition::Unknown);
283 let object = value
284 .as_object_mut()
285 .expect("runtime error projection is an object");
286 let details = object.entry("details").or_insert_with(|| json!({}));
287 if !details.is_object() {
288 *details = json!({});
289 }
290 details["lint_fix"] = json!(detail);
291 value["domain_disposition"] = json!("unknown");
294 value["retryable"] = json!(false);
295 return value;
296 }
297
298 RuntimeError::WithResolution { context } => {
299 let crate::error::ResolutionFailureContext { source, resolution } = context;
300 let mut value = runtime_error_value_inner(*source, disposition);
301 let details = value
302 .as_object_mut()
303 .expect("runtime error projection is an object")
304 .entry("details")
305 .or_insert_with(|| json!({}));
306 if !details.is_object() {
307 *details = json!({});
308 }
309 details["resolution"] = json!(resolution);
310 return value;
311 }
312 RuntimeError::BatchWrite { context } => {
313 let crate::error::BatchWriteContext {
314 source, admission, ..
315 } = *context;
316 let mut value = runtime_error_value_inner(*source, disposition);
317 if let Some(report) = admission {
318 if report.validation_complete {
319 if let Some(first) = report.refused.first() {
320 if !value["details"].is_object() {
321 value["details"] = json!({});
322 }
323 value["details"]["index"] = json!(first.index.to_string());
324 value["details"]["field"] = json!(first.field);
325 value["details"]["reason"] = json!(first.reason);
326 }
327 } else {
328 value["unchecked_indices"] = json!(report.unchecked_indices);
329 }
330 value["refused"] = json!(report.refused);
331 value["validation_complete"] = json!(report.validation_complete);
332 }
333 value["domain_disposition"] = json!(disposition.as_str());
334 return value;
335 }
336 RuntimeError::RefusedWithEvents { context } => {
337 let crate::error::RefusalEventContext { source, recordings } = context;
338 let mut value = runtime_error_value_inner(*source, disposition);
341 if !recordings.is_empty() {
342 value["refusal_recorded"] = json!(recordings.iter().all(|recording| {
343 matches!(recording, crate::RefusalEventRecording::Recorded { .. })
344 }));
345 value["refusal_events"] = json!(recordings);
346 }
347 return value;
348 }
349 RuntimeError::PermissionDenied {
350 verb,
351 reason,
352 receipt,
353 } => json!({
354 "kind": "runtime_error",
355 "code": "permission_denied",
356 "message": denial_message.unwrap_or_default(),
357 "verb": verb,
358 "reason": reason,
359 "audit_event_id": receipt.audit_event_id.map(|id| id.to_string()),
360 "audit_outcome": receipt.audit_outcome.wire_code(),
361 }),
362 RuntimeError::SecretDetected(matched) => {
363 let message = RuntimeError::SecretDetected(matched.clone()).to_string();
364 json!({
365 "kind": "runtime_error",
366 "code": "secret_detected",
367 "detector": matched.detector,
368 "location": matched.location,
369 "message": message,
370 })
371 }
372 RuntimeError::RefusedWithReceipt(refusal) => {
373 let crate::error::ReceiptRefusal {
374 code,
375 message,
376 receipt_id,
377 reason,
378 detail,
379 } = *refusal;
380 let mut error = match detail {
383 Value::Object(members) => members,
384 _ => serde_json::Map::new(),
385 };
386 error.insert("kind".into(), json!("runtime_error"));
387 error.insert("code".into(), json!(code));
388 error.insert("message".into(), json!(message));
389 error.insert("receipt_id".into(), json!(receipt_id));
390 error.insert("reason".into(), json!(reason));
391 Value::Object(error)
392 }
393 RuntimeError::AuditObligation {
394 failure,
395 domain_result,
396 } => {
397 let mut error = serde_json::Map::from_iter([
398 ("kind".into(), json!("obligation")),
399 ("code".into(), json!(failure.wire_code())),
400 ("message".into(), json!(failure.to_string())),
401 ]);
402 error.insert("domain_result".into(), domain_result);
403 Value::Object(error)
404 }
405 RuntimeError::Khive(k) => {
406 let mut value = serde_json::to_value(&k)
407 .unwrap_or_else(|_| json!({"kind": "internal", "message": k.to_string()}));
408 if k.kind() == khive_types::ErrorKind::InvalidInput
409 && k.details().and_then(|d| d.get("reason")) == Some("external_id_unverifiable")
410 {
411 value["code"] = json!("external_id_unverifiable");
412 }
413 value
414 }
415 RuntimeError::RemoteFetchError { remote, message } => json!({
416 "kind": "remote_fetch_error",
417 "remote": remote,
418 "message": message,
419 }),
420 missing @ RuntimeError::NotFound(_) => json!({
421 "kind": "not_found",
422 "code": null,
423 "details": null,
424 "message": missing.to_string(),
425 }),
426 other @ (RuntimeError::Storage(_)
427 | RuntimeError::Query(_)
428 | RuntimeError::InvalidInput(_)
429 | RuntimeError::UnknownVerb(_)
430 | RuntimeError::Unconfigured(_)
431 | RuntimeError::UnknownModel(_)
432 | RuntimeError::Embedding(_)
433 | RuntimeError::Ambiguous(_)
434 | RuntimeError::Fusion(_)
435 | RuntimeError::UnknownFusionStrategy(_)
436 | RuntimeError::Internal(_)
437 | RuntimeError::IncompatibleEventStore(_)
438 | RuntimeError::GuardedWriteFailed(_)
439 | RuntimeError::MissingPackDependency(_)
440 | RuntimeError::MissingPackDependencies(_)
441 | RuntimeError::CircularPackDependency(_)
442 | RuntimeError::PackRedeclared { .. }
443 | RuntimeError::VerbCollision { .. }
444 | RuntimeError::ReservedEnvelopeParam { .. }
445 | RuntimeError::GateUnavailable { .. }
446 | RuntimeError::NamespaceMismatch { .. }
447 | RuntimeError::AmbiguousPrefix { .. }
448 | RuntimeError::CrossBackendMergeUnsupported { .. }
449 | RuntimeError::UnknownRemote { .. }
450 | RuntimeError::RemoteCacheMissing { .. }
451 | RuntimeError::AmbiguousId { .. }
452 | RuntimeError::CrossNamespaceWrite { .. }
453 | RuntimeError::WriteBudgetExceeded { .. }
454 | RuntimeError::DeadlineExceeded { .. }
455 | RuntimeError::AttachmentCutoverRequired { .. }) => {
456 if let Some(capacity) = sqlite_capacity_failure(&other) {
457 let mut value = capacity.into_value(other.to_string());
458 if let Some(context) = other.writer_task_failure_context() {
459 value["request_state"] = json!(context.request_state.to_string());
460 value["task_terminated"] = json!(context.task_terminated);
461 }
462 value
463 } else if let Some(context) = other.writer_task_failure_context() {
464 json!({"kind":"storage", "code":context.stage, "stage":context.stage,
465 "message":other.to_string(), "retryable":context.retryable,
466 "request_state":context.request_state.to_string(), "task_terminated":context.task_terminated})
467 } else if let Some(context) = other.retryable_failure_context() {
468 let timeout_ms = u64::try_from(context.timeout.as_millis()).unwrap_or(u64::MAX);
469 let mut value = json!({"kind":"unavailable", "code":context.stage, "stage":context.stage,
470 "message":other.to_string(), "retryable":true, "timeout_ms":timeout_ms,
471 "capability":context.capability.map(storage_capability_wire_name),
472 "operation":context.operation, "scope":context.scope, "retry_after_ms":context.retry_after_ms});
473 if let Some(pool_identity) = context.pool_identity {
474 value["pool_identity"] = json!(pool_identity);
475 }
476 value
477 } else {
478 json!({"kind":"runtime_error", "message":other.to_string()})
479 }
480 }
481 };
482 let mut value = payload;
483 if let Some(failure) = sqlite_write_failure {
484 if !outcome_has_precedence
486 && !failure.settlement_unknown
487 && failure.primary_code != rusqlite::ffi::SQLITE_FULL
488 {
489 value["code"] = json!(failure.stage.as_str());
490 value["stage"] = json!(failure.stage.as_str());
491 if value["kind"] == "runtime_error" {
492 value["kind"] = json!("storage");
493 }
494 }
495 value["sqlite_write_stage"] = json!(failure.stage.as_str());
496 value["sqlite_primary_code"] = json!(failure.primary_code);
497 value["sqlite_extended_code"] = json!(failure.extended_code);
498 }
499 if let Some((retryable, _)) = receipt_projection {
500 value["retryable"] = json!(retryable);
501 }
502 value["domain_disposition"] = json!(disposition.as_str());
503 if let Some(named) = named_disposition {
504 value["domain_disposition"] = json!(named);
505 }
506 value
507}
508
509enum SqliteCapacityFailure {
510 Refused {
511 volume: String,
512 available_bytes: u64,
513 reserve_bytes: u64,
514 required_headroom_bytes: u64,
515 },
516 Unavailable {
517 phase: khive_storage::CapacityUnavailablePhase,
518 },
519 NativeFull {
520 primary_code: i32,
521 extended_code: i32,
522 },
523}
524
525impl SqliteCapacityFailure {
526 fn into_value(self, message: String) -> Value {
527 match self {
528 Self::Refused {
529 volume,
530 available_bytes,
531 reserve_bytes,
532 required_headroom_bytes,
533 } => json!({
534 "kind": "storage",
535 "code": "sqlite_capacity_refused",
536 "stage": "sqlite_capacity_refused",
537 "message": message,
538 "retryable": false,
539 "capability": "sql",
540 "volume": volume,
541 "available_bytes": available_bytes,
542 "reserve_bytes": reserve_bytes,
543 "required_headroom_bytes": required_headroom_bytes,
544 }),
545 Self::Unavailable { phase } => json!({
546 "kind": "storage",
547 "code": "sqlite_capacity_unavailable",
548 "stage": "sqlite_capacity_unavailable",
549 "message": message,
550 "retryable": phase == khive_storage::CapacityUnavailablePhase::LockContended,
551 "capability": "sql",
552 "phase": phase.as_str(),
553 }),
554 Self::NativeFull {
555 primary_code,
556 extended_code,
557 } => json!({
558 "kind": "storage",
559 "code": "sqlite_disk_full",
560 "stage": "sqlite_disk_full",
561 "message": message,
562 "retryable": false,
563 "capability": "sql",
564 "sqlite_primary_code": primary_code,
565 "sqlite_extended_code": extended_code,
566 }),
567 }
568 }
569}
570
571fn sqlite_capacity_failure(error: &RuntimeError) -> Option<SqliteCapacityFailure> {
572 match error {
573 RuntimeError::Storage(storage) => sqlite_capacity_failure_from_storage(storage),
574 _ => None,
575 }
576}
577
578fn sqlite_capacity_failure_from_storage(error: &StorageError) -> Option<SqliteCapacityFailure> {
579 match error {
580 StorageError::CapacityFloor {
581 capability: StorageCapability::Sql,
582 volume,
583 available_bytes,
584 floor_bytes,
585 required_headroom_bytes,
586 } => Some(SqliteCapacityFailure::Refused {
587 volume: volume.clone(),
588 available_bytes: *available_bytes,
589 reserve_bytes: *floor_bytes,
590 required_headroom_bytes: *required_headroom_bytes,
591 }),
592 StorageError::CapacityUnavailable {
593 capability: StorageCapability::Sql,
594 phase,
595 ..
596 } => Some(SqliteCapacityFailure::Unavailable { phase: *phase }),
597 StorageError::WriterTaskTerminated {
598 sqlite_full_codes: Some((primary_code, extended_code)),
599 ..
600 } if *primary_code == rusqlite::ffi::SQLITE_FULL
601 && *extended_code >= 0
602 && (*extended_code & 0xff) == *primary_code =>
603 {
604 Some(SqliteCapacityFailure::NativeFull {
605 primary_code: *primary_code,
606 extended_code: *extended_code,
607 })
608 }
609 StorageError::SqliteWrite { failure, .. }
610 if failure.primary_code == rusqlite::ffi::SQLITE_FULL =>
611 {
612 Some(SqliteCapacityFailure::NativeFull {
613 primary_code: failure.primary_code,
614 extended_code: failure.extended_code,
615 })
616 }
617 StorageError::WriterTaskRequestFailed { source, .. }
618 | StorageError::WriterOwnerUnavailable { source, .. }
619 | StorageError::SqliteWrite { source, .. } => sqlite_capacity_failure_from_storage(source),
620 StorageError::Driver { source, .. } => {
621 if let Some(sqlite) = source.downcast_ref::<khive_db::SqliteError>() {
622 sqlite_capacity_failure_from_sqlite(sqlite)
623 } else {
624 native_sqlite_full(source.as_ref())
625 }
626 }
627 _ => None,
628 }
629}
630
631fn sqlite_capacity_failure_from_sqlite(
632 error: &khive_db::SqliteError,
633) -> Option<SqliteCapacityFailure> {
634 match error {
635 khive_db::SqliteError::CapacityFloor {
636 volume,
637 available_bytes,
638 floor_bytes,
639 required_headroom_bytes,
640 } => Some(SqliteCapacityFailure::Refused {
641 volume: volume.clone(),
642 available_bytes: *available_bytes,
643 reserve_bytes: *floor_bytes,
644 required_headroom_bytes: *required_headroom_bytes,
645 }),
646 khive_db::SqliteError::CapacityUnavailable { phase, .. } => {
647 Some(SqliteCapacityFailure::Unavailable { phase: *phase })
648 }
649 khive_db::SqliteError::Rusqlite(sqlite)
650 | khive_db::SqliteError::Write { source: sqlite, .. } => native_sqlite_full(sqlite),
651 _ => None,
652 }
653}
654
655fn native_sqlite_full(error: &(dyn std::error::Error + 'static)) -> Option<SqliteCapacityFailure> {
656 let mut source = Some(error);
657 while let Some(current) = source {
658 if let Some(rusqlite::Error::SqliteFailure(code, _)) =
659 current.downcast_ref::<rusqlite::Error>()
660 {
661 if code.code == rusqlite::ErrorCode::DiskFull {
662 return Some(SqliteCapacityFailure::NativeFull {
663 primary_code: code.extended_code & 0xff,
664 extended_code: code.extended_code,
665 });
666 }
667 }
668 source = current.source();
669 }
670 None
671}
672
673fn storage_capability_wire_name(capability: StorageCapability) -> &'static str {
674 match capability {
675 StorageCapability::Sql => "sql",
676 StorageCapability::Notes => "notes",
677 StorageCapability::Entities => "entities",
678 StorageCapability::Graph => "graph",
679 StorageCapability::Events => "events",
680 StorageCapability::Vectors => "vectors",
681 StorageCapability::Sparse => "sparse",
682 StorageCapability::Text => "text",
683 StorageCapability::Blob => "blob",
684 StorageCapability::Attachments => "attachments",
685 }
686}
687
688#[cfg(test)]
689#[path = "error_projection_tests.rs"]
690mod tests;