1use khive_storage::{StorageCapability, StorageError};
7use serde_json::{json, Value};
8
9use crate::{DomainDisposition, RuntimeError};
10
11pub fn runtime_error_value(error: RuntimeError, disposition: DomainDisposition) -> Value {
18 let disposition = if disposition == DomainDisposition::Committed {
19 disposition
20 } else {
21 error.batch_write_disposition().unwrap_or(disposition)
22 };
23 runtime_error_value_inner(error, disposition)
24}
25
26pub enum FailureEntrySource {
28 Runtime {
30 error: RuntimeError,
31 boundary: DomainDisposition,
32 },
33 Value {
35 error: Value,
36 disposition: DomainDisposition,
37 },
38}
39
40pub struct FailureEntryParts {
42 error: Value,
43 disposition: DomainDisposition,
44}
45
46impl FailureEntryParts {
47 pub fn into_error(self) -> Value {
49 self.error
50 }
51
52 pub fn into_entry(self, tool: impl Into<String>) -> Value {
54 let mut entry = serde_json::Map::new();
55 entry.insert("ok".into(), Value::Bool(false));
56 entry.insert("tool".into(), Value::String(tool.into()));
57 entry.insert(
58 "domain_disposition".into(),
59 json!(self.disposition.as_str()),
60 );
61 entry.insert("error".into(), self.error);
62 Value::Object(entry)
63 }
64}
65
66pub fn failure_error_disposition(error: &Value) -> DomainDisposition {
70 match error.get("domain_disposition").and_then(Value::as_str) {
71 Some("committed") => DomainDisposition::Committed,
72 Some("not_committed") => DomainDisposition::NotCommitted,
73 _ => DomainDisposition::Unknown,
74 }
75}
76
77pub fn failure_entry_parts(source: FailureEntrySource) -> FailureEntryParts {
86 use crate::daemon::{drop_error_detail_iteratively, error_detail_value_within_limit};
87
88 let (error, boundary, disposition) = match source {
89 FailureEntrySource::Runtime { error, boundary } => {
90 let projected = runtime_error_value(error, boundary);
91 let disposition = failure_error_disposition(&projected);
92 (projected, boundary, disposition)
93 }
94 FailureEntrySource::Value { error, disposition } => (error, disposition, disposition),
95 };
96 let mut error = match error {
97 Value::Object(map) => map,
98 Value::String(message) => serde_json::Map::from_iter([
99 ("kind".into(), json!("runtime_error")),
100 ("message".into(), Value::String(message)),
101 ]),
102 other => {
103 drop_error_detail_iteratively(other);
104 serde_json::Map::from_iter([
105 ("kind".into(), json!("runtime_error")),
106 ("message".into(), json!("operation failed")),
107 ])
108 }
109 };
110 let committed = |value: DomainDisposition| value == DomainDisposition::Committed;
111 if let Some(result) = error.remove("domain_result") {
112 if !committed(boundary) {
113 drop_error_detail_iteratively(result);
114 } else if !error_detail_value_within_limit(&result) {
115 drop_error_detail_iteratively(result);
116 error.insert("code".into(), json!("result_too_deep"));
117 error.insert(
118 "message".into(),
119 json!("committed domain result omitted because it exceeds the nesting depth limit"),
120 );
121 } else if !committed(disposition) {
122 drop_error_detail_iteratively(result);
123 } else {
124 error.insert("domain_result".into(), result);
125 }
126 }
127 error.insert("domain_disposition".into(), json!(disposition.as_str()));
128 FailureEntryParts {
129 error: Value::Object(error),
130 disposition,
131 }
132}
133
134fn runtime_error_value_inner(error: RuntimeError, disposition: DomainDisposition) -> Value {
137 let sqlite_write_failure = match error.refusal_source() {
140 RuntimeError::Storage(error) => error.sqlite_write_failure(),
141 _ => None,
142 };
143 let outcome_has_precedence = error.writer_task_failure_context().is_some_and(|context| {
144 context.task_terminated
145 || context.request_state == khive_storage::WriterTaskRequestState::SideEffectsUnknown
146 });
147 let receipt_projection = crate::visibility_receipts::receipt_error_projection(&error);
148 let named_disposition = match &error {
149 _ if receipt_projection.is_some_and(|(_, replay)| replay) => Some("not_committed"),
150 error if error.is_stream_policy_refusal() => Some("not_committed"),
154 RuntimeError::RefusedWithReceipt(_) => Some("not_committed"),
157 RuntimeError::Khive(k) => match (k.kind(), k.details().and_then(|d| d.get("reason"))) {
158 (
159 khive_types::ErrorKind::Internal,
160 Some("post_commit_degraded" | "embedding_input_truncated"),
161 ) => Some("committed"),
162 (
164 khive_types::ErrorKind::Conflict,
165 Some("key_conflict" | "fence_conflict" | "idempotency_key_conflict"),
166 ) => Some("not_committed"),
167 (khive_types::ErrorKind::Unavailable, Some("key_holder_unresolved")) => Some("unknown"),
168 (khive_types::ErrorKind::Conflict, Some("seq_conflict")) => Some("not_committed"),
176 (khive_types::ErrorKind::Conflict, Some("unknown_op")) => Some("not_committed"),
177 (khive_types::ErrorKind::Conflict, Some("version_conflict" | "identity_conflict")) => {
178 Some("not_committed")
179 }
180 (khive_types::ErrorKind::Conflict, Some("expired" | "live_until_unreadable")) => {
181 Some("not_committed")
182 }
183 (khive_types::ErrorKind::NotFound, Some("stream_write_not_found")) => {
184 Some("not_committed")
185 }
186 (khive_types::ErrorKind::NotFound, Some("keyed_replay_holder_missing")) => {
189 Some("not_committed")
190 }
191 (
192 khive_types::ErrorKind::InvalidInput,
193 Some("member_unavailable" | "sequence_exhausted"),
194 ) => Some("not_committed"),
195 _ => None,
196 },
197 _ => None,
198 };
199 let denial_message =
202 matches!(error, RuntimeError::PermissionDenied { .. }).then(|| error.to_string());
203 let payload = match error {
204 RuntimeError::CanonicalUnsupportedOperation {
205 op_index,
206 operation,
207 } => {
208 return json!({
209 "kind": "runtime_error",
210 "code": "invalid_input",
211 "reason": "KnownUnimplemented",
212 "operation": operation,
213 "op_index": op_index,
214 "message": format!("canonical operation {op_index}: {operation} is KnownUnimplemented for canonical apply"),
215 "domain_disposition": "not_committed",
216 "retryable": false,
217 });
218 }
219 RuntimeError::CanonicalAdmission { op_index, source } => {
220 let mut value = runtime_error_value_inner(*source, DomainDisposition::NotCommitted);
224 value["op_index"] = json!(op_index);
225 value["domain_disposition"] = json!("not_committed");
226 value["retryable"] = json!(false);
227 return value;
228 }
229 RuntimeError::LintFix { source, detail } => {
230 let mut value = runtime_error_value_inner(*source, DomainDisposition::Unknown);
231 let object = value
232 .as_object_mut()
233 .expect("runtime error projection is an object");
234 let details = object.entry("details").or_insert_with(|| json!({}));
235 if !details.is_object() {
236 *details = json!({});
237 }
238 details["lint_fix"] = json!(detail);
239 value["domain_disposition"] = json!("unknown");
242 value["retryable"] = json!(false);
243 return value;
244 }
245
246 RuntimeError::WithResolution { context } => {
247 let crate::error::ResolutionFailureContext { source, resolution } = context;
248 let mut value = runtime_error_value_inner(*source, disposition);
249 let details = value
250 .as_object_mut()
251 .expect("runtime error projection is an object")
252 .entry("details")
253 .or_insert_with(|| json!({}));
254 if !details.is_object() {
255 *details = json!({});
256 }
257 details["resolution"] = json!(resolution);
258 return value;
259 }
260 RuntimeError::BatchWrite { context } => {
261 let crate::error::BatchWriteContext {
262 source, admission, ..
263 } = *context;
264 let mut value = runtime_error_value_inner(*source, disposition);
265 if let Some(report) = admission {
266 if report.validation_complete {
267 if let Some(first) = report.refused.first() {
268 if !value["details"].is_object() {
269 value["details"] = json!({});
270 }
271 value["details"]["index"] = json!(first.index.to_string());
272 value["details"]["field"] = json!(first.field);
273 value["details"]["reason"] = json!(first.reason);
274 }
275 } else {
276 value["unchecked_indices"] = json!(report.unchecked_indices);
277 }
278 value["refused"] = json!(report.refused);
279 value["validation_complete"] = json!(report.validation_complete);
280 }
281 value["domain_disposition"] = json!(disposition.as_str());
282 return value;
283 }
284 RuntimeError::RefusedWithEvents { context } => {
285 let crate::error::RefusalEventContext { source, recordings } = context;
286 let mut value = runtime_error_value_inner(*source, disposition);
289 if !recordings.is_empty() {
290 value["refusal_recorded"] = json!(recordings.iter().all(|recording| {
291 matches!(recording, crate::RefusalEventRecording::Recorded { .. })
292 }));
293 value["refusal_events"] = json!(recordings);
294 }
295 return value;
296 }
297 RuntimeError::PermissionDenied {
298 verb,
299 reason,
300 receipt,
301 } => json!({
302 "kind": "runtime_error",
303 "code": "permission_denied",
304 "message": denial_message.unwrap_or_default(),
305 "verb": verb,
306 "reason": reason,
307 "audit_event_id": receipt.audit_event_id.map(|id| id.to_string()),
308 "audit_outcome": receipt.audit_outcome.wire_code(),
309 }),
310 RuntimeError::SecretDetected(matched) => {
311 let message = RuntimeError::SecretDetected(matched.clone()).to_string();
312 json!({
313 "kind": "runtime_error",
314 "code": "secret_detected",
315 "detector": matched.detector,
316 "location": matched.location,
317 "message": message,
318 })
319 }
320 RuntimeError::RefusedWithReceipt(refusal) => {
321 let crate::error::ReceiptRefusal {
322 code,
323 message,
324 receipt_id,
325 reason,
326 detail,
327 } = *refusal;
328 let mut error = match detail {
331 Value::Object(members) => members,
332 _ => serde_json::Map::new(),
333 };
334 error.insert("kind".into(), json!("runtime_error"));
335 error.insert("code".into(), json!(code));
336 error.insert("message".into(), json!(message));
337 error.insert("receipt_id".into(), json!(receipt_id));
338 error.insert("reason".into(), json!(reason));
339 Value::Object(error)
340 }
341 RuntimeError::AuditObligation {
342 failure,
343 domain_result,
344 } => {
345 let mut error = serde_json::Map::from_iter([
346 ("kind".into(), json!("obligation")),
347 ("code".into(), json!(failure.wire_code())),
348 ("message".into(), json!(failure.to_string())),
349 ]);
350 error.insert("domain_result".into(), domain_result);
351 Value::Object(error)
352 }
353 RuntimeError::Khive(k) => {
354 let mut value = serde_json::to_value(&k)
355 .unwrap_or_else(|_| json!({"kind": "internal", "message": k.to_string()}));
356 if k.kind() == khive_types::ErrorKind::InvalidInput
357 && k.details().and_then(|d| d.get("reason")) == Some("external_id_unverifiable")
358 {
359 value["code"] = json!("external_id_unverifiable");
360 }
361 value
362 }
363 RuntimeError::RemoteFetchError { remote, message } => json!({
364 "kind": "remote_fetch_error",
365 "remote": remote,
366 "message": message,
367 }),
368 missing @ RuntimeError::NotFound(_) => json!({
369 "kind": "not_found",
370 "code": null,
371 "details": null,
372 "message": missing.to_string(),
373 }),
374 other @ (RuntimeError::Storage(_)
375 | RuntimeError::Query(_)
376 | RuntimeError::InvalidInput(_)
377 | RuntimeError::UnknownVerb(_)
378 | RuntimeError::Unconfigured(_)
379 | RuntimeError::UnknownModel(_)
380 | RuntimeError::Embedding(_)
381 | RuntimeError::Ambiguous(_)
382 | RuntimeError::Fusion(_)
383 | RuntimeError::UnknownFusionStrategy(_)
384 | RuntimeError::Internal(_)
385 | RuntimeError::IncompatibleEventStore(_)
386 | RuntimeError::GuardedWriteFailed(_)
387 | RuntimeError::MissingPackDependency(_)
388 | RuntimeError::MissingPackDependencies(_)
389 | RuntimeError::CircularPackDependency(_)
390 | RuntimeError::PackRedeclared { .. }
391 | RuntimeError::VerbCollision { .. }
392 | RuntimeError::ReservedEnvelopeParam { .. }
393 | RuntimeError::GateUnavailable { .. }
394 | RuntimeError::NamespaceMismatch { .. }
395 | RuntimeError::AmbiguousPrefix { .. }
396 | RuntimeError::CrossBackendMergeUnsupported { .. }
397 | RuntimeError::UnknownRemote { .. }
398 | RuntimeError::RemoteCacheMissing { .. }
399 | RuntimeError::AmbiguousId { .. }
400 | RuntimeError::CrossNamespaceWrite { .. }
401 | RuntimeError::WriteBudgetExceeded { .. }
402 | RuntimeError::DeadlineExceeded { .. }) => {
403 if let Some(capacity) = sqlite_capacity_failure(&other) {
404 let mut value = capacity.into_value(other.to_string());
405 if let Some(context) = other.writer_task_failure_context() {
406 value["request_state"] = json!(context.request_state.to_string());
407 value["task_terminated"] = json!(context.task_terminated);
408 }
409 value
410 } else if let Some(context) = other.writer_task_failure_context() {
411 json!({"kind":"storage", "code":context.stage, "stage":context.stage,
412 "message":other.to_string(), "retryable":context.retryable,
413 "request_state":context.request_state.to_string(), "task_terminated":context.task_terminated})
414 } else if let Some(context) = other.retryable_failure_context() {
415 let timeout_ms = u64::try_from(context.timeout.as_millis()).unwrap_or(u64::MAX);
416 let mut value = json!({"kind":"unavailable", "code":context.stage, "stage":context.stage,
417 "message":other.to_string(), "retryable":true, "timeout_ms":timeout_ms,
418 "capability":context.capability.map(storage_capability_wire_name),
419 "operation":context.operation, "scope":context.scope, "retry_after_ms":context.retry_after_ms});
420 if let Some(pool_identity) = context.pool_identity {
421 value["pool_identity"] = json!(pool_identity);
422 }
423 value
424 } else {
425 json!({"kind":"runtime_error", "message":other.to_string()})
426 }
427 }
428 };
429 let mut value = payload;
430 if let Some(failure) = sqlite_write_failure {
431 if !outcome_has_precedence
433 && !failure.settlement_unknown
434 && failure.primary_code != rusqlite::ffi::SQLITE_FULL
435 {
436 value["code"] = json!(failure.stage.as_str());
437 value["stage"] = json!(failure.stage.as_str());
438 if value["kind"] == "runtime_error" {
439 value["kind"] = json!("storage");
440 }
441 }
442 value["sqlite_write_stage"] = json!(failure.stage.as_str());
443 value["sqlite_primary_code"] = json!(failure.primary_code);
444 value["sqlite_extended_code"] = json!(failure.extended_code);
445 }
446 if let Some((retryable, _)) = receipt_projection {
447 value["retryable"] = json!(retryable);
448 }
449 value["domain_disposition"] = json!(disposition.as_str());
450 if let Some(named) = named_disposition {
451 value["domain_disposition"] = json!(named);
452 }
453 value
454}
455
456enum SqliteCapacityFailure {
457 Refused {
458 volume: String,
459 available_bytes: u64,
460 reserve_bytes: u64,
461 required_headroom_bytes: u64,
462 },
463 Unavailable {
464 phase: khive_storage::CapacityUnavailablePhase,
465 },
466 NativeFull {
467 primary_code: i32,
468 extended_code: i32,
469 },
470}
471
472impl SqliteCapacityFailure {
473 fn into_value(self, message: String) -> Value {
474 match self {
475 Self::Refused {
476 volume,
477 available_bytes,
478 reserve_bytes,
479 required_headroom_bytes,
480 } => json!({
481 "kind": "storage",
482 "code": "sqlite_capacity_refused",
483 "stage": "sqlite_capacity_refused",
484 "message": message,
485 "retryable": false,
486 "capability": "sql",
487 "volume": volume,
488 "available_bytes": available_bytes,
489 "reserve_bytes": reserve_bytes,
490 "required_headroom_bytes": required_headroom_bytes,
491 }),
492 Self::Unavailable { phase } => json!({
493 "kind": "storage",
494 "code": "sqlite_capacity_unavailable",
495 "stage": "sqlite_capacity_unavailable",
496 "message": message,
497 "retryable": phase == khive_storage::CapacityUnavailablePhase::LockContended,
498 "capability": "sql",
499 "phase": phase.as_str(),
500 }),
501 Self::NativeFull {
502 primary_code,
503 extended_code,
504 } => json!({
505 "kind": "storage",
506 "code": "sqlite_disk_full",
507 "stage": "sqlite_disk_full",
508 "message": message,
509 "retryable": false,
510 "capability": "sql",
511 "sqlite_primary_code": primary_code,
512 "sqlite_extended_code": extended_code,
513 }),
514 }
515 }
516}
517
518fn sqlite_capacity_failure(error: &RuntimeError) -> Option<SqliteCapacityFailure> {
519 match error {
520 RuntimeError::Storage(storage) => sqlite_capacity_failure_from_storage(storage),
521 _ => None,
522 }
523}
524
525fn sqlite_capacity_failure_from_storage(error: &StorageError) -> Option<SqliteCapacityFailure> {
526 match error {
527 StorageError::CapacityFloor {
528 capability: StorageCapability::Sql,
529 volume,
530 available_bytes,
531 floor_bytes,
532 required_headroom_bytes,
533 } => Some(SqliteCapacityFailure::Refused {
534 volume: volume.clone(),
535 available_bytes: *available_bytes,
536 reserve_bytes: *floor_bytes,
537 required_headroom_bytes: *required_headroom_bytes,
538 }),
539 StorageError::CapacityUnavailable {
540 capability: StorageCapability::Sql,
541 phase,
542 ..
543 } => Some(SqliteCapacityFailure::Unavailable { phase: *phase }),
544 StorageError::WriterTaskTerminated {
545 sqlite_full_codes: Some((primary_code, extended_code)),
546 ..
547 } if *primary_code == rusqlite::ffi::SQLITE_FULL
548 && *extended_code >= 0
549 && (*extended_code & 0xff) == *primary_code =>
550 {
551 Some(SqliteCapacityFailure::NativeFull {
552 primary_code: *primary_code,
553 extended_code: *extended_code,
554 })
555 }
556 StorageError::SqliteWrite { failure, .. }
557 if failure.primary_code == rusqlite::ffi::SQLITE_FULL =>
558 {
559 Some(SqliteCapacityFailure::NativeFull {
560 primary_code: failure.primary_code,
561 extended_code: failure.extended_code,
562 })
563 }
564 StorageError::WriterTaskRequestFailed { source, .. }
565 | StorageError::SqliteWrite { source, .. } => sqlite_capacity_failure_from_storage(source),
566 StorageError::Driver { source, .. } => {
567 if let Some(sqlite) = source.downcast_ref::<khive_db::SqliteError>() {
568 sqlite_capacity_failure_from_sqlite(sqlite)
569 } else {
570 native_sqlite_full(source.as_ref())
571 }
572 }
573 _ => None,
574 }
575}
576
577fn sqlite_capacity_failure_from_sqlite(
578 error: &khive_db::SqliteError,
579) -> Option<SqliteCapacityFailure> {
580 match error {
581 khive_db::SqliteError::CapacityFloor {
582 volume,
583 available_bytes,
584 floor_bytes,
585 required_headroom_bytes,
586 } => Some(SqliteCapacityFailure::Refused {
587 volume: volume.clone(),
588 available_bytes: *available_bytes,
589 reserve_bytes: *floor_bytes,
590 required_headroom_bytes: *required_headroom_bytes,
591 }),
592 khive_db::SqliteError::CapacityUnavailable { phase, .. } => {
593 Some(SqliteCapacityFailure::Unavailable { phase: *phase })
594 }
595 khive_db::SqliteError::Rusqlite(sqlite)
596 | khive_db::SqliteError::Write { source: sqlite, .. } => native_sqlite_full(sqlite),
597 _ => None,
598 }
599}
600
601fn native_sqlite_full(error: &(dyn std::error::Error + 'static)) -> Option<SqliteCapacityFailure> {
602 let mut source = Some(error);
603 while let Some(current) = source {
604 if let Some(rusqlite::Error::SqliteFailure(code, _)) =
605 current.downcast_ref::<rusqlite::Error>()
606 {
607 if code.code == rusqlite::ErrorCode::DiskFull {
608 return Some(SqliteCapacityFailure::NativeFull {
609 primary_code: code.extended_code & 0xff,
610 extended_code: code.extended_code,
611 });
612 }
613 }
614 source = current.source();
615 }
616 None
617}
618
619fn storage_capability_wire_name(capability: StorageCapability) -> &'static str {
620 match capability {
621 StorageCapability::Sql => "sql",
622 StorageCapability::Notes => "notes",
623 StorageCapability::Entities => "entities",
624 StorageCapability::Graph => "graph",
625 StorageCapability::Events => "events",
626 StorageCapability::Vectors => "vectors",
627 StorageCapability::Sparse => "sparse",
628 StorageCapability::Text => "text",
629 StorageCapability::Blob => "blob",
630 StorageCapability::Attachments => "attachments",
631 }
632}
633
634#[cfg(test)]
635#[path = "error_projection_tests.rs"]
636mod tests;