1use super::*;
4use std::{error::Error, fmt, io};
5
6const PAYLOAD_BYTES: usize = 6144;
7
8fn written_source_code(error: &(dyn Error + 'static), source_stage: saddle_core::DiagnosticStage,
9 stage: RootRequestEvent) -> DiagnosticCode {
10 let generic = || DiagnosticCode::new("framework.error").expect("static code");
11 if !matches!(source_stage, saddle_core::DiagnosticStage::RequestDecode)
12 || !matches!(stage, RootRequestEvent::Ingress) { return generic(); }
13 let Some(mut failure) = error.downcast_ref::<saddle_admission::InputDecodeFailure>() else { return generic(); };
14 let mut depth = 0;
15 while let saddle_admission::InputDecodeFailure::Context { failure: inner, .. } = failure {
16 if depth == 8 { return generic(); }
17 failure = inner; depth += 1;
18 }
19 let code = match failure {
20 saddle_admission::InputDecodeFailure::Syntax(_) => "framework.input_decode.syntax",
21 saddle_admission::InputDecodeFailure::Utf8(_) => "framework.input_decode.utf8",
22 saddle_admission::InputDecodeFailure::Parser => "framework.input_decode.parser",
23 _ => return generic(),
24 };
25 DiagnosticCode::new(code).expect("static code")
26}
27
28
29#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]
30#[serde(rename_all = "snake_case")]
31pub enum OriginalCaptureState {
32 CompleteEnqueued,
34 CompleteWritten,
36 OutputUnavailable,
37 OutputRejected,
38 FormattingFailed,
39 CauseCycle,
40 LegacyProjectionOnly,
41}
42
43#[derive(Serialize)]
44pub(crate) struct Segment<'a> {
45 schema_version: u8,
46 event: &'static str,
47 #[serde(skip_serializing_if = "Option::is_none")]
48 occurrence: Option<DiagnosticOccurrence>,
49 pub(crate) sequence: u64,
50 channel: &'static str,
51 cause_depth: usize,
52 pub(crate) payload: &'a str,
53 pub(crate) state: &'static str,
54}
55
56pub(crate) trait OriginalOutput {
57 fn submit(&self, segment: &Segment<'_>) -> DiagnosticSubmission;
58}
59impl OriginalOutput for EmergencyDiagnosticHandle {
60 fn submit(&self, segment: &Segment<'_>) -> DiagnosticSubmission {
61 self.submit_original_record(segment)
62 }
63}
64impl OriginalOutput for crate::BootstrapDiagnostics {
65 fn submit(&self, segment: &Segment<'_>) -> DiagnosticSubmission {
66 match self.write_original_record(segment) {
67 Ok(()) => DiagnosticSubmission::Enqueued,
68 Err(_) => DiagnosticSubmission::Closed,
69 }
70 }
71}
72struct ConfirmedOutput<'a>(&'a EmergencyDiagnosticHandle);
73impl OriginalOutput for ConfirmedOutput<'_> {
74 fn submit(&self, segment: &Segment<'_>) -> DiagnosticSubmission {
75 match self.0.write_original_record(segment) {
76 Ok(()) => DiagnosticSubmission::Enqueued,
77 Err(_) => DiagnosticSubmission::Closed,
78 }
79 }
80}
81
82struct Stream<'a, O: OriginalOutput> {
83 output: &'a O,
84 occurrence: Option<DiagnosticOccurrence>,
85 event: &'static str,
86 sequence: u64,
87 depth: usize,
88 channel: &'static str,
89 bytes: [u8; PAYLOAD_BYTES],
90 len: usize,
91 escaped: usize,
92 failure: Option<OriginalCaptureState>,
93 submission: DiagnosticSubmission,
94}
95impl<O: OriginalOutput> Stream<'_, O> {
96 fn emit(&mut self, state: &'static str) -> fmt::Result {
97 let payload = std::str::from_utf8(&self.bytes[..self.len]).map_err(|_| fmt::Error)?;
98 let submission = self.output.submit(&Segment {
99 schema_version: 1,
100 event: self.event,
101 occurrence: self.occurrence,
102 sequence: self.sequence,
103 channel: self.channel,
104 cause_depth: self.depth,
105 payload,
106 state,
107 });
108 if submission != DiagnosticSubmission::Enqueued {
109 self.submission = submission;
110 self.failure = Some(OriginalCaptureState::OutputRejected);
111 return Err(fmt::Error);
112 }
113 self.sequence += 1;
114 self.len = 0;
115 self.escaped = 0;
116 Ok(())
117 }
118 fn push(&mut self, text: &str) -> fmt::Result {
119 if self.failure.is_some() {
120 return Err(fmt::Error);
121 }
122 for c in text.chars() {
123 let cost = match c {
124 '"' | '\\' | '\n' | '\r' | '\t' | '\u{8}' | '\u{c}' => 2,
125 '\u{0}'..='\u{1f}' => 6,
126 _ => c.len_utf8(),
127 };
128 if self.escaped + cost > PAYLOAD_BYTES || self.len + c.len_utf8() > PAYLOAD_BYTES {
129 self.emit("continuation")?;
130 }
131 let size = c.len_utf8();
132 c.encode_utf8(&mut self.bytes[self.len..self.len + size]);
133 self.len += size;
134 self.escaped += cost;
135 }
136 Ok(())
137 }
138 fn json<T: Serialize>(&mut self, value: &T) -> fmt::Result {
139 serde_json::to_writer(&mut *self, value).map_err(|_| fmt::Error)?;
140 self.emit("field_end")
141 }
142 fn report_failure(&mut self, state: OriginalCaptureState) {
143 if self.failure == Some(OriginalCaptureState::OutputRejected) {
147 return;
148 }
149 self.failure = Some(state);
150 let marker = match state {
151 OriginalCaptureState::FormattingFailed => "formatting_failed",
152 OriginalCaptureState::CauseCycle => "cause_cycle",
153 _ => "capture_failed",
154 };
155 let _ = self.emit(marker);
156 }
157}
158pub(crate) fn stream_record<O: OriginalOutput>(output: &O,
162 encode: impl Fn(&mut dyn io::Write) -> serde_json::Result<()>) -> DiagnosticSubmission {
163 stream_record_identified(output, "context_record_segment", None, encode)
164}
165fn stream_record_identified<O: OriginalOutput>(output: &O, event: &'static str,
166 occurrence: Option<DiagnosticOccurrence>, encode: impl Fn(&mut dyn io::Write) -> serde_json::Result<()>) -> DiagnosticSubmission {
167 let mut stream = Stream { output, occurrence, event, sequence: 0,
168 depth: 0, channel: "record", bytes: [0; PAYLOAD_BYTES], len: 0, escaped: 0,
169 failure: None, submission: DiagnosticSubmission::Enqueued };
170 struct Lossy<'a, 'b, O: OriginalOutput>(&'a mut Stream<'b, O>);
171 impl<O: OriginalOutput> io::Write for Lossy<'_, '_, O> {
172 fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
173 if self.0.failure.is_none() {
174 if let Ok(text) = std::str::from_utf8(bytes) { let _ = self.0.push(text); }
175 else { self.0.failure = Some(OriginalCaptureState::FormattingFailed); }
176 }
177 Ok(bytes.len())
178 }
179 fn flush(&mut self) -> io::Result<()> { Ok(()) }
180 }
181 if encode(&mut Lossy(&mut stream)).is_err() { return DiagnosticSubmission::EncodingFailed; }
182 if stream.failure == Some(OriginalCaptureState::FormattingFailed) { return DiagnosticSubmission::EncodingFailed; }
183 if stream.failure.is_none() { let _ = stream.emit("record_complete"); }
184 stream.submission
185}
186
187impl<O: OriginalOutput> fmt::Write for Stream<'_, O> {
188 fn write_str(&mut self, text: &str) -> fmt::Result {
189 self.push(text)
190 }
191}
192impl<O: OriginalOutput> io::Write for Stream<'_, O> {
193 fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
194 let text = std::str::from_utf8(bytes).map_err(|_| io::ErrorKind::InvalidData)?;
197 self.push(text).map_err(|_| io::ErrorKind::Other)?;
198 Ok(bytes.len())
199 }
200 fn flush(&mut self) -> io::Result<()> {
201 Ok(())
202 }
203}
204
205#[derive(Serialize)]
206struct Header<'a> {
207 timestamp_unix_ms: u128,
208 context: OriginalContext<'a>,
209 stage: RootRequestEvent,
210 facts: SourceDetail<'a>,
211 outcome: RootOutcomeFacts,
212 original_contract: &'static str,
213 source_interface: &'static str,
214 #[serde(skip_serializing_if = "Option::is_none")]
215 protocol: Option<RemoteProtocolContext<'a>>,
216 #[serde(skip_serializing_if = "Option::is_none")]
217 remote_target: Option<RemoteTargetContext<'a>>,
218 #[serde(skip_serializing_if = "Option::is_none")]
219 grpc: Option<GrpcTargetContext<'a>>,
220}
221enum OriginalContext<'a> {
222 Request(&'a RequestExecutionView),
223 Unrooted(crate::AdmissionEventContext<'a>),
224}
225impl Serialize for OriginalContext<'_> {
226 fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
227 match self {
228 Self::Request(view) => view.unified_context().serialize(serializer),
229 Self::Unrooted(context) => context.serialize(serializer),
230 }
231 }
232}
233#[derive(Serialize)]
234pub struct RemoteTargetContext<'a> {
235 pub business_unit: &'a str,
236 pub function: &'a str,
237}
238#[derive(Serialize)]
242pub struct RemoteProtocolContext<'a> {
243 pub business_unit: &'a str,
244 pub function: &'a str,
245 pub technical_code: &'a str,
246 pub certainty: &'a str,
247}
248#[derive(Serialize)]
250pub struct GrpcTargetContext<'a> {
251 pub business_unit: &'a str,
252 pub dependency_alias: &'a str,
253 pub contract_sha256: &'a [u8;32],
254 pub service: &'a str,
255 pub method: &'a str,
256 pub authority_template: &'a str,
257 pub authority: Option<&'a str>,
258 pub attempt: u64,
259 pub deadline_remaining_ns: u128,
260 pub phase: &'a str,
261 pub send_progress: &'a str,
262 pub response_wire_bytes: Option<usize>,
263 #[serde(skip_serializing_if="Option::is_none")]
264 pub status: Option<GrpcStatusFacts<'a>>,
265}
266#[derive(Serialize)]
267pub struct GrpcStatusFacts<'a> {
268 pub code: i32,
269 pub message: &'a str,
270 pub details: &'a [u8],
271}
272enum RawSource<'a> {
273 Error(&'a (dyn Error + 'static)),
274 BorrowedError(&'a (dyn Error + 'a)),
275 Description(&'a dyn fmt::Display, &'a dyn fmt::Debug),
276}
277#[derive(Serialize)]
278struct IoFacts {
279 kind: Kind,
280 os_code: Option<i32>,
281}
282struct Kind(io::ErrorKind);
283impl Serialize for Kind {
284 fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
285 serializer.collect_str(&format_args!("{:?}", self.0))
286 }
287}
288
289impl RootDiagnosticScope<'_> {
290 #[track_caller]
293 pub fn source_error_written(
294 &self,
295 error: &(dyn Error + 'static),
296 source_stage: saddle_core::DiagnosticStage,
297 stage: RootRequestEvent,
298 axes: RootOutcomeFacts,
299 ) -> Result<super::WrittenRootFailure, RootRequestFailure> {
300 let code = written_source_code(error, source_stage, stage);
301 let facts = BoundedDiagnostic::capture(
302 DiagnosticCategory::UnexpectedError,
303 saddle_core::CaptureSite::FirstObserved,
304 saddle_core::BoundedDiagnosticCause::new(source_stage, code),
305 );
306 let receipt = self.original_record_mode(
307 RawSource::Error(error), SourceDetail::Bounded(&facts), code, stage, axes, true, None, None,
308 );
309 if receipt.original_capture == OriginalCaptureState::CompleteWritten {
310 Ok(super::WrittenRootFailure(receipt))
311 } else {
312 Err(receipt)
314 }
315 }
316 #[track_caller]
319 pub fn source_error(
320 &self,
321 error: &(dyn Error + 'static),
322 source_stage: saddle_core::DiagnosticStage,
323 stage: RootRequestEvent,
324 axes: RootOutcomeFacts,
325 ) -> RootRequestFailure {
326 let code = DiagnosticCode::new("framework.error").expect("static code");
327 let facts = BoundedDiagnostic::capture(
328 DiagnosticCategory::UnexpectedError,
329 saddle_core::CaptureSite::FirstObserved,
330 saddle_core::BoundedDiagnosticCause::new(source_stage, code),
331 );
332 self.source_error_with_facts(error, facts, code, stage, axes)
333 }
334
335 pub fn source_error_with_facts(
339 &self,
340 error: &(dyn Error + 'static),
341 facts: BoundedDiagnostic,
342 classification: DiagnosticCode,
343 stage: RootRequestEvent,
344 axes: RootOutcomeFacts,
345 ) -> RootRequestFailure {
346 self.original_record(
347 RawSource::Error(error),
348 SourceDetail::Bounded(&facts),
349 classification,
350 stage,
351 axes,
352 )
353 }
354
355 pub fn source_error_with_facts_written(
358 &self,
359 error: &(dyn Error + 'static),
360 facts: BoundedDiagnostic,
361 classification: DiagnosticCode,
362 stage: RootRequestEvent,
363 axes: RootOutcomeFacts,
364 ) -> Result<super::WrittenRootFailure, RootRequestFailure> {
365 let receipt = self.original_record_mode(
366 RawSource::Error(error), SourceDetail::Bounded(&facts),
367 classification, stage, axes, true, None, None,
368 );
369 if receipt.original_capture == OriginalCaptureState::CompleteWritten {
370 Ok(super::WrittenRootFailure(receipt))
371 } else {
372 Err(receipt)
373 }
374 }
375
376 pub fn source_borrowed_error_with_facts_written<E: Error>(
379 &self,
380 error: &E,
381 facts: BoundedDiagnostic,
382 classification: DiagnosticCode,
383 stage: RootRequestEvent,
384 axes: RootOutcomeFacts,
385 ) -> Result<super::WrittenRootFailure, RootRequestFailure> {
386 let receipt = self.original_record_mode(
387 RawSource::BorrowedError(error), SourceDetail::Bounded(&facts),
388 classification, stage, axes, true, None, None,
389 );
390 if receipt.original_capture == OriginalCaptureState::CompleteWritten {
391 Ok(super::WrittenRootFailure(receipt))
392 } else {
393 Err(receipt)
394 }
395 }
396
397 pub fn source_description<D: fmt::Display + fmt::Debug>(
401 &self,
402 description: &D,
403 facts: BoundedDiagnostic,
404 classification: DiagnosticCode,
405 stage: RootRequestEvent,
406 axes: RootOutcomeFacts,
407 ) -> RootRequestFailure {
408 self.original_record(
409 RawSource::Description(description, description),
410 SourceDetail::Bounded(&facts),
411 classification,
412 stage,
413 axes,
414 )
415 }
416
417 pub fn source_description_written<D: fmt::Display + fmt::Debug>(
420 &self,
421 description: &D,
422 facts: BoundedDiagnostic,
423 classification: DiagnosticCode,
424 stage: RootRequestEvent,
425 axes: RootOutcomeFacts,
426 ) -> Result<super::WrittenRootFailure, RootRequestFailure> {
427 self.written_record(
428 RawSource::Description(description, description),
429 SourceDetail::Bounded(&facts),
430 classification,
431 stage,
432 axes,
433 )
434 }
435
436 pub fn source_remote_protocol_written<D: fmt::Display + fmt::Debug>(
437 &self,
438 description: &D,
439 facts: BoundedDiagnostic,
440 classification: DiagnosticCode,
441 stage: RootRequestEvent,
442 axes: RootOutcomeFacts,
443 protocol: RemoteProtocolContext<'_>,
444 ) -> Result<super::WrittenRootFailure, RootRequestFailure> {
445 let receipt = self.original_record_mode(
446 RawSource::Description(description, description),
447 SourceDetail::Bounded(&facts), classification, stage, axes, true,
448 Some(protocol), None,
449 );
450 if receipt.original_capture == OriginalCaptureState::CompleteWritten {
451 Ok(super::WrittenRootFailure(receipt))
452 } else { Err(receipt) }
453 }
454
455 pub fn source_remote_error_written(
456 &self,
457 error: &(dyn Error + 'static),
458 facts: BoundedDiagnostic,
459 classification: DiagnosticCode,
460 stage: RootRequestEvent,
461 axes: RootOutcomeFacts,
462 target: RemoteTargetContext<'_>,
463 ) -> Result<super::WrittenRootFailure, RootRequestFailure> {
464 let receipt = self.original_record_mode(
465 RawSource::Error(error), SourceDetail::Bounded(&facts),
466 classification, stage, axes, true, None, Some(target),
467 );
468 if receipt.original_capture == OriginalCaptureState::CompleteWritten {
469 Ok(super::WrittenRootFailure(receipt))
470 } else { Err(receipt) }
471 }
472
473 pub fn source_existing_description<D: fmt::Display + fmt::Debug>(
476 &self,
477 description: &D,
478 diagnostic: Diagnostic,
479 classification: DiagnosticCode,
480 stage: RootRequestEvent,
481 axes: RootOutcomeFacts,
482 ) -> RootRequestFailure {
483 self.original_record(
484 RawSource::Description(description, description),
485 SourceDetail::Existing(&diagnostic),
486 classification,
487 stage,
488 axes,
489 )
490 }
491
492 pub fn source_existing_description_written<D: fmt::Display + fmt::Debug>(
493 &self,
494 description: &D,
495 diagnostic: Diagnostic,
496 classification: DiagnosticCode,
497 stage: RootRequestEvent,
498 axes: RootOutcomeFacts,
499 ) -> Result<super::WrittenRootFailure, RootRequestFailure> {
500 self.written_record(
501 RawSource::Description(description, description),
502 SourceDetail::Existing(&diagnostic),
503 classification,
504 stage,
505 axes,
506 )
507 }
508
509 pub fn source_existing_error(
510 &self,
511 error: &(dyn Error + 'static),
512 diagnostic: Diagnostic,
513 classification: DiagnosticCode,
514 stage: RootRequestEvent,
515 axes: RootOutcomeFacts,
516 ) -> RootRequestFailure {
517 self.original_record(
518 RawSource::Error(error),
519 SourceDetail::Existing(&diagnostic),
520 classification,
521 stage,
522 axes,
523 )
524 }
525
526 pub fn source_existing_error_written(
527 &self,
528 error: &(dyn Error + 'static),
529 diagnostic: Diagnostic,
530 classification: DiagnosticCode,
531 stage: RootRequestEvent,
532 axes: RootOutcomeFacts,
533 ) -> Result<super::WrittenRootFailure, RootRequestFailure> {
534 self.written_record(
535 RawSource::Error(error),
536 SourceDetail::Existing(&diagnostic),
537 classification,
538 stage,
539 axes,
540 )
541 }
542
543 fn written_record(
544 &self,
545 raw: RawSource<'_>,
546 facts: SourceDetail<'_>,
547 classification: DiagnosticCode,
548 stage: RootRequestEvent,
549 axes: RootOutcomeFacts,
550 ) -> Result<super::WrittenRootFailure, RootRequestFailure> {
551 let receipt = self.original_record_mode(raw, facts, classification, stage, axes, true, None, None);
552 if receipt.original_capture == OriginalCaptureState::CompleteWritten {
553 Ok(super::WrittenRootFailure(receipt))
554 } else {
555 Err(receipt)
556 }
557 }
558
559 fn original_record(
560 &self,
561 raw: RawSource<'_>,
562 facts: SourceDetail<'_>,
563 classification: DiagnosticCode,
564 stage: RootRequestEvent,
565 axes: RootOutcomeFacts,
566 ) -> RootRequestFailure {
567 self.original_record_mode(raw, facts, classification, stage, axes, false, None, None)
568 }
569 fn original_record_mode(
570 &self,
571 raw: RawSource<'_>,
572 facts: SourceDetail<'_>,
573 classification: DiagnosticCode,
574 stage: RootRequestEvent,
575 axes: RootOutcomeFacts,
576 confirmed: bool,
577 protocol: Option<RemoteProtocolContext<'_>>,
578 remote_target: Option<RemoteTargetContext<'_>>,
579 ) -> RootRequestFailure {
580 self.original_record_grpc(raw, facts, classification, stage, axes, confirmed, protocol, remote_target, None)
581 }
582 pub fn source_grpc_error_written<E: Error>(
583 &self, error: &E, facts: BoundedDiagnostic, classification: DiagnosticCode,
584 stage: RootRequestEvent, axes: RootOutcomeFacts, target: GrpcTargetContext<'_>,
585 ) -> Result<super::WrittenRootFailure, RootRequestFailure> {
586 let receipt=self.original_record_grpc(RawSource::BorrowedError(error),SourceDetail::Bounded(&facts),classification,stage,axes,true,None,None,Some(target));
587 if receipt.original_capture==OriginalCaptureState::CompleteWritten {Ok(super::WrittenRootFailure(receipt))}else{Err(receipt)}
588 }
589 fn original_record_grpc(
590 &self, raw:RawSource<'_>, facts:SourceDetail<'_>,classification:DiagnosticCode,
591 stage:RootRequestEvent,axes:RootOutcomeFacts,confirmed:bool,
592 protocol:Option<RemoteProtocolContext<'_>>,remote_target:Option<RemoteTargetContext<'_>>,
593 grpc:Option<GrpcTargetContext<'_>>,
594 ) -> RootRequestFailure {
595 let (occurrence, category) = match &facts {
596 SourceDetail::Bounded(d) => (d.occurrence(), d.category()),
597 SourceDetail::Existing(d) => (d.occurrence(), d.category()),
598 };
599 let source_interface = match raw {
600 RawSource::Error(_) | RawSource::BorrowedError(_) => "error_source",
601 RawSource::Description(_, _) => "unavailable_description_debug_only",
602 };
603 let (submission, original_capture) = match self.output {
604 None => (
605 DiagnosticSubmission::OutputUnavailable,
606 OriginalCaptureState::OutputUnavailable,
607 ),
608 Some(output) => {
609 let header = Header {
610 timestamp_unix_ms: timestamp(),
611 context: OriginalContext::Request(self.view),
612 stage,
613 facts,
614 outcome: axes,
615 original_contract: "display-debug-source/v1",
616 source_interface,
617 protocol,
618 remote_target,
619 grpc,
620 };
621 if confirmed {
622 let (submission, state) = capture(&ConfirmedOutput(output), occurrence, raw, &header);
623 if submission == DiagnosticSubmission::Enqueued && state == OriginalCaptureState::CompleteEnqueued {
624 (DiagnosticSubmission::Written, OriginalCaptureState::CompleteWritten)
625 } else {
626 (submission, state)
627 }
628 } else {
629 capture(output, occurrence, raw, &header)
630 }
631 },
632 };
633 RootRequestFailure {
634 source: self.view.clone(),
635 occurrence,
636 classification,
637 source_submission: submission,
638 category,
639 outcome: axes,
640 original_capture,
641 }
642 }
643}
644
645#[must_use = "capture facts describe submission, not durable output or resource completion"]
647pub struct UnrootedCaptureFacts {
648 occurrence: DiagnosticOccurrence,
649 submission: DiagnosticSubmission,
650 original_capture: OriginalCaptureState,
651}
652#[must_use = "carry the written source to its consumer"]
662pub struct WrittenUnrootedFailure(UnrootedCaptureFacts);
663impl WrittenUnrootedFailure {
664 fn from_facts(facts: UnrootedCaptureFacts) -> Result<Self, UnrootedCaptureFacts> {
665 if facts.original_capture == OriginalCaptureState::CompleteWritten {
666 Ok(Self(facts))
667 } else {
668 Err(facts)
669 }
670 }
671
672 pub fn occurrence(&self) -> DiagnosticOccurrence { self.0.occurrence }
673}
674impl UnrootedCaptureFacts {
675 pub fn occurrence(&self) -> DiagnosticOccurrence {
676 self.occurrence
677 }
678 pub fn submission(&self) -> DiagnosticSubmission {
679 self.submission
680 }
681 pub fn original_capture(&self) -> OriginalCaptureState {
682 self.original_capture
683 }
684}
685
686#[derive(Clone, Copy, Debug, Eq, PartialEq)]
688pub enum ComponentSourceKind { Start, Cleanup }
689
690pub struct ComponentSourceReceipt {
691 facts: UnrootedCaptureFacts,
692 kind: ComponentSourceKind,
693}
694impl ComponentSourceReceipt {
695 pub fn kind(&self) -> ComponentSourceKind { self.kind }
696 pub fn occurrence(&self) -> DiagnosticOccurrence { self.facts.occurrence() }
697 pub fn original_capture(&self) -> OriginalCaptureState { self.facts.original_capture() }
698}
699#[must_use = "carry the written component source to its consumer"]
708pub struct WrittenComponentFailure(ComponentSourceReceipt);
709impl WrittenComponentFailure {
710 fn from_receipt(receipt: ComponentSourceReceipt) -> Result<Self, ComponentSourceReceipt> {
711 if receipt.original_capture() == OriginalCaptureState::CompleteWritten {
712 Ok(Self(receipt))
713 } else {
714 Err(receipt)
715 }
716 }
717 pub fn occurrence(&self) -> DiagnosticOccurrence { self.0.occurrence() }
718 pub fn kind(&self) -> ComponentSourceKind { self.0.kind() }
719}
720
721pub struct UnrootedDiagnosticScope<'a> {
724 application: &'a saddle_core::ContextLabel,
725 lifecycle: saddle_core::RequestViewPhase,
726 output: Option<&'a EmergencyDiagnosticHandle>,
727}
728impl<'a> UnrootedDiagnosticScope<'a> {
729 pub fn new(
730 application: &'a saddle_core::ContextLabel,
731 lifecycle: saddle_core::RequestViewPhase,
732 output: Option<&'a EmergencyDiagnosticHandle>,
733 ) -> Self {
734 Self {
735 application,
736 lifecycle,
737 output,
738 }
739 }
740 #[track_caller]
741 pub fn source_error(
742 &self,
743 error: &(dyn Error + 'static),
744 source_stage: saddle_core::DiagnosticStage,
745 stage: RootRequestEvent,
746 axes: RootOutcomeFacts,
747 ) -> UnrootedCaptureFacts {
748 let facts = BoundedDiagnostic::capture(
749 DiagnosticCategory::UnexpectedError,
750 saddle_core::CaptureSite::FirstObserved,
751 saddle_core::BoundedDiagnosticCause::new(
752 source_stage,
753 DiagnosticCode::new("framework.error").unwrap(),
754 ),
755 );
756 self.original_record(RawSource::Error(error), &facts, stage, axes, false)
757 }
758 #[track_caller]
760 pub fn source_error_written(
761 &self,
762 error: &(dyn Error + 'static),
763 source_stage: saddle_core::DiagnosticStage,
764 stage: RootRequestEvent,
765 axes: RootOutcomeFacts,
766 ) -> UnrootedCaptureFacts {
767 let facts = BoundedDiagnostic::capture(
768 DiagnosticCategory::UnexpectedError,
769 saddle_core::CaptureSite::FirstObserved,
770 saddle_core::BoundedDiagnosticCause::new(
771 source_stage,
772 written_source_code(error, source_stage, stage),
773 ),
774 );
775 self.original_record(RawSource::Error(error), &facts, stage, axes, true)
776 }
777 #[track_caller]
780 pub fn source_error_recorded(
781 &self,
782 error: &(dyn Error + 'static),
783 source_stage: saddle_core::DiagnosticStage,
784 stage: RootRequestEvent,
785 axes: RootOutcomeFacts,
786 ) -> Result<WrittenUnrootedFailure, UnrootedCaptureFacts> {
787 WrittenUnrootedFailure::from_facts(
788 self.source_error_written(error, source_stage, stage, axes)
789 )
790 }
791 pub fn source_existing_error_recorded(
794 &self,
795 error: &(dyn Error + 'static),
796 diagnostic: &Diagnostic,
797 stage: RootRequestEvent,
798 axes: RootOutcomeFacts,
799 ) -> Result<WrittenUnrootedFailure, UnrootedCaptureFacts> {
800 WrittenUnrootedFailure::from_facts(self.original_record_detail(
801 RawSource::Error(error), SourceDetail::Existing(diagnostic), stage, axes, true,
802 ))
803 }
804 #[track_caller]
805 pub fn source_description<D: fmt::Display + fmt::Debug>(
806 &self,
807 description: &D,
808 source_stage: saddle_core::DiagnosticStage,
809 stage: RootRequestEvent,
810 axes: RootOutcomeFacts,
811 ) -> UnrootedCaptureFacts {
812 let facts = BoundedDiagnostic::capture(
813 DiagnosticCategory::UnexpectedError,
814 saddle_core::CaptureSite::FirstObserved,
815 saddle_core::BoundedDiagnosticCause::new(
816 source_stage,
817 DiagnosticCode::new("framework.error").unwrap(),
818 ),
819 );
820 self.original_record(
821 RawSource::Description(description, description),
822 &facts,
823 stage,
824 axes,
825 false,
826 )
827 }
828 #[track_caller]
830 pub fn source_description_written<D: fmt::Display + fmt::Debug>(
831 &self,
832 description: &D,
833 source_stage: saddle_core::DiagnosticStage,
834 stage: RootRequestEvent,
835 axes: RootOutcomeFacts,
836 ) -> UnrootedCaptureFacts {
837 let facts = BoundedDiagnostic::capture(
838 DiagnosticCategory::UnexpectedError,
839 saddle_core::CaptureSite::FirstObserved,
840 saddle_core::BoundedDiagnosticCause::new(
841 source_stage,
842 DiagnosticCode::new("framework.error").unwrap(),
843 ),
844 );
845 self.original_record(RawSource::Description(description, description), &facts, stage, axes, true)
846 }
847 fn original_record(
848 &self,
849 raw: RawSource<'_>,
850 facts: &BoundedDiagnostic,
851 stage: RootRequestEvent,
852 axes: RootOutcomeFacts,
853 confirmed: bool,
854 ) -> UnrootedCaptureFacts {
855 self.original_record_detail(raw, SourceDetail::Bounded(facts), stage, axes, confirmed)
856 }
857 fn original_record_detail(
858 &self,
859 raw: RawSource<'_>,
860 facts: SourceDetail<'_>,
861 stage: RootRequestEvent,
862 axes: RootOutcomeFacts,
863 confirmed: bool,
864 ) -> UnrootedCaptureFacts {
865 let occurrence = match facts {
866 SourceDetail::Bounded(diagnostic) => diagnostic.occurrence(),
867 SourceDetail::Existing(diagnostic) => diagnostic.occurrence(),
868 };
869 let source_interface = match raw {
870 RawSource::Error(_) | RawSource::BorrowedError(_) => "error_source",
871 RawSource::Description(_, _) => "unavailable_description_debug_only",
872 };
873 let (submission, original_capture) = match self.output {
874 None => (
875 DiagnosticSubmission::OutputUnavailable,
876 OriginalCaptureState::OutputUnavailable,
877 ),
878 Some(output) => {
879 let header = Header {
880 timestamp_unix_ms: timestamp(),
881 context: OriginalContext::Unrooted(crate::AdmissionEventContext::Unrooted {
882 application: self.application,
883 lifecycle: self.lifecycle,
884 }),
885 stage,
886 facts,
887 outcome: axes,
888 original_contract: "display-debug-source/v1",
889 source_interface,
890 protocol: None,
891 remote_target: None,
892 grpc: None,
893 };
894 if confirmed {
895 let (submission, state) = capture(&ConfirmedOutput(output), occurrence, raw, &header);
896 if submission == DiagnosticSubmission::Enqueued && state == OriginalCaptureState::CompleteEnqueued {
897 (DiagnosticSubmission::Written, OriginalCaptureState::CompleteWritten)
898 } else {
899 (submission, state)
900 }
901 } else {
902 capture(output, occurrence, raw, &header)
903 }
904 },
905 };
906 UnrootedCaptureFacts {
907 occurrence,
908 submission,
909 original_capture,
910 }
911 }
912}
913
914struct CycleDetector<'a> {
919 anchor: &'a (dyn Error + 'static),
920 power: usize,
921 distance: usize,
922}
923impl<'a> CycleDetector<'a> {
924 fn new(anchor: &'a (dyn Error + 'static)) -> Self {
925 Self {
926 anchor,
927 power: 1,
928 distance: 0,
929 }
930 }
931 fn repeats(&mut self, next: &'a (dyn Error + 'static)) -> bool {
932 if std::ptr::eq(self.anchor, next) {
936 return true;
937 }
938 self.distance += 1;
939 if self.distance == self.power {
940 self.anchor = next;
941 self.power = self.power.saturating_mul(2);
942 self.distance = 0;
943 }
944 false
945 }
946}
947
948pub fn write_json_error_debug<W: fmt::Write>(out: &mut W,
952 error: &serde_json::Error) -> fmt::Result {
953 struct FixedText<const N: usize> { bytes: [u8; N], len: usize }
954 impl<const N: usize> fmt::Write for FixedText<N> {
955 fn write_str(&mut self, value: &str) -> fmt::Result {
956 if self.len + value.len() > self.bytes.len() { return Err(fmt::Error); }
957 self.bytes[self.len..self.len + value.len()].copy_from_slice(value.as_bytes());
958 self.len += value.len();
959 Ok(())
960 }
961 }
962 let mut shown = FixedText { bytes: [0; 6144], len: 0 };
963 fmt::write(&mut shown, format_args!("{error}"))?;
964 let shown = std::str::from_utf8(&shown.bytes[..shown.len]).map_err(|_| fmt::Error)?;
965 let mut suffix = FixedText { bytes: [0; 64], len: 0 };
966 if error.line() != 0 {
967 fmt::write(&mut suffix,
968 format_args!(" at line {} column {}", error.line(), error.column()))?;
969 }
970 let suffix = std::str::from_utf8(&suffix.bytes[..suffix.len]).map_err(|_| fmt::Error)?;
971 let reason = shown.strip_suffix(suffix).ok_or(fmt::Error)?;
972 fmt::write(out, format_args!("Error({reason:?}, line: {}, column: {})",
973 error.line(), error.column()))
974}
975
976fn write_error_debug<O: OriginalOutput>(stream: &mut Stream<'_, O>,
977 error: &(dyn Error + 'static), bounded_input: bool) -> fmt::Result {
978 if bounded_input {
979 if let Some(json) = error.downcast_ref::<serde_json::Error>() {
980 return write_json_error_debug(stream, json);
981 }
982 if let Some(saddle_admission::InputDecodeFailure::Syntax(json)) =
983 error.downcast_ref::<saddle_admission::InputDecodeFailure>() {
984 stream.push("Syntax(")?;
985 write_json_error_debug(stream, json)?;
986 return stream.push(")");
987 }
988 }
989 fmt::write(stream, format_args!("{error:?}"))
990}
991
992fn capture<O: OriginalOutput>(
993 output: &O,
994 occurrence: DiagnosticOccurrence,
995 raw: RawSource<'_>,
996 header: &impl Serialize,
997) -> (DiagnosticSubmission, OriginalCaptureState) {
998 capture_with_mode(output, occurrence, raw, header, true)
1001}
1002
1003fn capture_with_mode<O: OriginalOutput>(
1004 output: &O,
1005 occurrence: DiagnosticOccurrence,
1006 raw: RawSource<'_>,
1007 header: &impl Serialize,
1008 bounded_input: bool,
1009) -> (DiagnosticSubmission, OriginalCaptureState) {
1010 let mut stream = Stream {
1011 output,
1012 occurrence: Some(occurrence),
1013 event: "request_error_original",
1014 sequence: 0,
1015 depth: 0,
1016 channel: "context",
1017 bytes: [0; PAYLOAD_BYTES],
1018 len: 0,
1019 escaped: 0,
1020 failure: None,
1021 submission: DiagnosticSubmission::Enqueued,
1022 };
1023 let result = (|| -> fmt::Result {
1024 stream.json(header)?;
1025 let error = match raw {
1026 RawSource::Error(error) => error,
1027 RawSource::BorrowedError(error) => {
1028 stream.channel = "description";
1029 if fmt::write(&mut stream, format_args!("{error}")).is_err() {
1030 let state = stream.failure.unwrap_or(OriginalCaptureState::FormattingFailed);
1031 stream.report_failure(state);
1032 return Err(fmt::Error);
1033 }
1034 stream.emit("field_end")?;
1035 stream.channel = "debug";
1036 if fmt::write(&mut stream, format_args!("{error:?}")).is_err() {
1037 let state = stream.failure.unwrap_or(OriginalCaptureState::FormattingFailed);
1038 stream.report_failure(state);
1039 return Err(fmt::Error);
1040 }
1041 stream.emit("field_end")?;
1042 stream.depth = 1;
1043 let Some(first) = error.source() else {
1044 stream.channel = "terminal";
1045 return stream.emit("exposed_chain_complete");
1046 };
1047 let mut cycle = CycleDetector::new(first);
1048 let mut next = Some(first);
1049 while let Some(current) = next {
1050 stream.channel = "description";
1051 if fmt::write(&mut stream, format_args!("{current}")).is_err() {
1052 let state = stream.failure.unwrap_or(OriginalCaptureState::FormattingFailed);
1053 stream.report_failure(state);
1054 return Err(fmt::Error);
1055 }
1056 stream.emit("field_end")?;
1057 stream.channel = "debug";
1058 if write_error_debug(&mut stream, current, bounded_input).is_err() {
1059 let state = stream.failure.unwrap_or(OriginalCaptureState::FormattingFailed);
1060 stream.report_failure(state);
1061 return Err(fmt::Error);
1062 }
1063 stream.emit("field_end")?;
1064 if let Some(io) = current.downcast_ref::<io::Error>() {
1065 stream.channel = "io_facts";
1066 stream.json(&IoFacts { kind: Kind(io.kind()), os_code: io.raw_os_error() })?;
1067 }
1068 next = current.source();
1069 stream.depth += 1;
1070 if next.is_some_and(|next| cycle.repeats(next)) {
1071 stream.channel = "description";
1072 stream.report_failure(OriginalCaptureState::CauseCycle);
1073 return Err(fmt::Error);
1074 }
1075 }
1076 stream.channel = "terminal";
1077 return stream.emit("exposed_chain_complete");
1078 }
1079 RawSource::Description(description, debug) => {
1080 stream.channel = "description";
1081 if fmt::write(&mut stream, format_args!("{description}")).is_err() {
1082 let state = stream
1083 .failure
1084 .unwrap_or(OriginalCaptureState::FormattingFailed);
1085 stream.report_failure(state);
1086 return Err(fmt::Error);
1087 }
1088 stream.emit("field_end")?;
1089 stream.channel = "debug";
1090 if fmt::write(&mut stream, format_args!("{debug:?}")).is_err() {
1091 let state = stream
1092 .failure
1093 .unwrap_or(OriginalCaptureState::FormattingFailed);
1094 stream.report_failure(state);
1095 return Err(fmt::Error);
1096 }
1097 stream.emit("field_end")?;
1098 stream.depth = 1;
1099 stream.channel = "terminal";
1100 return stream.emit("description_complete_source_unavailable");
1101 }
1102 };
1103 let mut cycle = CycleDetector::new(error);
1104 let mut next = Some(error);
1105 while let Some(current) = next {
1106 stream.channel = "description";
1107 if fmt::write(&mut stream, format_args!("{current}")).is_err() {
1108 let failure = stream
1109 .failure
1110 .unwrap_or(OriginalCaptureState::FormattingFailed);
1111 stream.report_failure(failure);
1112 return Err(fmt::Error);
1113 }
1114 stream.emit("field_end")?;
1115 stream.channel = "debug";
1116 if write_error_debug(&mut stream, current, bounded_input).is_err() {
1117 let state = stream
1118 .failure
1119 .unwrap_or(OriginalCaptureState::FormattingFailed);
1120 stream.report_failure(state);
1121 return Err(fmt::Error);
1122 }
1123 stream.emit("field_end")?;
1124 if let Some(io) = current.downcast_ref::<io::Error>() {
1125 stream.channel = "io_facts";
1126 stream.json(&IoFacts {
1127 kind: Kind(io.kind()),
1128 os_code: io.raw_os_error(),
1129 })?;
1130 }
1131 next = current.source();
1132 stream.depth += 1;
1133 if next.is_some_and(|next| cycle.repeats(next)) {
1134 stream.channel = "description";
1135 stream.report_failure(OriginalCaptureState::CauseCycle);
1136 return Err(fmt::Error);
1137 }
1138 }
1139 stream.channel = "terminal";
1140 stream.emit("exposed_chain_complete")
1141 })();
1142 if result.is_ok() {
1143 (stream.submission, OriginalCaptureState::CompleteEnqueued)
1144 } else {
1145 let state = stream
1146 .failure
1147 .unwrap_or(OriginalCaptureState::FormattingFailed);
1148 if stream.submission == DiagnosticSubmission::Enqueued {
1149 (DiagnosticSubmission::EncodingFailed, state)
1150 } else {
1151 (stream.submission, state)
1152 }
1153 }
1154}
1155
1156pub fn original_capture_layout() -> (std::alloc::Layout, std::alloc::Layout, usize) {
1158 (
1159 std::alloc::Layout::new::<Stream<'static, EmergencyDiagnosticHandle>>(),
1160 std::alloc::Layout::new::<CycleDetector<'static>>(),
1161 PAYLOAD_BYTES,
1162 )
1163}
1164
1165#[cfg(test)]
1166mod content_tests {
1167 use super::*;
1168 use std::cell::RefCell;
1169
1170 #[test]
1171 fn typed_written_input_code_is_borrowed_bounded_and_stage_specific() {
1172 use saddle_admission::InputDecodeFailure as F;
1173 use saddle_core::DiagnosticStage as D;
1174 let code = |e: &(dyn Error + 'static), source, stage| serde_json::to_value(written_source_code(e, source, stage)).unwrap();
1175 let syntax = F::Syntax(serde_json::from_str::<serde_json::Value>("{").unwrap_err());
1176 assert_eq!(code(&syntax,D::RequestDecode,RootRequestEvent::Ingress),"framework.input_decode.syntax");
1177 let bytes = vec![255]; let utf8 = F::Utf8(std::str::from_utf8(&bytes).unwrap_err());
1178 assert_eq!(code(&utf8,D::RequestDecode,RootRequestEvent::Ingress),"framework.input_decode.utf8");
1179 assert_eq!(code(&F::Parser,D::RequestDecode,RootRequestEvent::Ingress),"framework.input_decode.parser");
1180 let mut wrapped = F::Parser;
1181 for _ in 0..8 { wrapped = F::Context { request_id: "r".into(), failure: Box::new(wrapped) }; }
1182 assert_eq!(code(&wrapped,D::RequestDecode,RootRequestEvent::Ingress),"framework.input_decode.parser");
1183 wrapped = F::Context { request_id: "r".into(), failure: Box::new(wrapped) };
1184 assert_eq!(code(&wrapped,D::RequestDecode,RootRequestEvent::Ingress),"framework.error");
1185 assert_eq!(code(&F::Resource(saddle_admission::AdmissionError::ForeignManagedSource),D::RequestDecode,RootRequestEvent::Ingress),"framework.error");
1186 assert_eq!(code(&io::Error::other("Parser"),D::RequestDecode,RootRequestEvent::Ingress),"framework.error");
1187 assert_eq!(code(&F::Parser,D::RequestDecode,RootRequestEvent::Handler),"framework.error");
1188 }
1189
1190 #[test]
1191 fn typed_written_source_preserves_chain_and_missing_writer_is_unconfirmed() {
1192 use saddle_core::{ContextLabel, DiagnosticStage, RequestViewPhase};
1193 let directory = std::env::temp_dir().join(format!("typed-written-{}-{}",std::process::id(),
1194 std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()));
1195 std::fs::create_dir(&directory).unwrap();
1196 let mut output = crate::EmergencyDiagnostics::start(&crate::FileLoggingConfig::new(directory.clone(),crate::Rotation::Daily)).unwrap();
1197 let handle = output.handle(); let label = ContextLabel::checked("typed-written").unwrap();
1198 let error = saddle_admission::InputDecodeFailure::Context { request_id:"original-request".into(),
1199 failure:Box::new(saddle_admission::InputDecodeFailure::Syntax(serde_json::from_str::<serde_json::Value>("{").unwrap_err())) };
1200 let description=error.to_string(); let debug=format!("{error:?}");
1201 let scope=UnrootedDiagnosticScope::new(&label,RequestViewPhase::Reading,Some(&handle));
1202 assert!(scope.source_error_recorded(&error,DiagnosticStage::RequestDecode,RootRequestEvent::Ingress,RootOutcomeFacts::default()).is_ok());
1203 let body=std::fs::read_to_string(output.target()).unwrap();
1204 assert!(body.contains("framework.input_decode.syntax"));
1205 assert_eq!(error.to_string(),description);assert_eq!(format!("{error:?}"),debug);
1206 assert!(body.contains("original-request"));assert_eq!(serde_json::from_str::<serde_json::Value>(body.lines().last().unwrap()).unwrap()["channel"],"terminal");
1207 let missing=UnrootedDiagnosticScope::new(&label,RequestViewPhase::Reading,None);
1208 assert!(missing.source_error_recorded(&error,DiagnosticStage::RequestDecode,RootRequestEvent::Ingress,RootOutcomeFacts::default()).is_err());
1209 let deadline=std::time::Instant::now()+std::time::Duration::from_secs(5);
1210 while output.shutdown()==crate::DiagnosticShutdown::Pending && std::time::Instant::now()<deadline { std::thread::yield_now(); }
1211 assert_eq!(output.shutdown(),crate::DiagnosticShutdown::Finished);
1212 drop(output);std::fs::remove_dir_all(directory).unwrap();
1213 }
1214
1215 #[test]
1216 fn written_source_rejects_formatting_truncation_and_actual_file_failure() {
1217 #[derive(Debug)] struct Partial;
1218 impl fmt::Display for Partial { fn fmt(&self,f:&mut fmt::Formatter<'_>)->fmt::Result { f.write_str("partial-source")?; Err(fmt::Error) } }
1219 impl Error for Partial {}
1220 let directory=std::env::temp_dir().join(format!("typed-failed-{}-{}",std::process::id(),std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()));
1221 std::fs::create_dir(&directory).unwrap();
1222 let mut output=crate::EmergencyDiagnostics::start(&crate::FileLoggingConfig::new(directory.clone(),crate::Rotation::Daily)).unwrap();
1223 let handle=output.handle();let label=saddle_core::ContextLabel::checked("typed-failed").unwrap();
1224 let scope=UnrootedDiagnosticScope::new(&label,saddle_core::RequestViewPhase::Reading,Some(&handle));
1225 assert!(scope.source_error_recorded(&Partial,saddle_core::DiagnosticStage::RequestDecode,RootRequestEvent::Ingress,RootOutcomeFacts::default()).is_err());
1226 let deadline=std::time::Instant::now()+std::time::Duration::from_secs(5);
1227 while output.shutdown()==crate::DiagnosticShutdown::Pending && std::time::Instant::now()<deadline {std::thread::yield_now();}
1228 assert_eq!(output.shutdown(),crate::DiagnosticShutdown::Finished);
1229 assert!(scope.source_error_recorded(&saddle_admission::InputDecodeFailure::Parser,saddle_core::DiagnosticStage::RequestDecode,RootRequestEvent::Ingress,RootOutcomeFacts::default()).is_ok());
1231 drop(output);std::fs::remove_dir_all(directory).unwrap();
1232 #[cfg(unix)] {
1233 let child=std::process::Command::new("sh")
1234 .args(["-c", "trap '' XFSZ; ulimit -f 1; exec \"$@\"", "source-file-limit"])
1235 .arg(std::env::current_exe().unwrap())
1236 .args(["root_diagnostic::original::content_tests::written_source_file_limit_child", "--exact", "--nocapture"])
1237 .env("SADDLE_SOURCE_LIMIT_CONTROL", "1").output().unwrap();
1238 assert!(child.status.success(), "{}", String::from_utf8_lossy(&child.stdout));
1239 assert!(String::from_utf8_lossy(&child.stdout).contains("actual file write failed: unconfirmed"));
1240 }
1241 }
1242
1243 #[cfg(unix)]
1244 #[test]
1245 fn written_source_file_limit_child() {
1246 if std::env::var("SADDLE_SOURCE_LIMIT_CONTROL").as_deref()!=Ok("1") { return; }
1247 let directory=std::env::temp_dir().join(format!("source-limit-{}",std::process::id()));
1248 std::fs::create_dir(&directory).unwrap();
1249 let mut output=crate::EmergencyDiagnostics::start(&crate::FileLoggingConfig::new(directory.clone(),crate::Rotation::Daily)).unwrap();
1250 let handle=output.handle();let label=saddle_core::ContextLabel::checked("source-limit").unwrap();
1251 let scope=UnrootedDiagnosticScope::new(&label,saddle_core::RequestViewPhase::Reading,Some(&handle));
1252 let error=io::Error::other("x".repeat(16384));
1253 assert!(scope.source_error_recorded(&error,saddle_core::DiagnosticStage::RequestDecode,RootRequestEvent::Ingress,RootOutcomeFacts::default()).is_err());
1254 println!("actual file write failed: unconfirmed; original={}",output.target().display());
1255 let deadline=std::time::Instant::now()+std::time::Duration::from_secs(5);
1256 while output.shutdown()==crate::DiagnosticShutdown::Pending && std::time::Instant::now()<deadline {std::thread::yield_now();}
1257 }
1258
1259 #[test]
1260 fn json_parser_debug_stream_matches_native_bytes() {
1261 for input in ["{broken", "[", "{\"a\":1,}", "\"\\uD800\""] {
1262 let error = serde_json::from_str::<serde_json::Value>(input).unwrap_err();
1263 let mut streamed = String::new();
1264 write_json_error_debug(&mut streamed, &error).unwrap();
1265 assert_eq!(streamed, format!("{error:?}"));
1266 }
1267 }
1268
1269 #[test]
1270 fn owned_source_is_written_before_projection_and_mismatch_keeps_original() {
1271 use saddle_core::{CaptureSite, DiagnosticCause, DiagnosticStage, ErrorKind,
1272 RequestViewPhase, SaddleError};
1273 let directory = std::env::temp_dir().join(format!(
1274 "saddle-owned-source-{}-{}", std::process::id(),
1275 std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)
1276 .unwrap().as_nanos()));
1277 std::fs::create_dir(&directory).unwrap();
1278 let mut output = crate::EmergencyDiagnostics::start_checked(
1279 &crate::FileLoggingConfig::new(directory.clone(), crate::Rotation::Daily)).unwrap();
1280 let selected = output.source_output().expect("checked selected output");
1281 let label = saddle_core::ContextLabel::checked("owned-test").unwrap();
1282 let diagnostic = Diagnostic::capture(DiagnosticCategory::UnexpectedError,
1283 CaptureSite::FirstObserved,
1284 DiagnosticCause::new(DiagnosticStage::StartupService,
1285 DiagnosticCode::new("owned.test").unwrap()));
1286 let target = output.target().to_owned();
1287 let written = super::RecordedSaddleError::capture_unrooted(
1288 io::Error::other("original before projection 9271"), diagnostic,
1289 &label, RequestViewPhase::Admitted, &selected,
1290 RootRequestEvent::Admission, RootOutcomeFacts::default(),
1291 |_, diagnostic| {
1292 assert!(std::fs::read_to_string(&target).unwrap()
1293 .contains("original before projection 9271"));
1294 SaddleError::new(ErrorKind::Infrastructure, "owned.test", "safe")
1295 .with_diagnostic(diagnostic)
1296 }).unwrap_or_else(|_| panic!("written and paired"));
1297 assert_eq!(written.safe().code(), "owned.test");
1298 let component = Diagnostic::capture(DiagnosticCategory::UnexpectedError,
1299 CaptureSite::FirstObserved,
1300 DiagnosticCause::new(DiagnosticStage::ShutdownComponent,
1301 DiagnosticCode::new("owned.cleanup").unwrap()));
1302 let mismatch = super::RecordedSaddleError::capture_component(
1303 io::Error::other("owned cleanup original 9272"), component,
1304 saddle_core::ContextFact::Present(label.clone()), &selected,
1305 ComponentSourceKind::Cleanup,
1306 |_, _| SaddleError::new(ErrorKind::Infrastructure, "owned.cleanup", "safe")
1307 .with_diagnostic(Diagnostic::capture(DiagnosticCategory::UnexpectedError,
1308 CaptureSite::FirstObserved,
1309 DiagnosticCause::new(DiagnosticStage::ShutdownComponent,
1310 DiagnosticCode::new("owned.cleanup").unwrap()))),
1311 ).err().expect("wrong occurrence rejected");
1312 assert!(mismatch.safe().source_unavailable());
1313 assert_eq!(mismatch.source().to_string(), "owned cleanup original 9272");
1314 assert!(std::fs::read_to_string(&target).unwrap()
1315 .contains("owned cleanup original 9272"));
1316 drop(written);
1317 drop(selected);
1318 let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
1319 while output.shutdown() == crate::DiagnosticShutdown::Pending
1320 && std::time::Instant::now() < deadline { std::thread::yield_now(); }
1321 assert_eq!(output.shutdown(), crate::DiagnosticShutdown::Finished);
1322 drop(output);
1323 std::fs::remove_dir_all(directory).unwrap();
1324 }
1325
1326 #[test]
1327 fn sealed_unrooted_failure_requires_written_matching_original() {
1328 use saddle_core::{CaptureSite, DiagnosticCause, DiagnosticStage, ErrorKind,
1329 RequestViewPhase, SaddleError};
1330 let directory = std::env::temp_dir().join(format!(
1331 "saddle-sealed-source-{}-{}", std::process::id(),
1332 std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)
1333 .unwrap().as_nanos()));
1334 std::fs::create_dir(&directory).unwrap();
1335 let mut output = crate::EmergencyDiagnostics::start(
1336 &crate::FileLoggingConfig::new(directory.clone(), crate::Rotation::Daily)).unwrap();
1337 let handle = output.handle();
1338 let label = saddle_core::ContextLabel::checked("sealed-test").unwrap();
1339 let diagnostic = Diagnostic::capture(DiagnosticCategory::UnexpectedError,
1340 CaptureSite::FirstObserved,
1341 DiagnosticCause::new(DiagnosticStage::StartupService,
1342 DiagnosticCode::new("sealed.test").unwrap()));
1343 let original = io::Error::other("private original 1849");
1344 let scope = UnrootedDiagnosticScope::new(&label, RequestViewPhase::Admitted,
1345 Some(&handle));
1346 let written = scope.source_existing_error_recorded(&original, &diagnostic,
1347 RootRequestEvent::Admission, RootOutcomeFacts::default()).ok().expect("written");
1348 let safe = SaddleError::new(ErrorKind::Infrastructure, "sealed.test", "safe")
1349 .with_diagnostic(diagnostic);
1350 let recorded = super::RecordedSaddleError::from_written(safe, written).unwrap();
1351 assert!(!recorded.to_string().contains("private original 1849"));
1352 assert!(std::fs::read_to_string(output.target()).unwrap()
1353 .contains("private original 1849"));
1354
1355 let other = Diagnostic::capture(DiagnosticCategory::UnexpectedError,
1356 CaptureSite::FirstObserved,
1357 DiagnosticCause::new(DiagnosticStage::StartupService,
1358 DiagnosticCode::new("sealed.test").unwrap()));
1359 let mismatch = scope.source_existing_error_recorded(&original, &other,
1360 RootRequestEvent::Admission, RootOutcomeFacts::default()).ok().expect("written");
1361 let wrong = SaddleError::new(ErrorKind::Infrastructure, "sealed.test", "safe")
1362 .with_diagnostic(Diagnostic::capture(DiagnosticCategory::UnexpectedError,
1363 CaptureSite::FirstObserved,
1364 DiagnosticCause::new(DiagnosticStage::StartupService,
1365 DiagnosticCode::new("sealed.test").unwrap())));
1366 assert!(super::RecordedSaddleError::from_written(wrong, mismatch)
1367 .unwrap_err().source_unavailable());
1368 let missing = UnrootedDiagnosticScope::new(&label, RequestViewPhase::Admitted, None);
1369 let no_output = Diagnostic::capture(DiagnosticCategory::UnexpectedError,
1370 CaptureSite::FirstObserved,
1371 DiagnosticCause::new(DiagnosticStage::StartupService,
1372 DiagnosticCode::new("sealed.test").unwrap()));
1373 assert!(missing.source_existing_error_recorded(&original, &no_output,
1374 RootRequestEvent::Admission, RootOutcomeFacts::default()).is_err());
1375 let component = Diagnostic::capture(DiagnosticCategory::UnexpectedError,
1376 CaptureSite::FirstObserved,
1377 DiagnosticCause::new(DiagnosticStage::StartupService,
1378 DiagnosticCode::new("sealed.component").unwrap()));
1379 let component_written = process_component_start_recorded(Some(&handle),
1380 saddle_core::ContextFact::Unavailable, &component, &original)
1381 .ok().expect("component written");
1382 let component_safe = SaddleError::new(ErrorKind::Infrastructure,
1383 "sealed.component", "safe").with_diagnostic(component);
1384 let component_recorded = super::RecordedSaddleError::from_component_written(
1385 component_safe, component_written).unwrap();
1386 assert_eq!(component_recorded.component_kind(), Some(ComponentSourceKind::Start));
1387 drop(scope);
1388 drop(recorded);
1389 drop(component_recorded);
1390 drop(handle);
1391 let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
1392 while output.shutdown() == crate::DiagnosticShutdown::Pending
1393 && std::time::Instant::now() < deadline { std::thread::yield_now(); }
1394 assert_eq!(output.shutdown(), crate::DiagnosticShutdown::Finished);
1395 drop(output);
1396 std::fs::remove_dir_all(directory).unwrap();
1397 }
1398
1399 struct Node {
1400 text: String,
1401 next: Option<Box<Node>>,
1402 }
1403 impl fmt::Display for Node {
1404 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1405 f.write_str(&self.text)
1406 }
1407 }
1408 impl fmt::Debug for Node {
1409 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1410 f.write_str(&self.text)
1411 }
1412 }
1413 impl Error for Node {
1414 fn source(&self) -> Option<&(dyn Error + 'static)> {
1415 self.next.as_deref().map(|v| v as &dyn Error)
1416 }
1417 }
1418 struct Readback<'a> {
1419 expected: &'a [String],
1420 unrooted: bool,
1421 state: RefCell<(u64, usize, usize, bool)>,
1423 }
1424 impl OriginalOutput for Readback<'_> {
1425 fn submit(&self, segment: &Segment<'_>) -> DiagnosticSubmission {
1426 let encoded = serde_json::to_vec(segment).unwrap();
1429 assert!(encoded.len() < 8192);
1430 let row: serde_json::Value = serde_json::from_slice(&encoded).unwrap();
1431 let mut s = self.state.borrow_mut();
1432 assert_eq!(row["sequence"].as_u64().unwrap(), s.0);
1433 s.0 += 1;
1434 if segment.channel == "context" && segment.state == "field_end" {
1435 let header: serde_json::Value = serde_json::from_str(segment.payload).unwrap();
1436 if self.unrooted { assert_eq!(header["context"]["schema_version"], 2); }
1437 else {
1438 assert_eq!(header["context"]["schemaVersion"], "1.0");
1439 assert_eq!(header["context"]["_meta"]["fieldStates"]["/requestInfo/requestId"], "not_established");
1440 }
1441 }
1442 if matches!(segment.channel, "description" | "debug") {
1443 let expected = &self.expected[segment.cause_depth];
1444 let payload = row["payload"].as_str().unwrap();
1445 assert_eq!(payload, &expected[s.1..s.1 + payload.len()]);
1446 s.1 += payload.len();
1447 if segment.state == "field_end" {
1448 assert_eq!(s.1, expected.len(), "original tail missing");
1449 s.1 = 0;
1450 s.2 += 1;
1451 }
1452 }
1453 if segment.channel == "terminal" {
1454 assert_eq!(segment.state, "exposed_chain_complete");
1455 assert_eq!(segment.cause_depth, self.expected.len());
1456 assert_eq!(s.2, self.expected.len() * 2);
1457 s.3 = true;
1458 }
1459 DiagnosticSubmission::Enqueued
1460 }
1461 }
1462 fn assert_content(expected: Vec<String>) -> u64 {
1463 assert_content_context(expected, false)
1464 }
1465 fn assert_content_context(expected: Vec<String>, unrooted: bool) -> u64 {
1466 let mut head = None;
1467 for text in expected.iter().rev() {
1468 head = Some(Box::new(Node {
1469 text: text.clone(),
1470 next: head,
1471 }));
1472 }
1473 let publisher = saddle_core::RequestRootPublisher::create(
1474 saddle_core::ContextLabel::checked("content-test").unwrap(),
1475 saddle_core::ContextFact::NotEstablished,
1476 )
1477 .unwrap();
1478 let view = publisher
1479 .reference()
1480 .view(saddle_core::RequestLocalFacts::new(
1481 saddle_core::RequestViewPhase::Reading,
1482 ));
1483 let diagnostic = BoundedDiagnostic::capture(
1484 DiagnosticCategory::UnexpectedError,
1485 saddle_core::CaptureSite::FirstObserved,
1486 saddle_core::BoundedDiagnosticCause::new(
1487 saddle_core::DiagnosticStage::RequestDecode,
1488 DiagnosticCode::new("content.original").unwrap(),
1489 ),
1490 );
1491 let output = Readback {
1492 expected: &expected,
1493 unrooted,
1494 state: RefCell::new((0, 0, 0, false)),
1495 };
1496 let application = saddle_core::ContextLabel::checked("content-test").unwrap();
1497 let result = capture(
1498 &output,
1499 diagnostic.occurrence(),
1500 RawSource::Error(head.as_deref().unwrap()),
1501 &Header {
1502 timestamp_unix_ms: 0,
1503 context: if unrooted {
1504 OriginalContext::Unrooted(crate::AdmissionEventContext::Unrooted {
1505 application: &application,
1506 lifecycle: saddle_core::RequestViewPhase::Reading,
1507 })
1508 } else {
1509 OriginalContext::Request(&view)
1510 },
1511 stage: RootRequestEvent::Ingress,
1512 facts: SourceDetail::Bounded(&diagnostic),
1513 outcome: RootOutcomeFacts::default(),
1514 original_contract: "display-debug-source/v1",
1515 source_interface: "error_source",
1516 protocol: None,
1517 remote_target: None,
1518 grpc: None,
1519 },
1520 );
1521 assert_eq!(
1522 result,
1523 (
1524 DiagnosticSubmission::Enqueued,
1525 OriginalCaptureState::CompleteEnqueued
1526 )
1527 );
1528 assert!(output.state.borrow().3);
1529 output.state.into_inner().0
1530 }
1531 #[test]
1532 fn rooted_and_unrooted_share_lossless_long_content() {
1533 let expected = vec![
1534 format!("HEAD-䏿–‡\n\"{}\"-TAIL", "原文".repeat(2200)),
1535 "nested-cause\n完整".into(),
1536 ];
1537 let rooted = assert_content_context(expected.clone(), false);
1538 let unrooted = assert_content_context(expected, true);
1539 assert_eq!(rooted, unrooted);
1540 }
1541 #[test]
1542 fn finite_chain_beyond_old_256_keeps_every_cause() {
1543 let segments = assert_content(
1544 (0..300)
1545 .map(|n| format!("unknown-{n}-原文\n\"tail\""))
1546 .collect(),
1547 );
1548 assert!(segments > 600);
1549 println!("content readback: 300 original causes and Debug fields complete");
1550 }
1551 #[test]
1552 fn cycle_detector_handles_prefix_and_nontrivial_cycle() {
1553 let nodes: Vec<Node> = (0..4)
1554 .map(|n| Node {
1555 text: n.to_string(),
1556 next: None,
1557 })
1558 .collect();
1559 let mut cycle = CycleDetector::new(&nodes[0]);
1560 for node in [&nodes[1], &nodes[2], &nodes[3]] {
1562 assert!(!cycle.repeats(node));
1563 }
1564 let found = (0..32).any(|step| cycle.repeats(&nodes[1 + step % 3]));
1565 assert!(found);
1566 }
1567 #[test]
1568 fn finite_text_beyond_old_4096_segments_keeps_tail() {
1569 let text = format!("{}END-原文", "x".repeat(6144 * 4097));
1570 let segments = assert_content(vec![text]);
1571 assert!(segments > 8194);
1572 println!("content readback: {segments} segments, complete Display/Debug tails");
1573 }
1574}
1575
1576#[doc(hidden)]
1580pub fn database_background_error(
1581 output: Option<&EmergencyDiagnosticHandle>,
1582 datasource: u64,
1583 connection: Option<u64>,
1584 sweep: u64,
1585 phase: &'static str,
1586 maintenance: bool,
1587 error: &(dyn Error + 'static),
1588 facts: BoundedDiagnostic,
1589) -> UnrootedCaptureFacts {
1590 #[derive(Serialize)]
1591 struct Context {
1592 schema_version: u8,
1593 datasource: u64,
1594 connection: ContextFact<u64>,
1595 maintenance_sweep: u64,
1596 maintenance_phase: &'static str,
1597 request: ContextFact<()>,
1598 trace_id: ContextFact<()>,
1599 application: ContextFact<()>,
1600 }
1601 #[derive(Serialize)]
1602 struct BackgroundHeader<'a> {
1603 timestamp_unix_ms: u128,
1604 context: Context,
1605 stage: &'static str,
1606 facts: &'a BoundedDiagnostic,
1607 original_contract: &'static str,
1608 source_interface: &'static str,
1609 }
1610 let occurrence = facts.occurrence();
1611 let missing = if maintenance { ContextFact::NotApplicable } else { ContextFact::Unavailable };
1612 let header = BackgroundHeader {
1613 timestamp_unix_ms: timestamp(),
1614 context: Context { schema_version: 2, datasource,
1615 connection: connection.map(ContextFact::Present).unwrap_or(ContextFact::NotEstablished),
1616 maintenance_sweep: sweep, maintenance_phase: phase,
1617 request: missing, trace_id: missing, application: ContextFact::Unavailable },
1618 stage: if maintenance { "database_maintenance" } else { "database_connection" }, facts: &facts,
1619 original_contract: "display-debug-source/v1", source_interface: "error_source",
1620 };
1621 let (submission, original_capture) = match output {
1622 Some(output) => {
1623 let (submission, state) = capture(&ConfirmedOutput(output), occurrence, RawSource::Error(error), &header);
1624 if submission == DiagnosticSubmission::Enqueued && state == OriginalCaptureState::CompleteEnqueued {
1625 (DiagnosticSubmission::Written, OriginalCaptureState::CompleteWritten)
1626 } else {
1627 (submission, state)
1628 }
1629 },
1630 None => (DiagnosticSubmission::OutputUnavailable, OriginalCaptureState::OutputUnavailable),
1631 };
1632 UnrootedCaptureFacts { occurrence, submission, original_capture }
1633}
1634
1635#[doc(hidden)]
1639pub fn database_transaction_error(
1640 output: Option<&EmergencyDiagnosticHandle>,
1641 call: &saddle_core::CallContext,
1642 phase: &'static str,
1643 error: &(dyn Error + 'static),
1644 facts: &Diagnostic,
1645) -> UnrootedCaptureFacts {
1646 #[derive(Serialize)]
1647 struct Context<'a> {
1648 schema_version: u8,
1649 application: &'a str,
1650 module: &'a str,
1651 service: &'a str,
1652 operation: &'a str,
1653 trace_id: String,
1654 span_id: String,
1655 request: ContextFact<()>,
1656 transaction_phase: &'static str,
1657 }
1658 #[derive(Serialize)]
1659 struct Header<'a> {
1660 timestamp_unix_ms: u128,
1661 context: Context<'a>,
1662 stage: &'static str,
1663 facts: &'a Diagnostic,
1664 original_contract: &'static str,
1665 source_interface: &'static str,
1666 }
1667 let occurrence = facts.occurrence();
1668 let header = Header {
1669 timestamp_unix_ms: timestamp(),
1670 context: Context {
1671 schema_version: 2,
1672 application: call.application().as_str(),
1673 module: call.module().as_str(),
1674 service: call.service().as_str(),
1675 operation: call.operation().as_str(),
1676 trace_id: call.trace_id().to_string(),
1677 span_id: call.span_id().to_string(),
1678 request: ContextFact::Unavailable,
1679 transaction_phase: phase,
1680 },
1681 stage: if phase == "begin" || phase == "commit" { "database_transaction" }
1682 else { "database_transaction_cleanup" },
1683 facts,
1684 original_contract: "display-debug-source/v1",
1685 source_interface: "error_source",
1686 };
1687 let (submission, original_capture) = match output {
1688 Some(output) => {
1689 let (submission, state) = capture(
1690 &ConfirmedOutput(output), occurrence, RawSource::Error(error), &header);
1691 if submission == DiagnosticSubmission::Enqueued
1692 && state == OriginalCaptureState::CompleteEnqueued {
1693 (DiagnosticSubmission::Written, OriginalCaptureState::CompleteWritten)
1694 } else {
1695 (submission, state)
1696 }
1697 }
1698 None => (DiagnosticSubmission::OutputUnavailable, OriginalCaptureState::OutputUnavailable),
1699 };
1700 UnrootedCaptureFacts { occurrence, submission, original_capture }
1701}
1702
1703#[doc(hidden)]
1707pub fn database_operation_error(
1708 output: Option<&EmergencyDiagnosticHandle>,
1709 call: &saddle_core::CallContext,
1710 phase: &'static str,
1711 error: &(dyn Error + 'static),
1712 diagnostic: &Diagnostic,
1713) -> UnrootedCaptureFacts {
1714 database_operation_source(output, call, phase, RawSource::Error(error), diagnostic, diagnostic.occurrence(), false)
1715}
1716
1717#[doc(hidden)]
1720pub fn database_operation_borrowed_error<D: Error>(
1721 output: Option<&EmergencyDiagnosticHandle>,
1722 call: &saddle_core::CallContext,
1723 phase: &'static str,
1724 source: &D,
1725 diagnostic: &Diagnostic,
1726) -> UnrootedCaptureFacts {
1727 database_operation_source(output, call, phase,
1728 RawSource::BorrowedError(source), diagnostic, diagnostic.occurrence(), false)
1729}
1730
1731#[doc(hidden)]
1733pub fn database_operation_borrowed_input_bounded<D: Error>(
1734 output: Option<&EmergencyDiagnosticHandle>,
1735 call: &saddle_core::CallContext,
1736 phase: &'static str,
1737 source: &D,
1738 diagnostic: &BoundedDiagnostic,
1739) -> UnrootedCaptureFacts {
1740 database_operation_source(output, call, phase,
1741 RawSource::BorrowedError(source), diagnostic, diagnostic.occurrence(), true)
1742}
1743
1744fn database_operation_source<D: Serialize>(
1745 output: Option<&EmergencyDiagnosticHandle>,
1746 call: &saddle_core::CallContext,
1747 phase: &'static str,
1748 source: RawSource<'_>,
1749 diagnostic: &D,
1750 occurrence: DiagnosticOccurrence,
1751 bounded_input: bool,
1752) -> UnrootedCaptureFacts {
1753 struct DisplayId<T>(T);
1754 impl<T: fmt::Display> Serialize for DisplayId<T> {
1755 fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
1756 serializer.collect_str(&self.0)
1757 }
1758 }
1759 #[derive(Serialize)]
1760 struct Context<'a> {
1761 schema_version: u8,
1762 application: &'a str,
1763 module: &'a str,
1764 service: &'a str,
1765 operation: &'a str,
1766 trace_id: DisplayId<saddle_core::TraceId>,
1767 trace_correlation_id: &'a str,
1768 span_id: DisplayId<saddle_core::SpanId>,
1769 rpc_correlation_id: Option<&'a str>,
1770 request: ContextFact<()>,
1771 operation_phase: &'static str,
1772 }
1773 #[derive(Serialize)]
1774 struct Header<'a, D> {
1775 timestamp_unix_ms: u128,
1776 context: Context<'a>,
1777 stage: &'static str,
1778 facts: &'a D,
1779 original_contract: &'static str,
1780 source_interface: &'static str,
1781 }
1782 let header = Header {
1783 timestamp_unix_ms: timestamp(),
1784 context: Context {
1785 schema_version: 2,
1786 application: call.application().as_str(),
1787 module: call.module().as_str(),
1788 service: call.service().as_str(),
1789 operation: call.operation().as_str(),
1790 trace_id: DisplayId(call.trace_id()),
1791 trace_correlation_id: call.trace_correlation_id().as_str(),
1792 span_id: DisplayId(call.span_id()),
1793 rpc_correlation_id: call.rpc_correlation_id().map(|id| id.as_str()),
1794 request: ContextFact::Unavailable,
1795 operation_phase: phase,
1796 },
1797 stage: "database_operation",
1798 facts: diagnostic,
1799 original_contract: "display-debug-source/v1",
1800 source_interface: match &source {
1801 RawSource::Error(_) | RawSource::BorrowedError(_) => "error_source",
1802 RawSource::Description(_, _) => "description_debug_only",
1803 },
1804 };
1805 let (submission, original_capture) = match output {
1806 Some(output) => {
1807 let (submission, state) = capture_with_mode(
1808 &ConfirmedOutput(output), occurrence, source, &header, bounded_input);
1809 if submission == DiagnosticSubmission::Enqueued
1810 && state == OriginalCaptureState::CompleteEnqueued {
1811 (DiagnosticSubmission::Written, OriginalCaptureState::CompleteWritten)
1812 } else { (submission, state) }
1813 }
1814 None => (DiagnosticSubmission::OutputUnavailable, OriginalCaptureState::OutputUnavailable),
1815 };
1816 UnrootedCaptureFacts { occurrence, submission, original_capture }
1817}
1818
1819#[doc(hidden)]
1822#[track_caller]
1823pub fn process_startup_error(
1824 output: Option<&EmergencyDiagnosticHandle>,
1825 application: &saddle_core::ContextLabel,
1826 key: &'static str,
1827 diagnostic: &Diagnostic,
1828 error: &(dyn Error + 'static),
1829) -> UnrootedCaptureFacts {
1830 #[derive(Serialize)]
1831 struct StartupContext<'a> {
1832 schema_version: u8,
1833 application: &'a saddle_core::ContextLabel,
1834 lifecycle: &'static str,
1835 config_key: &'static str,
1836 request: ContextFact<()>,
1837 trace_id: ContextFact<()>,
1838 }
1839 #[derive(Serialize)]
1840 struct StartupHeader<'a> {
1841 timestamp_unix_ms: u128,
1842 context: StartupContext<'a>,
1843 stage: &'static str,
1844 facts: &'a Diagnostic,
1845 original_contract: &'static str,
1846 source_interface: &'static str,
1847 }
1848 let occurrence = diagnostic.occurrence();
1849 let header = StartupHeader {
1850 timestamp_unix_ms: timestamp(),
1851 context: StartupContext { schema_version: 2, application, lifecycle: "startup",
1852 config_key: key, request: ContextFact::NotEstablished,
1853 trace_id: ContextFact::NotEstablished },
1854 stage: "startup_listener", facts: diagnostic,
1855 original_contract: "display-debug-source/v1", source_interface: "error_source",
1856 };
1857 let (submission, original_capture) = match output {
1858 Some(output) => {
1859 let (submission, state) = capture(&ConfirmedOutput(output), occurrence, RawSource::Error(error), &header);
1860 if submission == DiagnosticSubmission::Enqueued && state == OriginalCaptureState::CompleteEnqueued {
1861 (DiagnosticSubmission::Written, OriginalCaptureState::CompleteWritten)
1862 } else { (submission, state) }
1863 }
1864 None => (DiagnosticSubmission::OutputUnavailable, OriginalCaptureState::OutputUnavailable),
1865 };
1866 UnrootedCaptureFacts { occurrence, submission, original_capture }
1867}
1868
1869#[doc(hidden)]
1872#[track_caller]
1873pub fn process_config_error(
1874 output: Option<&EmergencyDiagnosticHandle>,
1875 diagnostic: &Diagnostic,
1876 error: &(dyn Error + 'static),
1877) -> UnrootedCaptureFacts {
1878 match output {
1879 Some(output) => capture_config(&ConfirmedOutput(output), diagnostic, error),
1880 None => UnrootedCaptureFacts {
1881 occurrence: diagnostic.occurrence(),
1882 submission: DiagnosticSubmission::OutputUnavailable,
1883 original_capture: OriginalCaptureState::OutputUnavailable,
1884 },
1885 }
1886}
1887
1888#[doc(hidden)]
1890#[track_caller]
1891pub fn process_bootstrap_error(
1892 output: &crate::BootstrapDiagnostics,
1893 diagnostic: &Diagnostic,
1894 error: &(dyn Error + 'static),
1895) -> UnrootedCaptureFacts {
1896 capture_config(output, diagnostic, error)
1897}
1898
1899#[derive(Clone, Copy)]
1902pub enum ServiceRegistrySourceOutput<'a> {
1903 Bootstrap(&'a crate::BootstrapDiagnostics),
1904 Emergency(&'a EmergencyDiagnosticHandle),
1905}
1906
1907#[track_caller]
1908pub fn service_registry_source_error(
1909 output: Option<ServiceRegistrySourceOutput<'_>>,
1910 diagnostic: &Diagnostic,
1911 error: &(dyn Error + 'static),
1912 lifecycle: &'static str,
1913) -> UnrootedCaptureFacts {
1914 service_registry_source(output, diagnostic, RawSource::Error(error), lifecycle)
1915}
1916
1917#[track_caller]
1921pub fn service_production_fact_source_error(
1922 output: &crate::BootstrapDiagnostics,
1923 diagnostic: &Diagnostic,
1924 error: &(dyn Error + 'static),
1925 application: &saddle_core::ApplicationId,
1926) -> UnrootedCaptureFacts {
1927 service_compiled_build_source_error(output, diagnostic, error, application, "service_production_fact")
1928}
1929
1930#[track_caller]
1933pub fn service_compiled_build_source_error(
1934 output: &crate::BootstrapDiagnostics,
1935 diagnostic: &Diagnostic,
1936 error: &(dyn Error + 'static),
1937 application: &saddle_core::ApplicationId,
1938 source_stage: &'static str,
1939) -> UnrootedCaptureFacts {
1940 #[derive(Serialize)]
1941 struct FactContext<'a> {
1942 schema_version: u8,
1943 application: &'a str,
1944 lifecycle: &'static str,
1945 request: ContextFact<()>,
1946 trace_id: ContextFact<()>,
1947 }
1948 #[derive(Serialize)]
1949 struct FactHeader<'a> {
1950 timestamp_unix_ms: u128,
1951 context: FactContext<'a>,
1952 stage: &'static str,
1953 facts: &'a Diagnostic,
1954 original_contract: &'static str,
1955 source_interface: &'static str,
1956 }
1957 let occurrence = diagnostic.occurrence();
1958 let header = FactHeader {
1959 timestamp_unix_ms: timestamp(),
1960 context: FactContext {
1961 schema_version: 2, application: application.as_str(), lifecycle: "startup",
1962 request: ContextFact::NotEstablished, trace_id: ContextFact::NotEstablished,
1963 },
1964 stage: source_stage, facts: diagnostic,
1965 original_contract: "display-debug-source/v1", source_interface: "error_source",
1966 };
1967 let (submission, original_capture) = capture(output, occurrence, RawSource::Error(error), &header);
1968 let (submission, original_capture) = match (submission, original_capture) {
1969 (DiagnosticSubmission::Enqueued, OriginalCaptureState::CompleteEnqueued) =>
1970 (DiagnosticSubmission::Written, OriginalCaptureState::CompleteWritten),
1971 facts => facts,
1972 };
1973 UnrootedCaptureFacts { occurrence, submission, original_capture }
1974}
1975
1976#[track_caller]
1979pub fn service_registry_source_description(
1980 output: Option<ServiceRegistrySourceOutput<'_>>,
1981 diagnostic: &Diagnostic,
1982 description: &(impl fmt::Display + fmt::Debug),
1983 lifecycle: &'static str,
1984) -> UnrootedCaptureFacts {
1985 service_registry_source(output, diagnostic, RawSource::Description(description, description), lifecycle)
1986}
1987
1988#[track_caller]
1991pub fn service_route_source_description(
1992 output: Option<&EmergencyDiagnosticHandle>,
1993 diagnostic: &Diagnostic,
1994 description: &(impl fmt::Display + fmt::Debug),
1995 application: &saddle_core::ApplicationId,
1996 route: &str,
1997) -> UnrootedCaptureFacts {
1998 service_entry_source(output, diagnostic, RawSource::Description(description, description),
1999 application, route, None, "startup", "service_route_registration")
2000}
2001
2002#[track_caller]
2005pub fn service_entry_trace_source_error(
2006 output: Option<&EmergencyDiagnosticHandle>,
2007 diagnostic: &Diagnostic,
2008 error: &(dyn Error + 'static),
2009 application: &saddle_core::ApplicationId,
2010 route: &str,
2011 inbound_trace_id: &str,
2012) -> UnrootedCaptureFacts {
2013 service_entry_source(output, diagnostic, RawSource::Error(error), application, route,
2014 Some(inbound_trace_id), "request", "service_trace_validation")
2015}
2016
2017#[track_caller]
2018fn service_entry_source(
2019 output: Option<&EmergencyDiagnosticHandle>,
2020 diagnostic: &Diagnostic,
2021 raw: RawSource<'_>,
2022 application: &saddle_core::ApplicationId,
2023 route: &str,
2024 inbound_trace_id: Option<&str>,
2025 lifecycle: &'static str,
2026 stage: &'static str,
2027) -> UnrootedCaptureFacts {
2028 #[derive(Serialize)]
2029 struct RouteContext<'a> {
2030 schema_version: u8,
2031 application: &'a str,
2032 route: &'a str,
2033 inbound_trace_id: ContextFact<&'a str>,
2034 lifecycle: &'static str,
2035 request: ContextFact<()>,
2036 trace_id: ContextFact<()>,
2037 }
2038 #[derive(Serialize)]
2039 struct RouteHeader<'a> {
2040 timestamp_unix_ms: u128,
2041 context: RouteContext<'a>,
2042 stage: &'static str,
2043 facts: &'a Diagnostic,
2044 original_contract: &'static str,
2045 source_interface: &'static str,
2046 }
2047 let occurrence = diagnostic.occurrence();
2048 let header = RouteHeader {
2049 timestamp_unix_ms: timestamp(),
2050 context: RouteContext {
2051 schema_version: 2, application: application.as_str(), route,
2052 inbound_trace_id: inbound_trace_id.map(ContextFact::Present)
2053 .unwrap_or(ContextFact::NotEstablished),
2054 lifecycle, request: ContextFact::NotEstablished,
2055 trace_id: ContextFact::NotEstablished,
2056 },
2057 stage, facts: diagnostic,
2058 original_contract: "display-debug-source/v1",
2059 source_interface: match raw {
2060 RawSource::Error(_) | RawSource::BorrowedError(_) => "error_source",
2061 RawSource::Description(_, _) => "unavailable_description_debug_only",
2062 },
2063 };
2064 let (submission, original_capture) = match output {
2065 Some(output) => {
2066 let (submission, capture_state) = capture(&ConfirmedOutput(output), occurrence,
2067 raw, &header);
2068 match (submission, capture_state) {
2069 (DiagnosticSubmission::Enqueued, OriginalCaptureState::CompleteEnqueued) =>
2070 (DiagnosticSubmission::Written, OriginalCaptureState::CompleteWritten),
2071 facts => facts,
2072 }
2073 }
2074 None => (DiagnosticSubmission::OutputUnavailable, OriginalCaptureState::OutputUnavailable),
2075 };
2076 UnrootedCaptureFacts { occurrence, submission, original_capture }
2077}
2078
2079#[track_caller]
2080fn service_registry_source(
2081 output: Option<ServiceRegistrySourceOutput<'_>>,
2082 diagnostic: &Diagnostic,
2083 raw: RawSource<'_>,
2084 lifecycle: &'static str,
2085) -> UnrootedCaptureFacts {
2086 #[derive(Serialize)]
2087 struct RegistryContext {
2088 schema_version: u8,
2089 application: ContextFact<()>,
2090 lifecycle: &'static str,
2091 request: ContextFact<()>,
2092 trace_id: ContextFact<()>,
2093 }
2094 #[derive(Serialize)]
2095 struct RegistryHeader<'a> {
2096 timestamp_unix_ms: u128,
2097 context: RegistryContext,
2098 stage: &'static str,
2099 facts: &'a Diagnostic,
2100 original_contract: &'static str,
2101 source_interface: &'static str,
2102 }
2103 let occurrence = diagnostic.occurrence();
2104 let header = RegistryHeader {
2105 timestamp_unix_ms: timestamp(),
2106 context: RegistryContext {
2107 schema_version: 2,
2108 application: if lifecycle == "startup" { ContextFact::NotEstablished }
2109 else { ContextFact::Unavailable },
2110 lifecycle,
2111 request: if lifecycle == "startup" { ContextFact::NotEstablished }
2112 else { ContextFact::Unavailable },
2113 trace_id: if lifecycle == "startup" { ContextFact::NotEstablished }
2114 else { ContextFact::Unavailable },
2115 },
2116 stage: "service_registry", facts: diagnostic,
2117 original_contract: "display-debug-source/v1", source_interface: match raw {
2118 RawSource::Error(_) | RawSource::BorrowedError(_) => "error_source",
2119 RawSource::Description(_, _) => "unavailable_description_debug_only",
2120 },
2121 };
2122 let result = match output {
2123 Some(ServiceRegistrySourceOutput::Bootstrap(output)) =>
2124 Some(capture(output, occurrence, raw, &header)),
2125 Some(ServiceRegistrySourceOutput::Emergency(output)) =>
2126 Some(capture(&ConfirmedOutput(output), occurrence, raw, &header)),
2127 None => None,
2128 };
2129 let (submission, original_capture) = match result {
2130 Some((DiagnosticSubmission::Enqueued, OriginalCaptureState::CompleteEnqueued)) =>
2131 (DiagnosticSubmission::Written, OriginalCaptureState::CompleteWritten),
2132 Some(result) => result,
2133 None => (DiagnosticSubmission::OutputUnavailable, OriginalCaptureState::OutputUnavailable),
2134 };
2135 UnrootedCaptureFacts { occurrence, submission, original_capture }
2136}
2137
2138#[track_caller]
2142pub fn service_request_source_error(
2143 output: Option<&EmergencyDiagnosticHandle>,
2144 diagnostic: &Diagnostic,
2145 error: &(dyn Error + 'static),
2146 call: &saddle_core::CallContext,
2147 stage: &'static str,
2148) -> UnrootedCaptureFacts {
2149 service_request_source(output, diagnostic, RawSource::Error(error), call, stage)
2150}
2151
2152#[track_caller]
2153pub fn service_request_source_description(
2154 output: Option<&EmergencyDiagnosticHandle>,
2155 diagnostic: &Diagnostic,
2156 description: &(impl fmt::Display + fmt::Debug),
2157 call: &saddle_core::CallContext,
2158 stage: &'static str,
2159) -> UnrootedCaptureFacts {
2160 service_request_source(output, diagnostic, RawSource::Description(description, description), call, stage)
2161}
2162
2163#[track_caller]
2167pub fn runtime_admission_source_description(
2168 output: Option<&EmergencyDiagnosticHandle>,
2169 diagnostic: &Diagnostic,
2170 description: &(impl fmt::Display + fmt::Debug),
2171 call: &saddle_core::CallContext,
2172) -> UnrootedCaptureFacts {
2173 service_request_source(output, diagnostic, RawSource::Description(description, description),
2174 call, "runtime_request_admission")
2175}
2176
2177#[track_caller]
2180pub fn runtime_unrooted_admission_source_description(
2181 output: Option<&EmergencyDiagnosticHandle>,
2182 diagnostic: &Diagnostic,
2183 description: &(impl fmt::Display + fmt::Debug),
2184 application: Option<&saddle_core::ApplicationId>,
2185) -> UnrootedCaptureFacts {
2186 #[derive(Serialize)]
2187 struct AdmissionContext<'a> {
2188 schema_version: u8,
2189 application: ContextFact<&'a str>,
2190 lifecycle: &'static str,
2191 request: ContextFact<()>,
2192 trace_id: ContextFact<()>,
2193 }
2194 #[derive(Serialize)]
2195 struct AdmissionHeader<'a> {
2196 timestamp_unix_ms: u128,
2197 context: AdmissionContext<'a>,
2198 stage: &'static str,
2199 facts: &'a Diagnostic,
2200 original_contract: &'static str,
2201 source_interface: &'static str,
2202 }
2203 let occurrence = diagnostic.occurrence();
2204 let header = AdmissionHeader {
2205 timestamp_unix_ms: timestamp(),
2206 context: AdmissionContext {
2207 schema_version: 2,
2208 application: application.map(|id| ContextFact::Present(id.as_str()))
2209 .unwrap_or(ContextFact::Unavailable),
2210 lifecycle: "request", request: ContextFact::NotEstablished,
2211 trace_id: ContextFact::NotEstablished,
2212 },
2213 stage: "runtime_unrooted_admission", facts: diagnostic,
2214 original_contract: "display-debug-source/v1",
2215 source_interface: "unavailable_description_debug_only",
2216 };
2217 let (submission, original_capture) = match output {
2218 Some(output) => {
2219 let (submission, state) = capture(&ConfirmedOutput(output), occurrence,
2220 RawSource::Description(description, description), &header);
2221 if submission == DiagnosticSubmission::Enqueued
2222 && state == OriginalCaptureState::CompleteEnqueued {
2223 (DiagnosticSubmission::Written, OriginalCaptureState::CompleteWritten)
2224 } else { (submission, state) }
2225 }
2226 None => (DiagnosticSubmission::OutputUnavailable, OriginalCaptureState::OutputUnavailable),
2227 };
2228 UnrootedCaptureFacts { occurrence, submission, original_capture }
2229}
2230
2231#[track_caller]
2232fn service_request_source(
2233 output: Option<&EmergencyDiagnosticHandle>,
2234 diagnostic: &Diagnostic,
2235 raw: RawSource<'_>,
2236 call: &saddle_core::CallContext,
2237 stage: &'static str,
2238) -> UnrootedCaptureFacts {
2239 #[derive(Serialize)]
2240 struct ServiceContext<'a> {
2241 schema_version: u8,
2242 application: &'a str,
2243 module: &'a str,
2244 service: &'a str,
2245 operation: &'a str,
2246 trace_id: u128,
2247 trace_correlation_id: &'a str,
2248 span_id: u64,
2249 rpc_correlation_id: Option<&'a str>,
2250 request: ContextFact<()>,
2251 lifecycle: &'static str,
2252 }
2253 #[derive(Serialize)]
2254 struct ServiceHeader<'a> {
2255 timestamp_unix_ms: u128,
2256 context: ServiceContext<'a>,
2257 stage: &'static str,
2258 facts: &'a Diagnostic,
2259 original_contract: &'static str,
2260 source_interface: &'static str,
2261 }
2262 let occurrence = diagnostic.occurrence();
2263 let header = ServiceHeader {
2264 timestamp_unix_ms: timestamp(),
2265 context: ServiceContext {
2266 schema_version: 2,
2267 application: call.application().as_str(),
2268 module: call.module().as_str(),
2269 service: call.service().as_str(),
2270 operation: call.operation().as_str(),
2271 trace_id: call.trace_id().as_u128(),
2272 trace_correlation_id: call.trace_correlation_id().as_str(),
2273 span_id: call.span_id().as_u64(),
2274 rpc_correlation_id: call.rpc_correlation_id().map(|id| id.as_str()),
2275 request: ContextFact::Unavailable,
2276 lifecycle: "request",
2277 },
2278 stage, facts: diagnostic,
2279 original_contract: "display-debug-source/v1", source_interface: match raw {
2280 RawSource::Error(_) | RawSource::BorrowedError(_) => "error_source",
2281 RawSource::Description(_, _) => "unavailable_description_debug_only",
2282 },
2283 };
2284 let (submission, original_capture) = match output {
2285 Some(output) => {
2286 let (submission, state) = capture(
2287 &ConfirmedOutput(output), occurrence, raw, &header,
2288 );
2289 if submission == DiagnosticSubmission::Enqueued
2290 && state == OriginalCaptureState::CompleteEnqueued {
2291 (DiagnosticSubmission::Written, OriginalCaptureState::CompleteWritten)
2292 } else { (submission, state) }
2293 }
2294 None => (DiagnosticSubmission::OutputUnavailable, OriginalCaptureState::OutputUnavailable),
2295 };
2296 UnrootedCaptureFacts { occurrence, submission, original_capture }
2297}
2298
2299fn capture_config<O: OriginalOutput>(
2300 output: &O,
2301 diagnostic: &Diagnostic,
2302 error: &(dyn Error + 'static),
2303) -> UnrootedCaptureFacts {
2304 #[derive(Serialize)]
2305 struct ConfigContext {
2306 schema_version: u8,
2307 application: ContextFact<()>,
2308 lifecycle: &'static str,
2309 request: ContextFact<()>,
2310 trace_id: ContextFact<()>,
2311 }
2312 #[derive(Serialize)]
2313 struct ConfigHeader<'a> {
2314 timestamp_unix_ms: u128,
2315 context: ConfigContext,
2316 stage: &'static str,
2317 facts: &'a Diagnostic,
2318 original_contract: &'static str,
2319 source_interface: &'static str,
2320 }
2321 let occurrence = diagnostic.occurrence();
2322 let header = ConfigHeader {
2323 timestamp_unix_ms: timestamp(),
2324 context: ConfigContext { schema_version: 2, application: ContextFact::NotEstablished,
2325 lifecycle: "startup", request: ContextFact::NotEstablished,
2326 trace_id: ContextFact::NotEstablished },
2327 stage: "startup_config", facts: diagnostic,
2328 original_contract: "display-debug-source/v1", source_interface: "error_source",
2329 };
2330 let (submission, state) = capture(output, occurrence, RawSource::Error(error), &header);
2331 let (submission, original_capture) = if submission == DiagnosticSubmission::Enqueued
2332 && state == OriginalCaptureState::CompleteEnqueued {
2333 (DiagnosticSubmission::Written, OriginalCaptureState::CompleteWritten)
2334 } else { (submission, state) };
2335 UnrootedCaptureFacts { occurrence, submission, original_capture }
2336}
2337
2338#[doc(hidden)]
2341pub fn process_finalization_error(
2342 output: Option<&EmergencyDiagnosticHandle>,
2343 diagnostic: &Diagnostic,
2344 error: &(dyn Error + 'static),
2345) -> UnrootedCaptureFacts {
2346 process_lifecycle_error(output, ContextFact::Unavailable, "shutdown", "process_finalization", diagnostic, error)
2347}
2348
2349#[doc(hidden)]
2351pub fn process_component_start_error(
2352 output: Option<&EmergencyDiagnosticHandle>,
2353 application: ContextFact<saddle_core::ContextLabel>,
2354 diagnostic: &Diagnostic,
2355 error: &(dyn Error + 'static),
2356) -> ComponentSourceReceipt {
2357 ComponentSourceReceipt {
2358 facts: process_lifecycle_error(output, application, "startup", "component_start", diagnostic, error),
2359 kind: ComponentSourceKind::Start,
2360 }
2361}
2362
2363#[doc(hidden)]
2364pub fn process_component_start_recorded(
2365 output: Option<&EmergencyDiagnosticHandle>,
2366 application: ContextFact<saddle_core::ContextLabel>,
2367 diagnostic: &Diagnostic,
2368 error: &(dyn Error + 'static),
2369) -> Result<WrittenComponentFailure, ComponentSourceReceipt> {
2370 WrittenComponentFailure::from_receipt(process_component_start_error(
2371 output, application, diagnostic, error))
2372}
2373
2374#[doc(hidden)]
2376pub fn process_component_cleanup_error(
2377 output: Option<&EmergencyDiagnosticHandle>,
2378 application: ContextFact<saddle_core::ContextLabel>,
2379 diagnostic: &Diagnostic,
2380 error: &(dyn Error + 'static),
2381) -> ComponentSourceReceipt {
2382 ComponentSourceReceipt {
2383 facts: process_lifecycle_error(output, application, "shutdown", "component_cleanup", diagnostic, error),
2384 kind: ComponentSourceKind::Cleanup,
2385 }
2386}
2387
2388#[doc(hidden)]
2389pub fn process_component_cleanup_recorded(
2390 output: Option<&EmergencyDiagnosticHandle>,
2391 application: ContextFact<saddle_core::ContextLabel>,
2392 diagnostic: &Diagnostic,
2393 error: &(dyn Error + 'static),
2394) -> Result<WrittenComponentFailure, ComponentSourceReceipt> {
2395 WrittenComponentFailure::from_receipt(process_component_cleanup_error(
2396 output, application, diagnostic, error))
2397}
2398
2399#[doc(hidden)]
2401pub fn process_shutdown_signal_error(
2402 output: Option<&EmergencyDiagnosticHandle>,
2403 application: ContextFact<saddle_core::ContextLabel>,
2404 diagnostic: &Diagnostic,
2405 error: &(dyn Error + 'static),
2406) -> UnrootedCaptureFacts {
2407 process_lifecycle_error(output, application, "shutdown", "shutdown_signal", diagnostic, error)
2408}
2409
2410#[doc(hidden)]
2412pub fn process_request_drain_error(
2413 output: Option<&EmergencyDiagnosticHandle>,
2414 application: ContextFact<saddle_core::ContextLabel>,
2415 diagnostic: &Diagnostic,
2416 error: &(dyn Error + 'static),
2417) -> UnrootedCaptureFacts {
2418 process_lifecycle_error(output, application, "shutdown", "request_drain", diagnostic, error)
2419}
2420
2421#[doc(hidden)]
2423pub fn process_post_driver_error(
2424 output: Option<&EmergencyDiagnosticHandle>,
2425 application: ContextFact<saddle_core::ContextLabel>,
2426 diagnostic: &Diagnostic,
2427 error: &(dyn Error + 'static),
2428) -> UnrootedCaptureFacts {
2429 process_lifecycle_error(output, application, "shutdown", "post_driver", diagnostic, error)
2430}
2431
2432fn process_lifecycle_error(
2433 output: Option<&EmergencyDiagnosticHandle>,
2434 application: ContextFact<saddle_core::ContextLabel>,
2435 lifecycle: &'static str,
2436 stage: &'static str,
2437 diagnostic: &Diagnostic,
2438 error: &(dyn Error + 'static),
2439) -> UnrootedCaptureFacts {
2440 #[derive(Serialize)]
2441 struct FinalizationContext {
2442 schema_version: u8,
2443 application: ContextFact<saddle_core::ContextLabel>,
2444 lifecycle: &'static str,
2445 request: ContextFact<()>,
2446 trace_id: ContextFact<()>,
2447 }
2448 #[derive(Serialize)]
2449 struct FinalizationHeader<'a> {
2450 timestamp_unix_ms: u128,
2451 context: FinalizationContext,
2452 stage: &'static str,
2453 facts: &'a Diagnostic,
2454 original_contract: &'static str,
2455 source_interface: &'static str,
2456 }
2457 let occurrence = diagnostic.occurrence();
2458 let header = FinalizationHeader {
2459 timestamp_unix_ms: timestamp(),
2460 context: FinalizationContext { schema_version: 2,
2461 application, lifecycle,
2462 request: if lifecycle == "startup" { ContextFact::NotEstablished } else { ContextFact::NotApplicable },
2463 trace_id: if lifecycle == "startup" { ContextFact::NotEstablished } else { ContextFact::NotApplicable } },
2464 stage, facts: diagnostic,
2465 original_contract: "display-debug-source/v1", source_interface: "error_source",
2466 };
2467 let (submission, original_capture) = match output {
2468 Some(output) => {
2469 let (submission, state) = capture(&ConfirmedOutput(output), occurrence, RawSource::Error(error), &header);
2470 if submission == DiagnosticSubmission::Enqueued && state == OriginalCaptureState::CompleteEnqueued {
2471 (DiagnosticSubmission::Written, OriginalCaptureState::CompleteWritten)
2472 } else { (submission, state) }
2473 }
2474 None => (DiagnosticSubmission::OutputUnavailable, OriginalCaptureState::OutputUnavailable),
2475 };
2476 UnrootedCaptureFacts { occurrence, submission, original_capture }
2477}
2478
2479pub fn request_terminal_audit_error(
2482 output: Option<&EmergencyDiagnosticHandle>,
2483 context: Option<&saddle_core::request_context::RequestContextSnapshot>,
2484 diagnostic: &BoundedDiagnostic,
2485 source_occurrence: Option<DiagnosticOccurrence>,
2486 failure: &saddle_admission::ProfuseGwTerminalAuditFailure,
2487) -> UnrootedCaptureFacts {
2488 #[derive(Serialize)]
2489 struct AuditHeader<'a> {
2490 timestamp_unix_ms: u128,
2491 context: Option<&'a saddle_core::request_context::RequestContextSnapshot>,
2492 stage: &'static str,
2493 facts: &'a BoundedDiagnostic,
2494 source_occurrence: Option<DiagnosticOccurrence>,
2495 audit: Option<saddle_admission::RequestAuditSnapshot>,
2496 database: saddle_admission::ProfuseGwDatabaseDisposition,
2497 original_contract: &'static str,
2498 source_interface: &'static str,
2499 }
2500 let occurrence = diagnostic.occurrence();
2501 let header = AuditHeader {
2502 timestamp_unix_ms: timestamp(), context, stage: "request_terminal_audit",
2503 facts: diagnostic, source_occurrence, audit: failure.audit, database: failure.database,
2504 original_contract: "display-debug-source/v1", source_interface: "error_source",
2505 };
2506 let (submission, original_capture) = match output {
2507 Some(output) => {
2508 let (submission, state) = capture(&ConfirmedOutput(output), occurrence, RawSource::Error(failure), &header);
2509 if submission == DiagnosticSubmission::Enqueued && state == OriginalCaptureState::CompleteEnqueued {
2510 (DiagnosticSubmission::Written, OriginalCaptureState::CompleteWritten)
2511 } else { (submission, state) }
2512 }
2513 None => (DiagnosticSubmission::OutputUnavailable, OriginalCaptureState::OutputUnavailable),
2514 };
2515 UnrootedCaptureFacts { occurrence, submission, original_capture }
2516}
2517
2518pub fn request_terminal_audit_disposition(
2519 output: Option<&EmergencyDiagnosticHandle>,
2520 context: Option<&saddle_core::request_context::RequestContextSnapshot>,
2521 source_occurrence: Option<DiagnosticOccurrence>,
2522 captured: &UnrootedCaptureFacts,
2523 audit: Option<saddle_admission::RequestAuditSnapshot>,
2524) -> DiagnosticSubmission {
2525 #[derive(Serialize)]
2526 struct Record<'a> {
2527 schema_version: u8,
2528 timestamp_unix_ms: u128,
2529 event: &'static str,
2530 context: Option<&'a saddle_core::request_context::RequestContextSnapshot>,
2531 occurrence: DiagnosticOccurrence,
2532 source_occurrence: Option<DiagnosticOccurrence>,
2533 source_submission: DiagnosticSubmission,
2534 original_capture: OriginalCaptureState,
2535 disposition: &'static str,
2536 audit: Option<saddle_admission::RequestAuditSnapshot>,
2537 }
2538 let record = Record {
2539 schema_version: 2, timestamp_unix_ms: timestamp(), event: "request_audit_disposition",
2540 context, occurrence: captured.occurrence, source_occurrence,
2541 source_submission: captured.submission, original_capture: captured.original_capture,
2542 disposition: "process_untrusted", audit,
2543 };
2544 match output {
2545 Some(output) if output.write_original_record(&record).is_ok() => DiagnosticSubmission::Written,
2546 Some(_) => DiagnosticSubmission::OutputUnavailable,
2547 None => DiagnosticSubmission::OutputUnavailable,
2548 }
2549}
2550
2551pub fn request_terminal_audit_passed(
2553 output: Option<&EmergencyDiagnosticHandle>,
2554 context: Option<&saddle_core::request_context::RequestContextSnapshot>,
2555 source: DiagnosticOccurrence,
2556 audit: saddle_admission::RequestAuditSnapshot,
2557) -> DiagnosticSubmission {
2558 #[derive(Serialize)]
2559 struct Record<'a> {
2560 schema_version: u8,
2561 event: &'static str,
2562 occurrence: DiagnosticOccurrence,
2563 context: Option<&'a saddle_core::request_context::RequestContextSnapshot>,
2564 audit: saddle_admission::RequestAuditSnapshot,
2565 disposition: &'static str,
2566 }
2567 let record = Record { schema_version: 2, event: "request_audit_passed", occurrence: source,
2568 context, audit, disposition: "request_failed_process_healthy" };
2569 match output {
2570 Some(output) if output.write_original_record(&record).is_ok() => DiagnosticSubmission::Written,
2571 _ => DiagnosticSubmission::OutputUnavailable,
2572 }
2573}
2574
2575pub fn request_terminal_audit_cleanup(
2578 output: Option<&EmergencyDiagnosticHandle>,
2579 context: Option<&saddle_core::request_context::RequestContextSnapshot>,
2580 occurrence: DiagnosticOccurrence,
2581 snapshot: &saddle_admission::ProcessSnapshot,
2582) -> DiagnosticSubmission {
2583 #[derive(Serialize)]
2584 struct Record<'a> {
2585 schema_version: u8,
2586 event: &'static str,
2587 occurrence: DiagnosticOccurrence,
2588 context: Option<&'a saddle_core::request_context::RequestContextSnapshot>,
2589 active_accounts: usize,
2590 charged: usize,
2591 committed: usize,
2592 framework_charged: usize,
2593 task_charged: usize,
2594 healthy: bool,
2595 cleanup: &'static str,
2596 }
2597 let record = Record { schema_version: 2, event: "request_audit_cleanup", occurrence,
2598 context, active_accounts: snapshot.active_accounts, charged: snapshot.charged,
2599 committed: snapshot.committed, framework_charged: snapshot.framework_charged,
2600 task_charged: snapshot.task_charged, healthy: snapshot.healthy,
2601 cleanup: if snapshot.active_accounts == 0 && snapshot.charged == 0 && snapshot.task_charged == 0 {
2602 "requests_returned_framework_still_owned"
2603 } else { "request_resources_remaining" },
2604 };
2605 match output {
2606 Some(output) if output.write_original_record(&record).is_ok() => DiagnosticSubmission::Written,
2607 _ => DiagnosticSubmission::OutputUnavailable,
2608 }
2609}
2610
2611fn write_owned_terminal(output: Option<&EmergencyDiagnosticHandle>, event: &'static str,
2613 occurrence: DiagnosticOccurrence, record: &impl Serialize) -> DiagnosticSubmission {
2614 let Some(output) = output else { return DiagnosticSubmission::OutputUnavailable; };
2615 match stream_record_identified(&ConfirmedOutput(output), event, Some(occurrence),
2616 |writer| serde_json::to_writer(writer, record)) {
2617 DiagnosticSubmission::Enqueued => DiagnosticSubmission::Written,
2618 submission => submission,
2619 }
2620}
2621
2622pub fn request_terminal_audit_error_owned(output: Option<&EmergencyDiagnosticHandle>,
2623 context: Result<&crate::OwnedUnifiedContext, DiagnosticSubmission>, diagnostic: &BoundedDiagnostic,
2624 source_occurrence: Option<DiagnosticOccurrence>, failure: &saddle_admission::ProfuseGwTerminalAuditFailure)
2625 -> UnrootedCaptureFacts {
2626 #[derive(Serialize)]
2627 struct Header<'a> {
2628 timestamp_unix_ms: u128, context: Option<&'a crate::OwnedUnifiedContext>,
2629 context_capture: Option<DiagnosticSubmission>, stage: &'static str, facts: &'a BoundedDiagnostic,
2630 source_occurrence: Option<DiagnosticOccurrence>, audit: Option<saddle_admission::RequestAuditSnapshot>,
2631 database: saddle_admission::ProfuseGwDatabaseDisposition,
2632 original_contract: &'static str, source_interface: &'static str,
2633 }
2634 let rejected = context.err();
2635 let header = Header { timestamp_unix_ms: timestamp(), context: context.ok(), context_capture: rejected,
2636 stage: "request_terminal_audit", facts: diagnostic, source_occurrence, audit: failure.audit,
2637 database: failure.database, original_contract: "display-debug-source/v1", source_interface: "error_source" };
2638 let occurrence = diagnostic.occurrence();
2639 let (submission, state) = match output {
2640 Some(output) => capture(&ConfirmedOutput(output), occurrence, RawSource::Error(failure), &header),
2641 None => (DiagnosticSubmission::OutputUnavailable, OriginalCaptureState::OutputUnavailable),
2642 };
2643 let complete = submission == DiagnosticSubmission::Enqueued && state == OriginalCaptureState::CompleteEnqueued;
2644 UnrootedCaptureFacts { occurrence,
2645 submission: if complete { DiagnosticSubmission::Written } else { submission },
2646 original_capture: if rejected.is_some() { OriginalCaptureState::OutputRejected }
2647 else if complete { OriginalCaptureState::CompleteWritten } else { state } }
2648}
2649
2650pub fn request_terminal_audit_disposition_owned(output: Option<&EmergencyDiagnosticHandle>,
2651 context: Result<&crate::OwnedUnifiedContext, DiagnosticSubmission>, source_occurrence: Option<DiagnosticOccurrence>,
2652 captured: &UnrootedCaptureFacts, audit: Option<saddle_admission::RequestAuditSnapshot>) -> DiagnosticSubmission {
2653 #[derive(Serialize)]
2654 struct Record<'a> { #[serde(rename = "recordSchema")] record_schema: &'static str, event: &'static str,
2655 context: Option<&'a crate::OwnedUnifiedContext>, context_capture: Option<DiagnosticSubmission>,
2656 occurrence: DiagnosticOccurrence, source_occurrence: Option<DiagnosticOccurrence>,
2657 source_submission: DiagnosticSubmission, original_capture: OriginalCaptureState,
2658 disposition: &'static str, audit: Option<saddle_admission::RequestAuditSnapshot> }
2659 let rejected = context.err();
2660 let record = Record { record_schema: "1.0", event: "request_audit_disposition", context: context.ok(),
2661 context_capture: rejected, occurrence: captured.occurrence, source_occurrence,
2662 source_submission: captured.submission, original_capture: captured.original_capture,
2663 disposition: "process_untrusted", audit };
2664 let submission = write_owned_terminal(output, record.event, record.occurrence, &record);
2665 if rejected.is_some() { DiagnosticSubmission::OutputUnavailable } else { submission }
2666}
2667
2668pub fn request_terminal_audit_passed_owned(output: Option<&EmergencyDiagnosticHandle>,
2669 context: Result<&crate::OwnedUnifiedContext, DiagnosticSubmission>, source: DiagnosticOccurrence,
2670 audit: saddle_admission::RequestAuditSnapshot) -> DiagnosticSubmission {
2671 #[derive(Serialize)]
2672 struct Record<'a> { #[serde(rename = "recordSchema")] record_schema: &'static str, event: &'static str, occurrence: DiagnosticOccurrence,
2673 context: Option<&'a crate::OwnedUnifiedContext>, context_capture: Option<DiagnosticSubmission>,
2674 audit: saddle_admission::RequestAuditSnapshot, disposition: &'static str }
2675 let rejected = context.err();
2676 let record = Record { record_schema: "1.0", event: "request_audit_passed", occurrence: source,
2677 context: context.ok(), context_capture: rejected, audit, disposition: "request_failed_process_healthy" };
2678 let submission = write_owned_terminal(output, record.event, source, &record);
2679 if rejected.is_some() { DiagnosticSubmission::OutputUnavailable } else { submission }
2680}
2681
2682pub fn request_terminal_audit_cleanup_owned(output: Option<&EmergencyDiagnosticHandle>,
2683 context: Result<&crate::OwnedUnifiedContext, DiagnosticSubmission>, occurrence: DiagnosticOccurrence,
2684 snapshot: &saddle_admission::ProcessSnapshot) -> DiagnosticSubmission {
2685 #[derive(Serialize)]
2686 struct Record<'a> { #[serde(rename = "recordSchema")] record_schema: &'static str, event: &'static str, occurrence: DiagnosticOccurrence,
2687 context: Option<&'a crate::OwnedUnifiedContext>, context_capture: Option<DiagnosticSubmission>,
2688 active_accounts: usize, charged: usize, committed: usize, framework_charged: usize,
2689 task_charged: usize, healthy: bool, cleanup: &'static str }
2690 let rejected = context.err();
2691 let record = Record { record_schema: "1.0", event: "request_audit_cleanup", occurrence, context: context.ok(),
2692 context_capture: rejected, active_accounts: snapshot.active_accounts, charged: snapshot.charged,
2693 committed: snapshot.committed, framework_charged: snapshot.framework_charged,
2694 task_charged: snapshot.task_charged, healthy: snapshot.healthy,
2695 cleanup: if snapshot.active_accounts == 0 && snapshot.charged == 0 && snapshot.task_charged == 0 {
2696 "requests_returned_framework_still_owned"
2697 } else { "request_resources_remaining" } };
2698 let submission = write_owned_terminal(output, record.event, occurrence, &record);
2699 if rejected.is_some() { DiagnosticSubmission::OutputUnavailable } else { submission }
2700}
2701
2702pub(crate) fn write_encoding_selection(output: &EmergencyDiagnosticHandle, record: &impl Serialize) -> DiagnosticSubmission {
2705 struct Count(usize);
2706 impl std::io::Write for Count {
2707 fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> { self.0 = self.0.saturating_add(bytes.len()); Ok(bytes.len()) }
2708 fn flush(&mut self) -> std::io::Result<()> { Ok(()) }
2709 }
2710 let mut count = Count(0);
2711 if serde_json::to_writer(&mut count, record).is_err() { return DiagnosticSubmission::EncodingFailed; }
2712 if count.0 < 8192 {
2713 return if output.write_original_record(record).is_ok() { DiagnosticSubmission::Written } else { DiagnosticSubmission::OutputUnavailable };
2714 }
2715 match stream_record_identified(&ConfirmedOutput(output), "context_record_segment", None, |writer| serde_json::to_writer(writer, record)) {
2716 DiagnosticSubmission::Enqueued => DiagnosticSubmission::Written, submission => submission,
2717 }
2718}