1use crate::engine::{
2 AllowRequesterFailure, AnnounceNowFailure, CloseLinkFailure, EstablishLinkFailure,
3 IdentifyFailure, Journaled, LinkClosedReason, RequestPathFailure, RespondFailure,
4 RouteRemovalCause, SendGroupFailure, SendRequestFailure, SendResourceFailure,
5 SendSinglePacketFailure, SendToChannelFailure, SendToLinkFailure, SetResourceStrategyFailure,
6 Settlement,
7};
8use crate::routing::links::resources::table::ApplyHashmapUpdateError;
9use crate::routing::links::resources::ResourceFailureCause;
10
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12#[repr(u8)]
13pub enum RuntimeOperation {
14 AnnounceNow,
15 SendSinglePacket,
16 SendGroup,
17 RequestPath,
18 EstablishLink,
19 SendToLink,
20 Identify,
21 SendRequest,
22 Respond,
23 CloseLink,
24 SendResource,
25 SetResourceStrategy,
26 SendToChannel,
27 AllowRequester,
28}
29
30impl RuntimeOperation {
31 pub const ALL: [Self; 14] = [
32 Self::AnnounceNow,
33 Self::SendSinglePacket,
34 Self::SendGroup,
35 Self::RequestPath,
36 Self::EstablishLink,
37 Self::SendToLink,
38 Self::Identify,
39 Self::SendRequest,
40 Self::Respond,
41 Self::CloseLink,
42 Self::SendResource,
43 Self::SetResourceStrategy,
44 Self::SendToChannel,
45 Self::AllowRequester,
46 ];
47
48 const fn index(self) -> usize {
49 self as usize
50 }
51}
52
53#[derive(Debug, Clone, Copy, PartialEq, Eq)]
54#[repr(u8)]
55pub enum RuntimeOperationOutcome {
56 Succeeded,
57 Rejected,
58 WriteFailed,
59 Timeout,
60 Culled,
61 PeerRejected,
62 Sequencing,
63 DependencyFailed,
64 Backpressure,
65 Untrackable,
66 ResponseTooLarge,
67}
68
69impl RuntimeOperationOutcome {
70 pub const ALL: [Self; 11] = [
71 Self::Succeeded,
72 Self::Rejected,
73 Self::WriteFailed,
74 Self::Timeout,
75 Self::Culled,
76 Self::PeerRejected,
77 Self::Sequencing,
78 Self::DependencyFailed,
79 Self::Backpressure,
80 Self::Untrackable,
81 Self::ResponseTooLarge,
82 ];
83
84 const fn index(self) -> usize {
85 self as usize
86 }
87}
88
89#[derive(Debug, Clone, Copy, PartialEq, Eq)]
90pub struct RuntimeOperationCounts {
91 counts: [[u64; RuntimeOperationOutcome::ALL.len()]; RuntimeOperation::ALL.len()],
92}
93
94impl Default for RuntimeOperationCounts {
95 fn default() -> Self {
96 Self {
97 counts: [[0; RuntimeOperationOutcome::ALL.len()]; RuntimeOperation::ALL.len()],
98 }
99 }
100}
101
102impl RuntimeOperationCounts {
103 pub const fn get(&self, operation: RuntimeOperation, outcome: RuntimeOperationOutcome) -> u64 {
104 self.counts[operation.index()][outcome.index()]
105 }
106
107 pub fn iter(
108 &self,
109 ) -> impl Iterator<Item = (RuntimeOperation, RuntimeOperationOutcome, u64)> + '_ {
110 RuntimeOperation::ALL
111 .into_iter()
112 .flat_map(move |operation| {
113 RuntimeOperationOutcome::ALL
114 .into_iter()
115 .map(move |outcome| (operation, outcome, self.get(operation, outcome)))
116 })
117 }
118
119 fn record(&mut self, operation: RuntimeOperation, outcome: RuntimeOperationOutcome) {
120 let count = &mut self.counts[operation.index()][outcome.index()];
121 *count = count.saturating_add(1);
122 }
123}
124
125#[derive(Debug, Clone, Copy, PartialEq, Eq)]
126#[repr(u8)]
127pub enum RuntimeResourceFailure {
128 CancelledBySender,
129 HashmapBeyondPartCount,
130 HashmapSkipsAhead,
131 HashmapTooLong,
132 HashmapRagged,
133 RetriesExhausted,
134 LinkVanished,
135 TransferUnopenable,
136 TransferCorrupt,
137 ProofUnsendable,
138 DecompressionFailed,
139 DecompressionTimedOut,
140 OpenTimedOut,
141 MetadataOverrun,
142}
143
144impl RuntimeResourceFailure {
145 pub const ALL: [Self; 14] = [
146 Self::CancelledBySender,
147 Self::HashmapBeyondPartCount,
148 Self::HashmapSkipsAhead,
149 Self::HashmapTooLong,
150 Self::HashmapRagged,
151 Self::RetriesExhausted,
152 Self::LinkVanished,
153 Self::TransferUnopenable,
154 Self::TransferCorrupt,
155 Self::ProofUnsendable,
156 Self::DecompressionFailed,
157 Self::DecompressionTimedOut,
158 Self::OpenTimedOut,
159 Self::MetadataOverrun,
160 ];
161
162 const fn index(self) -> usize {
163 self as usize
164 }
165}
166
167#[derive(Debug, Clone, Copy, PartialEq, Eq)]
168pub struct RuntimeResourceFailureCounts {
169 counts: [u64; RuntimeResourceFailure::ALL.len()],
170}
171
172impl Default for RuntimeResourceFailureCounts {
173 fn default() -> Self {
174 Self {
175 counts: [0; RuntimeResourceFailure::ALL.len()],
176 }
177 }
178}
179
180impl RuntimeResourceFailureCounts {
181 pub const fn get(&self, failure: RuntimeResourceFailure) -> u64 {
182 self.counts[failure.index()]
183 }
184
185 pub fn iter(&self) -> impl ExactSizeIterator<Item = (RuntimeResourceFailure, u64)> + '_ {
186 RuntimeResourceFailure::ALL
187 .into_iter()
188 .map(|failure| (failure, self.get(failure)))
189 }
190
191 fn record(&mut self, failure: RuntimeResourceFailure) {
192 let count = &mut self.counts[failure.index()];
193 *count = count.saturating_add(1);
194 }
195}
196
197#[derive(Debug, Clone, Copy, PartialEq, Eq)]
198#[repr(u8)]
199pub enum RuntimeLinkClosure {
200 Timeout,
201 PeerClosed,
202 MalformedRtt,
203}
204
205impl RuntimeLinkClosure {
206 pub const ALL: [Self; 3] = [Self::Timeout, Self::PeerClosed, Self::MalformedRtt];
207
208 const fn index(self) -> usize {
209 self as usize
210 }
211}
212
213#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
214pub struct RuntimeLinkClosureCounts {
215 counts: [u64; RuntimeLinkClosure::ALL.len()],
216}
217
218impl RuntimeLinkClosureCounts {
219 pub const fn get(&self, reason: RuntimeLinkClosure) -> u64 {
220 self.counts[reason.index()]
221 }
222
223 pub fn iter(&self) -> impl ExactSizeIterator<Item = (RuntimeLinkClosure, u64)> + '_ {
224 RuntimeLinkClosure::ALL
225 .into_iter()
226 .map(|reason| (reason, self.get(reason)))
227 }
228
229 fn record(&mut self, reason: RuntimeLinkClosure) {
230 let count = &mut self.counts[reason.index()];
231 *count = count.saturating_add(1);
232 }
233}
234
235#[derive(Debug, Clone, Copy, PartialEq, Eq)]
236#[repr(u8)]
237pub enum RuntimeRouteRemoval {
238 Expired,
239 Evicted,
240 InterfaceGone,
241 Dropped,
242}
243
244impl RuntimeRouteRemoval {
245 pub const ALL: [Self; 4] = [
246 Self::Expired,
247 Self::Evicted,
248 Self::InterfaceGone,
249 Self::Dropped,
250 ];
251
252 const fn index(self) -> usize {
253 self as usize
254 }
255}
256
257#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
258pub struct RuntimeRouteRemovalCounts {
259 counts: [u64; RuntimeRouteRemoval::ALL.len()],
260}
261
262impl RuntimeRouteRemovalCounts {
263 pub const fn get(&self, cause: RuntimeRouteRemoval) -> u64 {
264 self.counts[cause.index()]
265 }
266
267 pub fn iter(&self) -> impl ExactSizeIterator<Item = (RuntimeRouteRemoval, u64)> + '_ {
268 RuntimeRouteRemoval::ALL
269 .into_iter()
270 .map(|cause| (cause, self.get(cause)))
271 }
272
273 fn record(&mut self, cause: RuntimeRouteRemoval) {
274 let count = &mut self.counts[cause.index()];
275 *count = count.saturating_add(1);
276 }
277}
278
279#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
280pub struct ReliabilityMetricsSnapshot {
281 pub operations: RuntimeOperationCounts,
282 pub resource_failures: RuntimeResourceFailureCounts,
283 pub link_closures: RuntimeLinkClosureCounts,
284 pub link_interface_mismatches: u64,
285 pub route_removals: RuntimeRouteRemovalCounts,
286}
287
288impl ReliabilityMetricsSnapshot {
289 pub fn record_journaled(&mut self, journaled: &Journaled<'_>) {
290 match journaled {
291 Journaled::PersistenceFlushed { .. } | Journaled::PersistenceFlushFailed { .. } => {}
292 Journaled::CommandSettled { settlement, .. } => {
293 let settled = SettledOperation::from(settlement);
294 self.operations.record(settled.operation, settled.outcome);
295 }
296 Journaled::LinkClosed { reason, .. } => {
297 self.link_closures.record((*reason).into());
298 }
299 Journaled::LinkInterfaceMismatch { .. } => {
300 self.link_interface_mismatches = self.link_interface_mismatches.saturating_add(1);
301 }
302 Journaled::ResourceFailed { cause, .. } => {
303 self.resource_failures.record((*cause).into());
304 }
305 Journaled::RouteRemoved { cause, .. } => {
306 self.route_removals.record((*cause).into());
307 }
308 Journaled::AnnounceHeard { .. }
309 | Journaled::SelfRatchetRotated { .. }
310 | Journaled::AnnounceHeldDropped { .. }
311 | Journaled::Delivered(_)
312 | Journaled::LinkEstablished(_)
313 | Journaled::PeerIdentified { .. }
314 | Journaled::RequestReceived { .. }
315 | Journaled::ResponseReceived { .. }
316 | Journaled::ResponseSegmentReceived { .. }
317 | Journaled::ChannelMessageReceived { .. }
318 | Journaled::ResourceReceived { .. }
319 | Journaled::ResourceNeedsDecompression { .. }
320 | Journaled::ResourceSegmentReceived { .. }
321 | Journaled::ResourceAssembled { .. } => {}
322 }
323 }
324}
325
326#[derive(Debug, Clone, Copy, PartialEq, Eq)]
327struct SettledOperation {
328 operation: RuntimeOperation,
329 outcome: RuntimeOperationOutcome,
330}
331
332trait RuntimeOutcome {
333 fn runtime_outcome(&self) -> RuntimeOperationOutcome;
334}
335
336impl<Success, Failure> RuntimeOutcome for Result<Success, Failure>
337where
338 for<'failure> RuntimeOperationOutcome: From<&'failure Failure>,
339{
340 fn runtime_outcome(&self) -> RuntimeOperationOutcome {
341 match self {
342 Ok(_) => RuntimeOperationOutcome::Succeeded,
343 Err(failure) => RuntimeOperationOutcome::from(failure),
344 }
345 }
346}
347
348impl From<&Settlement> for SettledOperation {
349 fn from(settlement: &Settlement) -> Self {
350 use RuntimeOperation as Operation;
351
352 match settlement {
353 Settlement::AnnounceNow(result) => Self {
354 operation: Operation::AnnounceNow,
355 outcome: result.runtime_outcome(),
356 },
357 Settlement::SendSinglePacket(result) => Self {
358 operation: Operation::SendSinglePacket,
359 outcome: result.runtime_outcome(),
360 },
361 Settlement::SendGroup(result) => Self {
362 operation: Operation::SendGroup,
363 outcome: result.runtime_outcome(),
364 },
365 Settlement::RequestPath(result) => Self {
366 operation: Operation::RequestPath,
367 outcome: result.runtime_outcome(),
368 },
369 Settlement::EstablishLink(result) => Self {
370 operation: Operation::EstablishLink,
371 outcome: result.runtime_outcome(),
372 },
373 Settlement::SendToLink(result) => Self {
374 operation: Operation::SendToLink,
375 outcome: result.runtime_outcome(),
376 },
377 Settlement::Identify(result) => Self {
378 operation: Operation::Identify,
379 outcome: result.runtime_outcome(),
380 },
381 Settlement::SendRequest(result) => Self {
382 operation: Operation::SendRequest,
383 outcome: result.runtime_outcome(),
384 },
385 Settlement::Respond(result) => Self {
386 operation: Operation::Respond,
387 outcome: result.runtime_outcome(),
388 },
389 Settlement::CloseLink(result) => Self {
390 operation: Operation::CloseLink,
391 outcome: result.runtime_outcome(),
392 },
393 Settlement::SendResource(result) => Self {
394 operation: Operation::SendResource,
395 outcome: result.runtime_outcome(),
396 },
397 Settlement::SetResourceStrategy(result) => Self {
398 operation: Operation::SetResourceStrategy,
399 outcome: result.runtime_outcome(),
400 },
401 Settlement::SendToChannel(result) => Self {
402 operation: Operation::SendToChannel,
403 outcome: result.runtime_outcome(),
404 },
405 Settlement::AllowRequester(result) => Self {
406 operation: Operation::AllowRequester,
407 outcome: result.runtime_outcome(),
408 },
409 }
410 }
411}
412
413impl From<&AnnounceNowFailure> for RuntimeOperationOutcome {
414 fn from(failure: &AnnounceNowFailure) -> Self {
415 match failure {
416 AnnounceNowFailure::Rejected(_) => Self::Rejected,
417 AnnounceNowFailure::WriteFailed(_) => Self::WriteFailed,
418 }
419 }
420}
421
422impl From<&SendSinglePacketFailure> for RuntimeOperationOutcome {
423 fn from(failure: &SendSinglePacketFailure) -> Self {
424 match failure {
425 SendSinglePacketFailure::Rejected(_) => Self::Rejected,
426 SendSinglePacketFailure::WriteFailed(_) => Self::WriteFailed,
427 SendSinglePacketFailure::Culled => Self::Culled,
428 SendSinglePacketFailure::Timeout => Self::Timeout,
429 }
430 }
431}
432
433impl From<&SendGroupFailure> for RuntimeOperationOutcome {
434 fn from(failure: &SendGroupFailure) -> Self {
435 match failure {
436 SendGroupFailure::Rejected(_) => Self::Rejected,
437 SendGroupFailure::WriteFailed(_) => Self::WriteFailed,
438 }
439 }
440}
441
442impl From<&RequestPathFailure> for RuntimeOperationOutcome {
443 fn from(failure: &RequestPathFailure) -> Self {
444 match failure {
445 RequestPathFailure::WriteFailed(_) => Self::WriteFailed,
446 RequestPathFailure::Timeout => Self::Timeout,
447 RequestPathFailure::Culled => Self::Culled,
448 }
449 }
450}
451
452impl From<&EstablishLinkFailure> for RuntimeOperationOutcome {
453 fn from(failure: &EstablishLinkFailure) -> Self {
454 match failure {
455 EstablishLinkFailure::Rejected(_) => Self::Rejected,
456 EstablishLinkFailure::WriteFailed(_) => Self::WriteFailed,
457 EstablishLinkFailure::Timeout => Self::Timeout,
458 }
459 }
460}
461
462impl From<&SendToLinkFailure> for RuntimeOperationOutcome {
463 fn from(failure: &SendToLinkFailure) -> Self {
464 match failure {
465 SendToLinkFailure::Rejected(_) => Self::Rejected,
466 SendToLinkFailure::WriteFailed(_) => Self::WriteFailed,
467 SendToLinkFailure::Culled => Self::Culled,
468 SendToLinkFailure::Timeout => Self::Timeout,
469 }
470 }
471}
472
473impl From<&IdentifyFailure> for RuntimeOperationOutcome {
474 fn from(failure: &IdentifyFailure) -> Self {
475 match failure {
476 IdentifyFailure::Rejected(_) => Self::Rejected,
477 IdentifyFailure::WriteFailed => Self::WriteFailed,
478 }
479 }
480}
481
482impl From<&SendRequestFailure> for RuntimeOperationOutcome {
483 fn from(failure: &SendRequestFailure) -> Self {
484 match failure {
485 SendRequestFailure::Rejected(_) => Self::Rejected,
486 SendRequestFailure::WriteFailed => Self::WriteFailed,
487 SendRequestFailure::Culled => Self::Culled,
488 SendRequestFailure::Timeout => Self::Timeout,
489 SendRequestFailure::ResponseTooLarge => Self::ResponseTooLarge,
490 }
491 }
492}
493
494impl From<&RespondFailure> for RuntimeOperationOutcome {
495 fn from(failure: &RespondFailure) -> Self {
496 match failure {
497 RespondFailure::Rejected(_) => Self::Rejected,
498 RespondFailure::WriteFailed => Self::WriteFailed,
499 RespondFailure::Resource(inner) => Self::from(inner),
500 }
501 }
502}
503
504impl From<&CloseLinkFailure> for RuntimeOperationOutcome {
505 fn from(failure: &CloseLinkFailure) -> Self {
506 match failure {
507 CloseLinkFailure::Rejected(_) => Self::Rejected,
508 CloseLinkFailure::WriteFailed => Self::WriteFailed,
509 }
510 }
511}
512
513impl From<&SendResourceFailure> for RuntimeOperationOutcome {
514 fn from(failure: &SendResourceFailure) -> Self {
515 match failure {
516 SendResourceFailure::Rejected(_) => Self::Rejected,
517 SendResourceFailure::WriteFailed => Self::WriteFailed,
518 SendResourceFailure::RejectedByPeer => Self::PeerRejected,
519 SendResourceFailure::Sequencing => Self::Sequencing,
520 SendResourceFailure::Timeout => Self::Timeout,
521 SendResourceFailure::PredecessorFailed => Self::DependencyFailed,
522 }
523 }
524}
525
526impl From<&SetResourceStrategyFailure> for RuntimeOperationOutcome {
527 fn from(failure: &SetResourceStrategyFailure) -> Self {
528 match failure {
529 SetResourceStrategyFailure::Rejected(_) => Self::Rejected,
530 }
531 }
532}
533
534impl From<&SendToChannelFailure> for RuntimeOperationOutcome {
535 fn from(failure: &SendToChannelFailure) -> Self {
536 match failure {
537 SendToChannelFailure::Rejected(_) => Self::Rejected,
538 SendToChannelFailure::WriteFailed(_) => Self::WriteFailed,
539 SendToChannelFailure::WindowFull => Self::Backpressure,
540 SendToChannelFailure::Untrackable => Self::Untrackable,
541 SendToChannelFailure::Timeout => Self::Timeout,
542 }
543 }
544}
545
546impl From<&AllowRequesterFailure> for RuntimeOperationOutcome {
547 fn from(failure: &AllowRequesterFailure) -> Self {
548 match failure {
549 AllowRequesterFailure::Rejected(_) => Self::Rejected,
550 }
551 }
552}
553
554impl From<ResourceFailureCause> for RuntimeResourceFailure {
555 fn from(cause: ResourceFailureCause) -> Self {
556 match cause {
557 ResourceFailureCause::CancelledBySender => Self::CancelledBySender,
558 ResourceFailureCause::RefusedHashmapUpdate(refusal) => match refusal {
559 ApplyHashmapUpdateError::BeyondPartCount => Self::HashmapBeyondPartCount,
560 ApplyHashmapUpdateError::SkipsAhead => Self::HashmapSkipsAhead,
561 ApplyHashmapUpdateError::HashmapTooLong => Self::HashmapTooLong,
562 ApplyHashmapUpdateError::HashmapRagged => Self::HashmapRagged,
563 },
564 ResourceFailureCause::RetriesExhausted => Self::RetriesExhausted,
565 ResourceFailureCause::LinkVanished => Self::LinkVanished,
566 ResourceFailureCause::TransferUnopenable => Self::TransferUnopenable,
567 ResourceFailureCause::TransferCorrupt => Self::TransferCorrupt,
568 ResourceFailureCause::ProofUnsendable => Self::ProofUnsendable,
569 ResourceFailureCause::DecompressionFailed => Self::DecompressionFailed,
570 ResourceFailureCause::DecompressionTimedOut => Self::DecompressionTimedOut,
571 ResourceFailureCause::OpenTimedOut => Self::OpenTimedOut,
572 ResourceFailureCause::MetadataOverrun => Self::MetadataOverrun,
573 }
574 }
575}
576
577impl From<LinkClosedReason> for RuntimeLinkClosure {
578 fn from(reason: LinkClosedReason) -> Self {
579 match reason {
580 LinkClosedReason::Timeout => Self::Timeout,
581 LinkClosedReason::PeerClosed => Self::PeerClosed,
582 LinkClosedReason::MalformedRtt => Self::MalformedRtt,
583 }
584 }
585}
586
587impl From<RouteRemovalCause> for RuntimeRouteRemoval {
588 fn from(cause: RouteRemovalCause) -> Self {
589 match cause {
590 RouteRemovalCause::Expired => Self::Expired,
591 RouteRemovalCause::Evicted => Self::Evicted,
592 RouteRemovalCause::InterfaceGone => Self::InterfaceGone,
593 RouteRemovalCause::Dropped => Self::Dropped,
594 }
595 }
596}
597
598#[cfg(test)]
599mod tests {
600 use super::*;
601 use crate::engine::{CommandId, SendRequestFailure, SendResourceFailure};
602
603 #[test]
604 fn journaled_command_settlements_are_counted_before_delivery() {
605 let mut snapshot = ReliabilityMetricsSnapshot::default();
606 snapshot.record_journaled(&Journaled::CommandSettled {
607 id: CommandId(1),
608 settlement: Settlement::SendRequest(Err(SendRequestFailure::Timeout)),
609 });
610 snapshot.record_journaled(&Journaled::CommandSettled {
611 id: CommandId(2),
612 settlement: Settlement::SendResource(Err(SendResourceFailure::RejectedByPeer)),
613 });
614
615 assert_eq!(
616 snapshot.operations.get(
617 RuntimeOperation::SendRequest,
618 RuntimeOperationOutcome::Timeout
619 ),
620 1
621 );
622 assert_eq!(
623 snapshot.operations.get(
624 RuntimeOperation::SendResource,
625 RuntimeOperationOutcome::PeerRejected
626 ),
627 1
628 );
629 }
630
631 #[test]
632 fn bounded_reliability_dimensions_cover_every_named_value() {
633 assert_eq!(
634 RuntimeOperation::ALL.len() * RuntimeOperationOutcome::ALL.len(),
635 RuntimeOperationCounts::default().iter().count()
636 );
637 assert_eq!(
638 RuntimeResourceFailure::ALL.len(),
639 RuntimeResourceFailureCounts::default().iter().count()
640 );
641 assert_eq!(
642 RuntimeLinkClosure::ALL.len(),
643 RuntimeLinkClosureCounts::default().iter().count()
644 );
645 assert_eq!(
646 RuntimeRouteRemoval::ALL.len(),
647 RuntimeRouteRemovalCounts::default().iter().count()
648 );
649 }
650
651 #[test]
652 fn nested_resource_and_maintenance_causes_keep_their_diagnostic_shape() {
653 assert_eq!(
654 RuntimeResourceFailure::from(ResourceFailureCause::RefusedHashmapUpdate(
655 ApplyHashmapUpdateError::SkipsAhead
656 )),
657 RuntimeResourceFailure::HashmapSkipsAhead
658 );
659 assert_eq!(
660 RuntimeLinkClosure::from(LinkClosedReason::MalformedRtt),
661 RuntimeLinkClosure::MalformedRtt
662 );
663 assert_eq!(
664 RuntimeRouteRemoval::from(RouteRemovalCause::InterfaceGone),
665 RuntimeRouteRemoval::InterfaceGone
666 );
667 assert_eq!(
668 RuntimeRouteRemoval::from(RouteRemovalCause::Dropped),
669 RuntimeRouteRemoval::Dropped
670 );
671 }
672}