1use crate::engine::{
2 AllowRequesterFailure, AnnounceNowFailure, CloseLinkFailure, EstablishLinkFailure,
3 IdentifyFailure, Journaled, LinkClosedReason, RequestPathFailure, RespondFailure,
4 RouteRemovalCause, SendGroupFailure, SendRequestFailure, SendResourceFailure,
5 SendSinglePacketFailure, SendToChannelFailure, SendToLinkFailure, SetResourceStrategyFailure,
6 Settlement,
7};
8use crate::routing::links::resources::table::ApplyHashmapUpdateError;
9use crate::routing::links::resources::ResourceFailureCause;
10
11prns_macros::iterable_enum! {
12 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
13 #[repr(u8)]
14 pub enum RuntimeOperation {
15 AnnounceNow,
16 SendSinglePacket,
17 SendGroup,
18 RequestPath,
19 EstablishLink,
20 SendToLink,
21 Identify,
22 SendRequest,
23 Respond,
24 CloseLink,
25 SendResource,
26 SetResourceStrategy,
27 SendToChannel,
28 AllowRequester,
29 }
30}
31
32impl RuntimeOperation {
33 const fn index(self) -> usize {
34 self as usize
35 }
36}
37
38prns_macros::iterable_enum! {
39 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
40 #[repr(u8)]
41 pub enum RuntimeOperationOutcome {
42 Succeeded,
43 Rejected,
44 WriteFailed,
45 Timeout,
46 Culled,
47 PeerRejected,
48 Sequencing,
49 DependencyFailed,
50 Backpressure,
51 Untrackable,
52 ResponseTooLarge,
53 }
54}
55
56impl RuntimeOperationOutcome {
57 const fn index(self) -> usize {
58 self as usize
59 }
60}
61
62#[derive(Debug, Clone, Copy, PartialEq, Eq)]
63pub struct RuntimeOperationCounts {
64 counts: [[u64; RuntimeOperationOutcome::ALL.len()]; RuntimeOperation::ALL.len()],
65}
66
67impl Default for RuntimeOperationCounts {
68 fn default() -> Self {
69 Self {
70 counts: [[0; RuntimeOperationOutcome::ALL.len()]; RuntimeOperation::ALL.len()],
71 }
72 }
73}
74
75impl RuntimeOperationCounts {
76 pub const fn get(&self, operation: RuntimeOperation, outcome: RuntimeOperationOutcome) -> u64 {
77 self.counts[operation.index()][outcome.index()]
78 }
79
80 pub fn iter(
81 &self,
82 ) -> impl Iterator<Item = (RuntimeOperation, RuntimeOperationOutcome, u64)> + '_ {
83 RuntimeOperation::ALL
84 .into_iter()
85 .flat_map(move |operation| {
86 RuntimeOperationOutcome::ALL
87 .into_iter()
88 .map(move |outcome| (operation, outcome, self.get(operation, outcome)))
89 })
90 }
91
92 fn record(&mut self, operation: RuntimeOperation, outcome: RuntimeOperationOutcome) {
93 let count = &mut self.counts[operation.index()][outcome.index()];
94 *count = count.saturating_add(1);
95 }
96}
97
98prns_macros::iterable_enum! {
99 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
100 #[repr(u8)]
101 pub enum RuntimeResourceFailure {
102 CancelledBySender,
103 HashmapBeyondPartCount,
104 HashmapSkipsAhead,
105 HashmapTooLong,
106 HashmapRagged,
107 RetriesExhausted,
108 LinkVanished,
109 TransferUnopenable,
110 TransferCorrupt,
111 ProofUnsendable,
112 DecompressionFailed,
113 DecompressionTimedOut,
114 OpenTimedOut,
115 MetadataOverrun,
116 }
117}
118
119impl RuntimeResourceFailure {
120 const fn index(self) -> usize {
121 self as usize
122 }
123}
124
125#[derive(Debug, Clone, Copy, PartialEq, Eq)]
126pub struct RuntimeResourceFailureCounts {
127 counts: [u64; RuntimeResourceFailure::ALL.len()],
128}
129
130impl Default for RuntimeResourceFailureCounts {
131 fn default() -> Self {
132 Self {
133 counts: [0; RuntimeResourceFailure::ALL.len()],
134 }
135 }
136}
137
138impl RuntimeResourceFailureCounts {
139 pub const fn get(&self, failure: RuntimeResourceFailure) -> u64 {
140 self.counts[failure.index()]
141 }
142
143 pub fn iter(&self) -> impl ExactSizeIterator<Item = (RuntimeResourceFailure, u64)> + '_ {
144 RuntimeResourceFailure::ALL
145 .into_iter()
146 .map(|failure| (failure, self.get(failure)))
147 }
148
149 fn record(&mut self, failure: RuntimeResourceFailure) {
150 let count = &mut self.counts[failure.index()];
151 *count = count.saturating_add(1);
152 }
153}
154
155prns_macros::iterable_enum! {
156 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
157 #[repr(u8)]
158 pub enum RuntimeLinkClosure {
159 Timeout,
160 PeerClosed,
161 MalformedRtt,
162 }
163}
164
165impl RuntimeLinkClosure {
166 const fn index(self) -> usize {
167 self as usize
168 }
169}
170
171#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
172pub struct RuntimeLinkClosureCounts {
173 counts: [u64; RuntimeLinkClosure::ALL.len()],
174}
175
176impl RuntimeLinkClosureCounts {
177 pub const fn get(&self, reason: RuntimeLinkClosure) -> u64 {
178 self.counts[reason.index()]
179 }
180
181 pub fn iter(&self) -> impl ExactSizeIterator<Item = (RuntimeLinkClosure, u64)> + '_ {
182 RuntimeLinkClosure::ALL
183 .into_iter()
184 .map(|reason| (reason, self.get(reason)))
185 }
186
187 fn record(&mut self, reason: RuntimeLinkClosure) {
188 let count = &mut self.counts[reason.index()];
189 *count = count.saturating_add(1);
190 }
191}
192
193prns_macros::iterable_enum! {
194 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
195 #[repr(u8)]
196 pub enum RuntimeRouteRemoval {
197 Expired,
198 Evicted,
199 InterfaceGone,
200 Dropped,
201 }
202}
203
204impl RuntimeRouteRemoval {
205 const fn index(self) -> usize {
206 self as usize
207 }
208}
209
210#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
211pub struct RuntimeRouteRemovalCounts {
212 counts: [u64; RuntimeRouteRemoval::ALL.len()],
213}
214
215impl RuntimeRouteRemovalCounts {
216 pub const fn get(&self, cause: RuntimeRouteRemoval) -> u64 {
217 self.counts[cause.index()]
218 }
219
220 pub fn iter(&self) -> impl ExactSizeIterator<Item = (RuntimeRouteRemoval, u64)> + '_ {
221 RuntimeRouteRemoval::ALL
222 .into_iter()
223 .map(|cause| (cause, self.get(cause)))
224 }
225
226 fn record(&mut self, cause: RuntimeRouteRemoval) {
227 let count = &mut self.counts[cause.index()];
228 *count = count.saturating_add(1);
229 }
230}
231
232#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
233pub struct ReliabilityMetricsSnapshot {
234 pub operations: RuntimeOperationCounts,
235 pub resource_failures: RuntimeResourceFailureCounts,
236 pub link_closures: RuntimeLinkClosureCounts,
237 pub link_interface_mismatches: u64,
238 pub route_removals: RuntimeRouteRemovalCounts,
239}
240
241impl ReliabilityMetricsSnapshot {
242 pub fn record_journaled(&mut self, journaled: &Journaled<'_>) {
243 match journaled {
244 Journaled::PersistenceFlushed { .. } | Journaled::PersistenceFlushFailed { .. } => {}
245 Journaled::CommandSettled { settlement, .. } => {
246 let settled = SettledOperation::from(settlement);
247 self.operations.record(settled.operation, settled.outcome);
248 }
249 Journaled::LinkClosed { reason, .. } => {
250 self.link_closures.record((*reason).into());
251 }
252 Journaled::LinkInterfaceMismatch { .. } => {
253 self.link_interface_mismatches = self.link_interface_mismatches.saturating_add(1);
254 }
255 Journaled::ResourceFailed { cause, .. } => {
256 self.resource_failures.record((*cause).into());
257 }
258 Journaled::RouteRemoved { cause, .. } => {
259 self.route_removals.record((*cause).into());
260 }
261 Journaled::AnnounceHeard { .. }
262 | Journaled::SelfRatchetRotated { .. }
263 | Journaled::AnnounceHeldDropped { .. }
264 | Journaled::Delivered(_)
265 | Journaled::LinkEstablished(_)
266 | Journaled::PeerIdentified { .. }
267 | Journaled::RequestReceived { .. }
268 | Journaled::ResponseReceived { .. }
269 | Journaled::ResponseSegmentReceived { .. }
270 | Journaled::ChannelMessageReceived { .. }
271 | Journaled::ResourceReceived { .. }
272 | Journaled::ResourceNeedsDecompression { .. }
273 | Journaled::ResourceSegmentReceived { .. }
274 | Journaled::ResourceAssembled { .. } => {}
275 }
276 }
277}
278
279#[derive(Debug, Clone, Copy, PartialEq, Eq)]
280struct SettledOperation {
281 operation: RuntimeOperation,
282 outcome: RuntimeOperationOutcome,
283}
284
285trait RuntimeOutcome {
286 fn runtime_outcome(&self) -> RuntimeOperationOutcome;
287}
288
289impl<Success, Failure> RuntimeOutcome for Result<Success, Failure>
290where
291 for<'failure> RuntimeOperationOutcome: From<&'failure Failure>,
292{
293 fn runtime_outcome(&self) -> RuntimeOperationOutcome {
294 match self {
295 Ok(_) => RuntimeOperationOutcome::Succeeded,
296 Err(failure) => RuntimeOperationOutcome::from(failure),
297 }
298 }
299}
300
301impl From<&Settlement> for SettledOperation {
302 fn from(settlement: &Settlement) -> Self {
303 use RuntimeOperation as Operation;
304
305 match settlement {
306 Settlement::AnnounceNow(result) => Self {
307 operation: Operation::AnnounceNow,
308 outcome: result.runtime_outcome(),
309 },
310 Settlement::SendSinglePacket(result) => Self {
311 operation: Operation::SendSinglePacket,
312 outcome: result.runtime_outcome(),
313 },
314 Settlement::SendGroup(result) => Self {
315 operation: Operation::SendGroup,
316 outcome: result.runtime_outcome(),
317 },
318 Settlement::RequestPath(result) => Self {
319 operation: Operation::RequestPath,
320 outcome: result.runtime_outcome(),
321 },
322 Settlement::EstablishLink(result) => Self {
323 operation: Operation::EstablishLink,
324 outcome: result.runtime_outcome(),
325 },
326 Settlement::SendToLink(result) => Self {
327 operation: Operation::SendToLink,
328 outcome: result.runtime_outcome(),
329 },
330 Settlement::Identify(result) => Self {
331 operation: Operation::Identify,
332 outcome: result.runtime_outcome(),
333 },
334 Settlement::SendRequest(result) => Self {
335 operation: Operation::SendRequest,
336 outcome: result.runtime_outcome(),
337 },
338 Settlement::Respond(result) => Self {
339 operation: Operation::Respond,
340 outcome: result.runtime_outcome(),
341 },
342 Settlement::CloseLink(result) => Self {
343 operation: Operation::CloseLink,
344 outcome: result.runtime_outcome(),
345 },
346 Settlement::SendResource(result) => Self {
347 operation: Operation::SendResource,
348 outcome: result.runtime_outcome(),
349 },
350 Settlement::SetResourceStrategy(result) => Self {
351 operation: Operation::SetResourceStrategy,
352 outcome: result.runtime_outcome(),
353 },
354 Settlement::SendToChannel(result) => Self {
355 operation: Operation::SendToChannel,
356 outcome: result.runtime_outcome(),
357 },
358 Settlement::AllowRequester(result) => Self {
359 operation: Operation::AllowRequester,
360 outcome: result.runtime_outcome(),
361 },
362 }
363 }
364}
365
366impl From<&AnnounceNowFailure> for RuntimeOperationOutcome {
367 fn from(failure: &AnnounceNowFailure) -> Self {
368 match failure {
369 AnnounceNowFailure::Rejected(_) => Self::Rejected,
370 AnnounceNowFailure::WriteFailed(_) => Self::WriteFailed,
371 }
372 }
373}
374
375impl From<&SendSinglePacketFailure> for RuntimeOperationOutcome {
376 fn from(failure: &SendSinglePacketFailure) -> Self {
377 match failure {
378 SendSinglePacketFailure::Rejected(_) => Self::Rejected,
379 SendSinglePacketFailure::WriteFailed(_) => Self::WriteFailed,
380 SendSinglePacketFailure::Culled => Self::Culled,
381 SendSinglePacketFailure::Timeout => Self::Timeout,
382 }
383 }
384}
385
386impl From<&SendGroupFailure> for RuntimeOperationOutcome {
387 fn from(failure: &SendGroupFailure) -> Self {
388 match failure {
389 SendGroupFailure::Rejected(_) => Self::Rejected,
390 SendGroupFailure::WriteFailed(_) => Self::WriteFailed,
391 }
392 }
393}
394
395impl From<&RequestPathFailure> for RuntimeOperationOutcome {
396 fn from(failure: &RequestPathFailure) -> Self {
397 match failure {
398 RequestPathFailure::WriteFailed(_) => Self::WriteFailed,
399 RequestPathFailure::Timeout => Self::Timeout,
400 RequestPathFailure::Culled => Self::Culled,
401 }
402 }
403}
404
405impl From<&EstablishLinkFailure> for RuntimeOperationOutcome {
406 fn from(failure: &EstablishLinkFailure) -> Self {
407 match failure {
408 EstablishLinkFailure::Rejected(_) => Self::Rejected,
409 EstablishLinkFailure::WriteFailed(_) => Self::WriteFailed,
410 EstablishLinkFailure::Timeout => Self::Timeout,
411 }
412 }
413}
414
415impl From<&SendToLinkFailure> for RuntimeOperationOutcome {
416 fn from(failure: &SendToLinkFailure) -> Self {
417 match failure {
418 SendToLinkFailure::Rejected(_) => Self::Rejected,
419 SendToLinkFailure::WriteFailed(_) => Self::WriteFailed,
420 SendToLinkFailure::Culled => Self::Culled,
421 SendToLinkFailure::Timeout => Self::Timeout,
422 }
423 }
424}
425
426impl From<&IdentifyFailure> for RuntimeOperationOutcome {
427 fn from(failure: &IdentifyFailure) -> Self {
428 match failure {
429 IdentifyFailure::Rejected(_) => Self::Rejected,
430 IdentifyFailure::WriteFailed => Self::WriteFailed,
431 }
432 }
433}
434
435impl From<&SendRequestFailure> for RuntimeOperationOutcome {
436 fn from(failure: &SendRequestFailure) -> Self {
437 match failure {
438 SendRequestFailure::Rejected(_) => Self::Rejected,
439 SendRequestFailure::WriteFailed => Self::WriteFailed,
440 SendRequestFailure::Culled => Self::Culled,
441 SendRequestFailure::Timeout => Self::Timeout,
442 SendRequestFailure::ResponseTooLarge => Self::ResponseTooLarge,
443 SendRequestFailure::ResourceCapacity => Self::Backpressure,
444 }
445 }
446}
447
448impl From<&RespondFailure> for RuntimeOperationOutcome {
449 fn from(failure: &RespondFailure) -> Self {
450 match failure {
451 RespondFailure::Rejected(_) => Self::Rejected,
452 RespondFailure::WriteFailed => Self::WriteFailed,
453 RespondFailure::Resource(inner) => Self::from(inner),
454 }
455 }
456}
457
458impl From<&CloseLinkFailure> for RuntimeOperationOutcome {
459 fn from(failure: &CloseLinkFailure) -> Self {
460 match failure {
461 CloseLinkFailure::Rejected(_) => Self::Rejected,
462 CloseLinkFailure::WriteFailed => Self::WriteFailed,
463 }
464 }
465}
466
467impl From<&SendResourceFailure> for RuntimeOperationOutcome {
468 fn from(failure: &SendResourceFailure) -> Self {
469 match failure {
470 SendResourceFailure::Rejected(_) => Self::Rejected,
471 SendResourceFailure::WriteFailed => Self::WriteFailed,
472 SendResourceFailure::RejectedByPeer => Self::PeerRejected,
473 SendResourceFailure::Sequencing => Self::Sequencing,
474 SendResourceFailure::Timeout => Self::Timeout,
475 SendResourceFailure::PredecessorFailed => Self::DependencyFailed,
476 }
477 }
478}
479
480impl From<&SetResourceStrategyFailure> for RuntimeOperationOutcome {
481 fn from(failure: &SetResourceStrategyFailure) -> Self {
482 match failure {
483 SetResourceStrategyFailure::Rejected(_) => Self::Rejected,
484 }
485 }
486}
487
488impl From<&SendToChannelFailure> for RuntimeOperationOutcome {
489 fn from(failure: &SendToChannelFailure) -> Self {
490 match failure {
491 SendToChannelFailure::Rejected(_) => Self::Rejected,
492 SendToChannelFailure::WriteFailed(_) => Self::WriteFailed,
493 SendToChannelFailure::WindowFull => Self::Backpressure,
494 SendToChannelFailure::Untrackable => Self::Untrackable,
495 SendToChannelFailure::Timeout => Self::Timeout,
496 }
497 }
498}
499
500impl From<&AllowRequesterFailure> for RuntimeOperationOutcome {
501 fn from(failure: &AllowRequesterFailure) -> Self {
502 match failure {
503 AllowRequesterFailure::Rejected(_) => Self::Rejected,
504 }
505 }
506}
507
508impl From<ResourceFailureCause> for RuntimeResourceFailure {
509 fn from(cause: ResourceFailureCause) -> Self {
510 match cause {
511 ResourceFailureCause::CancelledBySender => Self::CancelledBySender,
512 ResourceFailureCause::RefusedHashmapUpdate(refusal) => match refusal {
513 ApplyHashmapUpdateError::BeyondPartCount => Self::HashmapBeyondPartCount,
514 ApplyHashmapUpdateError::SkipsAhead => Self::HashmapSkipsAhead,
515 ApplyHashmapUpdateError::HashmapTooLong => Self::HashmapTooLong,
516 ApplyHashmapUpdateError::HashmapRagged => Self::HashmapRagged,
517 },
518 ResourceFailureCause::RetriesExhausted => Self::RetriesExhausted,
519 ResourceFailureCause::LinkVanished => Self::LinkVanished,
520 ResourceFailureCause::TransferUnopenable => Self::TransferUnopenable,
521 ResourceFailureCause::TransferCorrupt => Self::TransferCorrupt,
522 ResourceFailureCause::ProofUnsendable => Self::ProofUnsendable,
523 ResourceFailureCause::DecompressionFailed => Self::DecompressionFailed,
524 ResourceFailureCause::DecompressionTimedOut => Self::DecompressionTimedOut,
525 ResourceFailureCause::OpenTimedOut => Self::OpenTimedOut,
526 ResourceFailureCause::MetadataOverrun => Self::MetadataOverrun,
527 }
528 }
529}
530
531impl From<LinkClosedReason> for RuntimeLinkClosure {
532 fn from(reason: LinkClosedReason) -> Self {
533 match reason {
534 LinkClosedReason::Timeout => Self::Timeout,
535 LinkClosedReason::PeerClosed => Self::PeerClosed,
536 LinkClosedReason::MalformedRtt => Self::MalformedRtt,
537 }
538 }
539}
540
541impl From<RouteRemovalCause> for RuntimeRouteRemoval {
542 fn from(cause: RouteRemovalCause) -> Self {
543 match cause {
544 RouteRemovalCause::Expired => Self::Expired,
545 RouteRemovalCause::Evicted => Self::Evicted,
546 RouteRemovalCause::InterfaceGone => Self::InterfaceGone,
547 RouteRemovalCause::Dropped => Self::Dropped,
548 }
549 }
550}
551
552#[cfg(test)]
553mod tests {
554 use super::*;
555 use crate::engine::{CommandId, SendRequestFailure, SendResourceFailure};
556
557 #[test]
558 fn journaled_command_settlements_are_counted_before_delivery() {
559 let mut snapshot = ReliabilityMetricsSnapshot::default();
560 snapshot.record_journaled(&Journaled::CommandSettled {
561 id: CommandId(1),
562 settlement: Settlement::SendRequest(Err(SendRequestFailure::Timeout)),
563 });
564 snapshot.record_journaled(&Journaled::CommandSettled {
565 id: CommandId(2),
566 settlement: Settlement::SendResource(Err(SendResourceFailure::RejectedByPeer)),
567 });
568
569 assert_eq!(
570 snapshot.operations.get(
571 RuntimeOperation::SendRequest,
572 RuntimeOperationOutcome::Timeout
573 ),
574 1
575 );
576 assert_eq!(
577 snapshot.operations.get(
578 RuntimeOperation::SendResource,
579 RuntimeOperationOutcome::PeerRejected
580 ),
581 1
582 );
583 }
584
585 #[test]
586 fn response_resource_capacity_is_reported_as_backpressure() {
587 assert_eq!(
588 RuntimeOperationOutcome::from(&SendRequestFailure::ResourceCapacity),
589 RuntimeOperationOutcome::Backpressure,
590 );
591 }
592
593 #[test]
594 fn bounded_reliability_dimensions_cover_every_named_value() {
595 assert_eq!(
596 RuntimeOperation::ALL.len() * RuntimeOperationOutcome::ALL.len(),
597 RuntimeOperationCounts::default().iter().count()
598 );
599 assert_eq!(
600 RuntimeResourceFailure::ALL.len(),
601 RuntimeResourceFailureCounts::default().iter().count()
602 );
603 assert_eq!(
604 RuntimeLinkClosure::ALL.len(),
605 RuntimeLinkClosureCounts::default().iter().count()
606 );
607 assert_eq!(
608 RuntimeRouteRemoval::ALL.len(),
609 RuntimeRouteRemovalCounts::default().iter().count()
610 );
611 }
612
613 #[test]
614 fn nested_resource_and_maintenance_causes_keep_their_diagnostic_shape() {
615 assert_eq!(
616 RuntimeResourceFailure::from(ResourceFailureCause::RefusedHashmapUpdate(
617 ApplyHashmapUpdateError::SkipsAhead
618 )),
619 RuntimeResourceFailure::HashmapSkipsAhead
620 );
621 assert_eq!(
622 RuntimeLinkClosure::from(LinkClosedReason::MalformedRtt),
623 RuntimeLinkClosure::MalformedRtt
624 );
625 assert_eq!(
626 RuntimeRouteRemoval::from(RouteRemovalCause::InterfaceGone),
627 RuntimeRouteRemoval::InterfaceGone
628 );
629 assert_eq!(
630 RuntimeRouteRemoval::from(RouteRemovalCause::Dropped),
631 RuntimeRouteRemoval::Dropped
632 );
633 }
634}