1use serde::{Deserialize, Deserializer, Serialize, Serializer};
9use std::fmt;
10
11pub const ATP_AUTOTUNE_METRIC_NAMES: [AtpAutotuneMetric; 14] = [
16 AtpAutotuneMetric::RttMicros,
17 AtpAutotuneMetric::LossPermille,
18 AtpAutotuneMetric::PtoMicros,
19 AtpAutotuneMetric::CongestionWindowBytes,
20 AtpAutotuneMetric::InFlightBytes,
21 AtpAutotuneMetric::SendBufferQueuedBytes,
22 AtpAutotuneMetric::ReceiveBufferQueuedBytes,
23 AtpAutotuneMetric::DiskReadLagMicros,
24 AtpAutotuneMetric::DiskWriteLagMicros,
25 AtpAutotuneMetric::EncodeBacklogSymbols,
26 AtpAutotuneMetric::DecodeBacklogSymbols,
27 AtpAutotuneMetric::RepairRoiPermille,
28 AtpAutotuneMetric::RelayCostMicrosPerMiB,
29 AtpAutotuneMetric::MigrationEvents,
30];
31
32#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
34pub enum AtpAutotuneMetric {
35 RttMicros,
37 LossPermille,
39 PtoMicros,
41 CongestionWindowBytes,
43 InFlightBytes,
45 SendBufferQueuedBytes,
47 ReceiveBufferQueuedBytes,
49 DiskReadLagMicros,
51 DiskWriteLagMicros,
53 EncodeBacklogSymbols,
55 DecodeBacklogSymbols,
57 RepairRoiPermille,
59 RelayCostMicrosPerMiB,
61 MigrationEvents,
63}
64
65impl AtpAutotuneMetric {
66 #[must_use]
68 pub const fn as_str(self) -> &'static str {
69 match self {
70 Self::RttMicros => "atp.autotune.rtt_micros",
71 Self::LossPermille => "atp.autotune.loss_permille",
72 Self::PtoMicros => "atp.autotune.pto_micros",
73 Self::CongestionWindowBytes => "atp.autotune.congestion_window_bytes",
74 Self::InFlightBytes => "atp.autotune.in_flight_bytes",
75 Self::SendBufferQueuedBytes => "atp.autotune.send_buffer_queued_bytes",
76 Self::ReceiveBufferQueuedBytes => "atp.autotune.receive_buffer_queued_bytes",
77 Self::DiskReadLagMicros => "atp.autotune.disk_read_lag_micros",
78 Self::DiskWriteLagMicros => "atp.autotune.disk_write_lag_micros",
79 Self::EncodeBacklogSymbols => "atp.autotune.encode_backlog_symbols",
80 Self::DecodeBacklogSymbols => "atp.autotune.decode_backlog_symbols",
81 Self::RepairRoiPermille => "atp.autotune.repair_roi_permille",
82 Self::RelayCostMicrosPerMiB => "atp.autotune.relay_cost_micros_per_mib",
83 Self::MigrationEvents => "atp.autotune.migration_events",
84 }
85 }
86
87 #[must_use]
89 pub fn from_name(name: &str) -> Option<Self> {
90 match name {
91 "atp.autotune.rtt_micros" => Some(Self::RttMicros),
92 "atp.autotune.loss_permille" => Some(Self::LossPermille),
93 "atp.autotune.pto_micros" => Some(Self::PtoMicros),
94 "atp.autotune.congestion_window_bytes" => Some(Self::CongestionWindowBytes),
95 "atp.autotune.in_flight_bytes" => Some(Self::InFlightBytes),
96 "atp.autotune.send_buffer_queued_bytes" => Some(Self::SendBufferQueuedBytes),
97 "atp.autotune.receive_buffer_queued_bytes" => Some(Self::ReceiveBufferQueuedBytes),
98 "atp.autotune.disk_read_lag_micros" => Some(Self::DiskReadLagMicros),
99 "atp.autotune.disk_write_lag_micros" => Some(Self::DiskWriteLagMicros),
100 "atp.autotune.encode_backlog_symbols" => Some(Self::EncodeBacklogSymbols),
101 "atp.autotune.decode_backlog_symbols" => Some(Self::DecodeBacklogSymbols),
102 "atp.autotune.repair_roi_permille" => Some(Self::RepairRoiPermille),
103 "atp.autotune.relay_cost_micros_per_mib" => Some(Self::RelayCostMicrosPerMiB),
104 "atp.autotune.migration_events" => Some(Self::MigrationEvents),
105 _ => None,
106 }
107 }
108}
109
110impl Serialize for AtpAutotuneMetric {
111 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
112 where
113 S: Serializer,
114 {
115 serializer.serialize_str(self.as_str())
116 }
117}
118
119impl<'de> Deserialize<'de> for AtpAutotuneMetric {
120 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
121 where
122 D: Deserializer<'de>,
123 {
124 let name = String::deserialize(deserializer)?;
125 Self::from_name(&name).ok_or_else(|| {
126 serde::de::Error::unknown_variant(
127 &name,
128 &[
129 "atp.autotune.rtt_micros",
130 "atp.autotune.loss_permille",
131 "atp.autotune.pto_micros",
132 "atp.autotune.congestion_window_bytes",
133 "atp.autotune.in_flight_bytes",
134 "atp.autotune.send_buffer_queued_bytes",
135 "atp.autotune.receive_buffer_queued_bytes",
136 "atp.autotune.disk_read_lag_micros",
137 "atp.autotune.disk_write_lag_micros",
138 "atp.autotune.encode_backlog_symbols",
139 "atp.autotune.decode_backlog_symbols",
140 "atp.autotune.repair_roi_permille",
141 "atp.autotune.relay_cost_micros_per_mib",
142 "atp.autotune.migration_events",
143 ],
144 )
145 })
146 }
147}
148
149#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
151pub struct AtpAutotuneMetricSample {
152 pub metric: AtpAutotuneMetric,
154 pub value: u64,
156}
157
158impl AtpAutotuneMetricSample {
159 #[must_use]
161 pub const fn new(metric: AtpAutotuneMetric, value: u64) -> Self {
162 Self { metric, value }
163 }
164}
165
166#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
171pub struct AtpAutotuneTelemetryReport {
172 pub trace_id: String,
174 pub workload_id: String,
176 pub sample_count: u32,
178 pub samples: Vec<AtpAutotuneMetricSample>,
180}
181
182impl AtpAutotuneTelemetryReport {
183 #[must_use]
185 pub fn new(trace_id: impl Into<String>, workload_id: impl Into<String>) -> Self {
186 Self {
187 trace_id: trace_id.into(),
188 workload_id: workload_id.into(),
189 sample_count: 0,
190 samples: Vec::new(),
191 }
192 }
193
194 #[must_use]
196 pub const fn with_sample_count(mut self, sample_count: u32) -> Self {
197 self.sample_count = sample_count;
198 self
199 }
200
201 #[must_use]
203 pub fn with_sample(mut self, metric: AtpAutotuneMetric, value: u64) -> Self {
204 self.samples
205 .push(AtpAutotuneMetricSample::new(metric, value));
206 self
207 }
208
209 #[must_use]
211 pub fn from_telemetry(telemetry: &AtpAutotuneTelemetry) -> Self {
212 telemetry.to_report()
213 }
214
215 pub fn into_telemetry(self) -> Result<AtpAutotuneTelemetry, AtpAutotuneTelemetryError> {
220 let sample_count = if self.sample_count == 0 {
221 u32::try_from(self.samples.len()).unwrap_or(u32::MAX)
222 } else {
223 self.sample_count
224 };
225 let mut telemetry = AtpAutotuneTelemetry::new(self.trace_id, self.workload_id)
226 .with_sample_count(sample_count);
227 for sample in self.samples {
228 telemetry.record_metric(sample.metric, sample.value)?;
229 }
230 Ok(telemetry)
231 }
232}
233
234#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
241pub struct AtpTransferPressureSnapshot {
242 pub trace_id: String,
244 pub transfer_id: String,
246 pub sample_count: u32,
248 pub rtt_micros: Option<u64>,
250 pub loss_permille: Option<u16>,
252 pub pto_micros: Option<u64>,
254 pub congestion_window_bytes: Option<u64>,
256 pub in_flight_bytes: Option<u64>,
258 pub send_buffer_queued_bytes: Option<u64>,
260 pub receive_buffer_queued_bytes: Option<u64>,
262 pub disk_read_lag_micros: Option<u64>,
264 pub disk_write_lag_micros: Option<u64>,
266 pub encode_backlog_symbols: Option<u32>,
268 pub decode_backlog_symbols: Option<u32>,
270 pub repair_symbols_sent: Option<u32>,
272 pub useful_repair_symbols: Option<u32>,
274 pub relay_cost_micros: Option<u64>,
276 pub relay_bytes: Option<u64>,
278 pub migration_events: Option<u32>,
280}
281
282impl AtpTransferPressureSnapshot {
283 #[must_use]
285 pub fn new(trace_id: impl Into<String>, transfer_id: impl Into<String>) -> Self {
286 Self {
287 trace_id: trace_id.into(),
288 transfer_id: transfer_id.into(),
289 sample_count: 0,
290 rtt_micros: None,
291 loss_permille: None,
292 pto_micros: None,
293 congestion_window_bytes: None,
294 in_flight_bytes: None,
295 send_buffer_queued_bytes: None,
296 receive_buffer_queued_bytes: None,
297 disk_read_lag_micros: None,
298 disk_write_lag_micros: None,
299 encode_backlog_symbols: None,
300 decode_backlog_symbols: None,
301 repair_symbols_sent: None,
302 useful_repair_symbols: None,
303 relay_cost_micros: None,
304 relay_bytes: None,
305 migration_events: None,
306 }
307 }
308
309 #[must_use]
311 pub const fn with_sample_count(mut self, sample_count: u32) -> Self {
312 self.sample_count = sample_count;
313 self
314 }
315
316 #[must_use]
318 pub fn repair_roi_permille(&self) -> Option<u16> {
319 let sent = self.repair_symbols_sent?;
320 if sent == 0 {
321 return None;
322 }
323 let useful = u64::from(self.useful_repair_symbols.unwrap_or(0));
324 let roi = useful.saturating_mul(1_000) / u64::from(sent);
325 Some(roi.min(u64::from(u16::MAX)) as u16)
326 }
327
328 #[must_use]
330 pub fn relay_cost_micros_per_mib(&self) -> Option<u64> {
331 let bytes = self.relay_bytes?;
332 if bytes == 0 {
333 return None;
334 }
335 let cost = self.relay_cost_micros?;
336 Some(cost.saturating_mul(1_048_576) / bytes)
337 }
338
339 #[must_use]
341 pub fn to_report(&self) -> AtpAutotuneTelemetryReport {
342 let mut report =
343 AtpAutotuneTelemetryReport::new(self.trace_id.clone(), self.transfer_id.clone())
344 .with_sample_count(self.sample_count);
345
346 if let Some(value) = self.rtt_micros {
347 report = report.with_sample(AtpAutotuneMetric::RttMicros, value);
348 }
349 if let Some(value) = self.loss_permille {
350 report = report.with_sample(AtpAutotuneMetric::LossPermille, u64::from(value));
351 }
352 if let Some(value) = self.pto_micros {
353 report = report.with_sample(AtpAutotuneMetric::PtoMicros, value);
354 }
355 if let Some(value) = self.congestion_window_bytes {
356 report = report.with_sample(AtpAutotuneMetric::CongestionWindowBytes, value);
357 }
358 if let Some(value) = self.in_flight_bytes {
359 report = report.with_sample(AtpAutotuneMetric::InFlightBytes, value);
360 }
361 if let Some(value) = self.send_buffer_queued_bytes {
362 report = report.with_sample(AtpAutotuneMetric::SendBufferQueuedBytes, value);
363 }
364 if let Some(value) = self.receive_buffer_queued_bytes {
365 report = report.with_sample(AtpAutotuneMetric::ReceiveBufferQueuedBytes, value);
366 }
367 if let Some(value) = self.disk_read_lag_micros {
368 report = report.with_sample(AtpAutotuneMetric::DiskReadLagMicros, value);
369 }
370 if let Some(value) = self.disk_write_lag_micros {
371 report = report.with_sample(AtpAutotuneMetric::DiskWriteLagMicros, value);
372 }
373 if let Some(value) = self.encode_backlog_symbols {
374 report = report.with_sample(AtpAutotuneMetric::EncodeBacklogSymbols, u64::from(value));
375 }
376 if let Some(value) = self.decode_backlog_symbols {
377 report = report.with_sample(AtpAutotuneMetric::DecodeBacklogSymbols, u64::from(value));
378 }
379 if let Some(value) = self.repair_roi_permille() {
380 report = report.with_sample(AtpAutotuneMetric::RepairRoiPermille, u64::from(value));
381 }
382 if let Some(value) = self.relay_cost_micros_per_mib() {
383 report = report.with_sample(AtpAutotuneMetric::RelayCostMicrosPerMiB, value);
384 }
385 if let Some(value) = self.migration_events {
386 report = report.with_sample(AtpAutotuneMetric::MigrationEvents, u64::from(value));
387 }
388
389 report
390 }
391
392 pub fn into_telemetry(self) -> Result<AtpAutotuneTelemetry, AtpAutotuneTelemetryError> {
394 self.to_report().into_telemetry()
395 }
396}
397
398#[derive(Debug, Clone, Copy, PartialEq, Eq)]
400pub enum AtpAutotuneTelemetryError {
401 MetricValueOutOfRange {
403 metric: AtpAutotuneMetric,
405 value: u64,
407 max: u64,
409 },
410}
411
412impl fmt::Display for AtpAutotuneTelemetryError {
413 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
414 match self {
415 Self::MetricValueOutOfRange { metric, value, max } => write!(
416 f,
417 "ATP autotune metric {} value {} exceeds maximum {}",
418 metric.as_str(),
419 value,
420 max
421 ),
422 }
423 }
424}
425
426impl std::error::Error for AtpAutotuneTelemetryError {}
427
428#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
430pub struct AtpAutotuneSettings {
431 pub in_flight_bytes: u64,
433 pub stream_count: u16,
435 pub chunk_size_bytes: u32,
437 pub repair_symbols_per_second: u32,
439}
440
441impl AtpAutotuneSettings {
442 #[must_use]
444 pub const fn new(
445 in_flight_bytes: u64,
446 stream_count: u16,
447 chunk_size_bytes: u32,
448 repair_symbols_per_second: u32,
449 ) -> Self {
450 Self {
451 in_flight_bytes,
452 stream_count,
453 chunk_size_bytes,
454 repair_symbols_per_second,
455 }
456 }
457}
458
459impl Default for AtpAutotuneSettings {
460 fn default() -> Self {
461 Self {
462 in_flight_bytes: 8 * 1_048_576,
463 stream_count: 4,
464 chunk_size_bytes: 256 * 1_024,
465 repair_symbols_per_second: 256,
466 }
467 }
468}
469
470#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
472pub struct AtpAutotuneLimits {
473 pub min_in_flight_bytes: u64,
475 pub max_in_flight_bytes: u64,
477 pub min_stream_count: u16,
479 pub max_stream_count: u16,
481 pub min_chunk_size_bytes: u32,
483 pub max_chunk_size_bytes: u32,
485 pub min_repair_symbols_per_second: u32,
487 pub max_repair_symbols_per_second: u32,
489}
490
491impl Default for AtpAutotuneLimits {
492 fn default() -> Self {
493 Self {
494 min_in_flight_bytes: 1_048_576,
495 max_in_flight_bytes: 512 * 1_048_576,
496 min_stream_count: 1,
497 max_stream_count: 64,
498 min_chunk_size_bytes: 64 * 1_024,
499 max_chunk_size_bytes: 8 * 1_048_576,
500 min_repair_symbols_per_second: 0,
501 max_repair_symbols_per_second: 16_384,
502 }
503 }
504}
505
506impl AtpAutotuneLimits {
507 #[must_use]
509 pub fn clamp(self, settings: AtpAutotuneSettings) -> AtpAutotuneSettings {
510 AtpAutotuneSettings {
511 in_flight_bytes: settings
512 .in_flight_bytes
513 .clamp(self.min_in_flight_bytes, self.max_in_flight_bytes),
514 stream_count: settings
515 .stream_count
516 .clamp(self.min_stream_count, self.max_stream_count),
517 chunk_size_bytes: settings
518 .chunk_size_bytes
519 .clamp(self.min_chunk_size_bytes, self.max_chunk_size_bytes),
520 repair_symbols_per_second: settings.repair_symbols_per_second.clamp(
521 self.min_repair_symbols_per_second,
522 self.max_repair_symbols_per_second,
523 ),
524 }
525 }
526}
527
528#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
530pub struct AtpAutotuneTelemetry {
531 pub trace_id: String,
533 pub workload_id: String,
535 pub sample_count: u32,
537 pub rtt_micros: Option<u64>,
539 pub loss_permille: Option<u16>,
541 pub pto_micros: Option<u64>,
543 pub congestion_window_bytes: Option<u64>,
545 pub in_flight_bytes: Option<u64>,
547 pub send_buffer_queued_bytes: Option<u64>,
549 pub receive_buffer_queued_bytes: Option<u64>,
551 pub disk_read_lag_micros: Option<u64>,
553 pub disk_write_lag_micros: Option<u64>,
555 pub encode_backlog_symbols: Option<u32>,
557 pub decode_backlog_symbols: Option<u32>,
559 pub repair_roi_permille: Option<u16>,
561 pub relay_cost_micros_per_mib: Option<u64>,
563 pub migration_events: Option<u32>,
565}
566
567impl AtpAutotuneTelemetry {
568 #[must_use]
570 pub fn new(trace_id: impl Into<String>, workload_id: impl Into<String>) -> Self {
571 Self {
572 trace_id: trace_id.into(),
573 workload_id: workload_id.into(),
574 sample_count: 0,
575 rtt_micros: None,
576 loss_permille: None,
577 pto_micros: None,
578 congestion_window_bytes: None,
579 in_flight_bytes: None,
580 send_buffer_queued_bytes: None,
581 receive_buffer_queued_bytes: None,
582 disk_read_lag_micros: None,
583 disk_write_lag_micros: None,
584 encode_backlog_symbols: None,
585 decode_backlog_symbols: None,
586 repair_roi_permille: None,
587 relay_cost_micros_per_mib: None,
588 migration_events: None,
589 }
590 }
591
592 #[must_use]
594 pub const fn with_sample_count(mut self, sample_count: u32) -> Self {
595 self.sample_count = sample_count;
596 self
597 }
598
599 #[must_use]
606 pub fn to_report(&self) -> AtpAutotuneTelemetryReport {
607 let mut report =
608 AtpAutotuneTelemetryReport::new(self.trace_id.clone(), self.workload_id.clone())
609 .with_sample_count(self.sample_count);
610
611 if let Some(value) = self.rtt_micros {
612 report = report.with_sample(AtpAutotuneMetric::RttMicros, value);
613 }
614 if let Some(value) = self.loss_permille {
615 report = report.with_sample(AtpAutotuneMetric::LossPermille, u64::from(value));
616 }
617 if let Some(value) = self.pto_micros {
618 report = report.with_sample(AtpAutotuneMetric::PtoMicros, value);
619 }
620 if let Some(value) = self.congestion_window_bytes {
621 report = report.with_sample(AtpAutotuneMetric::CongestionWindowBytes, value);
622 }
623 if let Some(value) = self.in_flight_bytes {
624 report = report.with_sample(AtpAutotuneMetric::InFlightBytes, value);
625 }
626 if let Some(value) = self.send_buffer_queued_bytes {
627 report = report.with_sample(AtpAutotuneMetric::SendBufferQueuedBytes, value);
628 }
629 if let Some(value) = self.receive_buffer_queued_bytes {
630 report = report.with_sample(AtpAutotuneMetric::ReceiveBufferQueuedBytes, value);
631 }
632 if let Some(value) = self.disk_read_lag_micros {
633 report = report.with_sample(AtpAutotuneMetric::DiskReadLagMicros, value);
634 }
635 if let Some(value) = self.disk_write_lag_micros {
636 report = report.with_sample(AtpAutotuneMetric::DiskWriteLagMicros, value);
637 }
638 if let Some(value) = self.encode_backlog_symbols {
639 report = report.with_sample(AtpAutotuneMetric::EncodeBacklogSymbols, u64::from(value));
640 }
641 if let Some(value) = self.decode_backlog_symbols {
642 report = report.with_sample(AtpAutotuneMetric::DecodeBacklogSymbols, u64::from(value));
643 }
644 if let Some(value) = self.repair_roi_permille {
645 report = report.with_sample(AtpAutotuneMetric::RepairRoiPermille, u64::from(value));
646 }
647 if let Some(value) = self.relay_cost_micros_per_mib {
648 report = report.with_sample(AtpAutotuneMetric::RelayCostMicrosPerMiB, value);
649 }
650 if let Some(value) = self.migration_events {
651 report = report.with_sample(AtpAutotuneMetric::MigrationEvents, u64::from(value));
652 }
653
654 report
655 }
656
657 pub fn record_metric(
659 &mut self,
660 metric: AtpAutotuneMetric,
661 value: u64,
662 ) -> Result<(), AtpAutotuneTelemetryError> {
663 match metric {
664 AtpAutotuneMetric::RttMicros => self.rtt_micros = Some(value),
665 AtpAutotuneMetric::LossPermille => {
666 self.loss_permille = Some(narrow_u16_metric(metric, value)?);
667 }
668 AtpAutotuneMetric::PtoMicros => self.pto_micros = Some(value),
669 AtpAutotuneMetric::CongestionWindowBytes => {
670 self.congestion_window_bytes = Some(value);
671 }
672 AtpAutotuneMetric::InFlightBytes => self.in_flight_bytes = Some(value),
673 AtpAutotuneMetric::SendBufferQueuedBytes => {
674 self.send_buffer_queued_bytes = Some(value);
675 }
676 AtpAutotuneMetric::ReceiveBufferQueuedBytes => {
677 self.receive_buffer_queued_bytes = Some(value);
678 }
679 AtpAutotuneMetric::DiskReadLagMicros => self.disk_read_lag_micros = Some(value),
680 AtpAutotuneMetric::DiskWriteLagMicros => self.disk_write_lag_micros = Some(value),
681 AtpAutotuneMetric::EncodeBacklogSymbols => {
682 self.encode_backlog_symbols = Some(narrow_u32_metric(metric, value)?);
683 }
684 AtpAutotuneMetric::DecodeBacklogSymbols => {
685 self.decode_backlog_symbols = Some(narrow_u32_metric(metric, value)?);
686 }
687 AtpAutotuneMetric::RepairRoiPermille => {
688 self.repair_roi_permille = Some(narrow_u16_metric(metric, value)?);
689 }
690 AtpAutotuneMetric::RelayCostMicrosPerMiB => {
691 self.relay_cost_micros_per_mib = Some(value);
692 }
693 AtpAutotuneMetric::MigrationEvents => {
694 self.migration_events = Some(narrow_u32_metric(metric, value)?);
695 }
696 }
697 Ok(())
698 }
699}
700
701fn narrow_u16_metric(
702 metric: AtpAutotuneMetric,
703 value: u64,
704) -> Result<u16, AtpAutotuneTelemetryError> {
705 u16::try_from(value).map_err(|_| AtpAutotuneTelemetryError::MetricValueOutOfRange {
706 metric,
707 value,
708 max: u64::from(u16::MAX),
709 })
710}
711
712fn narrow_u32_metric(
713 metric: AtpAutotuneMetric,
714 value: u64,
715) -> Result<u32, AtpAutotuneTelemetryError> {
716 u32::try_from(value).map_err(|_| AtpAutotuneTelemetryError::MetricValueOutOfRange {
717 metric,
718 value,
719 max: u64::from(u32::MAX),
720 })
721}
722
723#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
725pub enum AtpRepairPathMode {
726 Direct,
728 RelayOnly,
730 DirectAndRelay,
732}
733
734impl AtpRepairPathMode {
735 #[must_use]
737 pub const fn as_str(self) -> &'static str {
738 match self {
739 Self::Direct => "direct",
740 Self::RelayOnly => "relay_only",
741 Self::DirectAndRelay => "direct_and_relay",
742 }
743 }
744
745 const fn relay_available(self) -> bool {
746 matches!(self, Self::RelayOnly | Self::DirectAndRelay)
747 }
748}
749
750#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
752pub enum AtpRepairAction {
753 NoRepair,
755 ExactRetransmit,
757 ParityTrickle,
759 BurstRepair,
761 MultiPeerRepair,
763 RelayOnlyRepair,
765}
766
767impl AtpRepairAction {
768 #[must_use]
770 pub const fn as_str(self) -> &'static str {
771 match self {
772 Self::NoRepair => "no_repair",
773 Self::ExactRetransmit => "exact_retransmit",
774 Self::ParityTrickle => "parity_trickle",
775 Self::BurstRepair => "burst_repair",
776 Self::MultiPeerRepair => "multi_peer_repair",
777 Self::RelayOnlyRepair => "relay_only_repair",
778 }
779 }
780}
781
782#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
784pub enum AtpRepairMode {
785 Off,
787 Tail,
789 Lossy,
791 ResumeRepair,
793 RelayExpensive,
795 MobileUnstable,
797 SatelliteHighBdp,
799 Broadcast,
801 Swarm,
803}
804
805impl AtpRepairMode {
806 #[must_use]
808 pub const fn as_str(self) -> &'static str {
809 match self {
810 Self::Off => "off",
811 Self::Tail => "tail",
812 Self::Lossy => "lossy",
813 Self::ResumeRepair => "resume_repair",
814 Self::RelayExpensive => "relay_expensive",
815 Self::MobileUnstable => "mobile_unstable",
816 Self::SatelliteHighBdp => "satellite_high_bdp",
817 Self::Broadcast => "broadcast",
818 Self::Swarm => "swarm",
819 }
820 }
821}
822
823#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
825pub struct AtpRepairRoiInputs {
826 pub trace_id: String,
828 pub workload_id: String,
830 pub expected_time_saved_micros: u64,
832 pub encode_cpu_micros: u64,
834 pub decode_cpu_micros: u64,
836 pub bandwidth_overhead_bytes: u64,
838 pub memory_pressure_permille: u16,
840 pub stream_contention_permille: u16,
842 pub relay_cost_micros_per_mib: u64,
844 pub path_stability_permille: u16,
846 pub resume_value_permille: u16,
848 pub loss_permille: u16,
850 pub available_peer_count: u16,
852 pub path_mode: AtpRepairPathMode,
854 pub requested_mode: Option<AtpRepairMode>,
856 pub missing_tail_chunks: u16,
858 pub rtt_micros: u64,
860 pub path_migration_events: u16,
862 pub broadcast_peer_count: u16,
864}
865
866impl AtpRepairRoiInputs {
867 #[must_use]
869 pub fn from_autotune_telemetry(telemetry: &AtpAutotuneTelemetry) -> Self {
870 let loss_permille = telemetry.loss_permille.unwrap_or(0);
871 let rtt_micros = telemetry.rtt_micros.unwrap_or(0);
872 let pto_micros = telemetry
873 .pto_micros
874 .unwrap_or_else(|| rtt_micros.saturating_mul(2));
875 let migration_events = telemetry.migration_events.unwrap_or(0);
876 let loss_wait = permille_of(pto_micros, loss_permille);
877 let migration_wait = rtt_micros.saturating_mul(u64::from(migration_events));
878 let repair_roi = telemetry.repair_roi_permille.unwrap_or(0);
879 let expected_time_saved_micros = loss_wait
880 .max(migration_wait)
881 .saturating_add(permille_of(rtt_micros, repair_roi.min(1_000)));
882 let encode_cpu_micros = u64::from(telemetry.encode_backlog_symbols.unwrap_or(0)) * 32;
883 let decode_cpu_micros = u64::from(telemetry.decode_backlog_symbols.unwrap_or(0)) * 48;
884 let bandwidth_overhead_bytes = u64::from(loss_permille)
885 .saturating_mul(16 * 1_024)
886 .min(4 * 1_048_576);
887 let queued_bytes = telemetry
888 .send_buffer_queued_bytes
889 .unwrap_or(0)
890 .saturating_add(telemetry.receive_buffer_queued_bytes.unwrap_or(0));
891 let memory_pressure_permille = ratio_permille(queued_bytes, 64 * 1_048_576);
892 let stream_contention_permille =
893 match (telemetry.in_flight_bytes, telemetry.congestion_window_bytes) {
894 (Some(in_flight), Some(cwnd)) => ratio_permille(in_flight, cwnd),
895 _ => 0,
896 };
897 let instability = u64::from(loss_permille)
898 .saturating_mul(4)
899 .saturating_add(u64::from(migration_events).saturating_mul(250))
900 .min(1_000);
901 let path_stability_permille = u16::try_from(1_000 - instability).unwrap_or(0);
902 let resume_value_permille = repair_roi
903 .max(if telemetry.decode_backlog_symbols.unwrap_or(0) > 0 {
904 400
905 } else {
906 0
907 })
908 .min(1_000);
909 let path_mode = if migration_events > 0 {
910 AtpRepairPathMode::DirectAndRelay
911 } else {
912 AtpRepairPathMode::Direct
913 };
914 let missing_tail_chunks =
915 u16::try_from(telemetry.decode_backlog_symbols.unwrap_or(0)).unwrap_or(u16::MAX);
916 let path_migration_events = u16::try_from(migration_events).unwrap_or(u16::MAX);
917
918 Self {
919 trace_id: telemetry.trace_id.clone(),
920 workload_id: telemetry.workload_id.clone(),
921 expected_time_saved_micros,
922 encode_cpu_micros,
923 decode_cpu_micros,
924 bandwidth_overhead_bytes,
925 memory_pressure_permille,
926 stream_contention_permille,
927 relay_cost_micros_per_mib: telemetry.relay_cost_micros_per_mib.unwrap_or(u64::MAX),
928 path_stability_permille,
929 resume_value_permille,
930 loss_permille,
931 available_peer_count: 1,
932 path_mode,
933 requested_mode: None,
934 missing_tail_chunks,
935 rtt_micros,
936 path_migration_events,
937 broadcast_peer_count: 0,
938 }
939 }
940}
941
942#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
944pub enum AtpRepairDecisionFactorKind {
945 NetRoi,
947 MemoryPressure,
949 StreamContention,
951 RelayCost,
953 PathStability,
955 ResumeValue,
957 Loss,
959 PeerDiversity,
961 PathMode,
963 RepairMode,
965}
966
967impl AtpRepairDecisionFactorKind {
968 #[must_use]
970 pub const fn as_str(self) -> &'static str {
971 match self {
972 Self::NetRoi => "net_roi",
973 Self::MemoryPressure => "memory_pressure",
974 Self::StreamContention => "stream_contention",
975 Self::RelayCost => "relay_cost",
976 Self::PathStability => "path_stability",
977 Self::ResumeValue => "resume_value",
978 Self::Loss => "loss",
979 Self::PeerDiversity => "peer_diversity",
980 Self::PathMode => "path_mode",
981 Self::RepairMode => "repair_mode",
982 }
983 }
984}
985
986#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
988pub enum AtpRepairDecisionFactorEffect {
989 SupportsRepair,
991 BlocksRepair,
993 Cost,
995}
996
997impl AtpRepairDecisionFactorEffect {
998 #[must_use]
1000 pub const fn as_str(self) -> &'static str {
1001 match self {
1002 Self::SupportsRepair => "supports_repair",
1003 Self::BlocksRepair => "blocks_repair",
1004 Self::Cost => "cost",
1005 }
1006 }
1007}
1008
1009#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1011pub struct AtpRepairDecisionFactor {
1012 pub kind: AtpRepairDecisionFactorKind,
1014 pub observed: u64,
1016 pub threshold: u64,
1018 pub effect: AtpRepairDecisionFactorEffect,
1020}
1021
1022impl AtpRepairDecisionFactor {
1023 const fn new(
1024 kind: AtpRepairDecisionFactorKind,
1025 observed: u64,
1026 threshold: u64,
1027 effect: AtpRepairDecisionFactorEffect,
1028 ) -> Self {
1029 Self {
1030 kind,
1031 observed,
1032 threshold,
1033 effect,
1034 }
1035 }
1036}
1037
1038#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1040pub struct AtpRepairCoordinatorDecision {
1041 pub mode: AtpRepairMode,
1043 pub action: AtpRepairAction,
1045 pub mode_reason_code: String,
1047 pub mode_cooldown_micros: u64,
1049 pub reason_code: String,
1051 pub fail_closed: bool,
1053 pub gross_benefit_micros: u64,
1055 pub total_cost_micros: u64,
1057 pub net_roi_micros: i64,
1059 pub factors: Vec<AtpRepairDecisionFactor>,
1061}
1062
1063impl AtpRepairCoordinatorDecision {
1064 #[must_use]
1066 pub fn human_summary_lines(&self) -> Vec<String> {
1067 let mut lines = vec![
1068 format!("Repair mode: {}", self.mode.as_str()),
1069 format!("Repair action: {}", self.action.as_str()),
1070 format!("Repair mode reason: {}", self.mode_reason_code),
1071 format!("Repair mode cooldown micros: {}", self.mode_cooldown_micros),
1072 format!("Repair reason: {}", self.reason_code),
1073 format!("Repair fail closed: {}", self.fail_closed),
1074 format!(
1075 "Repair ROI: gross_benefit_micros={}, total_cost_micros={}, net_roi_micros={}",
1076 self.gross_benefit_micros, self.total_cost_micros, self.net_roi_micros
1077 ),
1078 ];
1079 for factor in &self.factors {
1080 lines.push(format!(
1081 "- repair {}: effect={}, observed={}, threshold={}",
1082 factor.kind.as_str(),
1083 factor.effect.as_str(),
1084 factor.observed,
1085 factor.threshold
1086 ));
1087 }
1088 lines
1089 }
1090}
1091
1092#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1094pub struct AtpRepairCoordinatorPolicy {
1095 pub min_positive_roi_micros: u64,
1097 pub parity_trickle_min_roi_micros: u64,
1099 pub burst_repair_min_roi_micros: u64,
1101 pub multi_peer_min_roi_micros: u64,
1103 pub bandwidth_cost_micros_per_mib: u64,
1105 pub max_relay_cost_micros_per_mib: u64,
1107 pub high_memory_pressure_permille: u16,
1109 pub high_stream_contention_permille: u16,
1111 pub unstable_path_permille: u16,
1113 pub parity_loss_permille: u16,
1115 pub burst_loss_permille: u16,
1117 pub multi_peer_min_peers: u16,
1119 pub resume_value_floor_permille: u16,
1121 pub tail_min_missing_chunks: u16,
1123 pub satellite_high_bdp_min_rtt_micros: u64,
1125 pub mobile_unstable_min_migrations: u16,
1127 pub broadcast_min_peers: u16,
1129 pub tail_cooldown_micros: u64,
1131 pub lossy_cooldown_micros: u64,
1133 pub resume_cooldown_micros: u64,
1135 pub relay_expensive_cooldown_micros: u64,
1137 pub mobile_unstable_cooldown_micros: u64,
1139 pub satellite_high_bdp_cooldown_micros: u64,
1141 pub broadcast_cooldown_micros: u64,
1143 pub swarm_cooldown_micros: u64,
1145}
1146
1147impl Default for AtpRepairCoordinatorPolicy {
1148 fn default() -> Self {
1149 Self {
1150 min_positive_roi_micros: 50_000,
1151 parity_trickle_min_roi_micros: 150_000,
1152 burst_repair_min_roi_micros: 1_000_000,
1153 multi_peer_min_roi_micros: 1_500_000,
1154 bandwidth_cost_micros_per_mib: 30_000,
1155 max_relay_cost_micros_per_mib: 500_000,
1156 high_memory_pressure_permille: 850,
1157 high_stream_contention_permille: 900,
1158 unstable_path_permille: 500,
1159 parity_loss_permille: 10,
1160 burst_loss_permille: 80,
1161 multi_peer_min_peers: 2,
1162 resume_value_floor_permille: 600,
1163 tail_min_missing_chunks: 1,
1164 satellite_high_bdp_min_rtt_micros: 500_000,
1165 mobile_unstable_min_migrations: 1,
1166 broadcast_min_peers: 8,
1167 tail_cooldown_micros: 100_000,
1168 lossy_cooldown_micros: 250_000,
1169 resume_cooldown_micros: 200_000,
1170 relay_expensive_cooldown_micros: 500_000,
1171 mobile_unstable_cooldown_micros: 750_000,
1172 satellite_high_bdp_cooldown_micros: 1_000_000,
1173 broadcast_cooldown_micros: 1_000_000,
1174 swarm_cooldown_micros: 500_000,
1175 }
1176 }
1177}
1178
1179#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
1181pub struct AtpRepairCoordinator {
1182 pub policy: AtpRepairCoordinatorPolicy,
1184}
1185
1186impl AtpRepairCoordinator {
1187 #[must_use]
1189 pub const fn new(policy: AtpRepairCoordinatorPolicy) -> Self {
1190 Self { policy }
1191 }
1192
1193 #[must_use]
1195 pub fn decide(self, inputs: &AtpRepairRoiInputs) -> AtpRepairCoordinatorDecision {
1196 let gross_benefit_micros = inputs
1197 .expected_time_saved_micros
1198 .saturating_add(permille_of(
1199 inputs.expected_time_saved_micros,
1200 inputs.resume_value_permille,
1201 ));
1202 let bandwidth_cost_micros = mul_div_u64(
1203 inputs.bandwidth_overhead_bytes,
1204 self.policy.bandwidth_cost_micros_per_mib,
1205 1_048_576,
1206 );
1207 let memory_cost_micros = permille_of(gross_benefit_micros, inputs.memory_pressure_permille);
1208 let stream_cost_micros =
1209 permille_of(gross_benefit_micros, inputs.stream_contention_permille);
1210 let total_cost_micros = inputs
1211 .encode_cpu_micros
1212 .saturating_add(inputs.decode_cpu_micros)
1213 .saturating_add(bandwidth_cost_micros)
1214 .saturating_add(memory_cost_micros)
1215 .saturating_add(stream_cost_micros);
1216 let net_roi_micros = signed_diff_to_i64(gross_benefit_micros, total_cost_micros);
1217 let mut factors = self.base_factors(inputs, net_roi_micros);
1218
1219 if inputs.memory_pressure_permille >= self.policy.high_memory_pressure_permille
1220 && inputs.resume_value_permille < self.policy.resume_value_floor_permille
1221 {
1222 factors.push(AtpRepairDecisionFactor::new(
1223 AtpRepairDecisionFactorKind::MemoryPressure,
1224 u64::from(inputs.memory_pressure_permille),
1225 u64::from(self.policy.high_memory_pressure_permille),
1226 AtpRepairDecisionFactorEffect::BlocksRepair,
1227 ));
1228 return build_repair_decision(
1229 AtpRepairMode::Off,
1230 AtpRepairAction::NoRepair,
1231 "repair_mode_blocked_by_memory_pressure",
1232 0,
1233 "blocked_by_memory_pressure",
1234 true,
1235 gross_benefit_micros,
1236 total_cost_micros,
1237 net_roi_micros,
1238 factors,
1239 );
1240 }
1241
1242 if inputs.stream_contention_permille >= self.policy.high_stream_contention_permille
1243 && inputs.resume_value_permille < self.policy.resume_value_floor_permille
1244 {
1245 factors.push(AtpRepairDecisionFactor::new(
1246 AtpRepairDecisionFactorKind::StreamContention,
1247 u64::from(inputs.stream_contention_permille),
1248 u64::from(self.policy.high_stream_contention_permille),
1249 AtpRepairDecisionFactorEffect::BlocksRepair,
1250 ));
1251 return build_repair_decision(
1252 AtpRepairMode::Off,
1253 AtpRepairAction::NoRepair,
1254 "repair_mode_blocked_by_stream_contention",
1255 0,
1256 "blocked_by_stream_contention",
1257 true,
1258 gross_benefit_micros,
1259 total_cost_micros,
1260 net_roi_micros,
1261 factors,
1262 );
1263 }
1264
1265 if net_roi_micros < i64_from_u64(self.policy.min_positive_roi_micros) {
1266 return build_repair_decision(
1267 AtpRepairMode::Off,
1268 AtpRepairAction::NoRepair,
1269 "repair_mode_roi_not_positive",
1270 0,
1271 "repair_roi_not_positive",
1272 true,
1273 gross_benefit_micros,
1274 total_cost_micros,
1275 net_roi_micros,
1276 factors,
1277 );
1278 }
1279
1280 let relay_viable = inputs.path_mode.relay_available()
1281 && inputs.relay_cost_micros_per_mib <= self.policy.max_relay_cost_micros_per_mib;
1282 if let Some(requested_mode) = inputs.requested_mode {
1283 let action = self.action_for_manual_mode(requested_mode, inputs, relay_viable);
1284 factors.push(AtpRepairDecisionFactor::new(
1285 AtpRepairDecisionFactorKind::RepairMode,
1286 repair_mode_rank(requested_mode),
1287 repair_mode_rank(requested_mode),
1288 AtpRepairDecisionFactorEffect::SupportsRepair,
1289 ));
1290 return build_repair_decision(
1291 requested_mode,
1292 action,
1293 "manual_repair_mode_requested",
1294 self.mode_cooldown_micros(requested_mode),
1295 Self::reason_for_action(action, requested_mode),
1296 false,
1297 gross_benefit_micros,
1298 total_cost_micros,
1299 net_roi_micros,
1300 factors,
1301 );
1302 }
1303
1304 if matches!(inputs.path_mode, AtpRepairPathMode::RelayOnly)
1305 || (relay_viable && inputs.path_stability_permille < self.policy.unstable_path_permille)
1306 {
1307 let (action, reason, fail_closed) = if relay_viable {
1308 (
1309 AtpRepairAction::RelayOnlyRepair,
1310 "relay_only_repair_roi_positive",
1311 false,
1312 )
1313 } else {
1314 (AtpRepairAction::NoRepair, "relay_cost_not_viable", true)
1315 };
1316 let (mode, mode_reason) = if fail_closed {
1317 (AtpRepairMode::Off, "repair_mode_relay_cost_not_viable")
1318 } else {
1319 (
1320 AtpRepairMode::MobileUnstable,
1321 "auto_mobile_unstable_path_churn",
1322 )
1323 };
1324 return build_repair_decision(
1325 mode,
1326 action,
1327 mode_reason,
1328 self.mode_cooldown_micros(mode),
1329 reason,
1330 fail_closed,
1331 gross_benefit_micros,
1332 total_cost_micros,
1333 net_roi_micros,
1334 factors,
1335 );
1336 }
1337
1338 if inputs.available_peer_count >= self.policy.multi_peer_min_peers
1339 && net_roi_micros >= i64_from_u64(self.policy.multi_peer_min_roi_micros)
1340 && inputs.loss_permille >= self.policy.burst_loss_permille
1341 {
1342 return build_repair_decision(
1343 AtpRepairMode::Swarm,
1344 AtpRepairAction::MultiPeerRepair,
1345 "auto_swarm_high_loss_peer_diversity",
1346 self.mode_cooldown_micros(AtpRepairMode::Swarm),
1347 "multi_peer_repair_roi_positive",
1348 false,
1349 gross_benefit_micros,
1350 total_cost_micros,
1351 net_roi_micros,
1352 factors,
1353 );
1354 }
1355
1356 if inputs.loss_permille >= self.policy.burst_loss_permille
1357 && net_roi_micros >= i64_from_u64(self.policy.burst_repair_min_roi_micros)
1358 {
1359 return build_repair_decision(
1360 AtpRepairMode::Lossy,
1361 AtpRepairAction::BurstRepair,
1362 "auto_lossy_burst_loss",
1363 self.mode_cooldown_micros(AtpRepairMode::Lossy),
1364 "burst_repair_roi_positive",
1365 false,
1366 gross_benefit_micros,
1367 total_cost_micros,
1368 net_roi_micros,
1369 factors,
1370 );
1371 }
1372
1373 let (mode, mode_reason) = self.select_auto_mode(inputs);
1374 if (inputs.loss_permille >= self.policy.parity_loss_permille
1375 || inputs.resume_value_permille >= self.policy.resume_value_floor_permille)
1376 && net_roi_micros >= i64_from_u64(self.policy.parity_trickle_min_roi_micros)
1377 {
1378 return build_repair_decision(
1379 mode,
1380 AtpRepairAction::ParityTrickle,
1381 mode_reason,
1382 self.mode_cooldown_micros(mode),
1383 "parity_trickle_roi_positive",
1384 false,
1385 gross_benefit_micros,
1386 total_cost_micros,
1387 net_roi_micros,
1388 factors,
1389 );
1390 }
1391
1392 build_repair_decision(
1393 mode,
1394 AtpRepairAction::ExactRetransmit,
1395 mode_reason,
1396 self.mode_cooldown_micros(mode),
1397 "exact_retransmit_roi_positive",
1398 false,
1399 gross_benefit_micros,
1400 total_cost_micros,
1401 net_roi_micros,
1402 factors,
1403 )
1404 }
1405
1406 fn base_factors(
1407 self,
1408 inputs: &AtpRepairRoiInputs,
1409 net_roi_micros: i64,
1410 ) -> Vec<AtpRepairDecisionFactor> {
1411 vec![
1412 AtpRepairDecisionFactor::new(
1413 AtpRepairDecisionFactorKind::NetRoi,
1414 u64_from_nonnegative_i64(net_roi_micros),
1415 self.policy.min_positive_roi_micros,
1416 if net_roi_micros >= i64_from_u64(self.policy.min_positive_roi_micros) {
1417 AtpRepairDecisionFactorEffect::SupportsRepair
1418 } else {
1419 AtpRepairDecisionFactorEffect::BlocksRepair
1420 },
1421 ),
1422 AtpRepairDecisionFactor::new(
1423 AtpRepairDecisionFactorKind::MemoryPressure,
1424 u64::from(inputs.memory_pressure_permille),
1425 u64::from(self.policy.high_memory_pressure_permille),
1426 AtpRepairDecisionFactorEffect::Cost,
1427 ),
1428 AtpRepairDecisionFactor::new(
1429 AtpRepairDecisionFactorKind::StreamContention,
1430 u64::from(inputs.stream_contention_permille),
1431 u64::from(self.policy.high_stream_contention_permille),
1432 AtpRepairDecisionFactorEffect::Cost,
1433 ),
1434 AtpRepairDecisionFactor::new(
1435 AtpRepairDecisionFactorKind::RelayCost,
1436 inputs.relay_cost_micros_per_mib,
1437 self.policy.max_relay_cost_micros_per_mib,
1438 if inputs.path_mode.relay_available()
1439 && inputs.relay_cost_micros_per_mib <= self.policy.max_relay_cost_micros_per_mib
1440 {
1441 AtpRepairDecisionFactorEffect::SupportsRepair
1442 } else {
1443 AtpRepairDecisionFactorEffect::Cost
1444 },
1445 ),
1446 AtpRepairDecisionFactor::new(
1447 AtpRepairDecisionFactorKind::PathStability,
1448 u64::from(inputs.path_stability_permille),
1449 u64::from(self.policy.unstable_path_permille),
1450 if inputs.path_stability_permille < self.policy.unstable_path_permille {
1451 AtpRepairDecisionFactorEffect::BlocksRepair
1452 } else {
1453 AtpRepairDecisionFactorEffect::SupportsRepair
1454 },
1455 ),
1456 AtpRepairDecisionFactor::new(
1457 AtpRepairDecisionFactorKind::ResumeValue,
1458 u64::from(inputs.resume_value_permille),
1459 u64::from(self.policy.resume_value_floor_permille),
1460 if inputs.resume_value_permille >= self.policy.resume_value_floor_permille {
1461 AtpRepairDecisionFactorEffect::SupportsRepair
1462 } else {
1463 AtpRepairDecisionFactorEffect::Cost
1464 },
1465 ),
1466 AtpRepairDecisionFactor::new(
1467 AtpRepairDecisionFactorKind::Loss,
1468 u64::from(inputs.loss_permille),
1469 u64::from(self.policy.parity_loss_permille),
1470 if inputs.loss_permille >= self.policy.parity_loss_permille {
1471 AtpRepairDecisionFactorEffect::SupportsRepair
1472 } else {
1473 AtpRepairDecisionFactorEffect::Cost
1474 },
1475 ),
1476 AtpRepairDecisionFactor::new(
1477 AtpRepairDecisionFactorKind::PeerDiversity,
1478 u64::from(inputs.available_peer_count),
1479 u64::from(self.policy.multi_peer_min_peers),
1480 if inputs.available_peer_count >= self.policy.multi_peer_min_peers {
1481 AtpRepairDecisionFactorEffect::SupportsRepair
1482 } else {
1483 AtpRepairDecisionFactorEffect::Cost
1484 },
1485 ),
1486 AtpRepairDecisionFactor::new(
1487 AtpRepairDecisionFactorKind::PathMode,
1488 path_mode_rank(inputs.path_mode),
1489 path_mode_rank(AtpRepairPathMode::DirectAndRelay),
1490 AtpRepairDecisionFactorEffect::Cost,
1491 ),
1492 AtpRepairDecisionFactor::new(
1493 AtpRepairDecisionFactorKind::RepairMode,
1494 0,
1495 repair_mode_rank(AtpRepairMode::Swarm),
1496 AtpRepairDecisionFactorEffect::Cost,
1497 ),
1498 ]
1499 }
1500
1501 fn select_auto_mode(self, inputs: &AtpRepairRoiInputs) -> (AtpRepairMode, &'static str) {
1502 if inputs.broadcast_peer_count >= self.policy.broadcast_min_peers {
1503 return (AtpRepairMode::Broadcast, "auto_broadcast_peer_fanout");
1504 }
1505 if inputs.available_peer_count >= self.policy.multi_peer_min_peers
1506 && inputs.loss_permille >= self.policy.burst_loss_permille
1507 {
1508 return (AtpRepairMode::Swarm, "auto_swarm_peer_diversity");
1509 }
1510 if inputs.path_migration_events >= self.policy.mobile_unstable_min_migrations
1511 || inputs.path_stability_permille < self.policy.unstable_path_permille
1512 {
1513 return (
1514 AtpRepairMode::MobileUnstable,
1515 "auto_mobile_unstable_path_churn",
1516 );
1517 }
1518 if inputs.path_mode.relay_available()
1519 && inputs.relay_cost_micros_per_mib > self.policy.max_relay_cost_micros_per_mib
1520 {
1521 return (
1522 AtpRepairMode::RelayExpensive,
1523 "auto_relay_expensive_cost_gate",
1524 );
1525 }
1526 if inputs.resume_value_permille >= self.policy.resume_value_floor_permille {
1527 return (AtpRepairMode::ResumeRepair, "auto_resume_value");
1528 }
1529 if inputs.rtt_micros >= self.policy.satellite_high_bdp_min_rtt_micros {
1530 return (
1531 AtpRepairMode::SatelliteHighBdp,
1532 "auto_satellite_high_bdp_rtt",
1533 );
1534 }
1535 if inputs.missing_tail_chunks >= self.policy.tail_min_missing_chunks {
1536 return (AtpRepairMode::Tail, "auto_tail_missing_chunks");
1537 }
1538 if inputs.loss_permille >= self.policy.parity_loss_permille {
1539 return (AtpRepairMode::Lossy, "auto_lossy_packet_loss");
1540 }
1541 (AtpRepairMode::Tail, "auto_tail_exact_retransmit")
1542 }
1543
1544 fn action_for_manual_mode(
1545 self,
1546 mode: AtpRepairMode,
1547 inputs: &AtpRepairRoiInputs,
1548 relay_viable: bool,
1549 ) -> AtpRepairAction {
1550 match mode {
1551 AtpRepairMode::Off => AtpRepairAction::NoRepair,
1552 AtpRepairMode::Tail | AtpRepairMode::RelayExpensive => AtpRepairAction::ExactRetransmit,
1553 AtpRepairMode::Lossy | AtpRepairMode::SatelliteHighBdp => {
1554 if inputs.loss_permille >= self.policy.burst_loss_permille {
1555 AtpRepairAction::BurstRepair
1556 } else {
1557 AtpRepairAction::ParityTrickle
1558 }
1559 }
1560 AtpRepairMode::ResumeRepair => AtpRepairAction::ParityTrickle,
1561 AtpRepairMode::MobileUnstable => {
1562 if relay_viable {
1563 AtpRepairAction::RelayOnlyRepair
1564 } else {
1565 AtpRepairAction::BurstRepair
1566 }
1567 }
1568 AtpRepairMode::Broadcast | AtpRepairMode::Swarm => AtpRepairAction::MultiPeerRepair,
1569 }
1570 }
1571
1572 const fn mode_cooldown_micros(self, mode: AtpRepairMode) -> u64 {
1573 match mode {
1574 AtpRepairMode::Off => 0,
1575 AtpRepairMode::Tail => self.policy.tail_cooldown_micros,
1576 AtpRepairMode::Lossy => self.policy.lossy_cooldown_micros,
1577 AtpRepairMode::ResumeRepair => self.policy.resume_cooldown_micros,
1578 AtpRepairMode::RelayExpensive => self.policy.relay_expensive_cooldown_micros,
1579 AtpRepairMode::MobileUnstable => self.policy.mobile_unstable_cooldown_micros,
1580 AtpRepairMode::SatelliteHighBdp => self.policy.satellite_high_bdp_cooldown_micros,
1581 AtpRepairMode::Broadcast => self.policy.broadcast_cooldown_micros,
1582 AtpRepairMode::Swarm => self.policy.swarm_cooldown_micros,
1583 }
1584 }
1585
1586 const fn reason_for_action(action: AtpRepairAction, mode: AtpRepairMode) -> &'static str {
1587 match (action, mode) {
1588 (AtpRepairAction::NoRepair, _) => "manual_repair_mode_off",
1589 (AtpRepairAction::ExactRetransmit, AtpRepairMode::RelayExpensive) => {
1590 "manual_relay_expensive_exact_retransmit"
1591 }
1592 (AtpRepairAction::ExactRetransmit, _) => "manual_tail_exact_retransmit",
1593 (AtpRepairAction::ParityTrickle, AtpRepairMode::ResumeRepair) => {
1594 "manual_resume_parity_trickle"
1595 }
1596 (AtpRepairAction::ParityTrickle, _) => "manual_parity_trickle",
1597 (AtpRepairAction::BurstRepair, AtpRepairMode::MobileUnstable) => {
1598 "manual_mobile_unstable_burst_repair"
1599 }
1600 (AtpRepairAction::BurstRepair, _) => "manual_burst_repair",
1601 (AtpRepairAction::MultiPeerRepair, AtpRepairMode::Broadcast) => {
1602 "manual_broadcast_repair"
1603 }
1604 (AtpRepairAction::MultiPeerRepair, _) => "manual_swarm_repair",
1605 (AtpRepairAction::RelayOnlyRepair, _) => "manual_relay_only_repair",
1606 }
1607 }
1608}
1609
1610fn build_repair_decision(
1611 mode: AtpRepairMode,
1612 action: AtpRepairAction,
1613 mode_reason_code: &str,
1614 mode_cooldown_micros: u64,
1615 reason_code: &str,
1616 fail_closed: bool,
1617 gross_benefit_micros: u64,
1618 total_cost_micros: u64,
1619 net_roi_micros: i64,
1620 factors: Vec<AtpRepairDecisionFactor>,
1621) -> AtpRepairCoordinatorDecision {
1622 AtpRepairCoordinatorDecision {
1623 mode,
1624 action,
1625 mode_reason_code: mode_reason_code.to_string(),
1626 mode_cooldown_micros,
1627 reason_code: reason_code.to_string(),
1628 fail_closed,
1629 gross_benefit_micros,
1630 total_cost_micros,
1631 net_roi_micros,
1632 factors,
1633 }
1634}
1635
1636fn repair_mode_rank(mode: AtpRepairMode) -> u64 {
1637 match mode {
1638 AtpRepairMode::Off => 0,
1639 AtpRepairMode::Tail => 1,
1640 AtpRepairMode::Lossy => 2,
1641 AtpRepairMode::ResumeRepair => 3,
1642 AtpRepairMode::RelayExpensive => 4,
1643 AtpRepairMode::MobileUnstable => 5,
1644 AtpRepairMode::SatelliteHighBdp => 6,
1645 AtpRepairMode::Broadcast => 7,
1646 AtpRepairMode::Swarm => 8,
1647 }
1648}
1649
1650fn path_mode_rank(path_mode: AtpRepairPathMode) -> u64 {
1651 match path_mode {
1652 AtpRepairPathMode::Direct => 1,
1653 AtpRepairPathMode::RelayOnly => 2,
1654 AtpRepairPathMode::DirectAndRelay => 3,
1655 }
1656}
1657
1658fn permille_of(value: u64, permille: u16) -> u64 {
1659 mul_div_u64(value, u64::from(permille), 1_000)
1660}
1661
1662fn ratio_permille(numerator: u64, denominator: u64) -> u16 {
1663 if denominator == 0 {
1664 return 1_000;
1665 }
1666 let ratio = mul_div_u64(numerator, 1_000, denominator).min(1_000);
1667 u16::try_from(ratio).unwrap_or(1_000)
1668}
1669
1670fn mul_div_u64(value: u64, multiplier: u64, divisor: u64) -> u64 {
1671 let divisor = u128::from(divisor.max(1));
1672 let divided = u128::from(value).saturating_mul(u128::from(multiplier)) / divisor;
1673 u64::try_from(divided).unwrap_or(u64::MAX)
1674}
1675
1676fn signed_diff_to_i64(benefit: u64, cost: u64) -> i64 {
1677 let diff = i128::from(benefit) - i128::from(cost);
1678 let clamped = diff.clamp(i128::from(i64::MIN), i128::from(i64::MAX));
1679 i64::try_from(clamped).unwrap_or(if diff.is_negative() {
1680 i64::MIN
1681 } else {
1682 i64::MAX
1683 })
1684}
1685
1686fn i64_from_u64(value: u64) -> i64 {
1687 i64::try_from(value).unwrap_or(i64::MAX)
1688}
1689
1690fn u64_from_nonnegative_i64(value: i64) -> u64 {
1691 u64::try_from(value).unwrap_or(0)
1692}
1693
1694#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1696pub enum AtpBottleneckKind {
1697 InsufficientTelemetry,
1699 ContradictoryTelemetry,
1701 NetworkLoss,
1703 NetworkLatency,
1705 CongestionWindow,
1707 SendBufferPressure,
1709 ReceiveBufferPressure,
1711 DiskReadLag,
1713 DiskWriteLag,
1715 EncodeBacklog,
1717 DecodeBacklog,
1719 RepairLowRoi,
1721 RelayCost,
1723 MigrationInstability,
1725}
1726
1727impl AtpBottleneckKind {
1728 #[must_use]
1730 pub const fn as_str(self) -> &'static str {
1731 match self {
1732 Self::InsufficientTelemetry => "insufficient_telemetry",
1733 Self::ContradictoryTelemetry => "contradictory_telemetry",
1734 Self::NetworkLoss => "network_loss",
1735 Self::NetworkLatency => "network_latency",
1736 Self::CongestionWindow => "congestion_window",
1737 Self::SendBufferPressure => "send_buffer_pressure",
1738 Self::ReceiveBufferPressure => "receive_buffer_pressure",
1739 Self::DiskReadLag => "disk_read_lag",
1740 Self::DiskWriteLag => "disk_write_lag",
1741 Self::EncodeBacklog => "encode_backlog",
1742 Self::DecodeBacklog => "decode_backlog",
1743 Self::RepairLowRoi => "repair_low_roi",
1744 Self::RelayCost => "relay_cost",
1745 Self::MigrationInstability => "migration_instability",
1746 }
1747 }
1748}
1749
1750#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1752pub struct AtpBottleneckSignal {
1753 pub kind: AtpBottleneckKind,
1755 pub metric: Option<AtpAutotuneMetric>,
1757 pub observed: u64,
1759 pub threshold: u64,
1761}
1762
1763impl AtpBottleneckSignal {
1764 fn new(
1765 kind: AtpBottleneckKind,
1766 metric: Option<AtpAutotuneMetric>,
1767 observed: u64,
1768 threshold: u64,
1769 ) -> Self {
1770 Self {
1771 kind,
1772 metric,
1773 observed,
1774 threshold,
1775 }
1776 }
1777}
1778
1779#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1781pub struct AtpAutotuneDecision {
1782 pub settings: AtpAutotuneSettings,
1784 pub bottlenecks: Vec<AtpBottleneckSignal>,
1786 pub fail_closed: bool,
1788 pub reason_code: String,
1790}
1791
1792pub const ATP_AUTOTUNE_DECISION_RECEIPT_SCHEMA_VERSION: &str = "atp-autotune-decision-receipt-v1";
1794
1795#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1797pub enum AtpAutotuneDecisionOutcome {
1798 ConservativeGrowth,
1800 PressureBackoff,
1802 HoldNoWin,
1804 MalformedTelemetry,
1806}
1807
1808#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1813#[serde(rename_all = "snake_case")]
1814pub enum AtpAutotuneReceiptStatus {
1815 Pass,
1817 Degraded,
1819 NoWin,
1821 Blocked,
1823 Malformed,
1825 StaleEvidence,
1827}
1828
1829impl AtpAutotuneReceiptStatus {
1830 #[must_use]
1832 pub const fn as_str(self) -> &'static str {
1833 match self {
1834 Self::Pass => "pass",
1835 Self::Degraded => "degraded",
1836 Self::NoWin => "no_win",
1837 Self::Blocked => "blocked",
1838 Self::Malformed => "malformed",
1839 Self::StaleEvidence => "stale_evidence",
1840 }
1841 }
1842}
1843
1844#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1846#[serde(rename_all = "snake_case")]
1847pub enum AtpAutotuneReceiptConfidence {
1848 Conservative,
1850 FailClosed,
1852 InsufficientEvidence,
1854 Rejected,
1856}
1857
1858impl AtpAutotuneReceiptConfidence {
1859 #[must_use]
1861 pub const fn as_str(self) -> &'static str {
1862 match self {
1863 Self::Conservative => "conservative",
1864 Self::FailClosed => "fail_closed",
1865 Self::InsufficientEvidence => "insufficient_evidence",
1866 Self::Rejected => "rejected",
1867 }
1868 }
1869}
1870
1871#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1873pub enum AtpAutotuneKnob {
1874 InFlightBytes,
1876 StreamCount,
1878 ChunkSizeBytes,
1880 RepairSymbolsPerSecond,
1882}
1883
1884impl AtpAutotuneKnob {
1885 #[must_use]
1887 pub const fn as_str(self) -> &'static str {
1888 match self {
1889 Self::InFlightBytes => "in_flight_bytes",
1890 Self::StreamCount => "stream_count",
1891 Self::ChunkSizeBytes => "chunk_size_bytes",
1892 Self::RepairSymbolsPerSecond => "repair_symbols_per_second",
1893 }
1894 }
1895}
1896
1897#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1899pub enum AtpAutotuneKnobDirection {
1900 Increase,
1902 Decrease,
1904 Hold,
1906}
1907
1908#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1910pub struct AtpAutotuneKnobChange {
1911 pub knob: AtpAutotuneKnob,
1913 pub previous: u64,
1915 pub next: u64,
1917 pub direction: AtpAutotuneKnobDirection,
1919 pub delta: u64,
1921}
1922
1923impl AtpAutotuneKnobChange {
1924 fn new(knob: AtpAutotuneKnob, previous: u64, next: u64) -> Self {
1925 let direction = match next.cmp(&previous) {
1926 std::cmp::Ordering::Greater => AtpAutotuneKnobDirection::Increase,
1927 std::cmp::Ordering::Less => AtpAutotuneKnobDirection::Decrease,
1928 std::cmp::Ordering::Equal => AtpAutotuneKnobDirection::Hold,
1929 };
1930 Self {
1931 knob,
1932 previous,
1933 next,
1934 direction,
1935 delta: previous.abs_diff(next),
1936 }
1937 }
1938}
1939
1940#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1942pub struct AtpAutotuneReceiptProofPointer {
1943 pub receipt_schema_version: String,
1945 pub trace_id: String,
1947 pub workload_id: String,
1949 pub sample_count: u32,
1951}
1952
1953impl AtpAutotuneReceiptProofPointer {
1954 fn from_receipt_fields(trace_id: &str, workload_id: &str, sample_count: u32) -> Self {
1955 Self {
1956 receipt_schema_version: String::from(ATP_AUTOTUNE_DECISION_RECEIPT_SCHEMA_VERSION),
1957 trace_id: trace_id.to_string(),
1958 workload_id: workload_id.to_string(),
1959 sample_count,
1960 }
1961 }
1962}
1963
1964#[derive(Debug, Clone, PartialEq, Eq)]
1966pub enum AtpAutotuneReceiptValidationError {
1967 UnsupportedSchemaVersion {
1969 expected: String,
1971 actual: String,
1973 },
1974 MissingTraceId,
1976 MissingWorkloadId,
1978 ProofPointerMismatch {
1980 field: String,
1982 },
1983}
1984
1985impl fmt::Display for AtpAutotuneReceiptValidationError {
1986 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1987 match self {
1988 Self::UnsupportedSchemaVersion { expected, actual } => {
1989 write!(
1990 f,
1991 "unsupported ATP autotune receipt schema {actual}, expected {expected}"
1992 )
1993 }
1994 Self::MissingTraceId => write!(f, "ATP autotune receipt trace_id is missing"),
1995 Self::MissingWorkloadId => write!(f, "ATP autotune receipt workload_id is missing"),
1996 Self::ProofPointerMismatch { field } => {
1997 write!(f, "ATP autotune receipt proof pointer mismatches {field}")
1998 }
1999 }
2000 }
2001}
2002
2003impl std::error::Error for AtpAutotuneReceiptValidationError {}
2004
2005#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
2007pub struct AtpAutotuneDecisionReceipt {
2008 pub schema_version: String,
2010 pub trace_id: String,
2012 pub workload_id: String,
2014 pub sample_count: u32,
2016 pub current_settings: AtpAutotuneSettings,
2018 pub decision: AtpAutotuneDecision,
2020 pub outcome: AtpAutotuneDecisionOutcome,
2022 pub consumer_status: AtpAutotuneReceiptStatus,
2024 pub confidence: AtpAutotuneReceiptConfidence,
2026 pub caveats: Vec<String>,
2028 pub omitted_sources: Vec<AtpAutotuneMetric>,
2030 pub stale_sources: Vec<String>,
2032 pub proof_pointer: AtpAutotuneReceiptProofPointer,
2034 pub changes: Vec<AtpAutotuneKnobChange>,
2036}
2037
2038impl AtpAutotuneDecisionReceipt {
2039 #[must_use]
2041 pub fn from_decision(
2042 telemetry: &AtpAutotuneTelemetry,
2043 current_settings: AtpAutotuneSettings,
2044 decision: AtpAutotuneDecision,
2045 ) -> Self {
2046 let changes = knob_changes(current_settings, decision.settings);
2047 let outcome = classify_decision_outcome(&decision, &changes);
2048 let consumer_status = classify_receipt_status(&decision, outcome);
2049 let confidence = receipt_confidence(&decision, consumer_status);
2050 let caveats = receipt_caveats(&decision, consumer_status);
2051 let omitted_sources = omitted_metric_sources(telemetry);
2052 let proof_pointer = AtpAutotuneReceiptProofPointer::from_receipt_fields(
2053 &telemetry.trace_id,
2054 &telemetry.workload_id,
2055 telemetry.sample_count,
2056 );
2057 Self {
2058 schema_version: String::from(ATP_AUTOTUNE_DECISION_RECEIPT_SCHEMA_VERSION),
2059 trace_id: telemetry.trace_id.clone(),
2060 workload_id: telemetry.workload_id.clone(),
2061 sample_count: telemetry.sample_count,
2062 current_settings,
2063 decision,
2064 outcome,
2065 consumer_status,
2066 confidence,
2067 caveats,
2068 omitted_sources,
2069 stale_sources: Vec::new(),
2070 proof_pointer,
2071 changes,
2072 }
2073 }
2074
2075 pub fn validate_for_consumers(&self) -> Result<(), AtpAutotuneReceiptValidationError> {
2077 if self.schema_version != ATP_AUTOTUNE_DECISION_RECEIPT_SCHEMA_VERSION {
2078 return Err(
2079 AtpAutotuneReceiptValidationError::UnsupportedSchemaVersion {
2080 expected: String::from(ATP_AUTOTUNE_DECISION_RECEIPT_SCHEMA_VERSION),
2081 actual: self.schema_version.clone(),
2082 },
2083 );
2084 }
2085 if self.trace_id.trim().is_empty() {
2086 return Err(AtpAutotuneReceiptValidationError::MissingTraceId);
2087 }
2088 if self.workload_id.trim().is_empty() {
2089 return Err(AtpAutotuneReceiptValidationError::MissingWorkloadId);
2090 }
2091 if self.proof_pointer.receipt_schema_version != ATP_AUTOTUNE_DECISION_RECEIPT_SCHEMA_VERSION
2092 {
2093 return Err(AtpAutotuneReceiptValidationError::ProofPointerMismatch {
2094 field: String::from("receipt_schema_version"),
2095 });
2096 }
2097 if self.proof_pointer.trace_id != self.trace_id {
2098 return Err(AtpAutotuneReceiptValidationError::ProofPointerMismatch {
2099 field: String::from("trace_id"),
2100 });
2101 }
2102 if self.proof_pointer.workload_id != self.workload_id {
2103 return Err(AtpAutotuneReceiptValidationError::ProofPointerMismatch {
2104 field: String::from("workload_id"),
2105 });
2106 }
2107 if self.proof_pointer.sample_count != self.sample_count {
2108 return Err(AtpAutotuneReceiptValidationError::ProofPointerMismatch {
2109 field: String::from("sample_count"),
2110 });
2111 }
2112 Ok(())
2113 }
2114
2115 #[must_use]
2117 pub fn selected_knobs(&self) -> Vec<AtpAutotuneKnob> {
2118 self.changes
2119 .iter()
2120 .filter(|change| change.direction != AtpAutotuneKnobDirection::Hold)
2121 .map(|change| change.knob)
2122 .collect()
2123 }
2124
2125 #[must_use]
2127 pub fn render_human_summary(&self, explain: bool) -> String {
2128 self.human_summary_lines(explain).join("\n")
2129 }
2130
2131 #[must_use]
2133 pub fn human_summary_lines(&self, explain: bool) -> Vec<String> {
2134 let mut lines = vec![
2135 format!("Trace ID: {}", self.trace_id),
2136 format!("Workload ID: {}", self.workload_id),
2137 format!("Samples: {}", self.sample_count),
2138 format!("Status: {}", self.consumer_status.as_str()),
2139 format!("Outcome: {:?}", self.outcome),
2140 format!("Reason: {}", self.decision.reason_code),
2141 format!("Confidence: {}", self.confidence.as_str()),
2142 format!("Fail closed: {}", self.decision.fail_closed),
2143 format!(
2144 "Next settings: in_flight_bytes={}, stream_count={}, chunk_size_bytes={}, repair_symbols_per_second={}",
2145 self.decision.settings.in_flight_bytes,
2146 self.decision.settings.stream_count,
2147 self.decision.settings.chunk_size_bytes,
2148 self.decision.settings.repair_symbols_per_second
2149 ),
2150 format!("Bottlenecks: {}", self.decision.bottlenecks.len()),
2151 ];
2152
2153 if !self.caveats.is_empty() {
2154 lines.push(format!("Caveats: {}", self.caveats.join(",")));
2155 }
2156 if !self.stale_sources.is_empty() {
2157 lines.push(format!("Stale sources: {}", self.stale_sources.join(",")));
2158 }
2159
2160 if explain {
2161 for change in &self.changes {
2162 if change.direction != AtpAutotuneKnobDirection::Hold {
2163 lines.push(format!(
2164 "- knob {}: {:?} {} -> {} (delta={})",
2165 change.knob.as_str(),
2166 change.direction,
2167 change.previous,
2168 change.next,
2169 change.delta
2170 ));
2171 }
2172 }
2173 for signal in &self.decision.bottlenecks {
2174 let metric = signal.metric.map_or("none", AtpAutotuneMetric::as_str);
2175 lines.push(format!(
2176 "- {}: metric={}, observed={}, threshold={}",
2177 signal.kind.as_str(),
2178 metric,
2179 signal.observed,
2180 signal.threshold
2181 ));
2182 }
2183 }
2184
2185 lines
2186 }
2187}
2188
2189pub const ATP_AUTOTUNE_APPLICATION_RECEIPT_SCHEMA_VERSION: &str =
2191 "atp-autotune-application-receipt-v1";
2192
2193#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
2195pub enum AtpAutotuneApplicationOutcome {
2196 AppliedPressureBackoff,
2198 AppliedConfirmedGrowth,
2200 DeferredGrowthHysteresis,
2202 HeldNoWin,
2204 RejectedMalformedTelemetry,
2206 RejectedStaleReceipt,
2208 RejectedSchemaVersion,
2210}
2211
2212#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
2214pub struct AtpAutotuneApplicationReceipt {
2215 pub schema_version: String,
2217 pub trace_id: String,
2219 pub workload_id: String,
2221 pub sample_count: u32,
2223 pub previous_settings: AtpAutotuneSettings,
2225 pub candidate_settings: AtpAutotuneSettings,
2227 pub applied_settings: AtpAutotuneSettings,
2229 pub applied: bool,
2231 pub outcome: AtpAutotuneApplicationOutcome,
2233 pub consumer_status: AtpAutotuneReceiptStatus,
2235 pub consecutive_growth_windows: u8,
2237 pub growth_confirmations_required: u8,
2239 pub reason_code: String,
2241 pub decision_receipt: AtpAutotuneDecisionReceipt,
2243}
2244
2245#[derive(Debug, Clone, PartialEq, Eq)]
2248pub enum AtpAutotuneApplicationReceiptValidationError {
2249 UnsupportedSchemaVersion {
2251 expected: String,
2253 actual: String,
2255 },
2256 DecisionReceiptInvalid {
2258 reason: AtpAutotuneReceiptValidationError,
2260 },
2261 DecisionReceiptMismatch {
2263 field: String,
2265 },
2266 AppliedFlagMismatch,
2268}
2269
2270impl fmt::Display for AtpAutotuneApplicationReceiptValidationError {
2271 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2272 match self {
2273 Self::UnsupportedSchemaVersion { expected, actual } => write!(
2274 f,
2275 "unsupported ATP autotune application receipt schema {actual}, expected {expected}"
2276 ),
2277 Self::DecisionReceiptInvalid { reason } => {
2278 write!(
2279 f,
2280 "ATP autotune application receipt embeds invalid decision receipt: {reason}"
2281 )
2282 }
2283 Self::DecisionReceiptMismatch { field } => {
2284 write!(
2285 f,
2286 "ATP autotune application receipt decision receipt mismatches {field}"
2287 )
2288 }
2289 Self::AppliedFlagMismatch => {
2290 write!(
2291 f,
2292 "ATP autotune application receipt applied flag mismatches settings"
2293 )
2294 }
2295 }
2296 }
2297}
2298
2299impl std::error::Error for AtpAutotuneApplicationReceiptValidationError {}
2300
2301impl AtpAutotuneApplicationReceipt {
2302 fn from_parts(
2303 previous_settings: AtpAutotuneSettings,
2304 candidate_settings: AtpAutotuneSettings,
2305 applied_settings: AtpAutotuneSettings,
2306 outcome: AtpAutotuneApplicationOutcome,
2307 consecutive_growth_windows: u8,
2308 growth_confirmations_required: u8,
2309 decision_receipt: AtpAutotuneDecisionReceipt,
2310 ) -> Self {
2311 let consumer_status = application_consumer_status(outcome);
2312 let reason_code = match outcome {
2313 AtpAutotuneApplicationOutcome::AppliedPressureBackoff => "applied_pressure_backoff",
2314 AtpAutotuneApplicationOutcome::AppliedConfirmedGrowth => "applied_confirmed_growth",
2315 AtpAutotuneApplicationOutcome::DeferredGrowthHysteresis => "deferred_growth_hysteresis",
2316 AtpAutotuneApplicationOutcome::HeldNoWin => "held_no_win",
2317 AtpAutotuneApplicationOutcome::RejectedMalformedTelemetry => {
2318 "rejected_malformed_telemetry"
2319 }
2320 AtpAutotuneApplicationOutcome::RejectedStaleReceipt => "rejected_stale_receipt",
2321 AtpAutotuneApplicationOutcome::RejectedSchemaVersion => "rejected_schema_version",
2322 };
2323 let applied = previous_settings != applied_settings;
2324 Self {
2325 schema_version: String::from(ATP_AUTOTUNE_APPLICATION_RECEIPT_SCHEMA_VERSION),
2326 trace_id: decision_receipt.trace_id.clone(),
2327 workload_id: decision_receipt.workload_id.clone(),
2328 sample_count: decision_receipt.sample_count,
2329 previous_settings,
2330 candidate_settings,
2331 applied_settings,
2332 applied,
2333 outcome,
2334 consumer_status,
2335 consecutive_growth_windows,
2336 growth_confirmations_required,
2337 reason_code: String::from(reason_code),
2338 decision_receipt,
2339 }
2340 }
2341
2342 pub fn validate_for_consumers(
2344 &self,
2345 ) -> Result<(), AtpAutotuneApplicationReceiptValidationError> {
2346 if self.schema_version != ATP_AUTOTUNE_APPLICATION_RECEIPT_SCHEMA_VERSION {
2347 return Err(
2348 AtpAutotuneApplicationReceiptValidationError::UnsupportedSchemaVersion {
2349 expected: String::from(ATP_AUTOTUNE_APPLICATION_RECEIPT_SCHEMA_VERSION),
2350 actual: self.schema_version.clone(),
2351 },
2352 );
2353 }
2354 self.decision_receipt
2355 .validate_for_consumers()
2356 .map_err(|reason| {
2357 AtpAutotuneApplicationReceiptValidationError::DecisionReceiptInvalid { reason }
2358 })?;
2359 if self.decision_receipt.trace_id != self.trace_id {
2360 return Err(
2361 AtpAutotuneApplicationReceiptValidationError::DecisionReceiptMismatch {
2362 field: String::from("trace_id"),
2363 },
2364 );
2365 }
2366 if self.decision_receipt.workload_id != self.workload_id {
2367 return Err(
2368 AtpAutotuneApplicationReceiptValidationError::DecisionReceiptMismatch {
2369 field: String::from("workload_id"),
2370 },
2371 );
2372 }
2373 if self.decision_receipt.sample_count != self.sample_count {
2374 return Err(
2375 AtpAutotuneApplicationReceiptValidationError::DecisionReceiptMismatch {
2376 field: String::from("sample_count"),
2377 },
2378 );
2379 }
2380 if self.applied != (self.previous_settings != self.applied_settings) {
2381 return Err(AtpAutotuneApplicationReceiptValidationError::AppliedFlagMismatch);
2382 }
2383 Ok(())
2384 }
2385}
2386
2387#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
2394pub struct AtpAutotuneApplicationState {
2395 pub settings: AtpAutotuneSettings,
2397 pub limits: AtpAutotuneLimits,
2399 pub growth_confirmations_required: u8,
2401 pub consecutive_growth_windows: u8,
2403}
2404
2405impl Default for AtpAutotuneApplicationState {
2406 fn default() -> Self {
2407 Self::new(AtpAutotuneSettings::default(), AtpAutotuneLimits::default())
2408 }
2409}
2410
2411impl AtpAutotuneApplicationState {
2412 #[must_use]
2414 pub fn new(settings: AtpAutotuneSettings, limits: AtpAutotuneLimits) -> Self {
2415 Self {
2416 settings: limits.clamp(settings),
2417 limits,
2418 growth_confirmations_required: 2,
2419 consecutive_growth_windows: 0,
2420 }
2421 }
2422
2423 #[must_use]
2425 pub fn with_growth_confirmations_required(mut self, required: u8) -> Self {
2426 self.growth_confirmations_required = required.max(1);
2427 self
2428 }
2429
2430 #[must_use]
2432 pub fn apply_policy_window(
2433 &mut self,
2434 policy: AtpAutotunePolicy,
2435 telemetry: &AtpAutotuneTelemetry,
2436 ) -> AtpAutotuneApplicationReceipt {
2437 let bounded_policy = AtpAutotunePolicy {
2438 limits: self.limits,
2439 ..policy
2440 };
2441 let receipt = bounded_policy.decide_with_receipt(self.settings, telemetry);
2442 self.apply_decision_receipt(receipt)
2443 }
2444
2445 #[must_use]
2447 pub fn apply_decision_receipt(
2448 &mut self,
2449 receipt: AtpAutotuneDecisionReceipt,
2450 ) -> AtpAutotuneApplicationReceipt {
2451 let previous = self.limits.clamp(self.settings);
2452 self.settings = previous;
2453
2454 if let Err(validation) = receipt.validate_for_consumers() {
2455 let candidate = self.limits.clamp(receipt.decision.settings);
2456 self.consecutive_growth_windows = 0;
2457 let outcome = match validation {
2458 AtpAutotuneReceiptValidationError::UnsupportedSchemaVersion { .. } => {
2459 AtpAutotuneApplicationOutcome::RejectedSchemaVersion
2460 }
2461 AtpAutotuneReceiptValidationError::MissingTraceId
2462 | AtpAutotuneReceiptValidationError::MissingWorkloadId
2463 | AtpAutotuneReceiptValidationError::ProofPointerMismatch { .. } => {
2464 AtpAutotuneApplicationOutcome::RejectedMalformedTelemetry
2465 }
2466 };
2467 return AtpAutotuneApplicationReceipt::from_parts(
2468 previous,
2469 candidate,
2470 previous,
2471 outcome,
2472 self.consecutive_growth_windows,
2473 self.growth_confirmations_required,
2474 receipt,
2475 );
2476 }
2477
2478 if receipt.current_settings != previous {
2479 self.consecutive_growth_windows = 0;
2480 return AtpAutotuneApplicationReceipt::from_parts(
2481 previous,
2482 self.limits.clamp(receipt.decision.settings),
2483 previous,
2484 AtpAutotuneApplicationOutcome::RejectedStaleReceipt,
2485 self.consecutive_growth_windows,
2486 self.growth_confirmations_required,
2487 receipt,
2488 );
2489 }
2490
2491 let candidate = self.limits.clamp(receipt.decision.settings);
2492 let outcome = match receipt.outcome {
2493 AtpAutotuneDecisionOutcome::PressureBackoff => {
2494 self.consecutive_growth_windows = 0;
2495 self.settings = candidate;
2496 AtpAutotuneApplicationOutcome::AppliedPressureBackoff
2497 }
2498 AtpAutotuneDecisionOutcome::ConservativeGrowth => {
2499 self.consecutive_growth_windows = self.consecutive_growth_windows.saturating_add(1);
2500 if self.consecutive_growth_windows >= self.growth_confirmations_required {
2501 self.settings = candidate;
2502 self.consecutive_growth_windows = 0;
2503 AtpAutotuneApplicationOutcome::AppliedConfirmedGrowth
2504 } else {
2505 AtpAutotuneApplicationOutcome::DeferredGrowthHysteresis
2506 }
2507 }
2508 AtpAutotuneDecisionOutcome::HoldNoWin => {
2509 self.consecutive_growth_windows = 0;
2510 AtpAutotuneApplicationOutcome::HeldNoWin
2511 }
2512 AtpAutotuneDecisionOutcome::MalformedTelemetry => {
2513 self.consecutive_growth_windows = 0;
2514 AtpAutotuneApplicationOutcome::RejectedMalformedTelemetry
2515 }
2516 };
2517
2518 AtpAutotuneApplicationReceipt::from_parts(
2519 previous,
2520 candidate,
2521 self.settings,
2522 outcome,
2523 self.consecutive_growth_windows,
2524 self.growth_confirmations_required,
2525 receipt,
2526 )
2527 }
2528}
2529
2530#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
2532pub struct AtpAutotunePolicy {
2533 pub limits: AtpAutotuneLimits,
2535 pub min_growth_samples: u32,
2537 pub loss_backoff_permille: u16,
2539 pub latency_hold_micros: u64,
2541 pub buffer_pressure_bytes: u64,
2543 pub disk_lag_micros: u64,
2545 pub cpu_backlog_symbols: u32,
2547 pub repair_roi_floor_permille: u16,
2549 pub relay_cost_micros_per_mib: u64,
2551}
2552
2553impl Default for AtpAutotunePolicy {
2554 fn default() -> Self {
2555 Self {
2556 limits: AtpAutotuneLimits::default(),
2557 min_growth_samples: 8,
2558 loss_backoff_permille: 25,
2559 latency_hold_micros: 250_000,
2560 buffer_pressure_bytes: 8 * 1_048_576,
2561 disk_lag_micros: 100_000,
2562 cpu_backlog_symbols: 4_096,
2563 repair_roi_floor_permille: 350,
2564 relay_cost_micros_per_mib: 500_000,
2565 }
2566 }
2567}
2568
2569impl AtpAutotunePolicy {
2570 #[must_use]
2572 pub fn decide(
2573 self,
2574 current: AtpAutotuneSettings,
2575 telemetry: &AtpAutotuneTelemetry,
2576 ) -> AtpAutotuneDecision {
2577 let mut settings = self.limits.clamp(current);
2578 let mut bottlenecks = Vec::new();
2579
2580 self.detect_bottlenecks(telemetry, &mut bottlenecks);
2581
2582 if telemetry.sample_count < self.min_growth_samples {
2583 bottlenecks.push(AtpBottleneckSignal::new(
2584 AtpBottleneckKind::InsufficientTelemetry,
2585 None,
2586 u64::from(telemetry.sample_count),
2587 u64::from(self.min_growth_samples),
2588 ));
2589 }
2590
2591 let fail_closed = !bottlenecks.is_empty();
2592 if fail_closed {
2593 settings = self.backoff(settings, telemetry, &bottlenecks);
2594 return AtpAutotuneDecision {
2595 settings: self.limits.clamp(settings),
2596 bottlenecks,
2597 fail_closed,
2598 reason_code: String::from("hold_or_backoff_on_pressure"),
2599 };
2600 }
2601
2602 AtpAutotuneDecision {
2603 settings: self.limits.clamp(self.grow(settings)),
2604 bottlenecks,
2605 fail_closed,
2606 reason_code: String::from("conservative_growth"),
2607 }
2608 }
2609
2610 #[must_use]
2612 pub fn decide_with_receipt(
2613 self,
2614 current: AtpAutotuneSettings,
2615 telemetry: &AtpAutotuneTelemetry,
2616 ) -> AtpAutotuneDecisionReceipt {
2617 let current_settings = self.limits.clamp(current);
2618 let decision = self.decide(current, telemetry);
2619 AtpAutotuneDecisionReceipt::from_decision(telemetry, current_settings, decision)
2620 }
2621
2622 fn detect_bottlenecks(
2623 self,
2624 telemetry: &AtpAutotuneTelemetry,
2625 bottlenecks: &mut Vec<AtpBottleneckSignal>,
2626 ) {
2627 if telemetry.trace_id.trim().is_empty() || telemetry.workload_id.trim().is_empty() {
2628 bottlenecks.push(AtpBottleneckSignal::new(
2629 AtpBottleneckKind::ContradictoryTelemetry,
2630 None,
2631 0,
2632 1,
2633 ));
2634 }
2635
2636 if let Some(loss) = telemetry.loss_permille
2637 && loss > self.loss_backoff_permille
2638 {
2639 bottlenecks.push(AtpBottleneckSignal::new(
2640 AtpBottleneckKind::NetworkLoss,
2641 Some(AtpAutotuneMetric::LossPermille),
2642 u64::from(loss),
2643 u64::from(self.loss_backoff_permille),
2644 ));
2645 }
2646
2647 if let Some(rtt) = telemetry.rtt_micros
2648 && rtt > self.latency_hold_micros
2649 {
2650 bottlenecks.push(AtpBottleneckSignal::new(
2651 AtpBottleneckKind::NetworkLatency,
2652 Some(AtpAutotuneMetric::RttMicros),
2653 rtt,
2654 self.latency_hold_micros,
2655 ));
2656 }
2657
2658 if let Some(pto) = telemetry.pto_micros
2659 && pto > self.latency_hold_micros
2660 {
2661 bottlenecks.push(AtpBottleneckSignal::new(
2662 AtpBottleneckKind::NetworkLatency,
2663 Some(AtpAutotuneMetric::PtoMicros),
2664 pto,
2665 self.latency_hold_micros,
2666 ));
2667 }
2668
2669 if let (Some(in_flight), Some(cwnd)) =
2670 (telemetry.in_flight_bytes, telemetry.congestion_window_bytes)
2671 && in_flight > cwnd
2672 {
2673 bottlenecks.push(AtpBottleneckSignal::new(
2674 AtpBottleneckKind::CongestionWindow,
2675 Some(AtpAutotuneMetric::InFlightBytes),
2676 in_flight,
2677 cwnd,
2678 ));
2679 }
2680
2681 self.detect_queue_bottleneck(
2682 telemetry.send_buffer_queued_bytes,
2683 AtpBottleneckKind::SendBufferPressure,
2684 AtpAutotuneMetric::SendBufferQueuedBytes,
2685 bottlenecks,
2686 );
2687 self.detect_queue_bottleneck(
2688 telemetry.receive_buffer_queued_bytes,
2689 AtpBottleneckKind::ReceiveBufferPressure,
2690 AtpAutotuneMetric::ReceiveBufferQueuedBytes,
2691 bottlenecks,
2692 );
2693 self.detect_lag_bottleneck(
2694 telemetry.disk_read_lag_micros,
2695 AtpBottleneckKind::DiskReadLag,
2696 AtpAutotuneMetric::DiskReadLagMicros,
2697 bottlenecks,
2698 );
2699 self.detect_lag_bottleneck(
2700 telemetry.disk_write_lag_micros,
2701 AtpBottleneckKind::DiskWriteLag,
2702 AtpAutotuneMetric::DiskWriteLagMicros,
2703 bottlenecks,
2704 );
2705 self.detect_cpu_bottleneck(
2706 telemetry.encode_backlog_symbols,
2707 AtpBottleneckKind::EncodeBacklog,
2708 AtpAutotuneMetric::EncodeBacklogSymbols,
2709 bottlenecks,
2710 );
2711 self.detect_cpu_bottleneck(
2712 telemetry.decode_backlog_symbols,
2713 AtpBottleneckKind::DecodeBacklog,
2714 AtpAutotuneMetric::DecodeBacklogSymbols,
2715 bottlenecks,
2716 );
2717
2718 if let Some(roi) = telemetry.repair_roi_permille
2719 && roi < self.repair_roi_floor_permille
2720 {
2721 bottlenecks.push(AtpBottleneckSignal::new(
2722 AtpBottleneckKind::RepairLowRoi,
2723 Some(AtpAutotuneMetric::RepairRoiPermille),
2724 u64::from(roi),
2725 u64::from(self.repair_roi_floor_permille),
2726 ));
2727 }
2728
2729 if let Some(cost) = telemetry.relay_cost_micros_per_mib
2730 && cost > self.relay_cost_micros_per_mib
2731 {
2732 bottlenecks.push(AtpBottleneckSignal::new(
2733 AtpBottleneckKind::RelayCost,
2734 Some(AtpAutotuneMetric::RelayCostMicrosPerMiB),
2735 cost,
2736 self.relay_cost_micros_per_mib,
2737 ));
2738 }
2739
2740 if let Some(events) = telemetry.migration_events
2741 && events > 0
2742 {
2743 bottlenecks.push(AtpBottleneckSignal::new(
2744 AtpBottleneckKind::MigrationInstability,
2745 Some(AtpAutotuneMetric::MigrationEvents),
2746 u64::from(events),
2747 0,
2748 ));
2749 }
2750 }
2751
2752 fn detect_queue_bottleneck(
2753 self,
2754 observed: Option<u64>,
2755 kind: AtpBottleneckKind,
2756 metric: AtpAutotuneMetric,
2757 bottlenecks: &mut Vec<AtpBottleneckSignal>,
2758 ) {
2759 if let Some(bytes) = observed
2760 && bytes > self.buffer_pressure_bytes
2761 {
2762 bottlenecks.push(AtpBottleneckSignal::new(
2763 kind,
2764 Some(metric),
2765 bytes,
2766 self.buffer_pressure_bytes,
2767 ));
2768 }
2769 }
2770
2771 fn detect_lag_bottleneck(
2772 self,
2773 observed: Option<u64>,
2774 kind: AtpBottleneckKind,
2775 metric: AtpAutotuneMetric,
2776 bottlenecks: &mut Vec<AtpBottleneckSignal>,
2777 ) {
2778 if let Some(micros) = observed
2779 && micros > self.disk_lag_micros
2780 {
2781 bottlenecks.push(AtpBottleneckSignal::new(
2782 kind,
2783 Some(metric),
2784 micros,
2785 self.disk_lag_micros,
2786 ));
2787 }
2788 }
2789
2790 fn detect_cpu_bottleneck(
2791 self,
2792 observed: Option<u32>,
2793 kind: AtpBottleneckKind,
2794 metric: AtpAutotuneMetric,
2795 bottlenecks: &mut Vec<AtpBottleneckSignal>,
2796 ) {
2797 if let Some(symbols) = observed
2798 && symbols > self.cpu_backlog_symbols
2799 {
2800 bottlenecks.push(AtpBottleneckSignal::new(
2801 kind,
2802 Some(metric),
2803 u64::from(symbols),
2804 u64::from(self.cpu_backlog_symbols),
2805 ));
2806 }
2807 }
2808
2809 fn backoff(
2810 self,
2811 mut settings: AtpAutotuneSettings,
2812 telemetry: &AtpAutotuneTelemetry,
2813 bottlenecks: &[AtpBottleneckSignal],
2814 ) -> AtpAutotuneSettings {
2815 let reduce_transport = bottlenecks.iter().any(|signal| {
2816 matches!(
2817 signal.kind,
2818 AtpBottleneckKind::NetworkLoss
2819 | AtpBottleneckKind::NetworkLatency
2820 | AtpBottleneckKind::CongestionWindow
2821 | AtpBottleneckKind::SendBufferPressure
2822 | AtpBottleneckKind::ReceiveBufferPressure
2823 | AtpBottleneckKind::RelayCost
2824 | AtpBottleneckKind::MigrationInstability
2825 )
2826 });
2827 if reduce_transport {
2828 settings.in_flight_bytes = decrease_by_quarter(settings.in_flight_bytes);
2829 settings.stream_count = settings.stream_count.saturating_sub(1).max(1);
2830 }
2831
2832 let reduce_chunk = bottlenecks.iter().any(|signal| {
2833 matches!(
2834 signal.kind,
2835 AtpBottleneckKind::DiskReadLag
2836 | AtpBottleneckKind::DiskWriteLag
2837 | AtpBottleneckKind::EncodeBacklog
2838 | AtpBottleneckKind::DecodeBacklog
2839 )
2840 });
2841 if reduce_chunk {
2842 settings.chunk_size_bytes = decrease_by_quarter_u32(settings.chunk_size_bytes);
2843 }
2844
2845 if bottlenecks
2846 .iter()
2847 .any(|signal| signal.kind == AtpBottleneckKind::RepairLowRoi)
2848 {
2849 settings.repair_symbols_per_second =
2850 decrease_by_quarter_u32(settings.repair_symbols_per_second);
2851 } else if telemetry
2852 .loss_permille
2853 .is_some_and(|loss| loss > self.loss_backoff_permille)
2854 {
2855 settings.repair_symbols_per_second = increase_by_quarter_u32(
2856 settings.repair_symbols_per_second.max(1),
2857 self.limits.max_repair_symbols_per_second,
2858 );
2859 }
2860
2861 settings
2862 }
2863
2864 fn grow(self, mut settings: AtpAutotuneSettings) -> AtpAutotuneSettings {
2865 settings.in_flight_bytes =
2866 increase_by_eighth(settings.in_flight_bytes, self.limits.max_in_flight_bytes);
2867 settings.stream_count = settings
2868 .stream_count
2869 .saturating_add(1)
2870 .min(self.limits.max_stream_count);
2871 settings.chunk_size_bytes =
2872 increase_by_eighth_u32(settings.chunk_size_bytes, self.limits.max_chunk_size_bytes);
2873 settings
2874 }
2875}
2876
2877fn decrease_by_quarter(value: u64) -> u64 {
2878 value.saturating_sub(value / 4).max(1)
2879}
2880
2881fn decrease_by_quarter_u32(value: u32) -> u32 {
2882 value.saturating_sub(value / 4).max(1)
2883}
2884
2885fn increase_by_eighth(value: u64, max: u64) -> u64 {
2886 value.saturating_add(value / 8).min(max)
2887}
2888
2889fn increase_by_eighth_u32(value: u32, max: u32) -> u32 {
2890 value.saturating_add(value / 8).min(max)
2891}
2892
2893fn increase_by_quarter_u32(value: u32, max: u32) -> u32 {
2894 value.saturating_add(value / 4).min(max)
2895}
2896
2897fn knob_changes(
2898 current: AtpAutotuneSettings,
2899 next: AtpAutotuneSettings,
2900) -> Vec<AtpAutotuneKnobChange> {
2901 vec![
2902 AtpAutotuneKnobChange::new(
2903 AtpAutotuneKnob::InFlightBytes,
2904 current.in_flight_bytes,
2905 next.in_flight_bytes,
2906 ),
2907 AtpAutotuneKnobChange::new(
2908 AtpAutotuneKnob::StreamCount,
2909 u64::from(current.stream_count),
2910 u64::from(next.stream_count),
2911 ),
2912 AtpAutotuneKnobChange::new(
2913 AtpAutotuneKnob::ChunkSizeBytes,
2914 u64::from(current.chunk_size_bytes),
2915 u64::from(next.chunk_size_bytes),
2916 ),
2917 AtpAutotuneKnobChange::new(
2918 AtpAutotuneKnob::RepairSymbolsPerSecond,
2919 u64::from(current.repair_symbols_per_second),
2920 u64::from(next.repair_symbols_per_second),
2921 ),
2922 ]
2923}
2924
2925fn classify_decision_outcome(
2926 decision: &AtpAutotuneDecision,
2927 changes: &[AtpAutotuneKnobChange],
2928) -> AtpAutotuneDecisionOutcome {
2929 if decision
2930 .bottlenecks
2931 .iter()
2932 .any(|signal| signal.kind == AtpBottleneckKind::ContradictoryTelemetry)
2933 {
2934 return AtpAutotuneDecisionOutcome::MalformedTelemetry;
2935 }
2936
2937 if changes
2938 .iter()
2939 .any(|change| change.direction == AtpAutotuneKnobDirection::Decrease)
2940 {
2941 return AtpAutotuneDecisionOutcome::PressureBackoff;
2942 }
2943
2944 if !decision.fail_closed
2945 && changes
2946 .iter()
2947 .any(|change| change.direction == AtpAutotuneKnobDirection::Increase)
2948 {
2949 return AtpAutotuneDecisionOutcome::ConservativeGrowth;
2950 }
2951
2952 AtpAutotuneDecisionOutcome::HoldNoWin
2953}
2954
2955fn classify_receipt_status(
2956 decision: &AtpAutotuneDecision,
2957 outcome: AtpAutotuneDecisionOutcome,
2958) -> AtpAutotuneReceiptStatus {
2959 if decision
2960 .bottlenecks
2961 .iter()
2962 .any(|signal| signal.kind == AtpBottleneckKind::ContradictoryTelemetry)
2963 {
2964 return AtpAutotuneReceiptStatus::Malformed;
2965 }
2966 if decision
2967 .bottlenecks
2968 .iter()
2969 .any(|signal| signal.kind == AtpBottleneckKind::InsufficientTelemetry)
2970 {
2971 return AtpAutotuneReceiptStatus::Blocked;
2972 }
2973
2974 match outcome {
2975 AtpAutotuneDecisionOutcome::ConservativeGrowth => AtpAutotuneReceiptStatus::Pass,
2976 AtpAutotuneDecisionOutcome::PressureBackoff => AtpAutotuneReceiptStatus::Degraded,
2977 AtpAutotuneDecisionOutcome::HoldNoWin => AtpAutotuneReceiptStatus::NoWin,
2978 AtpAutotuneDecisionOutcome::MalformedTelemetry => AtpAutotuneReceiptStatus::Malformed,
2979 }
2980}
2981
2982fn receipt_confidence(
2983 decision: &AtpAutotuneDecision,
2984 status: AtpAutotuneReceiptStatus,
2985) -> AtpAutotuneReceiptConfidence {
2986 match status {
2987 AtpAutotuneReceiptStatus::Pass => AtpAutotuneReceiptConfidence::Conservative,
2988 AtpAutotuneReceiptStatus::NoWin if !decision.fail_closed => {
2989 AtpAutotuneReceiptConfidence::Conservative
2990 }
2991 AtpAutotuneReceiptStatus::Degraded | AtpAutotuneReceiptStatus::NoWin => {
2992 AtpAutotuneReceiptConfidence::FailClosed
2993 }
2994 AtpAutotuneReceiptStatus::Blocked => AtpAutotuneReceiptConfidence::InsufficientEvidence,
2995 AtpAutotuneReceiptStatus::Malformed | AtpAutotuneReceiptStatus::StaleEvidence => {
2996 AtpAutotuneReceiptConfidence::Rejected
2997 }
2998 }
2999}
3000
3001fn receipt_caveats(
3002 decision: &AtpAutotuneDecision,
3003 status: AtpAutotuneReceiptStatus,
3004) -> Vec<String> {
3005 match status {
3006 AtpAutotuneReceiptStatus::Pass => {
3007 vec![String::from("growth_requires_application_hysteresis")]
3008 }
3009 AtpAutotuneReceiptStatus::NoWin => vec![String::from("bounded_no_win")],
3010 AtpAutotuneReceiptStatus::Blocked => vec![String::from("insufficient_evidence")],
3011 AtpAutotuneReceiptStatus::Malformed => vec![String::from("malformed_evidence")],
3012 AtpAutotuneReceiptStatus::StaleEvidence => vec![String::from("stale_evidence")],
3013 AtpAutotuneReceiptStatus::Degraded => decision
3014 .bottlenecks
3015 .iter()
3016 .map(|signal| signal.kind.as_str().to_string())
3017 .collect(),
3018 }
3019}
3020
3021fn omitted_metric_sources(telemetry: &AtpAutotuneTelemetry) -> Vec<AtpAutotuneMetric> {
3022 ATP_AUTOTUNE_METRIC_NAMES
3023 .iter()
3024 .copied()
3025 .filter(|metric| !telemetry_has_metric(telemetry, *metric))
3026 .collect()
3027}
3028
3029fn telemetry_has_metric(telemetry: &AtpAutotuneTelemetry, metric: AtpAutotuneMetric) -> bool {
3030 match metric {
3031 AtpAutotuneMetric::RttMicros => telemetry.rtt_micros.is_some(),
3032 AtpAutotuneMetric::LossPermille => telemetry.loss_permille.is_some(),
3033 AtpAutotuneMetric::PtoMicros => telemetry.pto_micros.is_some(),
3034 AtpAutotuneMetric::CongestionWindowBytes => telemetry.congestion_window_bytes.is_some(),
3035 AtpAutotuneMetric::InFlightBytes => telemetry.in_flight_bytes.is_some(),
3036 AtpAutotuneMetric::SendBufferQueuedBytes => telemetry.send_buffer_queued_bytes.is_some(),
3037 AtpAutotuneMetric::ReceiveBufferQueuedBytes => {
3038 telemetry.receive_buffer_queued_bytes.is_some()
3039 }
3040 AtpAutotuneMetric::DiskReadLagMicros => telemetry.disk_read_lag_micros.is_some(),
3041 AtpAutotuneMetric::DiskWriteLagMicros => telemetry.disk_write_lag_micros.is_some(),
3042 AtpAutotuneMetric::EncodeBacklogSymbols => telemetry.encode_backlog_symbols.is_some(),
3043 AtpAutotuneMetric::DecodeBacklogSymbols => telemetry.decode_backlog_symbols.is_some(),
3044 AtpAutotuneMetric::RepairRoiPermille => telemetry.repair_roi_permille.is_some(),
3045 AtpAutotuneMetric::RelayCostMicrosPerMiB => telemetry.relay_cost_micros_per_mib.is_some(),
3046 AtpAutotuneMetric::MigrationEvents => telemetry.migration_events.is_some(),
3047 }
3048}
3049
3050fn application_consumer_status(outcome: AtpAutotuneApplicationOutcome) -> AtpAutotuneReceiptStatus {
3051 match outcome {
3052 AtpAutotuneApplicationOutcome::AppliedPressureBackoff => AtpAutotuneReceiptStatus::Degraded,
3053 AtpAutotuneApplicationOutcome::AppliedConfirmedGrowth => AtpAutotuneReceiptStatus::Pass,
3054 AtpAutotuneApplicationOutcome::DeferredGrowthHysteresis => {
3055 AtpAutotuneReceiptStatus::Blocked
3056 }
3057 AtpAutotuneApplicationOutcome::HeldNoWin => AtpAutotuneReceiptStatus::NoWin,
3058 AtpAutotuneApplicationOutcome::RejectedMalformedTelemetry
3059 | AtpAutotuneApplicationOutcome::RejectedSchemaVersion => {
3060 AtpAutotuneReceiptStatus::Malformed
3061 }
3062 AtpAutotuneApplicationOutcome::RejectedStaleReceipt => {
3063 AtpAutotuneReceiptStatus::StaleEvidence
3064 }
3065 }
3066}
3067
3068#[cfg(test)]
3069mod tests {
3070 use super::*;
3071
3072 fn healthy_telemetry() -> AtpAutotuneTelemetry {
3073 AtpAutotuneTelemetry::new("trace-a", "workload-a").with_sample_count(16)
3074 }
3075
3076 fn repair_inputs() -> AtpRepairRoiInputs {
3077 AtpRepairRoiInputs {
3078 trace_id: String::from("trace-repair"),
3079 workload_id: String::from("workload-repair"),
3080 expected_time_saved_micros: 400_000,
3081 encode_cpu_micros: 10_000,
3082 decode_cpu_micros: 10_000,
3083 bandwidth_overhead_bytes: 64 * 1_024,
3084 memory_pressure_permille: 50,
3085 stream_contention_permille: 25,
3086 relay_cost_micros_per_mib: 100_000,
3087 path_stability_permille: 950,
3088 resume_value_permille: 0,
3089 loss_permille: 1,
3090 available_peer_count: 1,
3091 path_mode: AtpRepairPathMode::Direct,
3092 requested_mode: None,
3093 missing_tail_chunks: 0,
3094 rtt_micros: 50_000,
3095 path_migration_events: 0,
3096 broadcast_peer_count: 0,
3097 }
3098 }
3099
3100 #[test]
3101 fn metric_names_are_stable_and_namespaced() {
3102 let names: Vec<_> = ATP_AUTOTUNE_METRIC_NAMES
3103 .iter()
3104 .map(|metric| metric.as_str())
3105 .collect();
3106
3107 assert_eq!(names.len(), 14);
3108 assert!(names.iter().all(|name| name.starts_with("atp.autotune.")));
3109 assert_eq!(names[0], "atp.autotune.rtt_micros");
3110 assert_eq!(names[13], "atp.autotune.migration_events");
3111 }
3112
3113 #[test]
3114 fn metric_json_uses_stable_names() -> serde_json::Result<()> {
3115 let encoded = serde_json::to_string(&AtpAutotuneMetric::LossPermille)?;
3116 assert_eq!(encoded, r#""atp.autotune.loss_permille""#);
3117
3118 let decoded: AtpAutotuneMetric = serde_json::from_str(&encoded)?;
3119 assert_eq!(decoded, AtpAutotuneMetric::LossPermille);
3120 Ok(())
3121 }
3122
3123 #[test]
3124 fn telemetry_report_collects_stable_metric_samples() -> Result<(), Box<dyn std::error::Error>> {
3125 let report = AtpAutotuneTelemetryReport::new("trace-report", "workload-report")
3126 .with_sample_count(16)
3127 .with_sample(AtpAutotuneMetric::RttMicros, 42_000)
3128 .with_sample(AtpAutotuneMetric::LossPermille, 7)
3129 .with_sample(AtpAutotuneMetric::EncodeBacklogSymbols, 128)
3130 .with_sample(AtpAutotuneMetric::RelayCostMicrosPerMiB, 250_000);
3131
3132 let encoded = serde_json::to_string(&report)?;
3133 assert!(encoded.contains("atp.autotune.rtt_micros"));
3134
3135 let decoded: AtpAutotuneTelemetryReport = serde_json::from_str(&encoded)?;
3136 let telemetry = decoded.into_telemetry()?;
3137
3138 assert_eq!(telemetry.trace_id, "trace-report");
3139 assert_eq!(telemetry.workload_id, "workload-report");
3140 assert_eq!(telemetry.sample_count, 16);
3141 assert_eq!(telemetry.rtt_micros, Some(42_000));
3142 assert_eq!(telemetry.loss_permille, Some(7));
3143 assert_eq!(telemetry.encode_backlog_symbols, Some(128));
3144 assert_eq!(telemetry.relay_cost_micros_per_mib, Some(250_000));
3145 Ok(())
3146 }
3147
3148 #[test]
3149 fn telemetry_report_rejects_out_of_range_metric_samples() {
3150 let report = AtpAutotuneTelemetryReport::new("trace-report", "workload-report")
3151 .with_sample(AtpAutotuneMetric::LossPermille, u64::from(u16::MAX) + 1);
3152
3153 let error = report.into_telemetry();
3154
3155 assert_eq!(
3156 error,
3157 Err(AtpAutotuneTelemetryError::MetricValueOutOfRange {
3158 metric: AtpAutotuneMetric::LossPermille,
3159 value: u64::from(u16::MAX) + 1,
3160 max: u64::from(u16::MAX),
3161 })
3162 );
3163 }
3164
3165 #[test]
3166 fn telemetry_window_exports_stable_sample_report_order() {
3167 let mut telemetry =
3168 AtpAutotuneTelemetry::new("trace-window", "workload-window").with_sample_count(32);
3169 telemetry.loss_permille = Some(5);
3170 telemetry.rtt_micros = Some(40_000);
3171 telemetry.congestion_window_bytes = Some(64 * 1_048_576);
3172 telemetry.migration_events = Some(2);
3173
3174 let report = telemetry.to_report();
3175
3176 assert_eq!(report.trace_id, "trace-window");
3177 assert_eq!(report.workload_id, "workload-window");
3178 assert_eq!(report.sample_count, 32);
3179 assert_eq!(
3180 report.samples,
3181 vec![
3182 AtpAutotuneMetricSample::new(AtpAutotuneMetric::RttMicros, 40_000),
3183 AtpAutotuneMetricSample::new(AtpAutotuneMetric::LossPermille, 5),
3184 AtpAutotuneMetricSample::new(
3185 AtpAutotuneMetric::CongestionWindowBytes,
3186 64 * 1_048_576,
3187 ),
3188 AtpAutotuneMetricSample::new(AtpAutotuneMetric::MigrationEvents, 2),
3189 ]
3190 );
3191 }
3192
3193 #[test]
3194 fn telemetry_window_roundtrips_through_sample_report() -> Result<(), Box<dyn std::error::Error>>
3195 {
3196 let telemetry = AtpAutotuneTelemetry {
3197 trace_id: String::from("trace-roundtrip"),
3198 workload_id: String::from("workload-roundtrip"),
3199 sample_count: 16,
3200 rtt_micros: Some(41_000),
3201 loss_permille: Some(3),
3202 pto_micros: Some(125_000),
3203 congestion_window_bytes: Some(96 * 1_048_576),
3204 in_flight_bytes: Some(32 * 1_048_576),
3205 send_buffer_queued_bytes: Some(2 * 1_048_576),
3206 receive_buffer_queued_bytes: Some(1_048_576),
3207 disk_read_lag_micros: Some(10_000),
3208 disk_write_lag_micros: Some(12_000),
3209 encode_backlog_symbols: Some(128),
3210 decode_backlog_symbols: Some(64),
3211 repair_roi_permille: Some(800),
3212 relay_cost_micros_per_mib: Some(250_000),
3213 migration_events: Some(1),
3214 };
3215
3216 let report = AtpAutotuneTelemetryReport::from_telemetry(&telemetry);
3217
3218 assert_eq!(report.samples.len(), ATP_AUTOTUNE_METRIC_NAMES.len());
3219 assert_eq!(
3220 report.samples[0],
3221 AtpAutotuneMetricSample::new(AtpAutotuneMetric::RttMicros, 41_000)
3222 );
3223 assert_eq!(
3224 report.samples[13],
3225 AtpAutotuneMetricSample::new(AtpAutotuneMetric::MigrationEvents, 1)
3226 );
3227 assert_eq!(report.into_telemetry()?, telemetry);
3228 Ok(())
3229 }
3230
3231 #[test]
3232 fn telemetry_window_zero_sample_count_roundtrip_uses_exported_sample_count()
3233 -> Result<(), Box<dyn std::error::Error>> {
3234 let mut telemetry = AtpAutotuneTelemetry::new("trace-inferred", "workload-inferred");
3235 telemetry.rtt_micros = Some(25_000);
3236 telemetry.loss_permille = Some(1);
3237
3238 let roundtrip = telemetry.to_report().into_telemetry()?;
3239
3240 assert_eq!(roundtrip.trace_id, telemetry.trace_id);
3241 assert_eq!(roundtrip.workload_id, telemetry.workload_id);
3242 assert_eq!(roundtrip.sample_count, 2);
3243 assert_eq!(roundtrip.rtt_micros, Some(25_000));
3244 assert_eq!(roundtrip.loss_permille, Some(1));
3245 Ok(())
3246 }
3247
3248 #[test]
3249 fn transfer_pressure_snapshot_exports_runtime_metrics_and_derived_costs()
3250 -> Result<(), Box<dyn std::error::Error>> {
3251 let mut snapshot =
3252 AtpTransferPressureSnapshot::new("trace-transfer", "transfer-42").with_sample_count(12);
3253 snapshot.rtt_micros = Some(44_000);
3254 snapshot.loss_permille = Some(9);
3255 snapshot.pto_micros = Some(120_000);
3256 snapshot.congestion_window_bytes = Some(96 * 1_048_576);
3257 snapshot.in_flight_bytes = Some(32 * 1_048_576);
3258 snapshot.send_buffer_queued_bytes = Some(512 * 1_024);
3259 snapshot.receive_buffer_queued_bytes = Some(256 * 1_024);
3260 snapshot.disk_read_lag_micros = Some(8_000);
3261 snapshot.disk_write_lag_micros = Some(9_000);
3262 snapshot.encode_backlog_symbols = Some(64);
3263 snapshot.decode_backlog_symbols = Some(32);
3264 snapshot.repair_symbols_sent = Some(400);
3265 snapshot.useful_repair_symbols = Some(250);
3266 snapshot.relay_cost_micros = Some(300_000);
3267 snapshot.relay_bytes = Some(2 * 1_048_576);
3268 snapshot.migration_events = Some(1);
3269
3270 assert_eq!(snapshot.repair_roi_permille(), Some(625));
3271 assert_eq!(snapshot.relay_cost_micros_per_mib(), Some(150_000));
3272
3273 let report = snapshot.to_report();
3274 assert_eq!(report.trace_id, "trace-transfer");
3275 assert_eq!(report.workload_id, "transfer-42");
3276 assert_eq!(report.sample_count, 12);
3277 assert_eq!(report.samples.len(), ATP_AUTOTUNE_METRIC_NAMES.len());
3278 assert_eq!(
3279 report.samples[11],
3280 AtpAutotuneMetricSample::new(AtpAutotuneMetric::RepairRoiPermille, 625)
3281 );
3282 assert_eq!(
3283 report.samples[12],
3284 AtpAutotuneMetricSample::new(AtpAutotuneMetric::RelayCostMicrosPerMiB, 150_000)
3285 );
3286
3287 let telemetry = report.into_telemetry()?;
3288 assert_eq!(telemetry.trace_id, "trace-transfer");
3289 assert_eq!(telemetry.workload_id, "transfer-42");
3290 assert_eq!(telemetry.sample_count, 12);
3291 assert_eq!(telemetry.loss_permille, Some(9));
3292 assert_eq!(telemetry.repair_roi_permille, Some(625));
3293 assert_eq!(telemetry.relay_cost_micros_per_mib, Some(150_000));
3294 assert_eq!(telemetry.migration_events, Some(1));
3295 Ok(())
3296 }
3297
3298 #[test]
3299 fn transfer_pressure_snapshot_omits_denominator_based_metrics_when_empty()
3300 -> Result<(), Box<dyn std::error::Error>> {
3301 let mut snapshot = AtpTransferPressureSnapshot::new("trace-empty", "transfer-empty");
3302 snapshot.repair_symbols_sent = Some(0);
3303 snapshot.useful_repair_symbols = Some(10);
3304 snapshot.relay_cost_micros = Some(1_000);
3305 snapshot.relay_bytes = Some(0);
3306 snapshot.migration_events = Some(2);
3307
3308 assert_eq!(snapshot.repair_roi_permille(), None);
3309 assert_eq!(snapshot.relay_cost_micros_per_mib(), None);
3310
3311 let report = snapshot.to_report();
3312 assert_eq!(
3313 report.samples,
3314 vec![AtpAutotuneMetricSample::new(
3315 AtpAutotuneMetric::MigrationEvents,
3316 2,
3317 )]
3318 );
3319
3320 let telemetry = report.into_telemetry()?;
3321 assert_eq!(telemetry.sample_count, 1);
3322 assert_eq!(telemetry.repair_roi_permille, None);
3323 assert_eq!(telemetry.relay_cost_micros_per_mib, None);
3324 assert_eq!(telemetry.migration_events, Some(2));
3325 Ok(())
3326 }
3327
3328 #[test]
3329 fn healthy_window_grows_conservatively() {
3330 let policy = AtpAutotunePolicy::default();
3331 let current = AtpAutotuneSettings::default();
3332 let decision = policy.decide(current, &healthy_telemetry());
3333
3334 assert!(!decision.fail_closed);
3335 assert_eq!(decision.reason_code, "conservative_growth");
3336 assert_eq!(
3337 decision.settings.in_flight_bytes,
3338 current.in_flight_bytes + current.in_flight_bytes / 8
3339 );
3340 assert_eq!(decision.settings.stream_count, current.stream_count + 1);
3341 assert_eq!(
3342 decision.settings.chunk_size_bytes,
3343 current.chunk_size_bytes + current.chunk_size_bytes / 8
3344 );
3345 assert_eq!(
3346 decision.settings.repair_symbols_per_second,
3347 current.repair_symbols_per_second
3348 );
3349 }
3350
3351 #[test]
3352 fn insufficient_samples_hold_existing_settings() {
3353 let policy = AtpAutotunePolicy::default();
3354 let current = AtpAutotuneSettings::default();
3355 let telemetry = AtpAutotuneTelemetry::new("trace-a", "workload-a").with_sample_count(2);
3356 let decision = policy.decide(current, &telemetry);
3357
3358 assert!(decision.fail_closed);
3359 assert_eq!(decision.settings, current);
3360 assert_eq!(
3361 decision.bottlenecks[0].kind,
3362 AtpBottleneckKind::InsufficientTelemetry
3363 );
3364 }
3365
3366 #[test]
3367 fn loss_backs_off_transport_and_raises_repair_rate() {
3368 let policy = AtpAutotunePolicy::default();
3369 let current = AtpAutotuneSettings::default();
3370 let mut telemetry = healthy_telemetry();
3371 telemetry.loss_permille = Some(100);
3372
3373 let decision = policy.decide(current, &telemetry);
3374
3375 assert!(decision.fail_closed);
3376 assert!(
3377 decision
3378 .bottlenecks
3379 .iter()
3380 .any(|signal| signal.kind == AtpBottleneckKind::NetworkLoss)
3381 );
3382 assert!(decision.settings.in_flight_bytes < current.in_flight_bytes);
3383 assert_eq!(decision.settings.stream_count, current.stream_count - 1);
3384 assert!(decision.settings.repair_symbols_per_second > current.repair_symbols_per_second);
3385 }
3386
3387 #[test]
3388 fn low_repair_roi_reduces_repair_rate_without_transport_backoff() {
3389 let policy = AtpAutotunePolicy::default();
3390 let current = AtpAutotuneSettings::default();
3391 let mut telemetry = healthy_telemetry();
3392 telemetry.repair_roi_permille = Some(100);
3393
3394 let decision = policy.decide(current, &telemetry);
3395
3396 assert!(decision.fail_closed);
3397 assert_eq!(decision.settings.in_flight_bytes, current.in_flight_bytes);
3398 assert_eq!(decision.settings.stream_count, current.stream_count);
3399 assert!(decision.settings.repair_symbols_per_second < current.repair_symbols_per_second);
3400 }
3401
3402 #[test]
3403 fn relay_cost_backs_off_transport_without_repair_backoff() {
3404 let policy = AtpAutotunePolicy::default();
3405 let current = AtpAutotuneSettings::default();
3406 let mut telemetry = healthy_telemetry();
3407 telemetry.relay_cost_micros_per_mib = Some(policy.relay_cost_micros_per_mib + 1);
3408
3409 let decision = policy.decide(current, &telemetry);
3410
3411 assert!(decision.fail_closed);
3412 assert!(
3413 decision
3414 .bottlenecks
3415 .iter()
3416 .any(|signal| signal.kind == AtpBottleneckKind::RelayCost)
3417 );
3418 assert!(decision.settings.in_flight_bytes < current.in_flight_bytes);
3419 assert_eq!(decision.settings.stream_count, current.stream_count - 1);
3420 assert_eq!(
3421 decision.settings.repair_symbols_per_second,
3422 current.repair_symbols_per_second
3423 );
3424 }
3425
3426 #[test]
3427 fn repair_coordinator_clean_path_defaults_to_no_repair() {
3428 let coordinator = AtpRepairCoordinator::default();
3429 let mut inputs = repair_inputs();
3430 inputs.expected_time_saved_micros = 20_000;
3431 inputs.bandwidth_overhead_bytes = 0;
3432
3433 let decision = coordinator.decide(&inputs);
3434
3435 assert_eq!(decision.action, AtpRepairAction::NoRepair);
3436 assert_eq!(decision.mode, AtpRepairMode::Off);
3437 assert!(decision.fail_closed);
3438 assert_eq!(decision.reason_code, "repair_roi_not_positive");
3439 assert!(
3440 decision
3441 .factors
3442 .iter()
3443 .any(|factor| factor.kind == AtpRepairDecisionFactorKind::NetRoi)
3444 );
3445 }
3446
3447 #[test]
3448 fn repair_coordinator_selects_exact_retransmit_for_low_loss_positive_roi() {
3449 let decision = AtpRepairCoordinator::default().decide(&repair_inputs());
3450
3451 assert_eq!(decision.action, AtpRepairAction::ExactRetransmit);
3452 assert_eq!(decision.mode, AtpRepairMode::Tail);
3453 assert!(!decision.fail_closed);
3454 assert_eq!(decision.reason_code, "exact_retransmit_roi_positive");
3455 assert!(decision.net_roi_micros > 0);
3456 }
3457
3458 #[test]
3459 fn repair_coordinator_selects_parity_trickle_for_moderate_loss() {
3460 let mut inputs = repair_inputs();
3461 inputs.loss_permille = 25;
3462 inputs.expected_time_saved_micros = 700_000;
3463 inputs.resume_value_permille = 200;
3464
3465 let decision = AtpRepairCoordinator::default().decide(&inputs);
3466
3467 assert_eq!(decision.action, AtpRepairAction::ParityTrickle);
3468 assert_eq!(decision.mode, AtpRepairMode::Lossy);
3469 assert_eq!(decision.reason_code, "parity_trickle_roi_positive");
3470 }
3471
3472 #[test]
3473 fn repair_coordinator_selects_burst_and_multi_peer_for_high_loss() {
3474 let coordinator = AtpRepairCoordinator::default();
3475 let mut burst = repair_inputs();
3476 burst.loss_permille = 120;
3477 burst.expected_time_saved_micros = 2_000_000;
3478 burst.resume_value_permille = 500;
3479
3480 let burst_decision = coordinator.decide(&burst);
3481 assert_eq!(burst_decision.action, AtpRepairAction::BurstRepair);
3482 assert_eq!(burst_decision.mode, AtpRepairMode::Lossy);
3483
3484 let mut multi_peer = burst;
3485 multi_peer.available_peer_count = 4;
3486 let multi_peer_decision = coordinator.decide(&multi_peer);
3487 assert_eq!(multi_peer_decision.action, AtpRepairAction::MultiPeerRepair);
3488 assert_eq!(multi_peer_decision.mode, AtpRepairMode::Swarm);
3489 assert_eq!(
3490 multi_peer_decision.reason_code,
3491 "multi_peer_repair_roi_positive"
3492 );
3493 }
3494
3495 #[test]
3496 fn repair_coordinator_selects_relay_only_when_direct_path_is_unstable() {
3497 let mut inputs = repair_inputs();
3498 inputs.path_mode = AtpRepairPathMode::DirectAndRelay;
3499 inputs.path_stability_permille = 200;
3500 inputs.loss_permille = 40;
3501 inputs.expected_time_saved_micros = 1_000_000;
3502 inputs.relay_cost_micros_per_mib = 100_000;
3503
3504 let decision = AtpRepairCoordinator::default().decide(&inputs);
3505
3506 assert_eq!(decision.action, AtpRepairAction::RelayOnlyRepair);
3507 assert_eq!(decision.mode, AtpRepairMode::MobileUnstable);
3508 assert_eq!(decision.reason_code, "relay_only_repair_roi_positive");
3509 assert!(
3510 decision
3511 .human_summary_lines()
3512 .join("\n")
3513 .contains("Repair mode: mobile_unstable")
3514 );
3515 }
3516
3517 #[test]
3518 fn repair_coordinator_blocks_repair_under_high_memory_pressure() {
3519 let mut inputs = repair_inputs();
3520 inputs.expected_time_saved_micros = 2_000_000;
3521 inputs.memory_pressure_permille = 950;
3522 inputs.resume_value_permille = 100;
3523
3524 let decision = AtpRepairCoordinator::default().decide(&inputs);
3525
3526 assert_eq!(decision.action, AtpRepairAction::NoRepair);
3527 assert_eq!(decision.mode, AtpRepairMode::Off);
3528 assert_eq!(decision.reason_code, "blocked_by_memory_pressure");
3529 assert!(decision.fail_closed);
3530 }
3531
3532 #[test]
3533 fn repair_coordinator_honors_manual_resume_mode_after_budget_gates() {
3534 let mut inputs = repair_inputs();
3535 inputs.requested_mode = Some(AtpRepairMode::ResumeRepair);
3536 inputs.resume_value_permille = 700;
3537 inputs.expected_time_saved_micros = 900_000;
3538
3539 let decision = AtpRepairCoordinator::default().decide(&inputs);
3540
3541 assert_eq!(decision.mode, AtpRepairMode::ResumeRepair);
3542 assert_eq!(decision.action, AtpRepairAction::ParityTrickle);
3543 assert_eq!(decision.mode_reason_code, "manual_repair_mode_requested");
3544 assert_eq!(
3545 decision.mode_cooldown_micros,
3546 AtpRepairCoordinatorPolicy::default().resume_cooldown_micros
3547 );
3548 }
3549
3550 #[test]
3551 fn repair_coordinator_auto_selects_relay_expensive_mode() {
3552 let mut inputs = repair_inputs();
3553 inputs.path_mode = AtpRepairPathMode::DirectAndRelay;
3554 inputs.relay_cost_micros_per_mib =
3555 AtpRepairCoordinatorPolicy::default().max_relay_cost_micros_per_mib + 1;
3556 inputs.expected_time_saved_micros = 800_000;
3557
3558 let decision = AtpRepairCoordinator::default().decide(&inputs);
3559
3560 assert_eq!(decision.mode, AtpRepairMode::RelayExpensive);
3561 assert_eq!(decision.action, AtpRepairAction::ExactRetransmit);
3562 assert_eq!(decision.mode_reason_code, "auto_relay_expensive_cost_gate");
3563 }
3564
3565 #[test]
3566 fn repair_coordinator_auto_selects_satellite_broadcast_and_tail_modes() {
3567 let coordinator = AtpRepairCoordinator::default();
3568 let mut satellite = repair_inputs();
3569 satellite.rtt_micros =
3570 AtpRepairCoordinatorPolicy::default().satellite_high_bdp_min_rtt_micros;
3571 satellite.expected_time_saved_micros = 900_000;
3572 satellite.loss_permille = 20;
3573 assert_eq!(
3574 coordinator.decide(&satellite).mode,
3575 AtpRepairMode::SatelliteHighBdp
3576 );
3577
3578 let mut broadcast = repair_inputs();
3579 broadcast.broadcast_peer_count = AtpRepairCoordinatorPolicy::default().broadcast_min_peers;
3580 broadcast.expected_time_saved_micros = 2_000_000;
3581 assert_eq!(
3582 coordinator.decide(&broadcast).mode,
3583 AtpRepairMode::Broadcast
3584 );
3585
3586 let mut tail = repair_inputs();
3587 tail.missing_tail_chunks = AtpRepairCoordinatorPolicy::default().tail_min_missing_chunks;
3588 tail.expected_time_saved_micros = 300_000;
3589 assert_eq!(coordinator.decide(&tail).mode, AtpRepairMode::Tail);
3590 }
3591
3592 #[test]
3593 fn repair_roi_inputs_derive_traceable_status_values_from_autotune() {
3594 let mut telemetry = healthy_telemetry();
3595 telemetry.rtt_micros = Some(50_000);
3596 telemetry.pto_micros = Some(300_000);
3597 telemetry.loss_permille = Some(100);
3598 telemetry.decode_backlog_symbols = Some(32);
3599 telemetry.repair_roi_permille = Some(700);
3600 telemetry.migration_events = Some(1);
3601 telemetry.relay_cost_micros_per_mib = Some(100_000);
3602
3603 let inputs = AtpRepairRoiInputs::from_autotune_telemetry(&telemetry);
3604 let decision = AtpRepairCoordinator::default().decide(&inputs);
3605
3606 assert_eq!(inputs.trace_id, "trace-a");
3607 assert_eq!(inputs.workload_id, "workload-a");
3608 assert_eq!(inputs.path_mode, AtpRepairPathMode::DirectAndRelay);
3609 assert_eq!(inputs.missing_tail_chunks, 32);
3610 assert_eq!(inputs.path_migration_events, 1);
3611 assert_eq!(inputs.rtt_micros, 50_000);
3612 assert!(inputs.expected_time_saved_micros > 0);
3613 assert_ne!(decision.action, AtpRepairAction::NoRepair);
3614 }
3615
3616 #[test]
3617 fn buffer_and_disk_pressure_reduce_different_knobs() {
3618 let policy = AtpAutotunePolicy::default();
3619 let current = AtpAutotuneSettings::default();
3620 let mut telemetry = healthy_telemetry();
3621 telemetry.send_buffer_queued_bytes = Some(policy.buffer_pressure_bytes + 1);
3622 telemetry.disk_write_lag_micros = Some(policy.disk_lag_micros + 1);
3623
3624 let decision = policy.decide(current, &telemetry);
3625
3626 assert!(decision.fail_closed);
3627 assert!(decision.settings.in_flight_bytes < current.in_flight_bytes);
3628 assert!(decision.settings.chunk_size_bytes < current.chunk_size_bytes);
3629 assert!(
3630 decision
3631 .bottlenecks
3632 .iter()
3633 .any(|signal| signal.kind == AtpBottleneckKind::SendBufferPressure)
3634 );
3635 assert!(
3636 decision
3637 .bottlenecks
3638 .iter()
3639 .any(|signal| signal.kind == AtpBottleneckKind::DiskWriteLag)
3640 );
3641 }
3642
3643 #[test]
3644 fn empty_ids_fail_closed() {
3645 let policy = AtpAutotunePolicy::default();
3646 let current = AtpAutotuneSettings::default();
3647 let telemetry = AtpAutotuneTelemetry::new("", " ").with_sample_count(16);
3648
3649 let decision = policy.decide(current, &telemetry);
3650
3651 assert!(decision.fail_closed);
3652 assert!(
3653 decision
3654 .bottlenecks
3655 .iter()
3656 .any(|signal| signal.kind == AtpBottleneckKind::ContradictoryTelemetry)
3657 );
3658 }
3659
3660 #[test]
3661 fn limits_are_enforced_on_growth_and_backoff() {
3662 let policy = AtpAutotunePolicy {
3663 limits: AtpAutotuneLimits {
3664 min_in_flight_bytes: 4,
3665 max_in_flight_bytes: 10,
3666 min_stream_count: 2,
3667 max_stream_count: 3,
3668 min_chunk_size_bytes: 8,
3669 max_chunk_size_bytes: 12,
3670 min_repair_symbols_per_second: 2,
3671 max_repair_symbols_per_second: 4,
3672 },
3673 ..AtpAutotunePolicy::default()
3674 };
3675 let current = AtpAutotuneSettings::new(100, 99, 100, 99);
3676 let decision = policy.decide(current, &healthy_telemetry());
3677
3678 assert_eq!(decision.settings.in_flight_bytes, 10);
3679 assert_eq!(decision.settings.stream_count, 3);
3680 assert_eq!(decision.settings.chunk_size_bytes, 12);
3681 assert_eq!(decision.settings.repair_symbols_per_second, 4);
3682 }
3683
3684 #[test]
3685 fn decision_receipt_records_stable_knob_changes_and_outcome() {
3686 let policy = AtpAutotunePolicy::default();
3687 let current = AtpAutotuneSettings::default();
3688 let mut telemetry = healthy_telemetry();
3689 telemetry.loss_permille = Some(100);
3690
3691 let receipt = policy.decide_with_receipt(current, &telemetry);
3692
3693 assert_eq!(
3694 receipt.schema_version,
3695 ATP_AUTOTUNE_DECISION_RECEIPT_SCHEMA_VERSION
3696 );
3697 assert_eq!(receipt.trace_id, "trace-a");
3698 assert_eq!(receipt.workload_id, "workload-a");
3699 assert_eq!(receipt.sample_count, 16);
3700 assert_eq!(receipt.current_settings, current);
3701 assert_eq!(receipt.outcome, AtpAutotuneDecisionOutcome::PressureBackoff);
3702 assert_eq!(receipt.consumer_status, AtpAutotuneReceiptStatus::Degraded);
3703 assert_eq!(receipt.confidence, AtpAutotuneReceiptConfidence::FailClosed);
3704 assert_eq!(receipt.caveats, vec![String::from("network_loss")]);
3705 assert!(receipt.stale_sources.is_empty());
3706 assert_eq!(
3707 receipt.proof_pointer,
3708 AtpAutotuneReceiptProofPointer {
3709 receipt_schema_version: String::from(ATP_AUTOTUNE_DECISION_RECEIPT_SCHEMA_VERSION),
3710 trace_id: String::from("trace-a"),
3711 workload_id: String::from("workload-a"),
3712 sample_count: 16,
3713 }
3714 );
3715 assert!(receipt.validate_for_consumers().is_ok());
3716 assert!(
3717 receipt
3718 .omitted_sources
3719 .contains(&AtpAutotuneMetric::RttMicros)
3720 );
3721 assert!(
3722 !receipt
3723 .omitted_sources
3724 .contains(&AtpAutotuneMetric::LossPermille)
3725 );
3726 assert_eq!(receipt.changes.len(), 4);
3727 assert_eq!(receipt.changes[0].knob, AtpAutotuneKnob::InFlightBytes);
3728 assert_eq!(
3729 receipt.changes[0].direction,
3730 AtpAutotuneKnobDirection::Decrease
3731 );
3732 assert_eq!(receipt.changes[1].knob.as_str(), "stream_count");
3733 assert_eq!(
3734 receipt.changes[3].direction,
3735 AtpAutotuneKnobDirection::Increase
3736 );
3737 assert_eq!(
3738 receipt.selected_knobs(),
3739 vec![
3740 AtpAutotuneKnob::InFlightBytes,
3741 AtpAutotuneKnob::StreamCount,
3742 AtpAutotuneKnob::RepairSymbolsPerSecond,
3743 ]
3744 );
3745
3746 let encoded = serde_json::to_string(&receipt).expect("receipt JSON");
3747 assert!(encoded.contains(r#""consumer_status":"degraded""#));
3748 assert!(encoded.contains(r#""confidence":"fail_closed""#));
3749 let decoded: AtpAutotuneDecisionReceipt =
3750 serde_json::from_str(&encoded).expect("roundtrip receipt JSON");
3751 assert_eq!(decoded, receipt);
3752
3753 let human = receipt.render_human_summary(true);
3754 assert!(human.contains("Status: degraded"));
3755 assert!(human.contains("Confidence: fail_closed"));
3756 assert!(human.contains("- network_loss: metric=atp.autotune.loss_permille"));
3757 }
3758
3759 #[test]
3760 fn decision_receipt_classifies_malformed_and_no_win_outcomes() {
3761 let policy = AtpAutotunePolicy {
3762 limits: AtpAutotuneLimits {
3763 min_in_flight_bytes: 8 * 1_048_576,
3764 max_in_flight_bytes: 8 * 1_048_576,
3765 min_stream_count: 4,
3766 max_stream_count: 4,
3767 min_chunk_size_bytes: 256 * 1_024,
3768 max_chunk_size_bytes: 256 * 1_024,
3769 min_repair_symbols_per_second: 256,
3770 max_repair_symbols_per_second: 256,
3771 },
3772 ..AtpAutotunePolicy::default()
3773 };
3774
3775 let malformed = policy.decide_with_receipt(
3776 AtpAutotuneSettings::default(),
3777 &AtpAutotuneTelemetry::new("", "workload-a").with_sample_count(16),
3778 );
3779 assert_eq!(
3780 malformed.outcome,
3781 AtpAutotuneDecisionOutcome::MalformedTelemetry
3782 );
3783 assert_eq!(
3784 malformed.consumer_status,
3785 AtpAutotuneReceiptStatus::Malformed
3786 );
3787 assert_eq!(
3788 malformed.validate_for_consumers(),
3789 Err(AtpAutotuneReceiptValidationError::MissingTraceId)
3790 );
3791
3792 let bounded =
3793 policy.decide_with_receipt(AtpAutotuneSettings::default(), &healthy_telemetry());
3794 assert_eq!(bounded.outcome, AtpAutotuneDecisionOutcome::HoldNoWin);
3795 assert_eq!(bounded.consumer_status, AtpAutotuneReceiptStatus::NoWin);
3796 assert_eq!(
3797 bounded.confidence,
3798 AtpAutotuneReceiptConfidence::Conservative
3799 );
3800 assert_eq!(bounded.caveats, vec![String::from("bounded_no_win")]);
3801 assert!(
3802 bounded
3803 .changes
3804 .iter()
3805 .all(|change| change.direction == AtpAutotuneKnobDirection::Hold)
3806 );
3807 }
3808
3809 #[test]
3810 fn decision_receipt_status_distinguishes_blocked_and_malformed_consumers() {
3811 let policy = AtpAutotunePolicy::default();
3812 let current = AtpAutotuneSettings::default();
3813 let blocked = policy.decide_with_receipt(
3814 current,
3815 &AtpAutotuneTelemetry::new("trace-blocked", "workload-blocked").with_sample_count(2),
3816 );
3817
3818 assert_eq!(blocked.outcome, AtpAutotuneDecisionOutcome::HoldNoWin);
3819 assert_eq!(blocked.consumer_status, AtpAutotuneReceiptStatus::Blocked);
3820 assert_eq!(
3821 blocked.confidence,
3822 AtpAutotuneReceiptConfidence::InsufficientEvidence
3823 );
3824 assert_eq!(blocked.caveats, vec![String::from("insufficient_evidence")]);
3825 assert!(blocked.validate_for_consumers().is_ok());
3826
3827 let mut unsupported = blocked.clone();
3828 unsupported.schema_version = String::from("atp-autotune-decision-receipt-v0");
3829 assert_eq!(
3830 unsupported.validate_for_consumers(),
3831 Err(
3832 AtpAutotuneReceiptValidationError::UnsupportedSchemaVersion {
3833 expected: String::from(ATP_AUTOTUNE_DECISION_RECEIPT_SCHEMA_VERSION),
3834 actual: String::from("atp-autotune-decision-receipt-v0"),
3835 },
3836 )
3837 );
3838
3839 let mut bad_pointer = blocked;
3840 bad_pointer.proof_pointer.trace_id = String::from("other-trace");
3841 assert_eq!(
3842 bad_pointer.validate_for_consumers(),
3843 Err(AtpAutotuneReceiptValidationError::ProofPointerMismatch {
3844 field: String::from("trace_id"),
3845 })
3846 );
3847 }
3848
3849 #[test]
3850 fn application_state_defers_growth_until_hysteresis_is_satisfied() {
3851 let policy = AtpAutotunePolicy::default();
3852 let mut state = AtpAutotuneApplicationState::default();
3853 let initial = state.settings;
3854
3855 let first = state.apply_policy_window(policy, &healthy_telemetry());
3856 assert_eq!(
3857 first.outcome,
3858 AtpAutotuneApplicationOutcome::DeferredGrowthHysteresis
3859 );
3860 assert_eq!(first.consumer_status, AtpAutotuneReceiptStatus::Blocked);
3861 assert!(!first.applied);
3862 assert_eq!(state.settings, initial);
3863 assert_eq!(state.consecutive_growth_windows, 1);
3864
3865 let second = state.apply_policy_window(policy, &healthy_telemetry());
3866 assert_eq!(
3867 second.outcome,
3868 AtpAutotuneApplicationOutcome::AppliedConfirmedGrowth
3869 );
3870 assert_eq!(second.consumer_status, AtpAutotuneReceiptStatus::Pass);
3871 assert!(second.applied);
3872 assert!(state.settings.in_flight_bytes > initial.in_flight_bytes);
3873 assert_eq!(state.consecutive_growth_windows, 0);
3874 }
3875
3876 #[test]
3877 fn application_state_applies_pressure_backoff_immediately() {
3878 let policy = AtpAutotunePolicy::default();
3879 let mut state = AtpAutotuneApplicationState::default();
3880 let initial = state.settings;
3881 let mut telemetry = healthy_telemetry();
3882 telemetry.loss_permille = Some(100);
3883
3884 let receipt = state.apply_policy_window(policy, &telemetry);
3885
3886 assert_eq!(
3887 receipt.outcome,
3888 AtpAutotuneApplicationOutcome::AppliedPressureBackoff
3889 );
3890 assert_eq!(receipt.consumer_status, AtpAutotuneReceiptStatus::Degraded);
3891 assert!(receipt.applied);
3892 assert!(state.settings.in_flight_bytes < initial.in_flight_bytes);
3893 assert!(state.settings.repair_symbols_per_second > initial.repair_symbols_per_second);
3894 assert_eq!(state.consecutive_growth_windows, 0);
3895 }
3896
3897 #[test]
3898 fn application_receipt_validates_consumer_identity_and_applied_flag() {
3899 let policy = AtpAutotunePolicy::default();
3900 let mut state = AtpAutotuneApplicationState::default();
3901 let mut telemetry = healthy_telemetry();
3902 telemetry.loss_permille = Some(100);
3903
3904 let receipt = state.apply_policy_window(policy, &telemetry);
3905
3906 assert!(receipt.validate_for_consumers().is_ok());
3907
3908 let mut unsupported_schema = receipt.clone();
3909 unsupported_schema.schema_version = String::from("atp-autotune-application-receipt-v0");
3910 assert_eq!(
3911 unsupported_schema.validate_for_consumers(),
3912 Err(
3913 AtpAutotuneApplicationReceiptValidationError::UnsupportedSchemaVersion {
3914 expected: String::from(ATP_AUTOTUNE_APPLICATION_RECEIPT_SCHEMA_VERSION),
3915 actual: String::from("atp-autotune-application-receipt-v0"),
3916 },
3917 )
3918 );
3919
3920 let mut mismatched_trace = receipt.clone();
3921 mismatched_trace.trace_id = String::from("other-trace");
3922 assert_eq!(
3923 mismatched_trace.validate_for_consumers(),
3924 Err(
3925 AtpAutotuneApplicationReceiptValidationError::DecisionReceiptMismatch {
3926 field: String::from("trace_id"),
3927 },
3928 )
3929 );
3930
3931 let mut mismatched_applied = receipt;
3932 mismatched_applied.applied = false;
3933 assert_eq!(
3934 mismatched_applied.validate_for_consumers(),
3935 Err(AtpAutotuneApplicationReceiptValidationError::AppliedFlagMismatch)
3936 );
3937 }
3938
3939 #[test]
3940 fn application_state_resets_pending_growth_after_noisy_pressure() {
3941 let policy = AtpAutotunePolicy::default();
3942 let mut state = AtpAutotuneApplicationState::default();
3943 let first = state.apply_policy_window(policy, &healthy_telemetry());
3944 assert_eq!(
3945 first.outcome,
3946 AtpAutotuneApplicationOutcome::DeferredGrowthHysteresis
3947 );
3948 assert_eq!(state.consecutive_growth_windows, 1);
3949
3950 let mut telemetry = healthy_telemetry();
3951 telemetry.send_buffer_queued_bytes = Some(policy.buffer_pressure_bytes + 1);
3952 let noisy = state.apply_policy_window(policy, &telemetry);
3953
3954 assert_eq!(
3955 noisy.outcome,
3956 AtpAutotuneApplicationOutcome::AppliedPressureBackoff
3957 );
3958 assert_eq!(state.consecutive_growth_windows, 0);
3959
3960 let next_clean = state.apply_policy_window(policy, &healthy_telemetry());
3961 assert_eq!(
3962 next_clean.outcome,
3963 AtpAutotuneApplicationOutcome::DeferredGrowthHysteresis
3964 );
3965 assert_eq!(state.consecutive_growth_windows, 1);
3966 }
3967
3968 #[test]
3969 fn application_state_rejects_stale_receipts_without_mutation() {
3970 let policy = AtpAutotunePolicy::default();
3971 let mut state = AtpAutotuneApplicationState::default();
3972 let stale_current = AtpAutotuneSettings::new(1, 1, 64 * 1_024, 0);
3973 let stale_receipt = policy.decide_with_receipt(stale_current, &healthy_telemetry());
3974 let before = state.settings;
3975
3976 let applied = state.apply_decision_receipt(stale_receipt);
3977
3978 assert_eq!(
3979 applied.outcome,
3980 AtpAutotuneApplicationOutcome::RejectedStaleReceipt
3981 );
3982 assert_eq!(
3983 applied.consumer_status,
3984 AtpAutotuneReceiptStatus::StaleEvidence
3985 );
3986 assert!(!applied.applied);
3987 assert_eq!(state.settings, before);
3988 assert_eq!(state.consecutive_growth_windows, 0);
3989 }
3990
3991 #[test]
3992 fn application_state_rejects_malformed_receipts_without_mutation() {
3993 let policy = AtpAutotunePolicy::default();
3994 let mut state = AtpAutotuneApplicationState::default();
3995 let malformed = policy.decide_with_receipt(
3996 state.settings,
3997 &AtpAutotuneTelemetry::new("", "workload-a").with_sample_count(16),
3998 );
3999 let before = state.settings;
4000
4001 let applied = state.apply_decision_receipt(malformed);
4002
4003 assert_eq!(
4004 applied.outcome,
4005 AtpAutotuneApplicationOutcome::RejectedMalformedTelemetry
4006 );
4007 assert_eq!(applied.consumer_status, AtpAutotuneReceiptStatus::Malformed);
4008 assert!(!applied.applied);
4009 assert_eq!(state.settings, before);
4010 assert_eq!(state.consecutive_growth_windows, 0);
4011 assert!(matches!(
4012 applied.validate_for_consumers(),
4013 Err(
4014 AtpAutotuneApplicationReceiptValidationError::DecisionReceiptInvalid {
4015 reason: AtpAutotuneReceiptValidationError::MissingTraceId,
4016 },
4017 )
4018 ));
4019 }
4020}