use std::collections::VecDeque;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use super::{CompletionSample, ConcurrencyController, ErrorKind};
use crate::metrics::IOCounters;
#[derive(Debug, Clone)]
pub(crate) struct AdaptiveConfig {
pub(crate) initial_concurrency: usize,
pub(crate) max_concurrency: Option<usize>,
pub(crate) window: WindowConfig,
pub(crate) slow_start: SlowStartConfig,
pub(crate) stable: StableConfig,
}
#[derive(Debug, Clone)]
pub(crate) struct WindowConfig {
pub(crate) duration: Duration,
pub(crate) smoothing_window: usize,
}
#[derive(Debug, Clone)]
pub(crate) struct SlowStartConfig {
pub(crate) evaluation_interval: Duration,
pub(crate) acceptance_margin: f64,
pub(crate) max_growth: usize,
pub(crate) exit_threshold: usize,
}
#[derive(Debug, Clone)]
pub(crate) struct StableConfig {
pub(crate) acceptance_margin: f64,
pub(crate) probe_pct: f64,
pub(crate) min_probe_size: usize,
pub(crate) max_probe_size: usize,
pub(crate) history_size: usize,
pub(crate) shedding_transition_threshold: usize,
}
impl Default for AdaptiveConfig {
fn default() -> Self {
Self {
initial_concurrency: 32,
max_concurrency: None,
window: WindowConfig {
duration: Duration::from_millis(500),
smoothing_window: 4,
},
slow_start: SlowStartConfig {
evaluation_interval: Duration::from_millis(300),
acceptance_margin: 0.10,
max_growth: 32,
exit_threshold: 4,
},
stable: StableConfig {
acceptance_margin: 0.02,
probe_pct: 0.02,
min_probe_size: 5,
max_probe_size: 200,
history_size: 10,
shedding_transition_threshold: 3,
},
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Phase {
SlowStart = 0,
StableProbing = 1,
StableShedding = 2,
}
impl Phase {
fn from_usize(v: usize) -> Self {
debug_assert!(v <= 2, "invalid phase value: {v}");
match v {
0 => Phase::SlowStart,
2 => Phase::StableShedding,
_ => Phase::StableProbing,
}
}
}
#[derive(Debug)]
struct AlgorithmState {
accepted_concurrency: usize,
probing_concurrency: usize,
recent_goodput: VecDeque<f64>,
raw_goodput: VecDeque<f64>,
consecutive_probes: usize,
last_evaluation: Instant,
}
#[derive(Debug)]
pub(crate) struct AdaptiveConcurrencyController {
config: AdaptiveConfig,
target: AtomicUsize,
in_flight: AtomicUsize,
peak_in_flight: AtomicUsize,
io_counters: Arc<IOCounters>,
phase: AtomicUsize,
state: Mutex<AlgorithmState>,
}
impl AdaptiveConcurrencyController {
pub(crate) fn new(config: AdaptiveConfig, io_counters: Arc<IOCounters>) -> Self {
let initial = config.initial_concurrency;
Self {
target: AtomicUsize::new(initial),
in_flight: AtomicUsize::new(0),
peak_in_flight: AtomicUsize::new(0),
io_counters,
phase: AtomicUsize::new(Phase::SlowStart as usize),
state: Mutex::new(AlgorithmState {
accepted_concurrency: initial,
probing_concurrency: initial,
recent_goodput: VecDeque::with_capacity(config.stable.history_size),
raw_goodput: VecDeque::with_capacity(config.window.smoothing_window),
consecutive_probes: 0,
last_evaluation: Instant::now(),
}),
config,
}
}
fn maybe_update_window(&self) {
let Ok(mut state) = self.state.try_lock() else {
return;
};
let phase = Phase::from_usize(self.phase.load(Ordering::Relaxed));
let interval = match phase {
Phase::SlowStart => self.config.slow_start.evaluation_interval,
_ => self.config.window.duration,
};
if state.last_evaluation.elapsed() < interval {
return;
}
state.last_evaluation = Instant::now();
let peak = self.peak_in_flight.swap(0, Ordering::Relaxed);
if self.io_counters.is_idle() {
tracing::trace!(
target: crate::telemetry::TARGET_CONCURRENCY,
in_flight = self.in_flight.load(Ordering::Relaxed),
target = self.target.load(Ordering::Relaxed),
"idle — skipping evaluation",
);
return;
}
let goodput = self.io_counters.network_throughput();
state.raw_goodput.push_back(goodput);
if state.raw_goodput.len() > self.config.window.smoothing_window {
state.raw_goodput.pop_front();
}
let smoothed = state.raw_goodput.iter().copied().fold(0.0f64, f64::max);
tracing::trace!(
target: crate::telemetry::TARGET_CONCURRENCY,
goodput_mbps = goodput / 1_000_000.0,
smoothed_mbps = smoothed / 1_000_000.0,
peak,
"window evaluated"
);
let new_target = self.run_algorithm(&mut state, smoothed, peak);
let clamped = match self.config.max_concurrency {
Some(max) => new_target.min(max),
None => new_target,
}
.max(1);
let old_target = self.target.swap(clamped, Ordering::Relaxed);
let phase = Phase::from_usize(self.phase.load(Ordering::Relaxed));
let gbps = crate::types::ByteUnit::Gigabit.as_bytes_u64() as f64;
tracing::trace!(
target: crate::telemetry::TARGET_CONCURRENCY,
goodput_gbps = goodput / gbps,
smoothed_gbps = smoothed / gbps,
accepted = state.accepted_concurrency,
target = clamped,
?phase,
peak,
"evaluation"
);
if old_target != clamped {
tracing::debug!(
target: crate::telemetry::TARGET_CONCURRENCY,
old_target,
new_target = clamped,
accepted = state.accepted_concurrency,
?phase,
"concurrency target updated"
);
}
}
fn transition_to(&self, state: &mut AlgorithmState, new_phase: Phase) {
let old_phase = Phase::from_usize(self.phase.load(Ordering::Relaxed));
self.phase.store(new_phase as usize, Ordering::Relaxed);
if old_phase == Phase::SlowStart {
state.recent_goodput.clear();
state.raw_goodput.clear();
}
state.consecutive_probes = 0;
tracing::debug!(target: crate::telemetry::TARGET_CONCURRENCY, from = ?old_phase, to = ?new_phase, accepted = state.accepted_concurrency, "concurrency phase transition");
}
fn run_algorithm(
&self,
state: &mut AlgorithmState,
goodput: f64,
peak_in_flight: usize,
) -> usize {
state.recent_goodput.push_back(goodput);
if state.recent_goodput.len() > self.config.stable.history_size {
state.recent_goodput.pop_front();
}
if state.recent_goodput.len() < 2 {
if Phase::from_usize(self.phase.load(Ordering::Relaxed)) == Phase::SlowStart {
state.probing_concurrency =
state.accepted_concurrency + self.config.slow_start.max_growth;
}
return self.clamp_probe(state.probing_concurrency, state.accepted_concurrency);
}
if peak_in_flight < state.accepted_concurrency {
tracing::trace!(target: crate::telemetry::TARGET_CONCURRENCY,
peak_in_flight,
accepted = state.accepted_concurrency,
"probe skipped, not exercised"
);
return self.clamp_probe(state.probing_concurrency, state.accepted_concurrency);
}
match Phase::from_usize(self.phase.load(Ordering::Relaxed)) {
Phase::SlowStart => self.run_slow_start(state, goodput),
Phase::StableProbing => self.run_stable_probing(state, goodput),
Phase::StableShedding => self.run_stable_shedding(state, goodput),
}
}
fn run_slow_start(&self, state: &mut AlgorithmState, measured: f64) -> usize {
let best = state
.recent_goodput
.iter()
.rev()
.skip(1)
.copied()
.fold(0.0f64, f64::max);
let ratio = ratio(measured, best);
let accept = ratio >= 1.0 + self.config.slow_start.acceptance_margin;
if accept {
state.consecutive_probes = 0;
tracing::trace!(target: crate::telemetry::TARGET_CONCURRENCY, phase = ?Phase::SlowStart, probing = state.probing_concurrency, accepted = state.accepted_concurrency, ratio, "probe accepted");
state.accepted_concurrency = state.probing_concurrency;
state.probing_concurrency =
state.accepted_concurrency + self.config.slow_start.max_growth;
} else {
state.consecutive_probes += 1;
tracing::trace!(target: crate::telemetry::TARGET_CONCURRENCY, phase = ?Phase::SlowStart, probing = state.probing_concurrency, accepted = state.accepted_concurrency, ratio, consecutive = state.consecutive_probes, threshold = self.config.slow_start.acceptance_margin, "SlowStart plateau");
if state.consecutive_probes >= self.config.slow_start.exit_threshold {
self.transition_to(state, Phase::StableProbing);
state.probing_concurrency = (state.accepted_concurrency as f64
* (1.0 + self.config.stable.probe_pct))
.ceil() as usize;
}
}
self.clamp_probe(state.probing_concurrency, state.accepted_concurrency)
}
fn run_stable_probing(&self, state: &mut AlgorithmState, measured: f64) -> usize {
let best_recent = state
.recent_goodput
.iter()
.rev()
.skip(1)
.copied()
.fold(0.0f64, f64::max);
let ratio = ratio(measured, best_recent);
let accept = ratio >= 1.0 + self.config.stable.acceptance_margin;
if accept {
tracing::trace!(target: crate::telemetry::TARGET_CONCURRENCY, phase = ?Phase::StableProbing, probing = state.probing_concurrency, accepted = state.accepted_concurrency, ratio, "probe accepted");
state.accepted_concurrency = state.probing_concurrency;
state.consecutive_probes = 0;
} else {
tracing::trace!(target: crate::telemetry::TARGET_CONCURRENCY, phase = ?Phase::StableProbing, probing = state.probing_concurrency, accepted = state.accepted_concurrency, ratio, threshold = self.config.stable.acceptance_margin, "probe rejected");
state.consecutive_probes += 1;
if state.consecutive_probes >= self.config.stable.shedding_transition_threshold {
self.transition_to(state, Phase::StableShedding);
}
}
state.probing_concurrency = (state.accepted_concurrency as f64
* (1.0 + self.config.stable.probe_pct))
.ceil() as usize;
self.clamp_probe(state.probing_concurrency, state.accepted_concurrency)
}
fn run_stable_shedding(&self, state: &mut AlgorithmState, measured: f64) -> usize {
let max_recent = state
.recent_goodput
.iter()
.fold(0.0f64, |acc, &x| acc.max(x));
let ratio = ratio(measured, max_recent);
let accept = ratio >= 1.0 - self.config.stable.acceptance_margin;
if accept {
tracing::trace!(target: crate::telemetry::TARGET_CONCURRENCY, phase = ?Phase::StableShedding, probing = state.probing_concurrency, accepted = state.accepted_concurrency, ratio, "probe accepted");
state.accepted_concurrency = state.probing_concurrency;
state.consecutive_probes += 1;
let reduction = self.config.stable.probe_pct * (state.consecutive_probes + 1) as f64;
state.probing_concurrency =
(state.accepted_concurrency as f64 * (1.0 - reduction)) as usize;
} else {
tracing::trace!(target: crate::telemetry::TARGET_CONCURRENCY, phase = ?Phase::StableShedding, probing = state.probing_concurrency, accepted = state.accepted_concurrency, ratio, threshold = self.config.stable.acceptance_margin, "probe rejected");
self.transition_to(state, Phase::StableProbing);
state.probing_concurrency = (state.accepted_concurrency as f64
* (1.0 + self.config.stable.probe_pct))
.ceil() as usize;
}
self.clamp_probe(state.probing_concurrency, state.accepted_concurrency)
}
fn clamp_probe(&self, probe: usize, accepted: usize) -> usize {
probe
.max(self.config.stable.min_probe_size)
.min(accepted + self.config.stable.max_probe_size)
.max(1)
}
}
fn ratio(measured: f64, baseline: f64) -> f64 {
if baseline > 0.0 {
measured / baseline
} else {
0.0
}
}
impl ConcurrencyController for AdaptiveConcurrencyController {
fn target(&self) -> usize {
self.target.load(Ordering::Relaxed)
}
fn on_dispatch(&self) {
let n = self.in_flight.fetch_add(1, Ordering::Relaxed) + 1;
self.peak_in_flight.fetch_max(n, Ordering::Relaxed);
}
fn on_completion(&self, sample: Option<&CompletionSample>) {
#[allow(deprecated)]
let _ = self
.in_flight
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |n| {
Some(n.saturating_sub(1))
});
let Some(sample) = sample else {
return;
};
if sample.error == Some(ErrorKind::Throttle) {
if let Ok(mut state) = self.state.lock() {
if Phase::from_usize(self.phase.load(Ordering::Relaxed)) != Phase::StableShedding {
self.transition_to(&mut state, Phase::StableShedding);
}
}
}
self.maybe_update_window();
}
}
#[cfg(test)]
mod tests {
use super::super::{CompletionSample, ErrorKind};
use super::*;
use crate::metrics::{IOCounters, IoSample};
use std::sync::Arc;
struct TestController(AdaptiveConcurrencyController);
impl TestController {
fn new(config: AdaptiveConfig) -> Self {
let io_counters = Arc::new(IOCounters::new(config.window.duration));
Self(AdaptiveConcurrencyController::new(config, io_counters))
}
fn record_goodput(&self, goodput: f64) -> usize {
let mut state = self.0.state.lock().unwrap();
let peak = state.accepted_concurrency;
self.0.run_algorithm(&mut state, goodput, peak)
}
fn record_goodput_with_peak(&self, goodput: f64, peak: usize) -> usize {
let mut state = self.0.state.lock().unwrap();
self.0.run_algorithm(&mut state, goodput, peak)
}
fn phase(&self) -> Phase {
Phase::from_usize(self.0.phase.load(Ordering::Relaxed))
}
fn accepted(&self) -> usize {
self.0.state.lock().unwrap().accepted_concurrency
}
fn consecutive_probes(&self) -> usize {
self.0.state.lock().unwrap().consecutive_probes
}
fn history_len(&self) -> usize {
self.0.state.lock().unwrap().recent_goodput.len()
}
fn exit_slow_start(&self) {
self.record_goodput(1000.0); let threshold = self.0.config.slow_start.exit_threshold;
for _ in 0..threshold {
self.record_goodput(1000.0);
}
assert_eq!(self.phase(), Phase::StableProbing);
}
}
fn test_config() -> AdaptiveConfig {
AdaptiveConfig::default()
}
fn sample(_bytes_sent: u64) -> CompletionSample {
CompletionSample { error: None }
}
#[test]
fn bootstrap_records_first_measurement_without_running_algorithm() {
let tc = TestController::new(test_config());
let target = tc.record_goodput(1000.0);
assert_eq!(tc.phase(), Phase::SlowStart);
assert_eq!(target, 64);
assert_eq!(tc.accepted(), 32); assert_eq!(tc.history_len(), 1);
}
#[test]
fn slow_start_stays_in_phase_on_throughput_improvement() {
let tc = TestController::new(test_config());
tc.record_goodput(1000.0); tc.record_goodput(1200.0); assert_eq!(tc.phase(), Phase::SlowStart);
assert_eq!(tc.accepted(), 64); }
#[test]
fn slow_start_rejects_insufficient_improvement() {
let tc = TestController::new(test_config());
tc.record_goodput(1000.0); tc.record_goodput(1050.0); assert_eq!(tc.phase(), Phase::SlowStart); tc.record_goodput(1050.0); assert_eq!(tc.phase(), Phase::SlowStart);
tc.record_goodput(1050.0); assert_eq!(tc.phase(), Phase::SlowStart);
tc.record_goodput(1050.0); assert_eq!(tc.phase(), Phase::StableProbing);
assert_eq!(tc.accepted(), 32);
}
#[test]
fn stable_probing_accepts_on_improvement() {
let tc = TestController::new(test_config());
tc.exit_slow_start();
tc.record_goodput(1000.0); tc.record_goodput(1030.0); assert_eq!(tc.phase(), Phase::StableProbing);
assert_eq!(tc.accepted(), 33); assert_eq!(tc.consecutive_probes(), 0); }
#[test]
fn stable_probing_rejects_to_shedding_after_threshold() {
let tc = TestController::new(test_config());
tc.exit_slow_start();
tc.record_goodput(1000.0); tc.record_goodput(1000.0); tc.record_goodput(1000.0); tc.record_goodput(1000.0); assert_eq!(tc.phase(), Phase::StableShedding);
}
#[test]
fn stable_shedding_accepts_when_throughput_near_peak() {
let tc = TestController::new(test_config());
tc.exit_slow_start();
tc.record_goodput(1000.0); tc.record_goodput(1000.0); tc.record_goodput(1000.0); tc.record_goodput(1000.0); tc.record_goodput(985.0); assert_eq!(tc.phase(), Phase::StableShedding);
assert_eq!(tc.consecutive_probes(), 1); }
#[test]
fn stable_shedding_rejects_to_probing_when_throughput_drops() {
let tc = TestController::new(test_config());
tc.exit_slow_start();
tc.record_goodput(1000.0); tc.record_goodput(1000.0); tc.record_goodput(1000.0); tc.record_goodput(1000.0); tc.record_goodput(900.0); assert_eq!(tc.phase(), Phase::StableProbing);
}
#[test]
fn throttle_forces_shedding() {
let config = AdaptiveConfig {
window: WindowConfig {
duration: Duration::from_nanos(10),
smoothing_window: 4,
},
..Default::default()
};
let tc = TestController::new(config);
let mut s = sample(8_000_000);
s.error = Some(ErrorKind::Throttle);
tc.0.on_completion(Some(&s));
assert_eq!(tc.phase(), Phase::StableShedding);
}
#[test]
fn max_concurrency_clamps_target() {
let config = AdaptiveConfig {
initial_concurrency: 100,
max_concurrency: Some(150),
window: WindowConfig {
duration: Duration::from_nanos(10),
smoothing_window: 4,
},
..Default::default()
};
let tc = TestController::new(config);
for _ in 0..20 {
tc.0.io_counters.record(&IoSample {
network_tx: 8_000_000,
..Default::default()
});
tc.0.on_completion(Some(&sample(8_000_000)));
}
std::thread::sleep(Duration::from_millis(1));
for _ in 0..20 {
tc.0.io_counters.record(&IoSample {
network_tx: 16_000_000,
..Default::default()
});
tc.0.on_completion(Some(&sample(16_000_000)));
}
assert!(tc.0.target() <= 150);
}
#[test]
fn min_probe_size_enforced() {
let config = AdaptiveConfig {
initial_concurrency: 100,
stable: StableConfig {
min_probe_size: 50,
..AdaptiveConfig::default().stable
},
..Default::default()
};
let tc = TestController::new(config);
let clamped = tc.0.clamp_probe(3, 100);
assert_eq!(clamped, 50);
}
#[test]
fn exercised_check_rejects_unexercised_probe() {
let tc = TestController::new(test_config());
tc.exit_slow_start();
tc.record_goodput(1000.0); let target = tc.record_goodput_with_peak(5000.0, 5); assert_eq!(tc.accepted(), 32); assert!(target >= 32); }
#[test]
fn exercised_check_passes_when_exercised() {
let tc = TestController::new(test_config());
tc.exit_slow_start();
tc.record_goodput(1000.0); tc.record_goodput_with_peak(1030.0, 32); assert_eq!(tc.accepted(), 33); }
#[test]
fn slow_start_additive_growth() {
let tc = TestController::new(test_config());
let target = tc.record_goodput(1000.0);
assert_eq!(target, 64);
assert_eq!(tc.accepted(), 32);
let target = tc.record_goodput(1200.0);
assert_eq!(tc.accepted(), 64);
assert_eq!(target, 96);
}
#[test]
fn slow_start_exits_on_plateau() {
let tc = TestController::new(test_config());
tc.record_goodput(1000.0); tc.record_goodput(1050.0); tc.record_goodput(1050.0); tc.record_goodput(1050.0); tc.record_goodput(1050.0); assert_eq!(tc.phase(), Phase::StableProbing);
assert_eq!(tc.accepted(), 32);
}
#[test]
fn idle_window_skips_evaluation() {
let config = AdaptiveConfig {
window: WindowConfig {
duration: Duration::from_nanos(10),
smoothing_window: 4,
},
..Default::default()
};
let io_counters = Arc::new(IOCounters::new(Duration::from_secs(1)));
let controller = AdaptiveConcurrencyController::new(config, Arc::clone(&io_counters));
controller.on_dispatch();
controller.on_completion(Some(&sample(0)));
let state = controller.state.lock().unwrap();
assert_eq!(state.recent_goodput.len(), 0, "idle window was evaluated");
}
#[test]
fn history_cleared_on_slow_start_exit() {
let tc = TestController::new(test_config());
tc.exit_slow_start();
assert_eq!(tc.phase(), Phase::StableProbing);
assert_eq!(tc.history_len(), 0); }
#[test]
fn history_preserved_between_stable_phases() {
let tc = TestController::new(test_config());
tc.exit_slow_start();
tc.record_goodput(1000.0); tc.record_goodput(1000.0); tc.record_goodput(1000.0); let history_before = tc.history_len();
assert!(history_before >= 3);
tc.record_goodput(1000.0); assert_eq!(tc.phase(), Phase::StableShedding);
assert!(
tc.history_len() > history_before,
"history lost on StableProbing → StableShedding: {} -> {}",
history_before,
tc.history_len()
);
}
#[test]
fn stable_probing_does_not_creep_on_flat_throughput() {
let tc = TestController::new(test_config());
tc.exit_slow_start();
let initial_accepted = tc.accepted();
for _ in 0..20 {
tc.record_goodput(1000.0);
}
let final_accepted = tc.accepted();
assert!(
final_accepted <= initial_accepted + 5,
"StableProbing crept from {} to {} on flat throughput",
initial_accepted,
final_accepted
);
}
#[test]
fn shedding_retains_peak_knowledge_from_probing() {
let tc = TestController::new(test_config());
tc.exit_slow_start();
tc.record_goodput(5000.0); tc.record_goodput(5000.0); tc.record_goodput(5000.0); tc.record_goodput(5000.0); assert_eq!(tc.phase(), Phase::StableShedding);
tc.record_goodput(4000.0);
assert_eq!(
tc.phase(),
Phase::StableProbing,
"StableShedding accepted 4000 vs peak 5000 — lost peak knowledge"
);
}
}