use crate::disk::error::{Error, Result};
use std::future::Future;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use tokio::task::JoinSet;
#[allow(
dead_code,
reason = "default operation label for the test-only AsyncBatchProcessor::new (backlog#1823)"
)]
const BATCH_PROCESSOR_OPERATION_CUSTOM: &str = "custom";
const BATCH_PROCESSOR_OPERATION_READ: &str = "read";
const BATCH_PROCESSOR_OPERATION_WRITE: &str = "write";
const BATCH_PROCESSOR_OPERATION_METADATA: &str = "metadata";
const ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE: &str = "RUSTFS_BATCH_PROCESSOR_ADAPTIVE";
const BATCH_PROCESSOR_ADAPTIVE_OFF: &str = "off";
const BATCH_PROCESSOR_ADAPTIVE_OBSERVE: &str = "observe";
const BATCH_PROCESSOR_ADAPTIVE_ON: &str = "on";
const BATCH_SUGGESTION_REASON_IMPROVING: &str = "improving";
const BATCH_SUGGESTION_REASON_DEGRADING: &str = "degrading";
const BATCH_SUGGESTION_REASON_STABLE: &str = "stable";
const BATCH_SUGGESTION_REASON_COOLDOWN: &str = "cooldown";
const BATCH_OBSERVATION_IMPROVING_LATENCY: Duration = Duration::from_millis(50);
const BATCH_OBSERVATION_DEGRADING_LATENCY: Duration = Duration::from_millis(250);
const BATCH_OBSERVATION_COOLDOWN: Duration = Duration::from_secs(30);
const BATCH_ADAPTIVE_MAX_CONCURRENCY_FACTOR: usize = 4;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum BatchProcessorAdaptiveMode {
Off,
Observe,
On,
}
impl BatchProcessorAdaptiveMode {
fn should_observe(self) -> bool {
matches!(self, Self::Observe | Self::On)
}
fn should_apply(self) -> bool {
matches!(self, Self::On)
}
}
fn parse_batch_processor_adaptive_mode(raw: &str) -> BatchProcessorAdaptiveMode {
match raw.trim() {
value if value.eq_ignore_ascii_case(BATCH_PROCESSOR_ADAPTIVE_OBSERVE) => BatchProcessorAdaptiveMode::Observe,
value if value.eq_ignore_ascii_case(BATCH_PROCESSOR_ADAPTIVE_ON) => BatchProcessorAdaptiveMode::On,
value if value.eq_ignore_ascii_case(BATCH_PROCESSOR_ADAPTIVE_OFF) => BatchProcessorAdaptiveMode::Off,
_ => BatchProcessorAdaptiveMode::Off,
}
}
fn batch_processor_adaptive_mode_from_env() -> BatchProcessorAdaptiveMode {
rustfs_utils::get_env_opt_str(ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE)
.as_deref()
.map(parse_batch_processor_adaptive_mode)
.unwrap_or(BatchProcessorAdaptiveMode::Off)
}
fn batch_processor_adaptive_mode() -> BatchProcessorAdaptiveMode {
static MODE: std::sync::LazyLock<BatchProcessorAdaptiveMode> =
std::sync::LazyLock::new(batch_processor_adaptive_mode_from_env);
*MODE
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct BatchConcurrencySuggestion {
concurrency: usize,
reason: &'static str,
}
#[derive(Debug, Clone, Copy)]
struct BatchObservation {
batch_size: usize,
success_count: usize,
error_count: usize,
timeout_count: usize,
max_queue_wait: Duration,
execution_latency: Duration,
}
#[derive(Debug)]
struct BatchObservationState {
last_suggestion_at: Option<Instant>,
last_suggested_concurrency: usize,
}
impl BatchObservationState {
fn new(configured_concurrency: usize) -> Self {
Self {
last_suggestion_at: None,
last_suggested_concurrency: configured_concurrency,
}
}
fn suggest(
&mut self,
current_concurrency: usize,
configured_concurrency: usize,
observation: BatchObservation,
now: Instant,
) -> BatchConcurrencySuggestion {
let raw = calculate_batch_concurrency_suggestion(current_concurrency, configured_concurrency, observation);
if raw.concurrency == current_concurrency {
self.last_suggested_concurrency = current_concurrency;
return raw;
}
if self
.last_suggestion_at
.is_some_and(|last_suggestion_at| now.duration_since(last_suggestion_at) < BATCH_OBSERVATION_COOLDOWN)
{
return BatchConcurrencySuggestion {
concurrency: self.last_suggested_concurrency,
reason: BATCH_SUGGESTION_REASON_COOLDOWN,
};
}
self.last_suggestion_at = Some(now);
self.last_suggested_concurrency = raw.concurrency;
raw
}
}
fn calculate_batch_concurrency_suggestion(
current_concurrency: usize,
configured_concurrency: usize,
observation: BatchObservation,
) -> BatchConcurrencySuggestion {
let current_concurrency = current_concurrency.max(1);
let configured_concurrency = configured_concurrency.max(1);
if observation.batch_size == 0 {
return BatchConcurrencySuggestion {
concurrency: current_concurrency,
reason: BATCH_SUGGESTION_REASON_STABLE,
};
}
if observation.timeout_count > 0
|| observation.error_count > 0
|| (observation.batch_size >= current_concurrency && observation.execution_latency >= BATCH_OBSERVATION_DEGRADING_LATENCY)
{
return BatchConcurrencySuggestion {
concurrency: decrease_batch_concurrency(current_concurrency),
reason: BATCH_SUGGESTION_REASON_DEGRADING,
};
}
if observation.success_count == observation.batch_size
&& observation.batch_size >= current_concurrency
&& observation.execution_latency <= BATCH_OBSERVATION_IMPROVING_LATENCY
{
return BatchConcurrencySuggestion {
concurrency: increase_batch_concurrency(current_concurrency, configured_concurrency, observation.batch_size),
reason: BATCH_SUGGESTION_REASON_IMPROVING,
};
}
BatchConcurrencySuggestion {
concurrency: current_concurrency,
reason: BATCH_SUGGESTION_REASON_STABLE,
}
}
fn decrease_batch_concurrency(current_concurrency: usize) -> usize {
(current_concurrency.saturating_mul(3) / 4).max(1)
}
fn increase_batch_concurrency(current_concurrency: usize, configured_concurrency: usize, batch_size: usize) -> usize {
let step = (current_concurrency / 4).max(1);
let upper_bound = configured_concurrency
.saturating_mul(BATCH_ADAPTIVE_MAX_CONCURRENCY_FACTOR)
.max(current_concurrency);
current_concurrency
.saturating_add(step)
.min(upper_bound)
.min(batch_size.max(current_concurrency))
}
fn is_timeout_error(err: &Error) -> bool {
matches!(err, Error::Timeout | Error::SourceStalled)
|| matches!(err, Error::Io(io_err) if io_err.kind() == std::io::ErrorKind::TimedOut)
}
pub struct AsyncBatchProcessor {
max_concurrent: usize,
operation: &'static str,
observation_state: Mutex<BatchObservationState>,
}
impl AsyncBatchProcessor {
#[allow(dead_code, reason = "constructor used only by this file's tests (backlog#1823)")]
pub fn new(max_concurrent: usize) -> Self {
Self::new_with_operation(max_concurrent, BATCH_PROCESSOR_OPERATION_CUSTOM)
}
fn new_with_operation(max_concurrent: usize, operation: &'static str) -> Self {
let max_concurrent = max_concurrent.max(1);
Self {
max_concurrent,
operation,
observation_state: Mutex::new(BatchObservationState::new(max_concurrent)),
}
}
fn execution_concurrency(&self, mode: BatchProcessorAdaptiveMode) -> usize {
if !mode.should_apply() {
return self.max_concurrent;
}
self.observation_state
.lock()
.map(|state| state.last_suggested_concurrency.max(1))
.unwrap_or(self.max_concurrent)
}
fn observe_batch_with_mode(
&self,
mode: BatchProcessorAdaptiveMode,
execution_concurrency: usize,
observation: BatchObservation,
) -> BatchConcurrencySuggestion {
let execution_concurrency = execution_concurrency.max(1);
if !mode.should_observe() {
return BatchConcurrencySuggestion {
concurrency: execution_concurrency,
reason: BATCH_SUGGESTION_REASON_STABLE,
};
}
let suggestion = match self.observation_state.lock() {
Ok(mut state) => state.suggest(execution_concurrency, self.max_concurrent, observation, Instant::now()),
Err(_) => calculate_batch_concurrency_suggestion(execution_concurrency, self.max_concurrent, observation),
};
rustfs_io_metrics::record_batch_processor_observation(rustfs_io_metrics::BatchProcessorObservation {
operation: self.operation,
batch_size: observation.batch_size,
configured_concurrency: execution_concurrency,
max_queue_wait_secs: observation.max_queue_wait.as_secs_f64(),
execution_latency_secs: observation.execution_latency.as_secs_f64(),
successes: observation.success_count,
errors: observation.error_count,
timeouts: observation.timeout_count,
suggested_concurrency: suggestion.concurrency,
suggestion_reason: suggestion.reason,
});
suggestion
}
pub async fn execute_batch<T, F>(&self, tasks: Vec<F>) -> Vec<Result<T>>
where
T: Send + 'static,
F: Future<Output = Result<T>> + Send + 'static,
{
if tasks.is_empty() {
return Vec::new();
}
let batch_size = tasks.len();
let batch_started_at = Instant::now();
let adaptive_mode = batch_processor_adaptive_mode();
let execution_concurrency = self.execution_concurrency(adaptive_mode);
let semaphore = Arc::new(tokio::sync::Semaphore::new(execution_concurrency));
let mut join_set = JoinSet::new();
let mut results = Vec::with_capacity(batch_size);
for _ in 0..batch_size {
results.push(Err(Error::other("Not completed")));
}
for (i, task) in tasks.into_iter().enumerate() {
let sem = semaphore.clone();
let queued_at = Instant::now();
join_set.spawn(async move {
let _permit = sem.acquire().await.map_err(|_| Error::other("Semaphore error"))?;
let queue_wait = queued_at.elapsed();
let result = task.await;
Ok::<(usize, Result<T>, Duration), Error>((i, result, queue_wait))
});
}
let mut max_queue_wait = Duration::ZERO;
while let Some(join_result) = join_set.join_next().await {
match join_result {
Ok(Ok((index, task_result, queue_wait))) => {
max_queue_wait = max_queue_wait.max(queue_wait);
if index < results.len() {
results[index] = task_result;
}
}
Ok(Err(e)) => {
tracing::warn!("Batch processor system error: {:?}", e);
}
Err(join_error) => {
tracing::warn!("Task panicked in batch processor: {:?}", join_error);
}
}
}
let mut success_count = 0;
let mut error_count = 0;
let mut timeout_count = 0;
for result in &results {
match result {
Ok(_) => success_count += 1,
Err(err) if is_timeout_error(err) => timeout_count += 1,
Err(_) => error_count += 1,
}
}
self.observe_batch_with_mode(
adaptive_mode,
execution_concurrency,
BatchObservation {
batch_size,
success_count,
error_count,
timeout_count,
max_queue_wait,
execution_latency: batch_started_at.elapsed(),
},
);
results
}
pub async fn execute_batch_with_quorum<T, F>(&self, tasks: Vec<F>, required_successes: usize) -> Result<Vec<T>>
where
T: Send + 'static,
F: Future<Output = Result<T>> + Send + 'static,
{
if required_successes == 0 {
return Ok(Vec::new());
}
if tasks.is_empty() {
return Err(Error::other(format!(
"Insufficient successful results: got 0, needed {required_successes}"
)));
}
let batch_size = tasks.len();
let batch_started_at = Instant::now();
let adaptive_mode = batch_processor_adaptive_mode();
let execution_concurrency = self.execution_concurrency(adaptive_mode);
let semaphore = Arc::new(tokio::sync::Semaphore::new(execution_concurrency));
let mut join_set = JoinSet::new();
let mut successes = Vec::new();
let mut pending_tasks = batch_size;
let mut first_error = None;
let mut error_count = 0;
let mut timeout_count = 0;
let mut max_queue_wait = Duration::ZERO;
for task in tasks {
let sem = semaphore.clone();
let queued_at = Instant::now();
join_set.spawn(async move {
let _permit = sem
.acquire()
.await
.map_err(|_| (Error::other("Semaphore error"), queued_at.elapsed()))?;
let queue_wait = queued_at.elapsed();
task.await.map(|value| (value, queue_wait)).map_err(|err| (err, queue_wait))
});
}
while let Some(join_result) = join_set.join_next().await {
pending_tasks = pending_tasks.saturating_sub(1);
match join_result {
Ok(Ok((value, queue_wait))) => {
max_queue_wait = max_queue_wait.max(queue_wait);
successes.push(value);
if successes.len() >= required_successes {
self.observe_batch_with_mode(
adaptive_mode,
execution_concurrency,
BatchObservation {
batch_size,
success_count: successes.len(),
error_count,
timeout_count,
max_queue_wait,
execution_latency: batch_started_at.elapsed(),
},
);
return Ok(successes);
}
}
Ok(Err((err, queue_wait))) => {
max_queue_wait = max_queue_wait.max(queue_wait);
if is_timeout_error(&err) {
timeout_count += 1;
} else {
error_count += 1;
}
if first_error.is_none() {
first_error = Some(err);
}
}
Err(join_error) => {
error_count += 1;
if first_error.is_none() {
first_error = Some(Error::other(format!("Task panicked in quorum batch processor: {join_error}")));
}
}
}
if successes.len() + pending_tasks < required_successes {
self.observe_batch_with_mode(
adaptive_mode,
execution_concurrency,
BatchObservation {
batch_size,
success_count: successes.len(),
error_count,
timeout_count,
max_queue_wait,
execution_latency: batch_started_at.elapsed(),
},
);
return Err(first_error.unwrap_or_else(|| {
Error::other(format!(
"Insufficient successful results: got {}, needed {}",
successes.len(),
required_successes
))
}));
}
}
self.observe_batch_with_mode(
adaptive_mode,
execution_concurrency,
BatchObservation {
batch_size,
success_count: successes.len(),
error_count,
timeout_count,
max_queue_wait,
execution_latency: batch_started_at.elapsed(),
},
);
Err(first_error.unwrap_or_else(|| {
Error::other(format!(
"Insufficient successful results: got {}, needed {}",
successes.len(),
required_successes
))
}))
}
}
pub struct GlobalBatchProcessors {
read_processor: AsyncBatchProcessor,
write_processor: AsyncBatchProcessor,
metadata_processor: AsyncBatchProcessor,
}
impl GlobalBatchProcessors {
pub fn new() -> Self {
Self {
read_processor: AsyncBatchProcessor::new_with_operation(16, BATCH_PROCESSOR_OPERATION_READ),
write_processor: AsyncBatchProcessor::new_with_operation(8, BATCH_PROCESSOR_OPERATION_WRITE),
metadata_processor: AsyncBatchProcessor::new_with_operation(12, BATCH_PROCESSOR_OPERATION_METADATA),
}
}
pub fn read_processor(&self) -> &AsyncBatchProcessor {
&self.read_processor
}
pub fn write_processor(&self) -> &AsyncBatchProcessor {
&self.write_processor
}
pub fn metadata_processor(&self) -> &AsyncBatchProcessor {
&self.metadata_processor
}
}
impl Default for GlobalBatchProcessors {
fn default() -> Self {
Self::new()
}
}
use std::sync::OnceLock;
static GLOBAL_PROCESSORS: OnceLock<GlobalBatchProcessors> = OnceLock::new();
pub fn get_global_processors() -> &'static GlobalBatchProcessors {
GLOBAL_PROCESSORS.get_or_init(GlobalBatchProcessors::new)
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Duration;
#[tokio::test]
async fn test_batch_processor_basic() {
let processor = AsyncBatchProcessor::new(4);
let tasks: Vec<_> = (0..10)
.map(|i| async move {
tokio::time::sleep(Duration::from_millis(10)).await;
Ok::<i32, Error>(i)
})
.collect();
let results = processor.execute_batch(tasks).await;
assert_eq!(results.len(), 10);
for (i, result) in results.iter().enumerate() {
assert!(result.is_ok());
assert_eq!(result.as_ref().unwrap(), &(i as i32));
}
}
#[tokio::test]
async fn test_batch_processor_with_errors() {
let processor = AsyncBatchProcessor::new(2);
let tasks: Vec<_> = (0..5)
.map(|i| async move {
tokio::time::sleep(Duration::from_millis(10)).await;
if i % 2 == 0 {
Ok::<i32, Error>(i)
} else {
Err(Error::other("Test error"))
}
})
.collect();
let results = processor.execute_batch(tasks).await;
assert_eq!(results.len(), 5);
for (i, result) in results.iter().enumerate() {
if i % 2 == 0 {
assert!(result.is_ok());
assert_eq!(result.as_ref().unwrap(), &(i as i32));
} else {
assert!(result.is_err());
}
}
}
#[tokio::test]
async fn test_batch_processor_quorum() {
let processor = AsyncBatchProcessor::new(4);
let tasks: Vec<_> = (0..10)
.map(|i| async move {
tokio::time::sleep(Duration::from_millis(10)).await;
if i < 3 {
Ok::<i32, Error>(i)
} else {
Err(Error::other("Test error"))
}
})
.collect();
let results = processor.execute_batch_with_quorum(tasks, 2).await;
assert!(results.is_ok());
let successes = results.unwrap();
assert!(successes.len() >= 2);
}
#[tokio::test(start_paused = true)]
async fn test_batch_processor_quorum_returns_before_slow_tail() {
let processor = AsyncBatchProcessor::new(4);
let started = tokio::time::Instant::now();
let tasks: Vec<_> = [(10_u64, Ok(1_i32)), (15, Ok(2)), (250, Ok(3))]
.into_iter()
.map(|(delay_ms, outcome)| async move {
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
outcome
})
.collect();
let results = processor
.execute_batch_with_quorum(tasks, 2)
.await
.expect("quorum should succeed");
assert_eq!(results.len(), 2);
assert!(started.elapsed() < Duration::from_millis(100));
}
#[tokio::test(start_paused = true)]
async fn test_batch_processor_quorum_fails_once_quorum_becomes_impossible() {
let processor = AsyncBatchProcessor::new(4);
let started = tokio::time::Instant::now();
let tasks: Vec<_> = vec![
(10_u64, Ok(1_i32)),
(15, Err(Error::other("first failure"))),
(20, Err(Error::other("second failure"))),
(250, Ok(4)),
]
.into_iter()
.map(|(delay_ms, outcome)| async move {
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
outcome
})
.collect();
let err = processor
.execute_batch_with_quorum(tasks, 3)
.await
.expect_err("quorum should fail once it becomes impossible");
assert!(err.to_string().contains("first failure"));
assert!(started.elapsed() < Duration::from_millis(120));
}
#[tokio::test]
async fn test_batch_processor_observe_only_keeps_configured_concurrency() {
let processor = AsyncBatchProcessor::new(2);
let tasks: Vec<_> = (0..4)
.map(|i| async move {
tokio::time::sleep(Duration::from_millis(5)).await;
Ok::<i32, Error>(i)
})
.collect();
let results = processor.execute_batch(tasks).await;
assert_eq!(results.len(), 4);
assert_eq!(processor.max_concurrent, 2);
}
#[test]
fn batch_processor_adaptive_mode_defaults_to_off_and_parses_supported_values() {
assert_eq!(parse_batch_processor_adaptive_mode(""), BatchProcessorAdaptiveMode::Off);
assert_eq!(parse_batch_processor_adaptive_mode("off"), BatchProcessorAdaptiveMode::Off);
assert_eq!(parse_batch_processor_adaptive_mode("observe"), BatchProcessorAdaptiveMode::Observe);
assert_eq!(parse_batch_processor_adaptive_mode("on"), BatchProcessorAdaptiveMode::On);
assert_eq!(parse_batch_processor_adaptive_mode("unknown"), BatchProcessorAdaptiveMode::Off);
assert!(!BatchProcessorAdaptiveMode::Off.should_observe());
assert!(BatchProcessorAdaptiveMode::Observe.should_observe());
assert!(BatchProcessorAdaptiveMode::On.should_observe());
assert!(!BatchProcessorAdaptiveMode::Observe.should_apply());
assert!(BatchProcessorAdaptiveMode::On.should_apply());
}
#[test]
fn batch_processor_adaptive_mode_env_gate_is_parsed_from_env() {
temp_env::with_var(ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE, None::<&str>, || {
assert_eq!(batch_processor_adaptive_mode_from_env(), BatchProcessorAdaptiveMode::Off);
});
temp_env::with_var(ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE, Some("observe"), || {
assert_eq!(batch_processor_adaptive_mode_from_env(), BatchProcessorAdaptiveMode::Observe);
});
}
#[test]
fn batch_processor_observation_gate_keeps_default_off_stable() {
let processor = AsyncBatchProcessor::new(8);
let suggestion = processor.observe_batch_with_mode(
BatchProcessorAdaptiveMode::Off,
processor.execution_concurrency(BatchProcessorAdaptiveMode::Off),
BatchObservation {
batch_size: 16,
success_count: 16,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
},
);
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_STABLE);
assert_eq!(suggestion.concurrency, 8);
}
#[test]
fn batch_processor_observation_gate_observe_records_suggestion() {
let processor = AsyncBatchProcessor::new(8);
let suggestion = processor.observe_batch_with_mode(
BatchProcessorAdaptiveMode::Observe,
processor.execution_concurrency(BatchProcessorAdaptiveMode::Observe),
BatchObservation {
batch_size: 16,
success_count: 16,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
},
);
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
assert!(suggestion.concurrency > 8);
assert_eq!(processor.execution_concurrency(BatchProcessorAdaptiveMode::Observe), 8);
}
#[test]
fn batch_processor_adaptive_on_applies_last_suggestion_to_next_batch() {
let processor = AsyncBatchProcessor::new(8);
let suggestion = processor.observe_batch_with_mode(
BatchProcessorAdaptiveMode::On,
processor.execution_concurrency(BatchProcessorAdaptiveMode::On),
BatchObservation {
batch_size: 16,
success_count: 16,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
},
);
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
assert!(suggestion.concurrency > 8);
assert_eq!(processor.execution_concurrency(BatchProcessorAdaptiveMode::On), suggestion.concurrency);
}
#[test]
fn batch_processor_suggestion_increases_for_fast_successful_batches() {
let suggestion = calculate_batch_concurrency_suggestion(
8,
8,
BatchObservation {
batch_size: 16,
success_count: 16,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
},
);
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
assert!(suggestion.concurrency > 8);
}
#[test]
fn batch_processor_suggestion_decreases_for_timeout_batches() {
let suggestion = calculate_batch_concurrency_suggestion(
8,
8,
BatchObservation {
batch_size: 16,
success_count: 14,
error_count: 0,
timeout_count: 2,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
},
);
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_DEGRADING);
assert!(suggestion.concurrency < 8);
assert!(suggestion.concurrency >= 1);
}
#[test]
fn batch_processor_suggestion_clamps_to_batch_size() {
let suggestion = calculate_batch_concurrency_suggestion(
64,
64,
BatchObservation {
batch_size: 65,
success_count: 65,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
},
);
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
assert_eq!(suggestion.concurrency, 65);
}
#[test]
fn batch_processor_suggestion_cooldown_prevents_rapid_oscillation() {
let mut state = BatchObservationState::new(8);
let now = Instant::now();
let observation = BatchObservation {
batch_size: 16,
success_count: 16,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
};
let first = state.suggest(8, 8, observation, now);
let second = state.suggest(8, 8, observation, now + Duration::from_secs(1));
assert_eq!(first.reason, BATCH_SUGGESTION_REASON_IMPROVING);
assert_eq!(second.reason, BATCH_SUGGESTION_REASON_COOLDOWN);
assert_eq!(second.concurrency, first.concurrency);
}
#[test]
fn batch_processor_suggestion_growth_is_capped_relative_to_configured_concurrency() {
const CONFIGURED: usize = 8;
let mut state = BatchObservationState::new(CONFIGURED);
let mut current = CONFIGURED;
let mut now = Instant::now();
let observation = BatchObservation {
batch_size: 1024,
success_count: 1024,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
};
for _ in 0..64 {
current = state.suggest(current, CONFIGURED, observation, now).concurrency;
now += BATCH_OBSERVATION_COOLDOWN;
}
assert_eq!(current, CONFIGURED * BATCH_ADAPTIVE_MAX_CONCURRENCY_FACTOR);
}
}