use super::*;
pub(crate) fn prune_runtime_profile_retry_backoff(runtime: &mut RuntimeRotationState, now: i64) {
runtime
.profile_retry_backoff_until
.retain(|_, until| *until > now);
}
pub(crate) fn prune_runtime_profile_transport_backoff(
runtime: &mut RuntimeRotationState,
now: i64,
) {
runtime
.profile_transport_backoff_until
.retain(|_, until| *until > now);
}
pub(crate) fn prune_runtime_profile_route_circuits(runtime: &mut RuntimeRotationState, now: i64) {
runtime
.profile_route_circuit_open_until
.retain(|key, until| {
if *until > now {
return true;
}
let health_key = runtime_profile_route_circuit_health_key(key);
runtime_profile_effective_health_score_from_map(
&runtime.profile_health,
&health_key,
now,
) > 0
});
}
pub(crate) fn prune_runtime_profile_selection_backoff(
runtime: &mut RuntimeRotationState,
now: i64,
) {
prune_runtime_profile_retry_backoff(runtime, now);
prune_runtime_profile_transport_backoff(runtime, now);
prune_runtime_profile_route_circuits(runtime, now);
}
pub(crate) fn runtime_profile_name_in_selection_backoff(
profile_name: &str,
retry_backoff_until: &BTreeMap<String, i64>,
transport_backoff_until: &BTreeMap<String, i64>,
route_circuit_open_until: &BTreeMap<String, i64>,
route_kind: RuntimeRouteKind,
now: i64,
) -> bool {
retry_backoff_until
.get(profile_name)
.copied()
.is_some_and(|until| until > now)
|| runtime_profile_transport_backoff_until_from_map(
transport_backoff_until,
profile_name,
route_kind,
now,
)
.is_some()
|| route_circuit_open_until
.get(&runtime_profile_route_circuit_key(profile_name, route_kind))
.copied()
.is_some_and(|until| until > now)
}
pub(crate) fn runtime_profile_backoff_sort_key(
profile_name: &str,
retry_backoff_until: &BTreeMap<String, i64>,
transport_backoff_until: &BTreeMap<String, i64>,
route_circuit_open_until: &BTreeMap<String, i64>,
route_kind: RuntimeRouteKind,
now: i64,
) -> (usize, i64, i64, i64) {
let retry_until = retry_backoff_until
.get(profile_name)
.copied()
.filter(|until| *until > now);
let transport_until = runtime_profile_transport_backoff_until_from_map(
transport_backoff_until,
profile_name,
route_kind,
now,
);
let circuit_until = route_circuit_open_until
.get(&runtime_profile_route_circuit_key(profile_name, route_kind))
.copied()
.filter(|until| *until > now);
match (circuit_until, transport_until, retry_until) {
(None, None, None) => (0, 0, 0, 0),
(Some(circuit_until), None, None) => (1, circuit_until, 0, 0),
(None, Some(transport_until), None) => (2, transport_until, 0, 0),
(None, None, Some(retry_until)) => (3, retry_until, 0, 0),
(Some(circuit_until), Some(transport_until), None) => (
4,
circuit_until.min(transport_until),
circuit_until.max(transport_until),
0,
),
(Some(circuit_until), None, Some(retry_until)) => (
5,
circuit_until.min(retry_until),
circuit_until.max(retry_until),
0,
),
(None, Some(transport_until), Some(retry_until)) => (
6,
transport_until.min(retry_until),
transport_until.max(retry_until),
0,
),
(Some(circuit_until), Some(transport_until), Some(retry_until)) => (
7,
circuit_until.min(transport_until.min(retry_until)),
circuit_until.max(transport_until.max(retry_until)),
retry_until,
),
}
}
pub(crate) fn runtime_profile_backoffs_snapshot(
runtime: &RuntimeRotationState,
) -> RuntimeProfileBackoffs {
RuntimeProfileBackoffs {
retry_backoff_until: runtime.profile_retry_backoff_until.clone(),
transport_backoff_until: runtime.profile_transport_backoff_until.clone(),
route_circuit_open_until: runtime.profile_route_circuit_open_until.clone(),
}
}
pub(crate) fn runtime_soften_persisted_backoff_map_for_startup(
backoffs: &mut BTreeMap<String, i64>,
now: i64,
max_future_seconds: i64,
) -> bool {
let max_until = now.saturating_add(max_future_seconds.max(0));
let mut changed = false;
backoffs.retain(|_, until| {
if *until <= now {
changed = true;
return false;
}
let next_until = (*until).min(max_until);
if next_until != *until {
changed = true;
}
*until = next_until;
true
});
changed
}
pub(crate) fn runtime_route_kind_from_label(label: &str) -> Option<RuntimeRouteKind> {
match label {
"responses" => Some(RuntimeRouteKind::Responses),
"compact" => Some(RuntimeRouteKind::Compact),
"websocket" => Some(RuntimeRouteKind::Websocket),
"standard" => Some(RuntimeRouteKind::Standard),
_ => None,
}
}
pub(crate) fn runtime_soften_persisted_backoffs_for_startup(
backoffs: &mut RuntimeProfileBackoffs,
profile_scores: &BTreeMap<String, RuntimeProfileHealth>,
now: i64,
) -> bool {
let mut changed = runtime_soften_persisted_backoff_map_for_startup(
&mut backoffs.transport_backoff_until,
now,
RUNTIME_PROFILE_TRANSPORT_BACKOFF_SECONDS,
);
changed = runtime_soften_persisted_route_circuits_for_startup(
&mut backoffs.route_circuit_open_until,
profile_scores,
now,
) || changed;
changed
}
pub(crate) fn mark_runtime_profile_retry_backoff(
shared: &RuntimeRotationProxyShared,
profile_name: &str,
) -> Result<()> {
let mut runtime = shared
.runtime
.lock()
.map_err(|_| anyhow::anyhow!("runtime auto-rotate state is poisoned"))?;
let now = Local::now().timestamp();
prune_runtime_profile_selection_backoff(&mut runtime, now);
runtime.profile_probe_cache.remove(profile_name);
let until = now.saturating_add(RUNTIME_PROFILE_RETRY_BACKOFF_SECONDS);
runtime
.profile_retry_backoff_until
.insert(profile_name.to_string(), until);
schedule_runtime_state_save_from_runtime(
shared,
&runtime,
&format!("profile_retry_backoff:{profile_name}"),
);
drop(runtime);
runtime_proxy_log(
shared,
format!("profile_retry_backoff profile={profile_name} until={until}"),
);
Ok(())
}
pub(crate) fn mark_runtime_profile_transport_backoff(
shared: &RuntimeRotationProxyShared,
profile_name: &str,
route_kind: RuntimeRouteKind,
context: &str,
) -> Result<()> {
let mut runtime = shared
.runtime
.lock()
.map_err(|_| anyhow::anyhow!("runtime auto-rotate state is poisoned"))?;
let now = Local::now().timestamp();
prune_runtime_profile_selection_backoff(&mut runtime, now);
runtime.profile_probe_cache.remove(profile_name);
let route_key = runtime_profile_transport_backoff_key(profile_name, route_kind);
let existing_remaining = runtime_profile_transport_backoff_until_from_map(
&runtime.profile_transport_backoff_until,
profile_name,
route_kind,
now,
)
.unwrap_or(now)
.saturating_sub(now);
let next_backoff_seconds = if existing_remaining > 0 {
existing_remaining.saturating_mul(2).clamp(
RUNTIME_PROFILE_TRANSPORT_BACKOFF_SECONDS,
RUNTIME_PROFILE_TRANSPORT_BACKOFF_MAX_SECONDS,
)
} else {
RUNTIME_PROFILE_TRANSPORT_BACKOFF_SECONDS
};
let until = now.saturating_add(next_backoff_seconds);
runtime
.profile_transport_backoff_until
.entry(route_key)
.and_modify(|current| *current = (*current).max(until))
.or_insert(until);
schedule_runtime_state_save_from_runtime(
shared,
&runtime,
&format!(
"profile_transport_backoff:{profile_name}:{}",
runtime_route_kind_label(route_kind)
),
);
drop(runtime);
runtime_proxy_log(
shared,
format!(
"profile_transport_backoff profile={profile_name} route={} until={until} seconds={next_backoff_seconds} context={context}",
runtime_route_kind_label(route_kind)
),
);
Ok(())
}
pub(crate) fn clear_runtime_profile_transport_backoff_for_route(
runtime: &mut RuntimeRotationState,
profile_name: &str,
route_kind: RuntimeRouteKind,
) -> bool {
let mut changed = runtime
.profile_transport_backoff_until
.remove(&runtime_profile_transport_backoff_key(
profile_name,
route_kind,
))
.is_some();
changed = runtime
.profile_transport_backoff_until
.remove(profile_name)
.is_some()
|| changed;
changed
}