use crate::credits::Credits;
use crate::error::BudgetKind;
use crate::params::RequestParams;
use crate::policy::backoff::Backoff;
use crate::policy::budget::Budget;
use crate::policy::ladder::{CountryPool, Ladder, Rung, Step};
use crate::policy::rule::{default_rules, Decision, Rule, Trigger};
use crate::response::Hint;
use crate::status::{ApiClass, ApiStatus, PageClass, PageStatus};
use std::time::Duration;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum Reached {
Api(ApiStatus),
TimedOut,
ConnectFailed,
}
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub struct Observed {
pub api: Reached,
pub page: Option<PageStatus>,
pub empty_content: bool,
pub cost: Credits,
pub elapsed: Duration,
pub retry_after: Option<Duration>,
pub sent_credentials: bool,
pub sent_session: bool,
}
impl Observed {
pub fn seen(api: u16, page: Option<u16>) -> Observed {
Observed {
api: Reached::Api(ApiStatus::new(api)),
page: page.map(PageStatus::new),
empty_content: false,
cost: Credits::ZERO,
elapsed: Duration::ZERO,
retry_after: None,
sent_credentials: false,
sent_session: false,
}
}
pub fn timed_out() -> Observed {
Observed {
api: Reached::TimedOut,
..Observed::seen(0, None)
}
}
pub fn connect_failed() -> Observed {
Observed {
api: Reached::ConnectFailed,
..Observed::seen(0, None)
}
}
pub fn blank(mut self) -> Observed {
self.empty_content = true;
self
}
pub fn costing(mut self, cost: Credits) -> Observed {
self.cost = cost;
self
}
pub fn taking(mut self, elapsed: Duration) -> Observed {
self.elapsed = elapsed;
self
}
pub fn retry_after(mut self, after: Duration) -> Observed {
self.retry_after = Some(after);
self
}
pub fn for_request(mut self, params: &RequestParams) -> Observed {
self.sent_credentials = params.cookies.is_some()
|| params
.headers
.as_ref()
.is_some_and(|headers| !headers.is_empty());
self.sent_session = params.session == Some(true);
self
}
pub fn was_billed(&self) -> bool {
match self.page {
Some(page) => page.consumes_credits(),
None => matches!(
self.api_class(),
Some(ApiClass::Ok) | Some(ApiClass::NoContent)
),
}
}
pub fn api_class(&self) -> Option<ApiClass> {
match self.api {
Reached::Api(status) => Some(status.class_with_retry_after(self.retry_after)),
Reached::TimedOut | Reached::ConnectFailed => None,
}
}
pub fn page_class(&self) -> Option<PageClass> {
self.page.map(PageStatus::class)
}
pub fn is_blank_success(&self) -> bool {
self.empty_content
&& matches!(self.api_class(), Some(ApiClass::Ok))
&& self.page.is_none_or(PageStatus::is_ok)
}
pub fn asked_to_slow_down(&self) -> bool {
matches!(
self.api_class(),
Some(ApiClass::RateLimited { .. }) | Some(ApiClass::Draining { .. })
) || self.page_class() == Some(PageClass::TargetRateLimited)
}
pub fn hint(&self) -> Hint {
match self.page_class() {
Some(class) => Hint::for_class(class),
None => Hint::Permanent,
}
}
}
impl Trigger {
pub fn matches(&self, observed: &Observed) -> bool {
match self {
Trigger::Timeout => observed.api == Reached::TimedOut,
Trigger::Connect => observed.api == Reached::ConnectFailed,
Trigger::EmptyContent => observed.is_blank_success(),
Trigger::Page(class) => observed.page_class() == Some(*class),
Trigger::Api(ApiClass::Ok) => {
observed.page.is_none() && matches!(observed.api_class(), Some(ApiClass::Ok))
}
Trigger::Api(class) => match observed.api_class() {
Some(seen) => std::mem::discriminant(&seen) == std::mem::discriminant(class),
None => false,
},
}
}
}
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub struct AttemptState {
pub budget: Budget,
pub attempts: u8,
pub retries_at_step: u8,
pub step: usize,
pub spent: Credits,
pub elapsed: Duration,
pub last_cost: Credits,
}
impl AttemptState {
pub fn new(budget: Budget) -> AttemptState {
AttemptState {
budget,
attempts: 0,
retries_at_step: 0,
step: 0,
spent: Credits::ZERO,
elapsed: Duration::ZERO,
last_cost: Credits::ZERO,
}
}
pub fn record(&mut self, observed: &Observed) {
let cost = if observed.cost.get().is_finite() && observed.cost.get() > 0.0 {
observed.cost
} else {
Credits::ZERO
};
self.attempts = self.attempts.saturating_add(1);
self.spent += cost;
self.last_cost = cost;
self.elapsed = self.elapsed.saturating_add(observed.elapsed);
}
pub fn follow(&mut self, next: &Next) {
match next {
Next::Retry { after } => {
self.retries_at_step = self.retries_at_step.saturating_add(1);
self.elapsed = self.elapsed.saturating_add(*after);
}
Next::Escalate { index, after, .. } => {
self.step = index.saturating_add(1);
self.retries_at_step = 0;
self.elapsed = self.elapsed.saturating_add(*after);
}
Next::Accept | Next::Stop(_) => {}
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub enum Next {
Accept,
Retry {
after: Duration,
},
Escalate {
index: usize,
step: Step,
after: Duration,
},
Stop(StopReason),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum StopReason {
Rejected {
hint: Hint,
},
OutOfCredits,
Budget(BudgetKind),
RetriesExhausted,
LadderExhausted,
Unhandled,
}
impl StopReason {
pub fn budget_kind(self) -> Option<BudgetKind> {
match self {
StopReason::Budget(kind) => Some(kind),
_ => None,
}
}
}
impl std::fmt::Display for StopReason {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
StopReason::Rejected { hint } => write!(f, "rejected, try {hint:?}"),
StopReason::OutOfCredits => f.write_str("no credits left on the account"),
StopReason::Budget(kind) => write!(f, "budget exceeded: {kind}"),
StopReason::RetriesExhausted => f.write_str("out of retries"),
StopReason::LadderExhausted => f.write_str("out of escalation steps"),
StopReason::Unhandled => f.write_str("no rule for what came back"),
}
}
}
pub const SESSION_LABEL: &str = "session";
pub fn target_slow_down() -> Duration {
match Hint::for_class(PageClass::TargetRateLimited) {
Hint::SlowDown { after } => after,
_ => Duration::from_secs(5),
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct Policy {
pub rules: Vec<Rule>,
pub ladder: Ladder,
pub backoff: Backoff,
pub max_attempts: u8,
}
impl Default for Policy {
fn default() -> Policy {
Policy::standard()
}
}
impl Policy {
pub fn standard() -> Policy {
Policy {
rules: default_rules(),
ladder: Ladder::standard(),
backoff: Backoff::default(),
max_attempts: crate::policy::budget::DEFAULT_ATTEMPTS,
}
}
pub fn for_target(url: &str) -> Policy {
Policy {
ladder: Ladder::for_target(url),
..Policy::standard()
}
}
pub fn with_countries(pool: &CountryPool) -> Policy {
Policy {
ladder: Ladder::with_countries(pool),
..Policy::standard()
}
}
pub fn with_backoff(mut self, backoff: Backoff) -> Policy {
self.backoff = backoff;
self
}
pub fn with_rules(mut self, rules: Vec<Rule>) -> Policy {
self.rules = rules;
self
}
pub fn with_max_attempts(mut self, max_attempts: u8) -> Policy {
self.max_attempts = max_attempts;
self
}
pub fn decide(&self, observed: &Observed, state: &AttemptState) -> Next {
if observed.api_class() == Some(ApiClass::InsufficientCredits) {
return Next::Stop(StopReason::OutOfCredits);
}
let Some(rule) = self.rules.iter().find(|rule| rule.when.matches(observed)) else {
return Next::Stop(StopReason::Unhandled);
};
match &rule.then {
Decision::Accept => Next::Accept,
Decision::Fail => match self.session_step(observed) {
Some(step) => self.escalate(observed, state, Some(step)),
None => Next::Stop(StopReason::Rejected {
hint: observed.hint(),
}),
},
Decision::RetrySame { max } => {
if state.retries_at_step < *max {
self.retry(observed, state)
} else if rule.when.retry_only() {
Next::Stop(StopReason::RetriesExhausted)
} else {
self.escalate(observed, state, None)
}
}
Decision::Escalate => self.escalate(observed, state, None),
Decision::EscalateTo(step) => self.escalate(observed, state, Some(step.clone())),
}
}
fn wait_before(&self, observed: &Observed, state: &AttemptState) -> Duration {
let curve = self
.backoff
.delay(u32::from(state.retries_at_step), observed.retry_after);
if observed.page_class() == Some(PageClass::TargetRateLimited)
&& observed.retry_after.is_none()
{
curve.max(target_slow_down().min(self.backoff.cap))
} else {
curve
}
}
fn basis(observed: &Observed, state: &AttemptState) -> Credits {
if observed.was_billed() {
state.last_cost
} else {
Credits::ZERO
}
}
pub fn session_step(&self, observed: &Observed) -> Option<Step> {
if observed.page_class() != Some(PageClass::NeedsLogin)
|| !observed.sent_credentials
|| observed.sent_session
{
return None;
}
Some(Step::new(SESSION_LABEL, 1.0, vec![Rung::Session(true)]))
}
fn retry(&self, observed: &Observed, state: &AttemptState) -> Next {
let after = self.wait_before(observed, state);
if let Err(kind) = self.room_for_another_call(state, after, Policy::basis(observed, state))
{
return Next::Stop(StopReason::Budget(kind));
}
Next::Retry { after }
}
fn escalate(&self, observed: &Observed, state: &AttemptState, forced: Option<Step>) -> Next {
let after = if observed.asked_to_slow_down() {
self.wait_before(observed, state)
} else {
Duration::ZERO
};
let (index, step) = match forced {
Some(step) => (self.ladder.len(), step),
None => {
let index = if observed.page_class() == Some(PageClass::TargetRateLimited) {
self.ladder.first_geo_from(state.step)
} else {
Some(state.step)
};
match index.and_then(|index| self.ladder.get(index).map(|step| (index, step))) {
Some((index, step)) => (index, step.clone()),
None => return Next::Stop(StopReason::LadderExhausted),
}
}
};
let estimate = step.estimate(Policy::basis(observed, state));
if let Err(kind) = self.room_for_another_call(state, after, estimate) {
return Next::Stop(StopReason::Budget(kind));
}
Next::Escalate { index, step, after }
}
fn room_for_another_call(
&self,
state: &AttemptState,
after: Duration,
estimate: Credits,
) -> Result<(), BudgetKind> {
if state.attempts >= self.max_attempts {
return Err(BudgetKind::Attempts);
}
state.budget.allows_attempt(state.attempts)?;
state.budget.allows_sleep(state.elapsed, after)?;
state.budget.allows_spend(state.spent, estimate)
}
}
#[cfg(test)]
mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::indexing_slicing,
clippy::string_slice
)]
use super::*;
use crate::policy::budget::ASSUMED_MINIMUM_COST;
fn state() -> AttemptState {
AttemptState::new(Budget::unlimited())
}
#[test]
fn a_served_page_is_accepted() {
let mut s = state();
let seen = Observed::seen(200, Some(200));
s.record(&seen);
assert_eq!(Policy::standard().decide(&seen, &s), Next::Accept);
}
#[test]
fn a_blank_served_page_climbs_instead() {
let mut s = state();
let seen = Observed::seen(200, Some(200)).blank().costing(Credits(2.0));
s.record(&seen);
match Policy::standard().decide(&seen, &s) {
Next::Escalate { step, .. } => assert_eq!(step.label, "browser"),
other => panic!("a blank page was not escalated: {other:?}"),
}
}
#[test]
fn nothing_to_return_is_a_result_not_a_blank_page() {
let mut s = state();
let seen = Observed::seen(204, None).blank();
s.record(&seen);
assert!(!seen.is_blank_success());
assert_eq!(Policy::standard().decide(&seen, &s), Next::Accept);
}
#[test]
fn a_call_that_succeeded_around_a_refused_fetch_is_judged_on_the_fetch() {
let seen = Observed::seen(200, Some(403));
assert!(!Trigger::Api(ApiClass::Ok).matches(&seen));
assert!(Trigger::Page(PageClass::Blocked).matches(&seen));
}
#[test]
fn a_rate_limit_written_without_a_wait_matches_one_that_carried_a_header() {
let seen = Observed::seen(429, None).retry_after(Duration::from_secs(9));
assert!(Trigger::Api(ApiClass::RateLimited { retry_after: None }).matches(&seen));
}
#[test]
fn a_site_rate_limit_jumps_to_the_country_step_rather_than_walking() {
let policy = Policy::standard();
let mut s = state();
let seen = Observed::seen(200, Some(429)).costing(Credits(1.0));
s.record(&seen);
let first = policy.decide(&seen, &s);
assert!(matches!(first, Next::Retry { .. }), "{first:?}");
s.follow(&first);
s.record(&seen);
match policy.decide(&seen, &s) {
Next::Escalate { index, step, .. } => {
assert_eq!(index, 3);
assert_eq!(step.label, "residential+geo");
}
other => panic!("{other:?}"),
}
}
#[test]
fn the_accounts_own_rate_limit_never_climbs() {
let policy = Policy::standard().with_max_attempts(u8::MAX);
let mut s = state();
let seen = Observed::seen(429, None);
s.record(&seen);
for _ in 0..3 {
let next = policy.decide(&seen, &s);
assert!(matches!(next, Next::Retry { .. }), "{next:?}");
s.follow(&next);
s.record(&seen);
}
assert_eq!(
policy.decide(&seen, &s),
Next::Stop(StopReason::RetriesExhausted)
);
}
#[test]
fn a_timeout_retries_once_then_climbs() {
let policy = Policy::standard();
let mut s = state();
let seen = Observed::timed_out();
s.record(&seen);
let first = policy.decide(&seen, &s);
assert!(matches!(first, Next::Retry { .. }), "{first:?}");
s.follow(&first);
s.record(&seen);
match policy.decide(&seen, &s) {
Next::Escalate { step, .. } => assert_eq!(step.label, "browser"),
other => panic!("{other:?}"),
}
}
#[test]
fn a_login_wall_stops_with_the_session_hint() {
let mut s = state();
let seen = Observed::seen(200, Some(401));
s.record(&seen);
assert_eq!(
Policy::standard().decide(&seen, &s),
Next::Stop(StopReason::Rejected {
hint: Hint::NeedsSession
})
);
}
#[test]
fn the_ladder_runs_out_before_the_policy_pretends_otherwise() {
let policy = Policy::standard().with_max_attempts(u8::MAX);
let mut s = state();
s.step = policy.ladder.len();
let seen = Observed::seen(200, Some(403));
s.record(&seen);
assert_eq!(
policy.decide(&seen, &s),
Next::Stop(StopReason::LadderExhausted)
);
}
#[test]
fn an_unnamed_status_stops_rather_than_spends() {
let mut s = state();
let seen = Observed::seen(418, None);
s.record(&seen);
assert_eq!(
Policy::standard().decide(&seen, &s),
Next::Stop(StopReason::Unhandled)
);
}
#[test]
fn the_policy_ceiling_stops_an_operation_a_generous_budget_would_not() {
let policy = Policy::standard().with_max_attempts(2);
let mut s = state();
s.attempts = 2;
let seen = Observed::seen(200, Some(403));
assert_eq!(
policy.decide(&seen, &s),
Next::Stop(StopReason::Budget(BudgetKind::Attempts))
);
}
#[test]
fn a_named_step_reports_the_end_of_the_ladder() {
let step = Step::new("named", 2.0, Vec::new());
let policy = Policy::standard().with_rules(vec![Rule::new(
Trigger::Page(PageClass::Blocked),
Decision::EscalateTo(step.clone()),
)]);
let mut s = state();
let seen = Observed::seen(200, Some(403));
s.record(&seen);
match policy.decide(&seen, &s) {
Next::Escalate {
index, step: got, ..
} => {
assert_eq!(index, policy.ladder.len());
assert_eq!(got, step);
}
other => panic!("{other:?}"),
}
}
#[test]
fn following_a_decision_moves_the_counters() {
let mut s = state();
s.follow(&Next::Retry {
after: Duration::from_secs(2),
});
assert_eq!(s.retries_at_step, 1);
assert_eq!(s.elapsed, Duration::from_secs(2));
s.follow(&Next::Escalate {
index: 2,
step: Step::new("x", 1.0, Vec::new()),
after: Duration::from_secs(1),
});
assert_eq!(s.step, 3);
assert_eq!(s.retries_at_step, 0);
assert_eq!(s.elapsed, Duration::from_secs(3));
}
#[test]
fn recording_an_attempt_adds_up_the_spend_and_the_clock() {
let mut s = state();
s.record(
&Observed::seen(200, Some(403))
.costing(Credits(3.0))
.taking(Duration::from_secs(1)),
);
s.record(
&Observed::seen(200, Some(403))
.costing(Credits(4.0))
.taking(Duration::from_secs(2)),
);
assert_eq!(s.attempts, 2);
assert_eq!(s.spent, Credits(7.0));
assert_eq!(s.last_cost, Credits(4.0));
assert_eq!(s.elapsed, Duration::from_secs(3));
}
#[test]
fn a_cost_that_cannot_be_a_bill_is_counted_as_nothing() {
let mut s = state();
s.record(&Observed::seen(200, Some(403)).costing(Credits(3.0)));
s.record(&Observed::seen(200, Some(403)).costing(Credits(-100.0)));
assert_eq!(s.spent, Credits(3.0));
assert_eq!(s.last_cost, Credits::ZERO);
s.record(&Observed::seen(200, Some(403)).costing(Credits(f64::NAN)));
assert_eq!(s.spent, Credits(3.0));
s.record(&Observed::seen(200, Some(403)).costing(Credits(f64::INFINITY)));
assert_eq!(s.spent, Credits(3.0));
assert_eq!(s.attempts, 4);
}
#[test]
fn the_unbilled_statuses_are_the_ones_the_response_layer_names() {
assert!(!Observed::seen(200, Some(500)).was_billed());
assert!(!Observed::seen(200, Some(503)).was_billed());
assert!(!Observed::seen(500, None).was_billed());
assert!(!Observed::timed_out().was_billed());
for page in [200u16, 403, 404, 429, 502, 504] {
assert!(
Observed::seen(200, Some(page)).was_billed(),
"target {page} should be billed"
);
}
assert!(Observed::seen(200, None).was_billed());
}
#[test]
fn an_escalation_after_a_free_attempt_is_priced_off_the_floor() {
let policy = Policy::standard().with_max_attempts(u8::MAX);
let floor = ASSUMED_MINIMUM_COST.get();
let mut s =
AttemptState::new(Budget::unlimited().with_per_page_credits(Credits(floor * 3.0)));
let seen = Observed::seen(200, Some(503));
s.record(&seen);
s.retries_at_step = 1;
assert_eq!(
policy.decide(&seen, &s),
Next::Stop(StopReason::Budget(BudgetKind::Credits))
);
}
#[test]
fn a_cost_reported_on_an_unbilled_attempt_is_not_used_as_a_basis() {
let policy = Policy::standard().with_max_attempts(u8::MAX);
let floor = ASSUMED_MINIMUM_COST.get();
let mut s =
AttemptState::new(Budget::unlimited().with_per_page_credits(Credits(floor * 10.0)));
let seen = Observed::seen(200, Some(503)).costing(Credits(floor * 4.0));
s.record(&seen);
s.retries_at_step = 1;
match policy.decide(&seen, &s) {
Next::Escalate { step, .. } => {
assert_eq!(step.label, "browser");
assert_eq!(step.estimate(Credits::ZERO), Credits(floor * 4.0));
}
other => panic!("the free attempt priced the next step: {other:?}"),
}
}
#[test]
fn a_stop_reason_prints_and_names_its_cap() {
assert_eq!(
StopReason::Budget(BudgetKind::Credits).budget_kind(),
Some(BudgetKind::Credits)
);
assert_eq!(StopReason::OutOfCredits.budget_kind(), None);
assert_eq!(
StopReason::Budget(BudgetKind::Time).to_string(),
"budget exceeded: time"
);
}
}