use core::time::Duration;
use std::sync::{Arc, Mutex};
use jiff::SignedDuration;
use crate::{
BoxedError,
drive::{
Delivery, Failure, Item, ItemId, LifecycleTap, NodeName, Observation, Payload, Station, Taps,
ack_channel, channel, open, reply,
},
};
use super::super::{
DeadlinePolicy, DropNewest, DropOldest, Missed, Occupancy, OverflowDecision, OverflowPolicy,
Policies, PollUpstream, QueryDeadline, RetryDecision, RetryPolicy, TickDecision, TickPolicy,
};
#[derive(Debug, Clone, PartialEq, Eq)]
enum Seen {
Arrived(Station),
Retrying {
attempt: u32,
after: Option<i128>,
},
Shed {
key: String,
decision: OverflowDecision,
},
GivenUp {
attempt: u32,
error: Option<String>,
},
Expired {
budget: i128,
},
PolicyFailed {
concern: String,
error: String,
},
Lapped {
poll: PollUpstream,
},
}
#[derive(Debug, Default)]
struct Watching {
seen: Mutex<Vec<(String, Seen)>>,
}
impl Watching {
fn seen(&self) -> Vec<(String, Seen)> {
self.seen.lock().expect("the watch").clone()
}
fn arms(&self) -> Vec<Seen> {
self.seen().into_iter().map(|(_, arm)| arm).collect()
}
}
impl LifecycleTap for Watching {
fn observe(&self, node: &NodeName, observation: &Observation) {
let arm = match observation {
Observation::Arrived(arrival) => Seen::Arrived(arrival.station()),
Observation::Retrying { attempt, after } => Seen::Retrying {
attempt: *attempt,
after: after.map(|after| after.as_millis()),
},
Observation::Shed { key, decision, .. } => Seen::Shed {
key: key.to_string(),
decision: *decision,
},
Observation::GivenUp { attempt, failure } => Seen::GivenUp {
attempt: *attempt,
error: failure.as_ref().map(ToString::to_string),
},
Observation::Expired { budget } => Seen::Expired {
budget: budget.as_millis(),
},
Observation::PolicyFailed { concern, error } => Seen::PolicyFailed {
concern: concern.as_str().to_owned(),
error: error.to_string(),
},
Observation::Lapped { poll } => Seen::Lapped { poll: *poll },
};
self
.seen
.lock()
.expect("the watch")
.push((node.as_str().to_owned(), arm));
}
}
fn watched() -> (Arc<Watching>, Taps) {
let tap = Arc::new(Watching::default());
let taps = Taps::new().with(tap.clone());
(tap, taps)
}
#[derive(Debug, derive_more::Display)]
#[display("the advisor is unwell")]
struct Unwell;
impl core::error::Error for Unwell {}
struct Mad;
impl TickPolicy for Mad {
fn on_missed(&self, _missed: Missed) -> Result<TickDecision, BoxedError> {
Err(Box::new(Unwell))
}
}
impl RetryPolicy for Mad {
fn on_failure(
&self,
_attempt: u32,
_error: &(dyn core::error::Error + 'static),
) -> Result<RetryDecision, BoxedError> {
Err(Box::new(Unwell))
}
}
impl OverflowPolicy for Mad {
fn on_full(&self, _occupancy: Occupancy) -> Result<OverflowDecision, BoxedError> {
Err(Box::new(Unwell))
}
}
impl DeadlinePolicy for Mad {
fn deadline(&self) -> Result<Option<SignedDuration>, BoxedError> {
Err(Box::new(Unwell))
}
}
#[derive(Debug, derive_more::Display)]
#[display("the store would not answer")]
struct Refused;
impl core::error::Error for Refused {}
#[test]
fn a_policy_that_refuses_is_recorded_and_the_shipped_default_is_taken() {
let (tap, taps) = watched();
let policies = Policies::new()
.with_tick_policy(Box::new(Mad))
.with_retry_policy(Box::new(Mad))
.with_overflow_policy(Box::new(Mad))
.with_deadline_policy(Box::new(Mad));
assert_eq!(
policies
.ticks("decode", taps.clone())
.on_missed(Missed::new(PollUpstream::new(0), 6)),
TickDecision::Coalesce
);
assert_eq!(
policies
.retries("decode", taps.clone())
.on_failure(1, &Refused),
RetryDecision::GiveUp
);
assert_eq!(
policies
.sheds("decode", taps.clone())
.on_full(Occupancy::bounded(4, 4)),
OverflowDecision::Block
);
assert_eq!(policies.deadlines("decode", taps).resolve(None), None);
let concerns: Vec<String> = tap
.arms()
.into_iter()
.filter_map(|arm| match arm {
Seen::PolicyFailed { concern, error } => {
assert_eq!(
error, "the advisor is unwell",
"the error keeps its own words"
);
Some(concern)
}
_ => None,
})
.collect();
assert_eq!(
concerns,
["tick", "retry", "overflow", "deadline"],
"one record per sick advisor, each naming which seat it sat in"
);
assert!(
tap.seen().iter().all(|(node, _)| node == "decode"),
"and every record names the node the gate speaks for"
);
}
#[test]
fn a_policy_that_decides_leaves_no_record_of_its_own() {
let (tap, taps) = watched();
let policies = Policies::new();
assert_eq!(
policies
.ticks("decode", taps.clone())
.on_missed(Missed::new(PollUpstream::new(0), 3)),
TickDecision::Coalesce
);
assert_eq!(
policies
.sheds("decode", taps)
.on_full(Occupancy::bounded(4, 4)),
OverflowDecision::Block
);
assert!(tap.seen().is_empty());
}
#[test]
fn a_settled_station_answers_give_up_without_asking_anybody() {
let (tap, taps) = watched();
struct Always;
impl RetryPolicy for Always {
fn on_failure(
&self,
_attempt: u32,
_error: &(dyn core::error::Error + 'static),
) -> Result<RetryDecision, BoxedError> {
Ok(RetryDecision::Retry { after: None })
}
}
let retries = Policies::new()
.with_retry_policy(Box::new(Always))
.retries("decode", taps);
for station in [Station::Persisted, Station::Processed, Station::Skipped] {
assert_eq!(
retries.settling(station, 1, None),
RetryDecision::GiveUp,
"an outcome is not a thing to try again"
);
}
assert!(
tap.seen().is_empty(),
"and a policy that was never asked leaves no record"
);
assert_eq!(
retries.settling(Station::Failed, 1, Some(&Failure::new(Refused))),
RetryDecision::Retry { after: None },
"the one station that holds the window is the one that asks"
);
}
#[test]
fn a_failing_settle_is_retried_twice_and_then_given_up_on() {
let (tap, taps) = watched();
struct Twice;
impl RetryPolicy for Twice {
fn on_failure(
&self,
attempt: u32,
error: &(dyn core::error::Error + 'static),
) -> Result<RetryDecision, BoxedError> {
assert_eq!(
error.to_string(),
"the store would not answer",
"the policy is shown the failure's own error and not a rendering of the station"
);
if attempt >= 3 {
return Ok(RetryDecision::GiveUp);
}
Ok(RetryDecision::Retry {
after: Some(SignedDuration::from_millis(i64::from(attempt) * 50)),
})
}
}
let retries = Policies::new()
.with_retry_policy(Box::new(Twice))
.retries("decode", taps);
let failure = Failure::new(Refused);
let decisions: Vec<RetryDecision> = (1..=3)
.map(|attempt| retries.settling(Station::Failed, attempt, Some(&failure)))
.collect();
assert_eq!(
decisions,
[
RetryDecision::Retry {
after: Some(SignedDuration::from_millis(50))
},
RetryDecision::Retry {
after: Some(SignedDuration::from_millis(100))
},
RetryDecision::GiveUp,
]
);
assert_eq!(
tap.arms(),
[
Seen::Retrying {
attempt: 1,
after: Some(50)
},
Seen::Retrying {
attempt: 2,
after: Some(100)
},
Seen::GivenUp {
attempt: 3,
error: Some("the store would not answer".to_owned())
},
],
"two scheduled retries and the stop that ended them — a stop is not a retry, and it is not \
silence either"
);
}
fn delivery(keys: &[u64]) -> Delivery<u64, ItemId> {
let (ack, answers) = ack_channel();
drop(answers);
Delivery::new(
Payload::Batch(
keys
.iter()
.map(|&key| Item::new(ItemId::new(key), key))
.collect(),
),
ack,
)
}
#[test]
fn the_shipped_overflow_policy_waits_and_sheds_nothing() {
let (tap, taps) = watched();
let (outbox, inbox) = channel::<ItemId, u64>(1);
let evictor = inbox.evictor();
let sheds = Policies::new().sheds("reels", taps);
assert!(
sheds
.deliver(&outbox, Some(&evictor), delivery(&[1]))
.is_sent(),
"the first goes in, the edge having room"
);
assert!(inbox.drain().count() >= 1);
assert!(
sheds
.deliver(&outbox, Some(&evictor), delivery(&[2]))
.is_sent()
);
assert!(
tap.seen().is_empty(),
"and nothing was shed, so there is no obituary"
);
}
#[test]
fn drop_newest_refuses_the_arrival_and_names_every_key_it_carried() {
let (tap, taps) = watched();
let (outbox, inbox) = channel::<ItemId, u64>(1);
let evictor = inbox.evictor();
let sheds = Policies::new()
.with_overflow_policy(Box::new(DropNewest))
.sheds("reels", taps);
assert!(
sheds
.deliver(&outbox, Some(&evictor), delivery(&[1]))
.is_sent()
);
let handover = sheds.deliver(&outbox, Some(&evictor), delivery(&[7, 8]));
assert!(handover.is_shed());
match handover {
super::Handover::Dropped(dropped) => {
assert_eq!(
dropped.payload().len(),
2,
"and it comes back to be settled"
)
}
other => panic!("drop-newest refuses the arrival, and this said {other:?}"),
}
assert_eq!(
tap.arms(),
[
Seen::Shed {
key: "item#7".to_owned(),
decision: OverflowDecision::DropNewest
},
Seen::Shed {
key: "item#8".to_owned(),
decision: OverflowDecision::DropNewest
},
],
"one obituary per ITEM, each carrying the key its node addresses it by"
);
assert_eq!(
inbox.len(),
1,
"and the queue was left exactly as it was — drop-newest touches nothing already in flight"
);
}
#[test]
fn drop_oldest_evicts_the_queued_item_and_the_arrival_goes_in() {
let (tap, taps) = watched();
let (outbox, inbox) = channel::<ItemId, u64>(1);
let evictor = inbox.evictor();
let sheds = Policies::new()
.with_overflow_policy(Box::new(DropOldest))
.sheds("reels", taps);
assert!(
sheds
.deliver(&outbox, Some(&evictor), delivery(&[1]))
.is_sent()
);
let handover = sheds.deliver(&outbox, Some(&evictor), delivery(&[2]));
assert!(handover.is_sent() && handover.is_shed());
match handover {
super::Handover::Evicted(evicted) => assert_eq!(
evicted.payload().keys().next(),
Some(&ItemId::new(1)),
"what came back is the OLD delivery, for its holder to settle"
),
other => panic!("drop-oldest evicts, and this said {other:?}"),
}
assert_eq!(
tap.arms(),
[Seen::Shed {
key: "item#1".to_owned(),
decision: OverflowDecision::DropOldest
}]
);
let queued: Vec<ItemId> = inbox
.drain()
.filter_map(|wake| match wake {
crate::Wake::Delivered(delivery) => delivery.payload().keys().next().copied(),
_ => None,
})
.collect();
assert_eq!(
queued,
[ItemId::new(2)],
"and the newest is what the consumer will see"
);
}
#[test]
fn a_departed_consumer_gets_the_delivery_back_and_nothing_is_mourned() {
let (tap, taps) = watched();
let (outbox, inbox) = channel::<ItemId, u64>(1);
let sheds = Policies::new()
.with_overflow_policy(Box::new(DropNewest))
.sheds("reels", taps);
assert!(sheds.deliver(&outbox, None, delivery(&[1])).is_sent());
drop(inbox);
let handover = sheds.deliver(&outbox, None, delivery(&[9]));
assert!(
matches!(handover, super::Handover::Gone(_)),
"the edge is full AND gone, and the second fact is the one that answers"
);
assert!(
!tap
.arms()
.iter()
.any(|arm| matches!(arm, Seen::Shed { .. })),
"a delivery no policy dropped has no obituary"
);
}
#[test]
fn drop_oldest_without_a_hand_loses_the_arrival_and_records_that_it_did() {
let (tap, taps) = watched();
let (outbox, inbox) = channel::<ItemId, u64>(1);
let sheds = Policies::new()
.with_overflow_policy(Box::new(DropOldest))
.sheds("reels", taps);
assert!(sheds.deliver(&outbox, None, delivery(&[1])).is_sent());
let handover = sheds.deliver(&outbox, None, delivery(&[2]));
assert!(matches!(handover, super::Handover::Dropped(_)));
assert_eq!(
tap.arms(),
[Seen::Shed {
key: "item#2".to_owned(),
decision: OverflowDecision::DropNewest
}],
"the record is of the shed and not of the wish"
);
assert_eq!(
inbox.len(),
1,
"and the queue is untouched — with no hand there was nothing to take out of it"
);
}
#[test]
fn drop_oldest_with_nothing_left_to_evict_loses_the_arrival() {
let (tap, taps) = watched();
let (outbox, inbox) = channel::<ItemId, u64>(0);
let evictor = inbox.evictor();
let sheds = Policies::new()
.with_overflow_policy(Box::new(DropOldest))
.sheds("reels", taps);
let handover = sheds.deliver(&outbox, Some(&evictor), delivery(&[5]));
assert!(matches!(handover, super::Handover::Dropped(_)));
assert_eq!(
tap.arms(),
[Seen::Shed {
key: "item#5".to_owned(),
decision: OverflowDecision::DropNewest
}]
);
}
#[test]
fn an_unbounded_edge_never_asks_the_policy_anything() {
let (tap, taps) = watched();
let (outbox, inbox) = open::<ItemId, u64>(crate::ChannelOptions::Unbounded);
let evictor = inbox.evictor();
let sheds = Policies::new()
.with_overflow_policy(Box::new(DropNewest))
.sheds("reels", taps);
for key in 0..64 {
assert!(matches!(
sheds.deliver(&outbox, Some(&evictor), delivery(&[key])),
super::Handover::Sent
));
}
assert!(tap.seen().is_empty());
assert_eq!(inbox.len(), 64);
}
#[test]
fn a_call_sites_own_budget_wins_and_the_policy_is_not_consulted() {
let (tap, taps) = watched();
let deadlines = Policies::new()
.with_deadline_policy(Box::new(Mad))
.deadlines("search", taps);
assert_eq!(
deadlines.resolve(Some(Duration::from_millis(25))),
Some(Duration::from_millis(25))
);
assert!(
tap.seen().is_empty(),
"a mad advisor that is never asked cannot even be reported — which is what `never over it` means"
);
}
#[test]
fn the_ambient_budget_answers_where_the_call_site_said_nothing() {
let (_tap, taps) = watched();
let deadlines = Policies::new()
.with_deadline_policy(Box::new(QueryDeadline::of(SignedDuration::from_millis(40))))
.deadlines("search", taps);
assert_eq!(deadlines.resolve(None), Some(Duration::from_millis(40)));
assert_eq!(
Policies::new()
.deadlines("search", Taps::new())
.resolve(None),
None,
"and the shipped default is today's behaviour: as long as it takes"
);
}
#[test]
fn a_negative_ambient_budget_is_no_wait_at_all() {
let deadlines = Policies::new()
.with_deadline_policy(Box::new(QueryDeadline::of(SignedDuration::from_secs(-5))))
.deadlines("search", Taps::new());
assert_eq!(deadlines.resolve(None), Some(Duration::ZERO));
}
#[test]
fn a_question_that_outlives_the_ambient_budget_times_out() {
let deadlines = Policies::new()
.with_deadline_policy(Box::new(QueryDeadline::of(SignedDuration::from_millis(20))))
.deadlines("search", Taps::new());
let (held, awaiting) = reply::<u64>();
let answer = deadlines.take(awaiting, None);
assert_eq!(
answer.expect_err("nobody answered inside the budget"),
crate::QueryError::TimedOut
);
drop(held);
let (held, awaiting) = reply::<u64>();
assert!(held.answer(7));
assert_eq!(
deadlines.take(awaiting, None).expect("answered in time"),
7,
"and an answer inside the budget is simply the answer"
);
}
#[test]
fn a_mad_deadline_advisor_leaves_the_door_waiting_as_it_always_did() {
let (tap, taps) = watched();
let deadlines = Policies::new()
.with_deadline_policy(Box::new(Mad))
.deadlines("search", taps);
let (held, awaiting) = reply::<u64>();
assert!(held.answer(3));
assert_eq!(deadlines.take(awaiting, None).expect("answered"), 3);
assert_eq!(
tap.arms(),
[Seen::PolicyFailed {
concern: "deadline".to_owned(),
error: "the advisor is unwell".to_owned()
}]
);
}