mod capture;
pub use self::capture::{Capture, CapturedEvent};
use std::cell::{Cell, RefCell};
use std::cmp::Reverse;
use std::collections::{BTreeMap, BTreeSet, BinaryHeap};
use std::fmt;
use std::io;
use std::net::SocketAddr;
use std::rc::Rc;
use std::time::Duration;
use hiss::curve::{Curve, DhCurve, p256::P256};
use hiss::provider::{CryptoKeyProvider, DhProvider};
use rand_chacha::ChaCha20Rng;
use rand_core::{Rng, SeedableRng};
use tokio::sync::Notify;
use tokio::time::Instant;
use crate::shell::wire::Wire;
pub const ENETUNREACH: i32 = if cfg!(any(target_os = "linux", target_os = "android")) {
101
} else {
51
};
const SEED_STRIDE: u64 = 0x9E37_79B9_7F4A_7C15;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Spied {
pub src: SocketAddr,
pub dst: SocketAddr,
pub bytes: Vec<u8>,
}
#[derive(Clone, Debug)]
pub struct Tap(Rc<RefCell<Vec<Spied>>>);
impl Tap {
pub fn len(&self) -> usize {
self.0.borrow().len()
}
pub fn is_empty(&self) -> bool {
self.0.borrow().is_empty()
}
pub fn drain(&self) -> Vec<Spied> {
self.0.borrow_mut().drain(..).collect()
}
pub fn datagrams(&self) -> Vec<(SocketAddr, SocketAddr, Vec<u8>)> {
self.0
.borrow()
.iter()
.map(|spied| (spied.src, spied.dst, spied.bytes.clone()))
.collect()
}
pub fn snapshot(&self) -> Vec<Spied> {
self.0.borrow().clone()
}
}
#[derive(Clone, Debug)]
pub struct SendFailure {
pub kind: io::ErrorKind,
pub raw_os: i32,
pub until: Instant,
}
impl SendFailure {
fn to_io_error(&self) -> io::Error {
if self.raw_os != 0 {
io::Error::from_raw_os_error(self.raw_os)
} else {
io::Error::from(self.kind)
}
}
}
#[derive(Clone, Debug)]
pub struct FlakyPolicy {
pub loss: f64,
pub duplicate: f64,
pub base_delay: Duration,
pub jitter: Duration,
pub drop_first: usize,
pub drop_at: BTreeSet<usize>,
pub send_failure: Option<SendFailure>,
failing: Rc<Cell<bool>>,
}
impl FlakyPolicy {
pub fn perfect() -> Self {
FlakyPolicy {
loss: 0.0,
duplicate: 0.0,
base_delay: Duration::ZERO,
jitter: Duration::ZERO,
drop_first: 0,
drop_at: BTreeSet::new(),
send_failure: None,
failing: Rc::new(Cell::new(false)),
}
}
pub fn fail_sends(&self, failing: bool) {
self.failing.set(failing);
}
pub fn is_failing(&self) -> bool {
self.failing.get()
}
pub fn drop_first(n: usize) -> Self {
FlakyPolicy {
drop_first: n,
..Self::perfect()
}
}
pub fn drop_at(indices: impl IntoIterator<Item = usize>) -> Self {
FlakyPolicy {
drop_at: indices.into_iter().collect(),
..Self::perfect()
}
}
pub fn lossy(rate: f64) -> Self {
FlakyPolicy {
loss: rate,
..Self::perfect()
}
}
#[must_use]
pub fn with_delay(self, base: Duration, jitter: Duration) -> Self {
FlakyPolicy {
base_delay: base,
jitter,
..self
}
}
#[must_use]
pub fn with_duplication(self, rate: f64) -> Self {
FlakyPolicy {
duplicate: rate,
..self
}
}
#[must_use]
pub fn failing_sends_until(self, until: Instant) -> Self {
FlakyPolicy {
send_failure: Some(SendFailure {
kind: io::ErrorKind::NetworkUnreachable,
raw_os: ENETUNREACH,
until,
}),
..self
}
}
}
#[derive(Debug, PartialEq, Eq)]
struct Queued {
deliver_at: Instant,
seq: u64,
src: SocketAddr,
bytes: Vec<u8>,
}
impl Ord for Queued {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.deliver_at
.cmp(&other.deliver_at)
.then(self.seq.cmp(&other.seq))
}
}
impl PartialOrd for Queued {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
struct EndpointState {
inbox: BinaryHeap<Reverse<Queued>>,
notify: Rc<Notify>,
}
struct Inner {
seed: u64,
endpoints: BTreeMap<SocketAddr, EndpointState>,
partitioned: BTreeSet<SocketAddr>,
blocked: BTreeSet<(SocketAddr, SocketAddr)>,
log: Rc<RefCell<Vec<Spied>>>,
sends: usize,
seq: u64,
ordinal: usize,
}
#[derive(Clone)]
pub struct Network(Rc<RefCell<Inner>>);
impl Default for Network {
fn default() -> Self {
Network::new()
}
}
impl Network {
pub fn seeded(seed: u64) -> Network {
Network(Rc::new(RefCell::new(Inner {
seed,
endpoints: BTreeMap::new(),
partitioned: BTreeSet::new(),
blocked: BTreeSet::new(),
log: Rc::new(RefCell::new(Vec::new())),
sends: 0,
seq: 0,
ordinal: 0,
})))
}
pub fn new() -> Network {
Network::seeded(0)
}
pub fn endpoint(&self, addr: SocketAddr) -> FlakyWire {
let (ordinal, seed, notify) = {
let mut inner = self.0.borrow_mut();
assert!(
!inner.endpoints.contains_key(&addr),
"{addr} is already registered on this network"
);
let notify = Rc::new(Notify::new());
inner.endpoints.insert(
addr,
EndpointState {
inbox: BinaryHeap::new(),
notify: Rc::clone(¬ify),
},
);
let ordinal = inner.ordinal;
inner.ordinal += 1;
(ordinal, inner.seed, notify)
};
let wire_seed = seed ^ (ordinal as u64).wrapping_mul(SEED_STRIDE);
FlakyWire {
addr: Cell::new(addr),
net: Rc::clone(&self.0),
rng: RefCell::new(ChaCha20Rng::seed_from_u64(wire_seed)),
policy: RefCell::new(FlakyPolicy::perfect()),
sent: Cell::new(0),
notify,
}
}
pub fn wire(&self, addr: SocketAddr) -> FlakyWire {
self.endpoint(addr)
}
pub fn wire_with(&self, addr: SocketAddr, policy: FlakyPolicy) -> FlakyWire {
let wire = self.endpoint(addr);
wire.set_policy(policy);
wire
}
pub fn partition(&self, addr: SocketAddr) {
self.0.borrow_mut().partitioned.insert(addr);
}
pub fn heal(&self, addr: SocketAddr) {
self.0.borrow_mut().partitioned.remove(&addr);
}
pub fn block_path(&self, from: SocketAddr, to: SocketAddr) {
self.0.borrow_mut().blocked.insert((from, to));
}
pub fn heal_path(&self, from: SocketAddr, to: SocketAddr) {
self.0.borrow_mut().blocked.remove(&(from, to));
}
pub fn tap(&self) -> Tap {
Tap(Rc::clone(&self.0.borrow().log))
}
pub fn sends(&self) -> usize {
self.0.borrow().sends
}
pub fn inject(&self, from: SocketAddr, to: SocketAddr, bytes: &[u8]) {
let mut inner = self.0.borrow_mut();
let now = Instant::now();
deliver(&mut inner, from, to, bytes.to_vec(), now);
}
}
fn deliver(inner: &mut Inner, src: SocketAddr, dst: SocketAddr, bytes: Vec<u8>, at: Instant) {
if inner.partitioned.contains(&dst) {
return;
}
let seq = inner.seq;
inner.seq += 1;
let Some(ep) = inner.endpoints.get_mut(&dst) else {
return;
};
ep.inbox.push(Reverse(Queued {
deliver_at: at,
seq,
src,
bytes,
}));
ep.notify.notify_waiters();
}
pub struct FlakyWire {
addr: Cell<SocketAddr>,
net: Rc<RefCell<Inner>>,
rng: RefCell<ChaCha20Rng>,
policy: RefCell<FlakyPolicy>,
sent: Cell<usize>,
notify: Rc<Notify>,
}
impl FlakyWire {
pub fn local_addr(&self) -> SocketAddr {
self.addr.get()
}
pub fn rebind(&self, new_addr: SocketAddr) {
let old = self.addr.get();
if old == new_addr {
return;
}
{
let mut inner = self.net.borrow_mut();
assert!(
!inner.endpoints.contains_key(&new_addr),
"{new_addr} is already registered on this network"
);
inner.endpoints.remove(&old);
inner.endpoints.insert(
new_addr,
EndpointState {
inbox: BinaryHeap::new(),
notify: Rc::clone(&self.notify),
},
);
inner.partitioned.remove(&old);
inner
.blocked
.retain(|(from, to)| *from != old && *to != old);
}
self.addr.set(new_addr);
self.notify.notify_waiters();
}
pub fn set_policy(&self, policy: FlakyPolicy) {
*self.policy.borrow_mut() = policy;
}
pub fn policy(&self) -> FlakyPolicy {
self.policy.borrow().clone()
}
fn draw_unit(&self) -> f64 {
let bits = self.rng.borrow_mut().next_u64() >> 11;
bits as f64 / (1u64 << 53) as f64
}
fn draw_delay(&self, policy: &FlakyPolicy) -> Duration {
let raw = self.rng.borrow_mut().next_u64();
let jitter_ns = policy.jitter.as_nanos() as u64;
if jitter_ns == 0 {
policy.base_delay
} else {
policy.base_delay + Duration::from_nanos(raw % jitter_ns)
}
}
}
impl Wire for FlakyWire {
async fn send_to(&self, buf: &[u8], addr: SocketAddr) -> io::Result<usize> {
let index = self.sent.get();
self.sent.set(index + 1);
self.net.borrow_mut().sends += 1;
let src = self.addr.get();
let policy = self.policy.borrow().clone();
if let Some(failure) = &policy.send_failure
&& Instant::now() < failure.until
{
return Err(failure.to_io_error());
}
if policy.is_failing() {
return Err(io::Error::from_raw_os_error(ENETUNREACH));
}
{
let net = self.net.borrow();
if net.partitioned.contains(&src) || net.blocked.contains(&(src, addr)) {
return Ok(buf.len());
}
}
self.net.borrow().log.borrow_mut().push(Spied {
src,
dst: addr,
bytes: buf.to_vec(),
});
let deliveries = if index < policy.drop_first || policy.drop_at.contains(&index) {
0
} else {
let lost = self.draw_unit() < policy.loss;
let duplicated = self.draw_unit() < policy.duplicate;
match (lost, duplicated) {
(true, _) => 0,
(false, false) => 1,
(false, true) => 2,
}
};
let now = Instant::now();
for _ in 0..deliveries {
let delay = self.draw_delay(&policy);
let mut net = self.net.borrow_mut();
deliver(&mut net, src, addr, buf.to_vec(), now + delay);
}
Ok(buf.len())
}
async fn recv_from(&self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)> {
loop {
let me = self.addr.get();
let next = {
let net = self.net.borrow();
net.endpoints
.get(&me)
.and_then(|ep| ep.inbox.peek().map(|Reverse(q)| q.deliver_at))
};
let Some(deliver_at) = next else {
self.notify.notified().await;
continue;
};
tokio::time::sleep_until(deliver_at).await;
let mut net = self.net.borrow_mut();
let Some(ep) = net.endpoints.get_mut(&self.addr.get()) else {
continue;
};
let due = ep
.inbox
.peek()
.is_some_and(|Reverse(q)| q.deliver_at <= Instant::now());
if !due {
continue;
}
let Reverse(queued) = ep.inbox.pop().expect("peeked above");
let n = queued.bytes.len().min(buf.len());
buf[..n].copy_from_slice(&queued.bytes[..n]);
return Ok((n, queued.src));
}
}
}
#[derive(Clone, Debug, Default)]
pub struct DhCounter(Rc<Cell<u32>>);
impl DhCounter {
pub fn new() -> Self {
DhCounter(Rc::new(Cell::new(0)))
}
pub fn get(&self) -> u32 {
self.0.get()
}
pub fn reset(&self) {
self.0.set(0);
}
pub fn provider<P>(&self, inner: P) -> CountingProvider<P> {
CountingProvider {
inner,
dhs: Rc::clone(&self.0),
}
}
}
pub struct CountingProvider<P> {
inner: P,
dhs: Rc<Cell<u32>>,
}
impl<P> CountingProvider<P> {
pub fn inner(&self) -> &P {
&self.inner
}
}
impl<P: CryptoKeyProvider<P256>> CryptoKeyProvider<P256> for CountingProvider<P> {
type Error = P::Error;
type PrivateKey = P::PrivateKey;
fn public_key(
&self,
key: &Self::PrivateKey,
) -> Result<<P256 as Curve>::PublicKey, Self::Error> {
self.inner.public_key(key)
}
fn generate_static_key(&mut self) -> Result<Self::PrivateKey, Self::Error> {
self.inner.generate_static_key()
}
fn generate_ephemeral_key(&mut self) -> Result<Self::PrivateKey, Self::Error> {
self.inner.generate_ephemeral_key()
}
}
impl<P: DhProvider<P256>> DhProvider<P256> for CountingProvider<P> {
fn dh(
&self,
key: &Self::PrivateKey,
peer: &<P256 as Curve>::PublicKey,
) -> Result<<P256 as DhCurve>::SharedSecret, Self::Error> {
self.dhs.set(self.dhs.get() + 1);
self.inner.dh(key, peer)
}
}
pub struct CountingIdentity<S = crate::packet::ReferenceSuite> {
inner: crate::identity::SoftwareIdentity<S, ChaCha20Rng>,
dhs: DhCounter,
}
impl<S> CountingIdentity<S> {
pub fn seeded(seed: [u8; 32]) -> Self {
let inner = crate::identity::SoftwareIdentity::generate(ChaCha20Rng::from_seed(seed))
.expect("a seeded ChaCha20 stream yields a valid P-256 scalar");
Self {
inner,
dhs: DhCounter::new(),
}
}
pub fn counter(&self) -> DhCounter {
self.dhs.clone()
}
pub fn dhs(&self) -> u32 {
self.dhs.get()
}
}
impl<S> crate::identity::Identity for CountingIdentity<S>
where
S: crate::packet::Handshake<Curve = P256>,
{
type Suite = S;
type Provider = CountingProvider<hiss::provider::EphemeralOnly<ChaCha20Rng>>;
type Error = crate::identity::SoftwareIdentityError;
fn public_static(&self) -> &<P256 as Curve>::PublicKey {
self.inner.public_static()
}
fn open(
&self,
) -> Result<
(
Self::Provider,
<Self::Provider as CryptoKeyProvider<P256>>::PrivateKey,
),
Self::Error,
> {
let (provider, key) = self.inner.open()?;
Ok((self.dhs.provider(provider), key))
}
}
pub type TestIdentity = CountingIdentity<crate::packet::ReferenceSuite>;
pub type TestEndpoint = crate::shell::Endpoint<TestIdentity>;
pub type TestConnection = crate::shell::Connection<crate::packet::ReferenceSuite>;
pub type TestSendStream = crate::shell::SendStream<crate::packet::ReferenceSuite>;
pub type TestRecvStream = crate::shell::RecvStream<crate::packet::ReferenceSuite>;
pub type TestBiStream = crate::shell::BiStream<crate::packet::ReferenceSuite>;
pub type TestIntro = crate::shell::Intro<TestIdentity>;
pub type TestConnecting = crate::shell::Connecting<TestIdentity>;
pub type TestPublicKey = crate::identity::PublicKeyOf<TestIdentity>;
#[derive(Clone)]
pub struct SharedWire(Rc<FlakyWire>);
impl SharedWire {
pub fn local_addr(&self) -> SocketAddr {
self.0.local_addr()
}
pub fn rebind(&self, to: SocketAddr) {
self.0.rebind(to);
}
pub fn set_policy(&self, policy: FlakyPolicy) {
self.0.set_policy(policy);
}
pub fn policy(&self) -> FlakyPolicy {
self.0.policy()
}
}
impl Wire for SharedWire {
async fn send_to(&self, buf: &[u8], addr: SocketAddr) -> io::Result<usize> {
self.0.send_to(buf, addr).await
}
async fn recv_from(&self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)> {
self.0.recv_from(buf).await
}
}
pub struct Peer {
pub endpoint: TestEndpoint,
pub wire: SharedWire,
pub dhs: DhCounter,
pub public_static: TestPublicKey,
}
impl Peer {
pub fn addr(&self) -> SocketAddr {
self.wire.local_addr()
}
pub fn rebind(&self, to: SocketAddr) {
self.wire.rebind(to);
}
}
pub struct Pair {
pub net: Network,
pub a: Peer,
pub b: Peer,
}
impl Pair {
pub fn seeded(seed: u64) -> Pair {
Pair::seeded_with(seed, crate::config::Config::new())
}
pub fn seeded_with(seed: u64, config: crate::config::Config) -> Pair {
let net = Network::seeded(seed);
let a = Peer::spawn(&net, addr_a(), seed, 0xA1, config.clone());
let b = Peer::spawn(&net, addr_b(), seed, 0xB2, config);
Pair { net, a, b }
}
pub async fn establish(&self) -> (TestConnection, TestConnection) {
let dial = async {
self.a
.endpoint
.connect(self.b.addr(), self.b.public_static)
.expect("connect")
.await
.expect("the dial completed")
};
tokio::pin!(dial);
let mut accepted: Option<TestConnection> = None;
let a_conn = 'outer: loop {
let ladder = async {
let intro = self.b.endpoint.accept().await.expect("an introduction");
let claimed = intro.read_identity().await.expect("read_identity");
let proven = claimed.authenticate().await.expect("authenticate");
proven.accept().await.expect("accept")
};
tokio::pin!(ladder);
tokio::select! {
biased;
conn = &mut ladder => {
accepted = Some(conn);
}
conn = &mut dial => {
if accepted.is_none() {
accepted = Some(ladder.await);
}
break 'outer conn;
}
}
};
let b_conn = accepted.expect("unreachable: set on both break paths");
(a_conn, b_conn)
}
}
impl Peer {
fn spawn(
net: &Network,
addr: SocketAddr,
seed: u64,
salt: u8,
config: crate::config::Config,
) -> Peer {
let wire = SharedWire(Rc::new(net.endpoint(addr)));
let identity: TestIdentity = CountingIdentity::seeded(derive_seed(seed, salt));
let dhs = identity.counter();
let public_static = *crate::identity::Identity::public_static(&identity);
let endpoint = crate::shell::Endpoint::builder()
.identity(identity)
.wire(wire.clone())
.config(config)
.rng_seed(derive_seed(seed, salt ^ 0xFF))
.build();
Peer {
endpoint,
wire,
dhs,
public_static,
}
}
}
pub fn addr_a() -> SocketAddr {
"10.0.0.1:4001".parse().expect("literal addr")
}
pub fn addr_b() -> SocketAddr {
"10.0.0.2:4002".parse().expect("literal addr")
}
pub fn addr_c() -> SocketAddr {
"10.0.0.3:4003".parse().expect("literal addr")
}
pub async fn local<F: std::future::Future>(body: F) -> F::Output {
tokio::task::LocalSet::new().run_until(body).await
}
pub async fn settle() {
for _ in 0..SETTLE_YIELDS {
tokio::task::yield_now().await;
}
}
const SETTLE_YIELDS: usize = 64;
fn derive_seed(seed: u64, salt: u8) -> [u8; 32] {
let mut out = [salt; 32];
out[..8].copy_from_slice(&seed.to_le_bytes());
out
}
impl fmt::Debug for Network {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let mut out = f.debug_struct("Network");
match self.0.try_borrow() {
Ok(inner) => out
.field("seed", &inner.seed)
.field("endpoints", &inner.endpoints.len())
.field("sends", &inner.sends),
Err(_) => out.field("state", &"borrowed"),
}
.finish_non_exhaustive()
}
}
impl fmt::Debug for FlakyWire {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("FlakyWire")
.field("addr", &self.addr.get())
.field("sent", &self.sent.get())
.finish_non_exhaustive()
}
}
impl fmt::Debug for SharedWire {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("SharedWire")
.field("wire", &*self.0)
.finish_non_exhaustive()
}
}
impl<P> fmt::Debug for CountingProvider<P> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("CountingProvider")
.field("dhs", &self.dhs.get())
.finish_non_exhaustive()
}
}
impl<S> fmt::Debug for CountingIdentity<S> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("CountingIdentity")
.field("dhs", &self.dhs.get())
.finish_non_exhaustive()
}
}
impl fmt::Debug for Peer {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Peer")
.field("addr", &self.addr())
.field("dhs", &self.dhs.get())
.finish_non_exhaustive()
}
}
impl fmt::Debug for Pair {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Pair")
.field("net", &self.net)
.field("a", &self.a)
.field("b", &self.b)
.finish_non_exhaustive()
}
}
#[cfg(test)]
mod tests {
use super::*;
async fn recv_ok(wire: &FlakyWire, buf: &mut [u8]) -> (usize, std::net::SocketAddr) {
tokio::time::timeout(Duration::from_secs(5), wire.recv_from(buf))
.await
.expect("recv_from hung: routing, send path or cancel-safety is broken")
.expect("recv")
}
fn addr(n: u8) -> SocketAddr {
format!("10.0.0.{n}:400{n}").parse().expect("literal addr")
}
#[tokio::test(start_paused = true)]
async fn a_byte_crosses_two_flaky_wires_under_injected_loss() {
let wall_t0 = std::time::Instant::now();
let net = Network::seeded(0xA11CE);
let a = net.endpoint("10.0.0.1:4001".parse().expect("literal addr"));
let b = net.endpoint("10.0.0.2:4002".parse().expect("literal addr"));
a.set_policy(
FlakyPolicy::drop_at([0, 1])
.with_delay(Duration::from_millis(50), Duration::from_millis(10)),
);
let t0 = Instant::now();
for _ in 0..3 {
a.send_to(b"!", b.local_addr()).await.expect("send");
}
let mut buf = [0u8; 1200];
let (n, src) = recv_ok(&b, &mut buf).await;
assert_eq!(&buf[..n], b"!");
assert_eq!(src, a.local_addr());
assert_eq!(net.sends(), 3, "all three left the wire");
assert_eq!(
net.tap().len(),
3,
"all three were tapped; two were dropped after"
);
let elapsed = Instant::now() - t0;
assert!(
(Duration::from_millis(50)..Duration::from_millis(60)).contains(&elapsed),
"delivery waited base_delay + jitter in virtual time, got {elapsed:?}",
);
assert!(
tokio::time::timeout(Duration::from_secs(1), b.recv_from(&mut buf))
.await
.is_err(),
"the two dropped datagrams never arrive",
);
assert!(
wall_t0.elapsed() < Duration::from_secs(1),
"the test spent real time; virtual time is not being used",
);
}
#[tokio::test(start_paused = true)]
async fn perfect_policy_delivers_everything_in_order() {
let net = Network::seeded(1);
let a = net.endpoint(addr(1));
let b = net.endpoint(addr(2));
a.set_policy(FlakyPolicy::perfect());
for i in 0..100u8 {
a.send_to(&[i], b.local_addr()).await.expect("send");
}
let mut buf = [0u8; 16];
for i in 0..100u8 {
let (n, src) = recv_ok(&b, &mut buf).await;
assert_eq!(n, 1);
assert_eq!(buf[0], i, "datagram {i} arrived out of order");
assert_eq!(src, a.local_addr());
}
assert_eq!(net.sends(), 100);
}
async fn trace(seed: u64) -> Vec<(Vec<u8>, Duration)> {
let net = Network::seeded(seed);
let a = net.endpoint(addr(1));
let b = net.endpoint(addr(2));
a.set_policy(
FlakyPolicy::lossy(0.3)
.with_duplication(0.1)
.with_delay(Duration::ZERO, Duration::from_millis(20)),
);
let t0 = Instant::now();
for i in 0..50u8 {
a.send_to(&[i], b.local_addr()).await.expect("send");
}
let mut out = Vec::new();
let mut buf = [0u8; 16];
while let Ok(Ok((n, _src))) =
tokio::time::timeout(Duration::from_secs(1), b.recv_from(&mut buf)).await
{
out.push((buf[..n].to_vec(), Instant::now() - t0));
}
out
}
#[tokio::test(start_paused = true)]
async fn seeded_runs_are_identical() {
let first = trace(7).await;
let second = trace(7).await;
assert!(!first.is_empty(), "the scenario delivered nothing at all");
assert_eq!(first, second, "the same seed produced a different trace");
}
#[tokio::test(start_paused = true)]
async fn different_seeds_diverge() {
let first = trace(7).await;
let second = trace(8).await;
assert_ne!(first, second, "two seeds produced identical traces");
}
#[tokio::test(start_paused = true)]
async fn recv_from_is_cancel_safe() {
let net = Network::seeded(3);
let a = net.endpoint(addr(1));
let b = net.endpoint(addr(2));
a.set_policy(FlakyPolicy::perfect().with_delay(Duration::from_millis(50), Duration::ZERO));
a.send_to(b"payload", b.local_addr()).await.expect("send");
let mut buf = [0u8; 32];
assert!(
tokio::time::timeout(Duration::from_millis(10), b.recv_from(&mut buf))
.await
.is_err(),
"the datagram is not due yet",
);
let (n, src) = recv_ok(&b, &mut buf).await;
assert_eq!(&buf[..n], b"payload");
assert_eq!(src, a.local_addr());
assert!(
tokio::time::timeout(Duration::from_secs(1), b.recv_from(&mut buf))
.await
.is_err(),
"the datagram was delivered twice",
);
}
#[tokio::test(start_paused = true)]
async fn jitter_reorders_and_the_heap_is_stable() {
let net = Network::seeded(11);
let a = net.endpoint(addr(1));
let b = net.endpoint(addr(2));
a.set_policy(FlakyPolicy::perfect().with_delay(Duration::ZERO, Duration::from_millis(100)));
for i in 0..30u8 {
a.send_to(&[i], b.local_addr()).await.expect("send");
}
let mut order = Vec::new();
let mut buf = [0u8; 16];
while let Ok(Ok((n, _))) =
tokio::time::timeout(Duration::from_secs(1), b.recv_from(&mut buf)).await
{
order.push(buf[..n][0]);
}
assert_eq!(order.len(), 30, "nothing was lost, only reordered");
let sent: Vec<u8> = (0..30).collect();
assert_ne!(order, sent, "jitter never reordered anything");
let mut sorted = order.clone();
sorted.sort_unstable();
assert_eq!(sorted, sent, "reordering must not invent or lose bytes");
let net = Network::seeded(11);
let a = net.endpoint(addr(3));
let b = net.endpoint(addr(4));
a.set_policy(FlakyPolicy::perfect());
for i in 0..30u8 {
a.send_to(&[i], b.local_addr()).await.expect("send");
}
for i in 0..30u8 {
let (n, _) = recv_ok(&b, &mut buf).await;
assert_eq!(buf[..n][0], i, "equal deadlines must deliver FIFO");
}
}
#[tokio::test(start_paused = true)]
async fn duplication_delivers_two_identical_copies() {
let net = Network::seeded(5);
let a = net.endpoint(addr(1));
let b = net.endpoint(addr(2));
a.set_policy(FlakyPolicy::perfect().with_duplication(1.0));
a.send_to(b"dup", b.local_addr()).await.expect("send");
let mut buf = [0u8; 16];
for _ in 0..2 {
let (n, src) = recv_ok(&b, &mut buf).await;
assert_eq!(&buf[..n], b"dup");
assert_eq!(src, a.local_addr());
}
assert!(
tokio::time::timeout(Duration::from_secs(1), b.recv_from(&mut buf))
.await
.is_err(),
"exactly two copies, not three",
);
assert_eq!(net.sends(), 1, "one send, two deliveries");
}
#[tokio::test(start_paused = true)]
async fn index_drops_do_not_perturb_the_probabilistic_draw_order() {
let net = Network::seeded(4242);
let a = net.endpoint(addr(70));
let b = net.endpoint(addr(71));
a.set_policy(FlakyPolicy {
loss: 0.5,
drop_at: [0, 7].into_iter().collect(),
..FlakyPolicy::perfect()
});
for i in 0..24u8 {
a.send_to(&[i], b.local_addr()).await.expect("send");
}
let mut got = Vec::new();
let mut buf = [0u8; 4];
while let Ok(Ok((n, _))) =
tokio::time::timeout(Duration::from_millis(200), b.recv_from(&mut buf)).await
{
got.push(buf[..n][0]);
}
assert!(
!got.contains(&0) && !got.contains(&7),
"index-dropped sends must never arrive; got {got:?}"
);
assert!(
!got.is_empty() && got.len() < 22,
"fixture must lose some and keep some, or it proves nothing \
(got {} of a possible 22)",
got.len()
);
assert_eq!(got, PINNED_DRAW_ORDER, "the RNG draw order changed");
}
const PINNED_DRAW_ORDER: &[u8] = &[3, 4, 12, 13, 14, 16, 17, 18, 19];
#[tokio::test(start_paused = true)]
async fn drop_at_is_exact() {
async fn run(seed: u64) -> Vec<u8> {
let net = Network::seeded(seed);
let a = net.endpoint(addr(1));
let b = net.endpoint(addr(2));
a.set_policy(FlakyPolicy::drop_at([1, 3]));
for i in 0..5u8 {
a.send_to(&[i], b.local_addr()).await.expect("send");
}
let mut got = Vec::new();
let mut buf = [0u8; 16];
while let Ok(Ok((n, _))) =
tokio::time::timeout(Duration::from_secs(1), b.recv_from(&mut buf)).await
{
got.push(buf[..n][0]);
}
got
}
assert_eq!(run(0).await, vec![0, 2, 4]);
assert_eq!(
run(0).await,
run(999_999).await,
"an index-based drop must not depend on the seed",
);
}
#[tokio::test(start_paused = true)]
async fn partition_blackholes_without_a_send_error() {
let net = Network::seeded(2);
let a = net.endpoint(addr(1));
let b = net.endpoint(addr(2));
net.partition(a.local_addr());
let n = a.send_to(b"x", b.local_addr()).await.expect("Ok, not Err");
assert_eq!(n, 1);
assert_eq!(net.sends(), 1, "the send was counted");
assert!(net.tap().is_empty(), "a blackholed send is not tapped");
let mut buf = [0u8; 16];
assert!(
tokio::time::timeout(Duration::from_secs(1), b.recv_from(&mut buf))
.await
.is_err(),
"nothing crosses a partition",
);
net.heal(a.local_addr());
net.block_path(a.local_addr(), b.local_addr());
a.send_to(b"y", b.local_addr()).await.expect("Ok, not Err");
assert!(
tokio::time::timeout(Duration::from_secs(1), b.recv_from(&mut buf))
.await
.is_err(),
"nothing crosses a blocked path",
);
net.heal_path(a.local_addr(), b.local_addr());
a.send_to(b"z", b.local_addr()).await.expect("send");
let (n, _) = recv_ok(&b, &mut buf).await;
assert_eq!(&buf[..n], b"z", "healing restores the path");
}
#[tokio::test(start_paused = true)]
async fn send_failure_is_an_err_and_then_heals() {
let net = Network::seeded(4);
let a = net.endpoint(addr(1));
let b = net.endpoint(addr(2));
let until = Instant::now() + Duration::from_secs(3);
a.set_policy(FlakyPolicy::perfect().failing_sends_until(until));
let err = a
.send_to(b"x", b.local_addr())
.await
.expect_err("the send must fail");
assert_eq!(err.kind(), io::ErrorKind::NetworkUnreachable);
assert_eq!(err.raw_os_error(), Some(ENETUNREACH));
assert_eq!(net.sends(), 1, "a failed send is still counted");
assert!(net.tap().is_empty(), "a failed send is not tapped");
let mut buf = [0u8; 16];
assert!(
tokio::time::timeout(Duration::from_secs(1), b.recv_from(&mut buf))
.await
.is_err(),
"nothing was queued",
);
tokio::time::sleep_until(until + Duration::from_millis(1)).await;
a.send_to(b"y", b.local_addr()).await.expect("healed");
let (n, src) = recv_ok(&b, &mut buf).await;
assert_eq!(&buf[..n], b"y");
assert_eq!(src, a.local_addr());
assert_eq!(net.tap().len(), 1, "only the healed send was tapped");
}
#[tokio::test(start_paused = true)]
async fn oversize_datagram_is_truncated_like_a_socket() {
let net = Network::seeded(6);
let a = net.endpoint(addr(1));
let b = net.endpoint(addr(2));
let big = vec![0xab; 2000];
a.send_to(&big, b.local_addr()).await.expect("send");
let mut buf = [0u8; crate::constants::MAX_DATAGRAM];
let (n, src) = recv_ok(&b, &mut buf).await;
assert_eq!(n, crate::constants::MAX_DATAGRAM);
assert_eq!(src, a.local_addr());
assert!(buf.iter().all(|byte| *byte == 0xab));
}
#[tokio::test(start_paused = true)]
async fn inject_forges_a_source() {
let net = Network::seeded(8);
let b = net.endpoint(addr(2));
let forged: SocketAddr = "203.0.113.9:9999".parse().expect("literal addr");
net.inject(forged, b.local_addr(), b"x");
let mut buf = [0u8; 16];
let (n, src) = recv_ok(&b, &mut buf).await;
assert_eq!(&buf[..n], b"x");
assert_eq!(src, forged, "the forged source is reported verbatim");
assert_eq!(net.sends(), 0, "an injection is not a wire's send");
assert!(net.tap().is_empty(), "an injection is not tapped");
}
#[tokio::test(start_paused = true)]
async fn unregistered_destination_is_dropped_not_an_error() {
let net = Network::seeded(9);
let a = net.endpoint(addr(1));
let nowhere: SocketAddr = "198.51.100.7:1".parse().expect("literal addr");
let n = a.send_to(b"x", nowhere).await.expect("Ok, not Err");
assert_eq!(n, 1);
assert_eq!(net.sends(), 1);
assert_eq!(net.tap().len(), 1, "it did leave the wire");
}
#[test]
fn dh_counter_counts_only_dh() {
use hiss::provider::EphemeralOnly;
let counter = DhCounter::new();
let mut provider = counter.provider(EphemeralOnly::new(ChaCha20Rng::seed_from_u64(42)));
let ephemeral = provider.generate_ephemeral_key().expect("keygen");
assert_eq!(counter.get(), 0, "key generation is not a DH");
let static_key = provider.generate_static_key().expect("keygen");
assert_eq!(counter.get(), 0, "key generation is not a DH");
let peer = provider.public_key(&static_key).expect("public_key");
assert_eq!(counter.get(), 0, "public_key is not a DH");
provider.dh(&ephemeral, &peer).expect("dh");
assert_eq!(counter.get(), 1);
let observer = counter.clone();
provider.dh(&static_key, &peer).expect("dh");
assert_eq!(observer.get(), 2);
assert_eq!(counter.get(), 2);
observer.reset();
assert_eq!(counter.get(), 0, "reset is shared too");
}
#[tokio::test(start_paused = true)]
async fn the_whole_module_is_not_send() {
let local = tokio::task::LocalSet::new();
local
.run_until(async {
let net = Network::seeded(12);
let a = net.endpoint(addr(1));
let b = net.endpoint(addr(2));
let b_addr = b.local_addr();
let sender = tokio::task::spawn_local(async move {
a.send_to(b"local", b_addr).await.expect("send")
});
assert_eq!(sender.await.expect("join"), 5);
let mut buf = [0u8; 16];
let (n, _src) = recv_ok(&b, &mut buf).await;
assert_eq!(&buf[..n], b"local");
fn _drives_any_wire<W: Wire>(_w: &W) {}
_drives_any_wire(&b);
})
.await;
}
#[tokio::test(start_paused = true)]
async fn a_rebound_wire_sends_from_and_receives_at_its_new_address() {
let net = Network::seeded(0x510AD);
let a = net.endpoint(addr(1));
let b = net.endpoint(addr(2));
a.send_to(b"before", addr(2)).await.expect("send");
let mut buf = [0u8; 16];
let (n, src) = recv_ok(&b, &mut buf).await;
assert_eq!(&buf[..n], b"before");
assert_eq!(src, addr(1), "the pre-move source");
a.rebind(addr(9));
assert_eq!(a.local_addr(), addr(9), "local_addr follows the rebind");
a.send_to(b"after", addr(2)).await.expect("send");
let (n, src) = recv_ok(&b, &mut buf).await;
assert_eq!(&buf[..n], b"after");
assert_eq!(src, addr(9), "§7.3: the source a peer would re-home to");
b.send_to(b"reply", addr(9)).await.expect("send");
let (n, src) = recv_ok(&a, &mut buf).await;
assert_eq!(&buf[..n], b"reply");
assert_eq!(src, addr(2));
}
#[tokio::test(start_paused = true)]
async fn the_vacated_address_delivers_nothing() {
let net = Network::seeded(0x510AE);
let a = net.endpoint(addr(1));
let b = net.endpoint(addr(2));
a.rebind(addr(9));
b.send_to(b"to the old mapping", addr(1))
.await
.expect("send reports success — a blackhole is not an error");
let mut buf = [0u8; 32];
assert!(
tokio::time::timeout(Duration::from_secs(5), a.recv_from(&mut buf))
.await
.is_err(),
"a datagram to the vacated address must never arrive"
);
}
#[tokio::test(start_paused = true)]
async fn a_datagram_in_flight_to_the_old_address_is_lost_on_rebind() {
let net = Network::seeded(0x510AF);
let a = net.wire_with(
addr(1),
FlakyPolicy::perfect().with_delay(Duration::from_secs(2), Duration::ZERO),
);
let b = net.endpoint(addr(2));
b.send_to(b"in flight", addr(1)).await.expect("send");
a.rebind(addr(9));
let mut buf = [0u8; 32];
assert!(
tokio::time::timeout(Duration::from_secs(10), a.recv_from(&mut buf))
.await
.is_err(),
"ruling 180: the old address's inbox is abandoned, not carried"
);
b.send_to(b"after the move", addr(9)).await.expect("send");
let (n, _src) = recv_ok(&a, &mut buf).await;
assert_eq!(&buf[..n], b"after the move");
}
#[tokio::test(start_paused = true)]
#[should_panic(expected = "is already registered on this network")]
async fn rebinding_onto_a_registered_address_panics() {
let net = Network::seeded(0x510B0);
let a = net.endpoint(addr(1));
let _b = net.endpoint(addr(2));
a.rebind(addr(2));
}
#[tokio::test(start_paused = true)]
async fn rebinding_to_the_same_address_keeps_the_inbox() {
let net = Network::seeded(0x510B1);
let a = net.wire_with(
addr(1),
FlakyPolicy::perfect().with_delay(Duration::from_secs(2), Duration::ZERO),
);
let b = net.endpoint(addr(2));
b.send_to(b"queued", addr(1)).await.expect("send");
a.rebind(addr(1));
let mut buf = [0u8; 32];
let (n, _src) = recv_ok(&a, &mut buf).await;
assert_eq!(
&buf[..n],
b"queued",
"a same-address rebind must not discard the inbox"
);
}
}