use std::net::IpAddr;
use crate::journal::cursor::Checkpoint;
use crate::journal::settle::Outcome;
use crate::model::ip::set::{IpSet, Positions};
use crate::model::target::{PlannedTarget, TargetIndex, TargetMap};
use crate::scanner::handle::ScanHandle;
use crate::scanner::order::Permutation;
use crate::scanner::session::ScanContext;
use rand::seq::SliceRandom;
use tokio::sync::mpsc;
pub fn dispatch_addresses(
ips: IpSet,
batch_size: usize,
seed: Option<u64>,
scan_handle: &ScanHandle,
) -> mpsc::Receiver<IpAddr> {
let batch_size = batch_size.max(1);
let (tx, rx) = mpsc::channel(batch_size.saturating_mul(2));
let scan_handle = scan_handle.clone();
tokio::spawn(async move {
let mut batch = Vec::with_capacity(batch_size);
let numbered = seed.map(|seed| {
let numbered = Positions::of(&ips);
let order = Permutation::new(seed, numbered.total());
(numbered, order)
});
let addresses: Box<dyn Iterator<Item = IpAddr> + Send + '_> = match &numbered {
Some((numbered, order)) => Box::new(
order
.iter()
.filter_map(|position| numbered.address_at(position))
.chain(numbered.unnumbered().iter().flat_map(|range| range.iter())),
),
None => ips.iter(),
};
for ip in addresses {
batch.push(ip);
if batch.len() < batch_size {
continue;
}
if !drain(&mut batch, &tx, &scan_handle).await {
return;
}
}
drain(&mut batch, &tx, &scan_handle).await;
});
rx
}
async fn drain<T>(batch: &mut Vec<T>, tx: &mpsc::Sender<T>, scan_handle: &ScanHandle) -> bool {
batch.shuffle(&mut rand::rng());
for item in batch.drain(..) {
if tx.send(item).await.is_err() || scan_handle.should_stop() {
return false;
}
}
true
}
pub const DEFAULT_BATCH: usize = 8192;
#[must_use]
pub struct Dispatcher {
target_map: TargetMap,
batch_size: usize,
settled: Checkpoint,
live: Option<IpSet>,
}
impl Dispatcher {
pub fn new(target_map: TargetMap) -> Self {
Self {
target_map,
batch_size: DEFAULT_BATCH,
settled: Checkpoint::default(),
live: None,
}
}
pub fn resuming(mut self, settled: Checkpoint) -> Self {
self.settled = settled;
self
}
pub fn only_live(mut self, live: IpSet) -> Self {
self.live = Some(live);
self
}
pub fn with_batch_size(mut self, batch_size: usize) -> Self {
self.batch_size = batch_size;
self
}
pub fn run(self, ctx: &ScanContext) -> mpsc::Receiver<PlannedTarget> {
let batch_size = self.batch_size.max(1);
let (tx, rx) = mpsc::channel(batch_size.saturating_mul(2));
let scan_handle = ctx.handle.clone();
let order = self.order(ctx.order_seed);
let ctx = ctx.clone();
tokio::spawn(async move {
let mut batch = Vec::with_capacity(batch_size);
let stream: Box<dyn Iterator<Item = PlannedTarget> + Send> = match &order {
Some((index, order)) => Box::new(order.iter().filter_map(|position| {
index
.target_at(position)
.map(|target| PlannedTarget::new(position, target))
})),
None => Box::new(
self.target_map
.iter()
.enumerate()
.map(|(position, target)| PlannedTarget::new(position as u64, target)),
),
};
for planned in stream {
if self.settled.is_settled(planned.position) {
continue;
}
if let Some(live) = &self.live
&& !live.contains(&planned.target.ip)
{
ctx.record_outcome(Outcome::Skipped {
position: planned.position,
});
continue;
}
batch.push(planned);
if batch.len() >= batch_size && !drain(&mut batch, &tx, &scan_handle).await {
return;
}
}
drain(&mut batch, &tx, &scan_handle).await;
});
rx
}
fn order(&self, seed: Option<u64>) -> Option<(TargetIndex, Permutation)> {
let seed = seed?;
let index = TargetIndex::of(&self.target_map);
if !index.is_complete() {
return None;
}
let order = Permutation::new(seed, index.total());
Some((index, order))
}
}
#[cfg(test)]
mod tests {
#[tokio::test]
async fn a_zero_batch_is_read_as_one_probe_rather_than_panicking() {
use crate::scanner::session::ScanSession;
let (_session, ctx) = ScanSession::new();
let mut rx = Dispatcher::new(TargetMap::new())
.with_batch_size(0)
.run(&ctx);
assert!(rx.recv().await.is_none(), "an empty plan yields nothing");
}
#[tokio::test]
async fn the_address_stream_holds_a_zero_batch_to_one_too() {
let handle = ScanHandle::new();
let mut rx = dispatch_addresses(IpSet::new(), 0, None, &handle);
assert!(rx.recv().await.is_none());
}
use super::*;
use crate::model::ip::set::IpSet;
use crate::model::port::PortSet;
use crate::model::target::Target;
use crate::model::target::TargetSet;
use crate::scanner::session::ScanSession;
use std::net::IpAddr;
fn context() -> (ScanSession, ScanContext) {
ScanSession::new()
}
#[tokio::test]
async fn dispatcher_emits_all_targets_shuffled() {
let mut target_map = TargetMap::new();
let ip_set: IpSet = "192.0.2.1-192.0.2.10".parse().unwrap();
let port_set: PortSet = "80".parse().unwrap();
let unit = TargetSet::new(ip_set, port_set);
target_map.units.push(unit);
let (_session, ctx) = context();
let dispatcher = Dispatcher::new(target_map).with_batch_size(4);
let mut rx = dispatcher.run(&ctx);
let mut received = Vec::new();
while let Some(target) = rx.recv().await {
received.push(target);
}
assert_eq!(received.len(), 10);
let is_ordered = received.windows(2).all(|w| match (w[0].ip(), w[1].ip()) {
(IpAddr::V4(a), IpAddr::V4(b)) => a.octets()[3] < b.octets()[3],
_ => false,
});
assert!(!is_ordered, "Targets were not shuffled");
}
#[tokio::test]
async fn the_dispatcher_emits_exactly_the_plans_enumeration() {
let mut target_map = TargetMap::new();
target_map.units.push(TargetSet::new(
"192.0.2.1-192.0.2.10".parse().unwrap(),
"80,443".parse().unwrap(),
));
target_map.units.push(TargetSet::new(
"198.51.100.1-198.51.100.3".parse().unwrap(),
"22".parse().unwrap(),
));
let expected: Vec<Target> = target_map.iter().collect();
let (_session, ctx) = context();
let mut rx = Dispatcher::new(target_map.clone())
.with_batch_size(4)
.run(&ctx);
let mut received = Vec::new();
while let Some(target) = rx.recv().await {
received.push(target);
}
let key = |t: &Target| (t.ip.to_string(), t.port, t.protocol);
let mut received_sorted: Vec<Target> =
received.iter().map(|planned| planned.target).collect();
let mut expected_sorted = expected.clone();
received_sorted.sort_by_key(key);
expected_sorted.sort_by_key(key);
assert_eq!(
received_sorted, expected_sorted,
"the dispatcher and the journal must walk one enumeration"
);
}
#[tokio::test]
async fn dispatcher_stops_early_on_abort() {
let mut target_map = TargetMap::new();
let ip_set: IpSet = "192.0.2.1-192.0.2.100".parse().unwrap();
let port_set: PortSet = "80".parse().unwrap();
let unit = TargetSet::new(ip_set, port_set);
target_map.units.push(unit);
let (_session, ctx) = context();
let dispatcher = Dispatcher::new(target_map).with_batch_size(10);
let mut rx = dispatcher.run(&ctx);
let mut count = 0;
while let Some(_target) = rx.recv().await {
count += 1;
if count == 15 {
ctx.handle.abort();
}
}
assert!((15..100).contains(&count));
}
#[tokio::test]
async fn liveness_never_moves_a_position() {
let plan = || {
let mut map = TargetMap::new();
map.add_unit(TargetSet::new(
"192.0.2.1-192.0.2.4".parse::<IpSet>().expect("a range"),
"80".parse::<PortSet>().expect("ports"),
));
map
};
let one = numbered(plan(), Some("192.0.2.4".parse().expect("a range"))).await;
let two = numbered(
plan(),
Some("192.0.2.2-192.0.2.4".parse().expect("a range")),
)
.await;
let all = numbered(plan(), None).await;
for emitted in [&one, &two] {
for (ip, position) in emitted {
assert_eq!(
all.get(ip),
Some(position),
"{ip} moved when the liveness answer changed"
);
}
}
assert_eq!(one.len(), 1, "one host answered");
assert_eq!(two.len(), 3);
}
#[tokio::test]
async fn a_target_whose_host_is_down_settles_rather_than_vanishing() {
let mut map = TargetMap::new();
map.add_unit(TargetSet::new(
"192.0.2.1-192.0.2.4".parse::<IpSet>().expect("a range"),
"80".parse::<PortSet>().expect("ports"),
));
let (_session, ctx) = context();
let mut rx = Dispatcher::new(map)
.only_live("192.0.2.4".parse::<IpSet>().expect("a range"))
.run(&ctx);
let mut emitted = 0;
while rx.recv().await.is_some() {
emitted += 1;
}
assert_eq!(emitted, 1, "only the live host is probed");
assert_eq!(
ctx.settlements().count(Outcome::Skipped { position: 0 }),
3,
"the three that were not probed have to be accounted for"
);
assert_eq!(
ctx.settlements().checkpoint().watermark,
3,
"and their positions are the ones below the live host"
);
}
fn ordered(seed: u64) -> (ScanSession, ScanContext) {
ScanSession::builder().ordering(Some(seed)).build()
}
fn wide(count: u8) -> TargetMap {
let mut map = TargetMap::new();
map.add_unit(TargetSet::new(
format!("192.0.2.0/{count}")
.parse::<IpSet>()
.expect("a prefix"),
"80".parse::<PortSet>().expect("ports"),
));
map
}
async fn emitted(map: TargetMap, ctx: &ScanContext, batch: usize) -> Vec<PlannedTarget> {
let mut rx = Dispatcher::new(map).with_batch_size(batch).run(ctx);
let mut asked = Vec::new();
while let Some(planned) = rx.recv().await {
asked.push(planned);
}
asked
}
#[tokio::test]
async fn a_seeded_scan_asks_about_every_target_exactly_once() {
let mut map = TargetMap::new();
map.add_unit(TargetSet::new(
"192.0.2.0/26".parse::<IpSet>().expect("a prefix"),
"80,443,u:53".parse::<PortSet>().expect("ports"),
));
map.add_unit(TargetSet::new(
"198.51.100.1-198.51.100.9"
.parse::<IpSet>()
.expect("a range"),
"22".parse::<PortSet>().expect("ports"),
));
let expected: Vec<Target> = map.iter().collect();
let (_session, ctx) = ordered(0x5EED);
let asked = emitted(map, &ctx, 32).await;
assert_eq!(asked.len(), expected.len());
let mut positions: Vec<u64> = asked.iter().map(|planned| planned.position).collect();
positions.sort_unstable();
assert!(
positions.iter().copied().eq(0..expected.len() as u64),
"a target was asked twice or not at all"
);
}
#[tokio::test]
async fn a_seeded_scan_numbers_by_the_plan_and_not_by_the_order() {
let mut map = TargetMap::new();
map.add_unit(TargetSet::new(
"192.0.2.0/28".parse::<IpSet>().expect("a prefix"),
"80,443".parse::<PortSet>().expect("ports"),
));
map.add_unit(TargetSet::new(
"198.51.100.1-198.51.100.5"
.parse::<IpSet>()
.expect("a range"),
"22,u:161".parse::<PortSet>().expect("ports"),
));
let plan: std::collections::HashMap<Target, u64> = map
.iter()
.enumerate()
.map(|(position, target)| (target, position as u64))
.collect();
let (_session, ctx) = ordered(0xC0FFEE);
for planned in emitted(map, &ctx, 16).await {
assert_eq!(
plan.get(&planned.target),
Some(&planned.position),
"{:?} was numbered by when it was asked about",
planned.target
);
}
}
#[tokio::test]
async fn a_seeded_scan_does_not_walk_the_plan_a_batch_at_a_time() {
const BATCH: usize = 64;
let (_session, ctx) = ordered(0x5EED);
let asked = emitted(wide(20), &ctx, BATCH).await;
assert_eq!(asked.len(), 4_096);
let beyond = asked[..BATCH]
.iter()
.filter(|planned| planned.position >= BATCH as u64)
.count();
assert!(
beyond > BATCH / 2,
"only {beyond} of the first {BATCH} targets came from beyond the \
first batch of the plan, which is a walk rather than a rearrangement"
);
}
#[tokio::test]
async fn a_resumed_seeded_scan_asks_only_what_is_left() {
use crate::journal::cursor::Checkpoint;
let settled = Checkpoint::new(1_000, [1_500, 2_000, 4_095]);
let (_session, ctx) = ordered(0x1234);
let mut rx = Dispatcher::new(wide(20))
.resuming(settled.clone())
.with_batch_size(64)
.run(&ctx);
let mut positions = Vec::new();
while let Some(planned) = rx.recv().await {
positions.push(planned.position);
}
positions.sort_unstable();
let expected: Vec<u64> = (0..4_096).filter(|p| !settled.is_settled(*p)).collect();
assert_eq!(positions, expected);
}
#[tokio::test]
async fn a_plan_too_wide_to_address_is_still_walked_whole() {
let mut map = TargetMap::new();
map.add_unit(TargetSet::new(
"192.0.2.0/30".parse::<IpSet>().expect("a prefix"),
"80".parse::<PortSet>().expect("ports"),
));
map.add_unit(TargetSet::new(
"2001:db8::/48".parse::<IpSet>().expect("a prefix"),
"80".parse::<PortSet>().expect("ports"),
));
let (_session, ctx) = ordered(0x5EED);
let mut rx = Dispatcher::new(map).with_batch_size(4).run(&ctx);
let mut asked = Vec::new();
for _ in 0..4 {
asked.push(rx.recv().await.expect("the narrow unit is emitted"));
}
ctx.handle.abort();
let mut positions: Vec<u64> = asked.iter().map(|planned| planned.position).collect();
positions.sort_unstable();
assert_eq!(positions, vec![0, 1, 2, 3]);
}
#[tokio::test]
async fn a_seeded_sweep_covers_the_whole_set_without_walking_it() {
const BATCH: usize = 32;
let set: IpSet = "192.0.2.0/24".parse().expect("a prefix");
let expected: Vec<IpAddr> = set.iter().collect();
let handle = ScanHandle::new();
let mut rx = dispatch_addresses(set, BATCH, Some(0x5EED), &handle);
let mut swept = Vec::new();
while let Some(ip) = rx.recv().await {
swept.push(ip);
}
assert_eq!(swept.len(), expected.len());
let beyond = swept[..BATCH]
.iter()
.filter(|ip| match ip {
IpAddr::V4(v4) => v4.octets()[3] as usize >= BATCH,
IpAddr::V6(_) => false,
})
.count();
assert!(
beyond > BATCH / 2,
"only {beyond} of the first {BATCH} addresses came from beyond the \
start of the range, which is a walk rather than a rearrangement"
);
let mut sorted = swept.clone();
sorted.sort();
assert_eq!(sorted, expected, "an address was swept twice or not at all");
}
async fn numbered(
map: TargetMap,
live: Option<IpSet>,
) -> std::collections::HashMap<IpAddr, u64> {
let (_session, ctx) = context();
let mut dispatcher = Dispatcher::new(map);
if let Some(live) = live {
dispatcher = dispatcher.only_live(live);
}
let mut rx = dispatcher.run(&ctx);
let mut found = std::collections::HashMap::new();
while let Some(planned) = rx.recv().await {
found.insert(planned.target.ip, planned.position);
}
found
}
}