use std::time::{Duration, Instant};
use crate::model::capture::CaptureCounts;
use crate::model::port::PortState;
use crate::report::ScannerKind;
use crate::report::WindowSummary;
use crate::report::{ATTEMPTS_COUNTED, BUCKET_BOUNDS_MS, ProbeStats, StopReason};
#[derive(Clone, Copy, Debug)]
pub(crate) struct Pacing {
pub(crate) window: WindowSummary,
pub(crate) silence: PortState,
pub(crate) unasked: u128,
}
pub struct ProbeAudit {
started: Instant,
pub(crate) sends_attempted: u64,
pub(crate) sends_failed: u64,
pub(crate) sends_witnessed: u64,
pub(crate) segments_seen: u64,
pub(crate) segments_off_target: u64,
pub(crate) replies_without_rtt: u64,
pub(crate) hosts_found: u64,
answered_on: [u64; ATTEMPTS_COUNTED],
answered_unattributed: u64,
first_reply: Option<Duration>,
last_reply: Option<Duration>,
buckets: [u64; BUCKET_BOUNDS_MS.len() + 1],
}
const RETRY_SHARE_SUGGESTING_LOSS: f64 = 0.35;
const MIN_ANSWERS_TO_JUDGE: u64 = 20;
const UNANSWERED_SHARE_SUGGESTING_LOSS: f64 = 0.10;
impl Default for ProbeAudit {
fn default() -> Self {
Self::new()
}
}
impl ProbeAudit {
pub fn new() -> Self {
Self {
started: Instant::now(),
sends_attempted: 0,
sends_failed: 0,
sends_witnessed: 0,
segments_seen: 0,
segments_off_target: 0,
replies_without_rtt: 0,
hosts_found: 0,
answered_on: [0; ATTEMPTS_COUNTED],
answered_unattributed: 0,
first_reply: None,
last_reply: None,
buckets: [0; BUCKET_BOUNDS_MS.len() + 1],
}
}
pub fn record_send(&mut self, sent: bool) {
self.sends_attempted += 1;
if !sent {
self.sends_failed += 1;
}
}
pub fn record_witnessed_send(&mut self) {
self.sends_witnessed += 1;
}
pub fn witnesses_its_sends(&self) -> bool {
self.sends_witnessed > 0
}
pub fn record_segment(&mut self) {
self.segments_seen += 1;
}
pub fn record_off_target(&mut self) {
self.segments_off_target += 1;
}
pub fn record_reply_without_rtt(&mut self) {
self.replies_without_rtt += 1;
}
pub fn record_host_found(&mut self, answered_attempt: Option<u8>) {
self.hosts_found += 1;
match answered_attempt {
Some(attempt) => {
let index = usize::from(attempt.saturating_sub(1));
self.answered_on[index.min(ATTEMPTS_COUNTED - 1)] += 1;
}
None => self.answered_unattributed += 1,
}
let offset = self.started.elapsed();
self.first_reply.get_or_insert(offset);
self.last_reply = Some(offset);
self.buckets[bucket_of(offset)] += 1;
}
pub fn elapsed(&self) -> Duration {
self.started.elapsed()
}
fn achieved_send_rate(&self) -> f64 {
crate::report::send_rate(self.sends_attempted, self.started.elapsed()).unwrap_or(0.0)
}
#[cfg(test)]
pub fn sends_witnessed(&self) -> u64 {
self.sends_witnessed
}
pub(crate) fn stats(
&self,
scanner: ScannerKind,
targets: u128,
reason: StopReason,
capture: Option<CaptureCounts>,
window: Option<WindowSummary>,
) -> ProbeStats {
ProbeStats {
scanner,
targets,
stop_reason: reason,
elapsed: self.elapsed(),
sends_attempted: self.sends_attempted,
sends_failed: self.sends_failed,
sends_witnessed: self.sends_witnessed,
segments_seen: self.segments_seen,
segments_off_target: self.segments_off_target,
replies_without_rtt: self.replies_without_rtt,
hosts_found: self.hosts_found,
answered_on: self.answered_on,
answered_unattributed: self.answered_unattributed,
first_reply: self.first_reply,
last_reply: self.last_reply,
found_at: self.buckets,
capture,
window,
}
}
pub(crate) fn report(
&self,
scanner: &str,
targets: u128,
reason: StopReason,
capture: Option<CaptureCounts>,
pacing: Option<Pacing>,
) {
let window = pacing.map(|pacing| pacing.window);
crate::info!(
verbosity = 3,
"audit[{scanner}] {found}/{targets} hosts in {elapsed:.0?}, stopped: {reason:?} \
| sent {sent} (failed {failed}, {rate:.0}/s) \
| captured {seen} (off-target {off}, no-rtt {no_rtt}){kernel} \
| found on {attempts}{window} \
| first {first}, last {last} \
| found-at {histogram}",
found = self.hosts_found,
elapsed = self.elapsed(),
sent = self.sends_attempted,
failed = self.sends_failed,
rate = self.achieved_send_rate(),
seen = self.segments_seen,
off = self.segments_off_target,
no_rtt = self.replies_without_rtt,
kernel = format_capture(capture),
attempts = self.attempt_distribution(),
window = format_window(window),
first = format_offset(self.first_reply),
last = format_offset(self.last_reply),
histogram = self.histogram(),
);
self.warn_if_degraded(scanner, targets, capture, pacing);
}
fn recovered_by_retry(&self) -> f64 {
let recovered: u64 = self.answered_on.iter().skip(1).sum();
match self.hosts_found {
0 => 0.0,
found => recovered as f64 / found as f64,
}
}
fn warn_if_degraded(
&self,
scanner: &str,
targets: u128,
capture: Option<CaptureCounts>,
pacing: Option<Pacing>,
) {
let window = pacing.map(|pacing| pacing.window);
let recovered = self.recovered_by_retry();
if recovered >= RETRY_SHARE_SUGGESTING_LOSS && self.hosts_found >= MIN_ANSWERS_TO_JUDGE {
let percent = recovered * 100.0;
match window {
Some(window) if window.adaptive => crate::warn!(
"{scanner}: {percent:.0}% of answers needed a retry (paced to {})",
window.capacity,
),
_ => crate::warn!(
"{scanner}: {percent:.0}% of answers needed a retry (rate too high)"
),
}
}
if let Some(Pacing {
window,
silence,
unasked,
}) = pacing
&& silence == PortState::Filtered
&& window.at_floor
&& targets > unasked
{
let asked = targets - unasked;
let unanswered = asked.saturating_sub(u128::from(self.hosts_found));
let share = unanswered as f64 / asked as f64;
if share >= UNANSWERED_SHARE_SUGGESTING_LOSS {
crate::warn!(
"{scanner}: {percent:.0}% unanswered at {} in flight (maybe loss)",
window.capacity,
percent = share * 100.0,
);
}
}
if let Some(counts) = capture
&& counts.dropped > 0
{
let frames = crate::logging::counted(counts.dropped.into(), "frame", "frames");
if u128::from(self.hosts_found) < targets {
crate::warn!("{scanner}: capture dropped {frames} (answers may be lost)");
} else {
crate::info!(
verbosity = 1,
"{scanner}: capture dropped {frames} (every target answered)"
);
}
}
}
fn attempt_distribution(&self) -> String {
let mut out = String::new();
for (index, count) in self.answered_on.iter().enumerate() {
if *count == 0 {
continue;
}
if !out.is_empty() {
out.push(' ');
}
if index == ATTEMPTS_COUNTED - 1 {
out.push_str(&format!("{}+:{count}", ATTEMPTS_COUNTED));
} else {
out.push_str(&format!("{}:{count}", index + 1));
}
}
if self.answered_unattributed > 0 {
if !out.is_empty() {
out.push(' ');
}
out.push_str(&format!("unattributed:{}", self.answered_unattributed));
}
if out.is_empty() {
out.push_str("(none)");
}
out
}
fn histogram(&self) -> String {
let mut out = String::new();
for (index, count) in self.buckets.iter().enumerate() {
if *count == 0 {
continue;
}
if !out.is_empty() {
out.push(' ');
}
match BUCKET_BOUNDS_MS.get(index) {
Some(bound) => out.push_str(&format!("<={bound}ms:{count}")),
None => out.push_str(&format!(">{}ms:{count}", BUCKET_BOUNDS_MS[index - 1])),
}
}
if out.is_empty() {
out.push_str("(none)");
}
out
}
}
fn bucket_of(offset: Duration) -> usize {
let ms = offset.as_millis() as u64;
BUCKET_BOUNDS_MS
.iter()
.position(|bound| ms <= *bound)
.unwrap_or(BUCKET_BOUNDS_MS.len())
}
fn format_offset(offset: Option<Duration>) -> String {
match offset {
Some(offset) => format!("{:.0?}", offset),
None => "-".to_string(),
}
}
fn format_capture(capture: Option<CaptureCounts>) -> String {
match capture {
Some(counts) => format!(
" | kernel {received} (dropped {dropped}, if-dropped {if_dropped})",
received = counts.received,
dropped = counts.dropped,
if_dropped = counts.if_dropped,
),
None => String::new(),
}
}
fn format_window(window: Option<WindowSummary>) -> String {
match window {
Some(window) => format!(" | window {window}"),
None => String::new(),
}
}
#[cfg(test)]
mod tests {
#[test]
fn answers_that_needed_a_retry_are_what_reveals_loss() {
let mut clean = ProbeAudit::new();
for _ in 0..50 {
clean.record_host_found(Some(1));
}
assert!(
clean.recovered_by_retry() < RETRY_SHARE_SUGGESTING_LOSS,
"every answer on the first ask is a scan that lost nothing"
);
let mut lossy = ProbeAudit::new();
for _ in 0..20 {
lossy.record_host_found(Some(1));
}
for _ in 0..30 {
lossy.record_host_found(Some(2));
}
assert!(
lossy.recovered_by_retry() >= RETRY_SHARE_SUGGESTING_LOSS,
"most answers arriving only on the second ask is the first ask failing"
);
}
#[test]
fn silence_at_the_windows_floor_is_called_loss_only_where_it_is_a_filter() {
let mut audit = ProbeAudit::new();
for _ in 0..20 {
audit.record_host_found(Some(1));
}
let window = WindowSummary {
capacity: 16,
peak: 256,
reductions: 5,
adaptive: true,
at_floor: true,
};
for (silence, warned) in [
(PortState::Filtered, true),
(PortState::OpenFiltered, false),
] {
let said = crate::logging::logged(|| {
audit.report(
"tcp-port",
40,
StopReason::AttemptsSpent,
None,
Some(Pacing {
window,
silence,
unasked: 0,
}),
);
});
let lines: Vec<&str> = said
.iter()
.filter(|line| line.message.contains("unanswered at"))
.map(|line| line.message.as_str())
.collect();
assert_eq!(!lines.is_empty(), warned, "{silence:?}: {said:?}");
}
}
#[test]
fn a_probe_never_sent_is_not_read_as_possible_loss() {
let audit = ProbeAudit::new();
let window = WindowSummary {
capacity: 16,
peak: 16,
reductions: 1,
adaptive: true,
at_floor: true,
};
for (unasked, warned) in [(0, true), (1, false)] {
let said = crate::logging::logged(|| {
audit.report(
"tcp-port",
1,
StopReason::AttemptsSpent,
None,
Some(Pacing {
window,
silence: PortState::Filtered,
unasked,
}),
);
});
let told = said
.iter()
.any(|line| line.message.contains("unanswered at"));
assert_eq!(told, warned, "{unasked} unasked: {said:?}");
}
}
#[test]
fn capture_drops_are_told_as_possible_loss_only_where_a_target_went_unanswered() {
let dropped = CaptureCounts {
received: 605,
dropped: 184,
if_dropped: 0,
stopped_early: 0,
};
let mut every_one_answered = ProbeAudit::new();
for _ in 0..40 {
every_one_answered.record_host_found(Some(1));
}
for (audit, targets, told) in [
(&ProbeAudit::new(), 200, true),
(&every_one_answered, 40, false),
] {
let said = crate::logging::logged(|| {
audit.report(
"local",
targets,
StopReason::AttemptsSpent,
Some(dropped),
None,
);
});
let drops: Vec<_> = said
.iter()
.filter(|line| line.message.contains("capture dropped 184 frames"))
.collect();
assert_eq!(drops.len(), 1, "{targets} targets: {said:?}");
assert_eq!(drops[0].verbosity == 0, told, "{targets} targets: {said:?}");
assert!(
!drops[0].message.contains("replies lost"),
"a drop is claimed as lost replies: {said:?}"
);
}
}
#[test]
fn a_run_too_small_to_judge_is_not_judged() {
let mut tiny = ProbeAudit::new();
tiny.record_host_found(Some(2));
assert!(tiny.recovered_by_retry() > RETRY_SHARE_SUGGESTING_LOSS);
assert!(
tiny.hosts_found < MIN_ANSWERS_TO_JUDGE,
"and the floor is what stops that being reported as a degraded scan"
);
}
use super::*;
#[test]
fn offsets_land_in_the_bucket_named_by_their_bound() {
assert_eq!(bucket_of(Duration::from_micros(200)), 0); assert_eq!(bucket_of(Duration::from_millis(1)), 0);
assert_eq!(bucket_of(Duration::from_millis(2)), 1);
assert_eq!(bucket_of(Duration::from_millis(3)), 2); assert_eq!(bucket_of(Duration::from_millis(1_000)), 8);
}
#[test]
fn anything_beyond_the_last_bound_overflows_into_the_final_bucket() {
assert_eq!(bucket_of(Duration::from_secs(30)), BUCKET_BOUNDS_MS.len());
let mut audit = ProbeAudit::new();
audit.buckets[BUCKET_BOUNDS_MS.len()] = 2;
assert_eq!(audit.histogram(), ">1000ms:2");
}
#[test]
fn an_empty_histogram_says_so_rather_than_rendering_blank() {
assert_eq!(ProbeAudit::new().histogram(), "(none)");
}
#[test]
fn a_found_host_sets_both_ends_of_the_reply_window() {
let mut audit = ProbeAudit::new();
assert!(audit.first_reply.is_none());
audit.record_host_found(Some(1));
audit.record_host_found(Some(2));
assert_eq!(audit.hosts_found, 2);
assert!(audit.first_reply.is_some());
assert!(audit.last_reply >= audit.first_reply);
}
#[test]
fn hosts_are_counted_against_the_attempt_that_revealed_them() {
let mut audit = ProbeAudit::new();
audit.record_host_found(Some(1));
audit.record_host_found(Some(1));
audit.record_host_found(Some(3));
audit.record_host_found(None);
assert_eq!(audit.attempt_distribution(), "1:2 3:1 unattributed:1");
}
#[test]
fn an_attempt_beyond_the_reported_range_falls_into_the_last_slot() {
let mut audit = ProbeAudit::new();
audit.record_host_found(Some(ATTEMPTS_COUNTED as u8));
audit.record_host_found(Some(200));
assert_eq!(audit.attempt_distribution(), "6+:2");
}
#[test]
fn a_run_that_found_nothing_says_so_rather_than_rendering_blank() {
assert_eq!(ProbeAudit::new().attempt_distribution(), "(none)");
}
#[test]
fn exported_stats_carry_every_counter_the_run_recorded() {
let mut audit = ProbeAudit::new();
audit.record_send(true);
audit.record_send(true);
audit.record_send(false);
audit.record_segment();
audit.record_segment();
audit.record_off_target();
audit.record_reply_without_rtt();
audit.record_host_found(Some(1));
audit.record_host_found(Some(3));
audit.record_host_found(None);
let capture = CaptureCounts {
received: 40,
dropped: 2,
if_dropped: 0,
stopped_early: 0,
};
let stats = audit.stats(
ScannerKind::Routed,
256,
StopReason::DeadlineExpired,
Some(capture),
None,
);
assert_eq!(stats.scanner(), ScannerKind::Routed);
assert_eq!(stats.targets(), 256);
assert_eq!(stats.stop_reason(), StopReason::DeadlineExpired);
assert_eq!(stats.sends_attempted(), 3);
assert_eq!(stats.sends_failed(), 1);
assert_eq!(stats.segments_seen(), 2);
assert_eq!(stats.segments_off_target(), 1);
assert_eq!(stats.replies_without_rtt(), 1);
assert_eq!(stats.hosts_found(), 3);
assert_eq!(stats.answered_on()[0], 1);
assert_eq!(stats.answered_on()[2], 1);
assert_eq!(stats.answered_unattributed(), 1);
assert!(stats.first_reply().is_some());
assert!(stats.last_reply() >= stats.first_reply());
assert_eq!(stats.capture(), Some(capture));
assert_eq!(stats.found_at().iter().sum::<u64>(), 3);
}
#[test]
fn exported_stats_keep_an_absent_capture_absent() {
let stats =
ProbeAudit::new().stats(ScannerKind::Routed, 1, StopReason::AllResponded, None, None);
assert_eq!(stats.capture(), None);
assert!(stats.stop_reason().is_complete());
}
#[test]
fn an_absent_capture_contributes_nothing_to_the_line() {
assert_eq!(format_capture(None), "");
}
#[test]
fn capture_counts_are_reported_next_to_the_scale_they_happened_at() {
let counts = CaptureCounts {
received: 881,
dropped: 17,
if_dropped: 0,
stopped_early: 0,
};
assert_eq!(
format_capture(Some(counts)),
" | kernel 881 (dropped 17, if-dropped 0)"
);
}
#[test]
fn a_failed_send_counts_as_attempted_too() {
let mut audit = ProbeAudit::new();
audit.record_send(true);
audit.record_send(false);
assert_eq!(audit.sends_attempted, 2);
assert_eq!(audit.sends_failed, 1);
}
}