use std::time::{Duration, Instant};
use crate::model::capture::CaptureCounts;
use crate::report::ScannerKind;
use crate::report::WindowSummary;
use crate::report::{ATTEMPTS_COUNTED, BUCKET_BOUNDS_MS, ProbeStats, StopReason};
pub struct ProbeAudit {
started: Instant,
pub(crate) sends_attempted: u64,
pub(crate) sends_failed: u64,
pub(crate) sends_witnessed: u64,
pub(crate) segments_seen: u64,
pub(crate) segments_off_target: u64,
pub(crate) replies_without_rtt: u64,
pub(crate) hosts_found: u64,
answered_on: [u64; ATTEMPTS_COUNTED],
answered_unattributed: u64,
first_reply: Option<Duration>,
last_reply: Option<Duration>,
buckets: [u64; BUCKET_BOUNDS_MS.len() + 1],
}
const RETRY_SHARE_SUGGESTING_LOSS: f64 = 0.35;
const MIN_ANSWERS_TO_JUDGE: u64 = 20;
const UNANSWERED_SHARE_SUGGESTING_LOSS: f64 = 0.10;
impl Default for ProbeAudit {
fn default() -> Self {
Self::new()
}
}
impl ProbeAudit {
pub fn new() -> Self {
Self {
started: Instant::now(),
sends_attempted: 0,
sends_failed: 0,
sends_witnessed: 0,
segments_seen: 0,
segments_off_target: 0,
replies_without_rtt: 0,
hosts_found: 0,
answered_on: [0; ATTEMPTS_COUNTED],
answered_unattributed: 0,
first_reply: None,
last_reply: None,
buckets: [0; BUCKET_BOUNDS_MS.len() + 1],
}
}
pub fn record_send(&mut self, sent: bool) {
self.sends_attempted += 1;
if !sent {
self.sends_failed += 1;
}
}
pub fn record_witnessed_send(&mut self) {
self.sends_witnessed += 1;
}
pub fn witnesses_its_sends(&self) -> bool {
self.sends_witnessed > 0
}
pub fn record_segment(&mut self) {
self.segments_seen += 1;
}
pub fn record_off_target(&mut self) {
self.segments_off_target += 1;
}
pub fn record_reply_without_rtt(&mut self) {
self.replies_without_rtt += 1;
}
pub fn record_host_found(&mut self, answered_attempt: Option<u8>) {
self.hosts_found += 1;
match answered_attempt {
Some(attempt) => {
let index = usize::from(attempt.saturating_sub(1));
self.answered_on[index.min(ATTEMPTS_COUNTED - 1)] += 1;
}
None => self.answered_unattributed += 1,
}
let offset = self.started.elapsed();
self.first_reply.get_or_insert(offset);
self.last_reply = Some(offset);
self.buckets[bucket_of(offset)] += 1;
}
pub fn elapsed(&self) -> Duration {
self.started.elapsed()
}
pub fn sends_attempted(&self) -> u64 {
self.sends_attempted
}
pub fn sends_failed(&self) -> u64 {
self.sends_failed
}
pub fn sends_witnessed(&self) -> u64 {
self.sends_witnessed
}
pub fn segments_seen(&self) -> u64 {
self.segments_seen
}
pub fn segments_off_target(&self) -> u64 {
self.segments_off_target
}
pub fn replies_without_rtt(&self) -> u64 {
self.replies_without_rtt
}
pub fn hosts_found(&self) -> u64 {
self.hosts_found
}
pub fn answered_on(&self) -> &[u64; ATTEMPTS_COUNTED] {
&self.answered_on
}
pub fn answered_unattributed(&self) -> u64 {
self.answered_unattributed
}
pub(crate) fn stats(
&self,
scanner: ScannerKind,
targets: u128,
reason: StopReason,
capture: Option<CaptureCounts>,
window: Option<WindowSummary>,
) -> ProbeStats {
ProbeStats {
scanner,
targets,
stop_reason: reason,
elapsed: self.elapsed(),
sends_attempted: self.sends_attempted,
sends_failed: self.sends_failed,
sends_witnessed: self.sends_witnessed,
segments_seen: self.segments_seen,
segments_off_target: self.segments_off_target,
replies_without_rtt: self.replies_without_rtt,
hosts_found: self.hosts_found,
answered_on: self.answered_on,
answered_unattributed: self.answered_unattributed,
first_reply: self.first_reply,
last_reply: self.last_reply,
found_at: self.buckets,
capture,
window,
}
}
pub(crate) fn report(
&self,
scanner: &str,
targets: u128,
reason: StopReason,
capture: Option<CaptureCounts>,
window: Option<WindowSummary>,
) {
crate::info!(
verbosity = 1,
"audit[{scanner}] {found}/{targets} hosts in {elapsed:.0?}, stopped: {reason:?} \
| sent {sent} (failed {failed}) \
| captured {seen} (off-target {off}, no-rtt {no_rtt}){kernel} \
| found on {attempts}{window} \
| first {first}, last {last} \
| found-at {histogram}",
found = self.hosts_found,
elapsed = self.elapsed(),
sent = self.sends_attempted,
failed = self.sends_failed,
seen = self.segments_seen,
off = self.segments_off_target,
no_rtt = self.replies_without_rtt,
kernel = format_capture(capture),
attempts = self.attempt_distribution(),
window = format_window(window),
first = format_offset(self.first_reply),
last = format_offset(self.last_reply),
histogram = self.histogram(),
);
self.warn_if_degraded(scanner, targets, capture, window);
}
fn recovered_by_retry(&self) -> f64 {
let recovered: u64 = self.answered_on.iter().skip(1).sum();
match self.hosts_found {
0 => 0.0,
found => recovered as f64 / found as f64,
}
}
fn warn_if_degraded(
&self,
scanner: &str,
targets: u128,
capture: Option<CaptureCounts>,
window: Option<WindowSummary>,
) {
let recovered = self.recovered_by_retry();
if recovered >= RETRY_SHARE_SUGGESTING_LOSS && self.hosts_found >= MIN_ANSWERS_TO_JUDGE {
let percent = recovered * 100.0;
match window {
Some(window) if window.adaptive => crate::warn!(
"{scanner}: {percent:.0}% of answers needed a retry, paced down to \
{} in flight",
window.capacity,
),
_ => crate::warn!(
"{scanner}: {percent:.0}% of answers needed a retry, lower --max-probe-rate"
),
}
}
if let Some(window) = window
&& window.at_floor
&& targets > 0
{
let unanswered = targets.saturating_sub(u128::from(self.hosts_found));
let share = unanswered as f64 / targets as f64;
if share >= UNANSWERED_SHARE_SUGGESTING_LOSS {
crate::warn!(
"{scanner}: paced down to {} in flight and {percent:.0}% still unanswered; \
those may be dropped probes rather than filtered ports",
window.capacity,
percent = share * 100.0,
);
}
}
if let Some(counts) = capture
&& counts.dropped > 0
{
crate::warn!(
"{scanner}: capture dropped {}, replies lost on this host",
crate::logging::counted(counts.dropped.into(), "frame", "frames"),
);
}
}
fn attempt_distribution(&self) -> String {
let mut out = String::new();
for (index, count) in self.answered_on.iter().enumerate() {
if *count == 0 {
continue;
}
if !out.is_empty() {
out.push(' ');
}
if index == ATTEMPTS_COUNTED - 1 {
out.push_str(&format!("{}+:{count}", ATTEMPTS_COUNTED));
} else {
out.push_str(&format!("{}:{count}", index + 1));
}
}
if self.answered_unattributed > 0 {
if !out.is_empty() {
out.push(' ');
}
out.push_str(&format!("unattributed:{}", self.answered_unattributed));
}
if out.is_empty() {
out.push_str("(none)");
}
out
}
fn histogram(&self) -> String {
let mut out = String::new();
for (index, count) in self.buckets.iter().enumerate() {
if *count == 0 {
continue;
}
if !out.is_empty() {
out.push(' ');
}
match BUCKET_BOUNDS_MS.get(index) {
Some(bound) => out.push_str(&format!("<={bound}ms:{count}")),
None => out.push_str(&format!(">{}ms:{count}", BUCKET_BOUNDS_MS[index - 1])),
}
}
if out.is_empty() {
out.push_str("(none)");
}
out
}
}
fn bucket_of(offset: Duration) -> usize {
let ms = offset.as_millis() as u64;
BUCKET_BOUNDS_MS
.iter()
.position(|bound| ms <= *bound)
.unwrap_or(BUCKET_BOUNDS_MS.len())
}
fn format_offset(offset: Option<Duration>) -> String {
match offset {
Some(offset) => format!("{:.0?}", offset),
None => "-".to_string(),
}
}
fn format_capture(capture: Option<CaptureCounts>) -> String {
match capture {
Some(counts) => format!(
" | kernel {received} (dropped {dropped}, if-dropped {if_dropped})",
received = counts.received,
dropped = counts.dropped,
if_dropped = counts.if_dropped,
),
None => String::new(),
}
}
fn format_window(window: Option<WindowSummary>) -> String {
match window {
Some(window) => format!(" | window {window}"),
None => String::new(),
}
}
#[cfg(test)]
mod tests {
#[test]
fn a_caller_outside_the_engine_can_record_and_read_a_run() {
let mut audit = ProbeAudit::new();
audit.record_send(true);
audit.record_send(false);
audit.record_segment();
audit.record_off_target();
audit.record_reply_without_rtt();
audit.record_host_found(Some(2));
audit.record_host_found(None);
assert_eq!(audit.sends_attempted(), 2);
assert_eq!(audit.sends_failed(), 1, "the shortfall started at home");
assert_eq!(audit.segments_seen(), 1);
assert_eq!(audit.segments_off_target(), 1);
assert_eq!(audit.replies_without_rtt(), 1);
assert_eq!(audit.hosts_found(), 2);
assert_eq!(
audit.answered_on()[1],
1,
"one target answered its second ask"
);
assert_eq!(audit.answered_unattributed(), 1);
}
#[test]
fn answers_that_needed_a_retry_are_what_reveals_loss() {
let mut clean = ProbeAudit::new();
for _ in 0..50 {
clean.record_host_found(Some(1));
}
assert!(
clean.recovered_by_retry() < RETRY_SHARE_SUGGESTING_LOSS,
"every answer on the first ask is a scan that lost nothing"
);
let mut lossy = ProbeAudit::new();
for _ in 0..20 {
lossy.record_host_found(Some(1));
}
for _ in 0..30 {
lossy.record_host_found(Some(2));
}
assert!(
lossy.recovered_by_retry() >= RETRY_SHARE_SUGGESTING_LOSS,
"most answers arriving only on the second ask is the first ask failing"
);
}
#[test]
fn a_run_too_small_to_judge_is_not_judged() {
let mut tiny = ProbeAudit::new();
tiny.record_host_found(Some(2));
assert!(tiny.recovered_by_retry() > RETRY_SHARE_SUGGESTING_LOSS);
assert!(
tiny.hosts_found < MIN_ANSWERS_TO_JUDGE,
"and the floor is what stops that being reported as a degraded scan"
);
}
use super::*;
#[test]
fn offsets_land_in_the_bucket_named_by_their_bound() {
assert_eq!(bucket_of(Duration::from_micros(200)), 0); assert_eq!(bucket_of(Duration::from_millis(1)), 0);
assert_eq!(bucket_of(Duration::from_millis(2)), 1);
assert_eq!(bucket_of(Duration::from_millis(3)), 2); assert_eq!(bucket_of(Duration::from_millis(1_000)), 8);
}
#[test]
fn anything_beyond_the_last_bound_overflows_into_the_final_bucket() {
assert_eq!(bucket_of(Duration::from_secs(30)), BUCKET_BOUNDS_MS.len());
let mut audit = ProbeAudit::new();
audit.buckets[BUCKET_BOUNDS_MS.len()] = 2;
assert_eq!(audit.histogram(), ">1000ms:2");
}
#[test]
fn an_empty_histogram_says_so_rather_than_rendering_blank() {
assert_eq!(ProbeAudit::new().histogram(), "(none)");
}
#[test]
fn a_found_host_sets_both_ends_of_the_reply_window() {
let mut audit = ProbeAudit::new();
assert!(audit.first_reply.is_none());
audit.record_host_found(Some(1));
audit.record_host_found(Some(2));
assert_eq!(audit.hosts_found, 2);
assert!(audit.first_reply.is_some());
assert!(audit.last_reply >= audit.first_reply);
}
#[test]
fn hosts_are_counted_against_the_attempt_that_revealed_them() {
let mut audit = ProbeAudit::new();
audit.record_host_found(Some(1));
audit.record_host_found(Some(1));
audit.record_host_found(Some(3));
audit.record_host_found(None);
assert_eq!(audit.attempt_distribution(), "1:2 3:1 unattributed:1");
}
#[test]
fn an_attempt_beyond_the_reported_range_falls_into_the_last_slot() {
let mut audit = ProbeAudit::new();
audit.record_host_found(Some(ATTEMPTS_COUNTED as u8));
audit.record_host_found(Some(200));
assert_eq!(audit.attempt_distribution(), "6+:2");
}
#[test]
fn a_run_that_found_nothing_says_so_rather_than_rendering_blank() {
assert_eq!(ProbeAudit::new().attempt_distribution(), "(none)");
}
#[test]
fn exported_stats_carry_every_counter_the_run_recorded() {
let mut audit = ProbeAudit::new();
audit.record_send(true);
audit.record_send(true);
audit.record_send(false);
audit.record_segment();
audit.record_segment();
audit.record_off_target();
audit.record_reply_without_rtt();
audit.record_host_found(Some(1));
audit.record_host_found(Some(3));
audit.record_host_found(None);
let capture = CaptureCounts {
received: 40,
dropped: 2,
if_dropped: 0,
stopped_early: 0,
};
let stats = audit.stats(
ScannerKind::Routed,
256,
StopReason::DeadlineExpired,
Some(capture),
None,
);
assert_eq!(stats.scanner(), ScannerKind::Routed);
assert_eq!(stats.targets(), 256);
assert_eq!(stats.stop_reason(), StopReason::DeadlineExpired);
assert_eq!(stats.sends_attempted(), 3);
assert_eq!(stats.sends_failed(), 1);
assert_eq!(stats.segments_seen(), 2);
assert_eq!(stats.segments_off_target(), 1);
assert_eq!(stats.replies_without_rtt(), 1);
assert_eq!(stats.hosts_found(), 3);
assert_eq!(stats.answered_on()[0], 1);
assert_eq!(stats.answered_on()[2], 1);
assert_eq!(stats.answered_unattributed(), 1);
assert!(stats.first_reply().is_some());
assert!(stats.last_reply() >= stats.first_reply());
assert_eq!(stats.capture(), Some(capture));
assert_eq!(stats.found_at().iter().sum::<u64>(), 3);
}
#[test]
fn exported_stats_keep_an_absent_capture_absent() {
let stats =
ProbeAudit::new().stats(ScannerKind::Routed, 1, StopReason::AllResponded, None, None);
assert_eq!(stats.capture(), None);
assert!(stats.stop_reason().is_complete());
}
#[test]
fn an_absent_capture_contributes_nothing_to_the_line() {
assert_eq!(format_capture(None), "");
}
#[test]
fn capture_counts_are_reported_next_to_the_scale_they_happened_at() {
let counts = CaptureCounts {
received: 881,
dropped: 17,
if_dropped: 0,
stopped_early: 0,
};
assert_eq!(
format_capture(Some(counts)),
" | kernel 881 (dropped 17, if-dropped 0)"
);
}
#[test]
fn a_failed_send_counts_as_attempted_too() {
let mut audit = ProbeAudit::new();
audit.record_send(true);
audit.record_send(false);
assert_eq!(audit.sends_attempted, 2);
assert_eq!(audit.sends_failed, 1);
}
}