use std::collections::{HashMap, HashSet, VecDeque};
use std::net::IpAddr;
use std::time::{Duration, Instant};
use pnet_packet::ip::IpNextHeaderProtocols;
use crate::config::ProbeTuning;
use crate::fingerprint::os::{self, SeriesClasses, SeriesSample, StackObservation, StackReply};
use crate::model::capture::IpObservation;
use crate::model::host::Host;
use crate::model::ip::scoped::ScopedIp;
use crate::model::port::{PortState, Protocol};
use crate::model::technique::TcpScanTechnique;
use crate::protocols::tcp;
use crate::report::ScannerKind;
use crate::report::StopReason;
use crate::scanner::audit::ProbeAudit;
use crate::scanner::session::ScanContext;
use crate::scanner::strategy::StrategyError;
use crate::system::interface::SourceResolver;
use crate::transport::capture::CapturedSegment;
use crate::transport::probe::{Emission, ProbeKind, ProbeTransport};
use crate::{counted, error, info, success};
pub const ACTIVE_SAMPLES: usize = 6;
pub const AGGRESSIVE_SAMPLES: usize = 12;
const SPACING: Duration = Duration::from_millis(100);
const BATCH: usize = 128;
const SEND_TICK: Duration = Duration::from_micros(250);
const LISTEN_AFTER_LAST: Duration = Duration::from_secs(2);
const RECV_TICK: Duration = Duration::from_millis(5);
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SeriesTarget {
pub address: ScopedIp,
pub open: Option<u16>,
pub closed: Option<u16>,
}
impl SeriesTarget {
pub fn for_host(address: ScopedIp, host: &Host) -> Option<Self> {
let tcp = || host.ports().filter(|port| port.protocol() == Protocol::Tcp);
let open = tcp()
.find(|port| port.state() == PortState::Open)
.map(|port| port.number());
let closed = tcp()
.find(|port| port.state() == PortState::Closed)
.map(|port| port.number());
(open.is_some() || closed.is_some()).then_some(Self {
address,
open,
closed,
})
}
fn ports(&self) -> impl Iterator<Item = u16> {
self.open.into_iter().chain(self.closed)
}
}
#[derive(Debug, Clone, Copy)]
struct Sent {
address: IpAddr,
}
#[derive(Debug, Default)]
struct Collected {
open: Track,
closed: Track,
}
#[derive(Debug, Default)]
struct Track {
first: Option<StackObservation>,
samples: Vec<SeriesSample>,
}
impl Track {
fn record(&mut self, observed: StackObservation, sample: SeriesSample) {
self.samples.push(sample);
self.first.get_or_insert(observed);
}
fn reading(&self) -> Option<(StackReply, SeriesClasses)> {
let first = self.first.clone()?;
Some((first.into(), SeriesClasses::from_samples(&self.samples)))
}
}
pub struct OsSeriesScanner {
ctx: ScanContext,
transport: ProbeTransport,
emission: Emission,
resolver: SourceResolver,
batches: VecDeque<Vec<SeriesTarget>>,
samples: usize,
sent: HashMap<u32, Sent>,
answered: HashSet<u32>,
collected: HashMap<IpAddr, Collected>,
audit: ProbeAudit,
send_failure: Option<String>,
named: usize,
}
impl OsSeriesScanner {
pub fn new(
ctx: ScanContext,
targets: Vec<SeriesTarget>,
samples: usize,
tuning: ProbeTuning,
) -> Result<Self, StrategyError> {
let transport = ProbeTransport::open_with(ProbeKind::TcpSyn, tuning.send_mode)?;
Ok(Self::with_transport(
ctx,
targets,
samples,
transport,
tuning.evasion.hop_limited_emission(),
))
}
pub fn with_transport(
ctx: ScanContext,
mut targets: Vec<SeriesTarget>,
samples: usize,
transport: ProbeTransport,
emission: Emission,
) -> Self {
targets.sort_unstable_by(|a, b| a.address.cmp(&b.address));
targets.dedup_by(|a, b| a.address == b.address);
let batches: VecDeque<Vec<SeriesTarget>> = targets
.chunks(BATCH)
.map(<[SeriesTarget]>::to_vec)
.collect();
Self {
ctx,
transport,
emission,
resolver: SourceResolver::from_system(),
batches,
samples: samples.max(2),
sent: HashMap::new(),
answered: HashSet::new(),
collected: HashMap::new(),
audit: ProbeAudit::new(),
send_failure: None,
named: 0,
}
}
async fn sweep(&mut self, batch: &[SeriesTarget]) {
let source_port: u16 = rand::random_range(50_000..u16::MAX);
let mut tick = tokio::time::interval(SEND_TICK);
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Burst);
for target in batch {
let Some(source) = self.resolver.resolve(target.address.addr()) else {
continue;
};
for port in target.ports() {
tick.tick().await;
self.send_one(source, target.address.addr(), source_port, port);
self.file_queued();
}
}
self.file_queued();
}
fn send_one(&mut self, source: IpAddr, address: IpAddr, source_port: u16, port: u16) {
let nonce: u32 = rand::random();
let segment = match tcp::build_probe(
TcpScanTechnique::Syn,
source,
address,
source_port,
port,
nonce,
) {
Ok(segment) => segment,
Err(e) => {
error!(
verbosity = 2,
"cannot build a series probe for {address}: {e}"
);
self.audit.record_send(false);
return;
}
};
match self
.transport
.tx
.send(&segment, source, address, None, self.emission)
{
Ok(()) => {
self.sent.insert(nonce, Sent { address });
self.audit.record_send(true);
}
Err(e) => {
error!(
verbosity = 2,
"failed to send a series probe to {address}: {e:#}"
);
self.send_failure = Some(format!("{e:#}"));
self.audit.record_send(false);
}
}
}
fn file_queued(&mut self) {
while let Ok(reply) = self.transport.rx.try_recv() {
self.audit.record_segment();
self.file(&reply);
}
}
async fn drain_until(&mut self, until: Instant, until_quiet: bool) {
while Instant::now() < until {
if self.ctx.handle.should_stop() {
return;
}
if until_quiet && self.answered.len() == self.sent.len() {
return;
}
let Ok(received) = tokio::time::timeout(RECV_TICK, self.transport.rx.recv()).await
else {
continue;
};
let Some(reply) = received else {
return;
};
self.audit.record_segment();
self.file(&reply);
}
}
fn file(&mut self, reply: &CapturedSegment) {
if reply.protocol != IpNextHeaderProtocols::Tcp {
self.audit.record_off_target();
return;
}
let at = Instant::now();
let Ok(segment) = tcp::parse(&reply.bytes) else {
self.audit.record_off_target();
return;
};
let nonce = tcp::echoed_nonce(TcpScanTechnique::Syn, &segment, 0);
let Some(&Sent { address }) = self.sent.get(&nonce) else {
self.audit.record_off_target();
return;
};
if !self.answered.insert(nonce) {
self.audit.record_reply_without_rtt();
return;
}
let Some(observation) = reply.observation else {
return;
};
if observation.is_fragment() {
return;
}
let Some(observed) = StackObservation::from_tcp(observation, &reply.bytes) else {
return;
};
let sample = SeriesSample {
at,
flags: observed.flags,
sequence: segment.sequence(),
ip_id: match observation {
IpObservation::V4(v4) => Some(v4.identification),
IpObservation::V6(_) => None,
},
tsval: observed.timestamps.map(|stamps| stamps.value),
};
let host = self.collected.entry(address).or_default();
let track = if sample.is_syn_ack() {
&mut host.open
} else {
&mut host.closed
};
track.record(observed, sample);
}
fn conclude(&mut self, batch: &[SeriesTarget]) {
for target in batch {
let Some(collected) = self.collected.remove(&target.address.addr()) else {
continue;
};
let readings: Vec<(StackReply, SeriesClasses)> =
[collected.open.reading(), collected.closed.reading()]
.into_iter()
.flatten()
.collect();
if readings.is_empty() {
continue;
}
self.audit.record_host_found(None);
let Some(verdict) = os::classify_series(os::RuleDb::global(), &readings) else {
continue;
};
success!(
verbosity = 2,
"series probe named {} as {}",
target.address,
verdict.label()
);
self.named += 1;
self.ctx.update_host(&target.address, |host| {
os::identify(host, [verdict.as_evidence()]);
});
}
self.collected.clear();
self.sent.clear();
self.answered.clear();
}
}
impl OsSeriesScanner {
pub async fn probe(&mut self) -> Result<(), StrategyError> {
let followed: u128 = self.batches.iter().map(|batch| batch.len() as u128).sum();
let mut reason = StopReason::AttemptsSpent;
while let Some(batch) = self.batches.pop_front() {
if let Some(cause) = self.ctx.handle.stopped() {
reason = cause.into();
break;
}
for _ in 0..self.samples {
let began = Instant::now();
self.sweep(&batch).await;
self.drain_until(began + SPACING, false).await;
if let Some(cause) = self.ctx.handle.stopped() {
reason = cause.into();
break;
}
}
self.drain_until(Instant::now() + LISTEN_AFTER_LAST, true)
.await;
self.conclude(&batch);
if matches!(reason, StopReason::Aborted | StopReason::TimedOut) {
break;
}
}
if self.audit.sends_failed > 0 {
self.ctx.record_failure(
ScannerKind::OsSeries,
format!(
"{} of {} series probes could not be sent: {}",
self.audit.sends_failed,
self.audit.sends_attempted,
self.send_failure.as_deref().unwrap_or("cause unrecorded"),
),
);
}
if self.named > 0 {
info!(
verbosity = 1,
"named {} from repeated probes",
counted(self.named as u128, "host", "hosts")
);
}
let capture = self.transport.capture_counts();
self.audit
.report("os-series", followed, reason, capture, None);
self.ctx.record_probe_stats(self.audit.stats(
ScannerKind::OsSeries,
followed,
reason,
capture,
None,
));
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::net::Ipv4Addr;
use std::sync::Arc;
use std::sync::atomic::{AtomicU16, Ordering};
use tokio::sync::mpsc;
use crate::model::capture::Ipv4Observation;
use crate::scanner::session::ScanSession;
use crate::transport::capture::CaptureStream;
use crate::transport::probe::{ProbeSender, SendError};
const TARGET: IpAddr = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10));
const OPEN: u16 = 22;
const CLOSED: u16 = 81;
struct Reply {
source_port: u16,
destination_port: u16,
sequence: u32,
acknowledgement: u32,
flags: u8,
window: u16,
options: Vec<u8>,
}
impl Reply {
fn bytes(&self) -> Vec<u8> {
let mut bytes = vec![0u8; 20 + self.options.len()];
bytes[0..2].copy_from_slice(&self.source_port.to_be_bytes());
bytes[2..4].copy_from_slice(&self.destination_port.to_be_bytes());
bytes[4..8].copy_from_slice(&self.sequence.to_be_bytes());
bytes[8..12].copy_from_slice(&self.acknowledgement.to_be_bytes());
bytes[12] = (((20 + self.options.len()) / 4) as u8) << 4;
bytes[13] = self.flags;
bytes[14..16].copy_from_slice(&self.window.to_be_bytes());
bytes[20..].copy_from_slice(&self.options);
bytes
}
}
fn linux_options(tsval: u32) -> Vec<u8> {
let mut options = Vec::with_capacity(20);
options.extend_from_slice(&[2, 4]);
options.extend_from_slice(&1460u16.to_be_bytes());
options.extend_from_slice(&[4, 2]);
options.extend_from_slice(&[8, 10]);
options.extend_from_slice(&tsval.to_be_bytes());
options.extend_from_slice(&0u32.to_be_bytes());
options.push(1);
options.extend_from_slice(&[3, 3, 7]);
options
}
fn captured(bytes: Vec<u8>, identification: u16) -> CapturedSegment {
CapturedSegment {
received_at: Instant::now(),
source: TARGET,
destination: None,
protocol: IpNextHeaderProtocols::Tcp,
observation: Some(IpObservation::V4(Ipv4Observation {
ttl: 64,
identification,
dont_fragment: true,
more_fragments: false,
dscp: 0,
ecn: 0,
})),
source_mac: None,
bytes,
}
}
struct Linux {
replies: mpsc::Sender<CapturedSegment>,
identifier: Arc<AtomicU16>,
booted: Instant,
refuses: bool,
}
impl ProbeSender for Linux {
fn send(
&self,
segment: &[u8],
_src: IpAddr,
_dst: IpAddr,
_zone: Option<u32>,
_emission: Emission,
) -> Result<(), SendError> {
let source_port = u16::from_be_bytes([segment[0], segment[1]]);
let destination_port = u16::from_be_bytes([segment[2], segment[3]]);
let nonce = u32::from_be_bytes([segment[4], segment[5], segment[6], segment[7]]);
let reply = if destination_port == OPEN {
let ticks = self.booted.elapsed().as_millis() as u32;
Reply {
source_port: destination_port,
destination_port: source_port,
sequence: rand::random(),
acknowledgement: nonce.wrapping_add(1),
flags: crate::protocols::tcp::flags::SYN | crate::protocols::tcp::flags::ACK,
window: 65160,
options: linux_options(ticks),
}
} else if self.refuses {
Reply {
source_port: destination_port,
destination_port: source_port,
sequence: 0,
acknowledgement: nonce.wrapping_add(1),
flags: crate::protocols::tcp::flags::RST | crate::protocols::tcp::flags::ACK,
window: 0,
options: Vec::new(),
}
} else {
return Ok(());
};
let identification = if destination_port == OPEN {
0
} else {
self.identifier.fetch_add(1, Ordering::Relaxed)
};
let _ = self
.replies
.try_send(captured(reply.bytes(), identification));
Ok(())
}
}
fn scanner(
ctx: &ScanContext,
target: SeriesTarget,
samples: usize,
refuses: bool,
) -> OsSeriesScanner {
let (tx, rx) = mpsc::channel(1024);
let link = Linux {
replies: tx,
identifier: Arc::new(AtomicU16::new(1000)),
booted: Instant::now(),
refuses,
};
let transport = ProbeTransport::from_parts(Box::new(link), rx as CaptureStream);
OsSeriesScanner::with_transport(
ctx.clone(),
vec![target],
samples,
transport,
Emission::routed(),
)
}
fn both_ports() -> SeriesTarget {
SeriesTarget {
address: ScopedIp::unscoped(TARGET),
open: Some(OPEN),
closed: Some(CLOSED),
}
}
#[test]
fn a_host_offers_the_lowest_port_of_each_kind_it_answered_on() {
use crate::model::port::Port;
let mut host = Host::new(TARGET);
host.add_port(Port::new(443, Protocol::Tcp, PortState::Open));
host.add_port(Port::new(22, Protocol::Tcp, PortState::Open));
host.add_port(Port::new(139, Protocol::Tcp, PortState::Closed));
host.add_port(Port::new(81, Protocol::Tcp, PortState::Closed));
let target = SeriesTarget::for_host(ScopedIp::unscoped(TARGET), &host)
.expect("both kinds of answer");
assert_eq!(target.open, Some(22));
assert_eq!(target.closed, Some(81));
}
#[test]
fn a_host_with_no_tcp_answer_is_not_a_target() {
use crate::model::port::Port;
let mut nothing = Host::new(TARGET);
assert!(SeriesTarget::for_host(ScopedIp::unscoped(TARGET), ¬hing).is_none());
nothing.add_port(Port::new(80, Protocol::Tcp, PortState::Filtered));
assert!(SeriesTarget::for_host(ScopedIp::unscoped(TARGET), ¬hing).is_none());
let mut udp = Host::new(TARGET);
udp.add_port(Port::new(53, Protocol::Udp, PortState::Open));
assert!(SeriesTarget::for_host(ScopedIp::unscoped(TARGET), &udp).is_none());
}
#[tokio::test(flavor = "current_thread")]
async fn a_followed_host_is_named_from_what_its_replies_added_up_to() {
let (session, ctx) = ScanSession::new();
let mut scanner = scanner(&ctx, both_ports(), 4, true);
scanner.probe().await.expect("the phase runs");
let host = session.hosts().get(TARGET).expect("the host is recorded");
let found = host.os().expect("a Linux-shaped series names Linux");
assert_eq!(found.family(), Some("Linux"));
}
#[tokio::test(flavor = "current_thread")]
async fn the_finding_carries_the_series_readings_and_not_just_one_reply() {
let (session, ctx) = ScanSession::new();
let mut scanner = scanner(&ctx, both_ports(), 4, true);
scanner.probe().await.expect("the phase runs");
let host = session.hosts().get(TARGET).expect("the host is recorded");
let evidence = host
.os()
.and_then(|os| os.evidence().map(str::to_owned))
.expect("a finding with its evidence");
assert!(
evidence.contains("isn=hashed"),
"a hashed generator is only visible across replies: {evidence}"
);
assert!(
evidence.contains("ts=ticking"),
"a clock rate needs two readings and an interval: {evidence}"
);
}
#[tokio::test(flavor = "current_thread")]
async fn the_two_reply_kinds_are_read_as_two_series() {
let (session, ctx) = ScanSession::new();
let mut scanner = scanner(&ctx, both_ports(), 4, true);
scanner.probe().await.expect("the phase runs");
let evidence = session
.hosts()
.get(TARGET)
.and_then(|host| host.os().and_then(|os| os.evidence().map(str::to_owned)))
.expect("a finding with its evidence");
assert!(
evidence.contains("id=zero"),
"the handshake path writes zero: {evidence}"
);
assert!(
evidence.contains("id=counting"),
"the reset path runs a counter, and pooling the two would hide it: {evidence}"
);
}
#[tokio::test(flavor = "current_thread")]
async fn a_host_with_only_an_open_port_is_still_read() {
let (session, ctx) = ScanSession::new();
let target = SeriesTarget {
address: ScopedIp::unscoped(TARGET),
open: Some(OPEN),
closed: None,
};
let mut scanner = scanner(&ctx, target, 4, false);
scanner.probe().await.expect("the phase runs");
let host = session.hosts().get(TARGET).expect("the host is recorded");
assert_eq!(host.os().and_then(|os| os.family()), Some("Linux"));
}
#[tokio::test(flavor = "current_thread")]
async fn each_sample_leaves_from_a_source_port_of_its_own() {
use crate::transport::probe::MockSender;
let (_session, ctx) = ScanSession::new();
let mock = MockSender::default();
let recorded = mock.sent.clone();
let (_tx, rx) = mpsc::channel(1024);
let transport = ProbeTransport::from_parts(Box::new(mock), rx as CaptureStream);
let samples = 4;
let target = SeriesTarget {
address: ScopedIp::unscoped(TARGET),
open: Some(OPEN),
closed: None,
};
let mut scanner = OsSeriesScanner::with_transport(
ctx.clone(),
vec![target],
samples,
transport,
Emission::routed(),
);
scanner.probe().await.expect("the phase runs");
let sent = recorded.lock().expect("the record is readable").clone();
assert_eq!(sent.len(), samples, "one probe per sample");
let ports: std::collections::HashSet<u16> = sent
.iter()
.map(|(segment, _, _)| u16::from_be_bytes([segment[0], segment[1]]))
.collect();
assert_eq!(
ports.len(),
samples,
"a repeated source port makes every sample after the first describe \
a connection the previous one opened"
);
}
#[tokio::test(flavor = "current_thread")]
async fn a_segment_this_scan_never_drew_is_not_a_reading() {
struct Silent;
impl ProbeSender for Silent {
fn send(
&self,
_s: &[u8],
_src: IpAddr,
_dst: IpAddr,
_zone: Option<u32>,
_emission: Emission,
) -> Result<(), SendError> {
Ok(())
}
}
let (session, ctx) = ScanSession::new();
let (tx, rx) = mpsc::channel(1024);
let transport = ProbeTransport::from_parts(Box::new(Silent), rx as CaptureStream);
let mut scanner = OsSeriesScanner::with_transport(
ctx.clone(),
vec![both_ports()],
2,
transport,
Emission::routed(),
);
let theirs = Reply {
source_port: OPEN,
destination_port: 40_000,
sequence: 12345,
acknowledgement: 0xDEAD_BEEF,
flags: crate::protocols::tcp::flags::SYN | crate::protocols::tcp::flags::ACK,
window: 65160,
options: linux_options(1000),
};
tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(20)).await;
let _ = tx.try_send(captured(theirs.bytes(), 0));
});
scanner.probe().await.expect("the phase runs");
assert!(
session.hosts().get(TARGET).is_none(),
"a segment answering no probe of ours records nothing whatsoever"
);
}
}