use std::collections::BTreeSet;
use std::sync::Arc;
use std::time::Duration;
use crate::config::DetectionEnvelope;
use crate::config::ServiceDetection;
use crate::config::limits::CONNECT_CONCURRENCY;
use crate::detect::compute::stage as compute_stage;
use crate::detect::compute::{
Budget, CapError, CapTapeRecord, Capabilities, DetectionRunRecord, LiveCapabilities,
RunOutcome, ScanInstant,
};
use crate::detect::flow::{Probe, ProbeRefusal, SocketProbe, stage};
use crate::detect::host::stage as host_stage;
use crate::detect::manifest::{
CapabilitySpec, DEFAULT_MAX_BYTES, DEFAULT_MAX_CONNECTIONS, DEFAULT_MAX_MILLIS,
};
use crate::fingerprint::{PortContext, Tunnel};
use crate::model::finding::Finding;
use crate::model::ip::scoped::ScopedIp;
use crate::model::port::{PortState, Protocol};
use crate::record::{DetectionIdRecord, wire};
use crate::report::ScannerKind;
use crate::scanner::pool::ProbePool;
use crate::scanner::session::{ScanContext, Tapes};
type PortResult = (ScopedIp, u16, Protocol, Vec<Finding>, Vec<(String, String)>);
pub async fn detect(ctx: &ScanContext, detection: ServiceDetection, envelope: DetectionEnvelope) {
if detection == ServiceDetection::Off {
return;
}
if envelope.ceiling().is_none() {
return;
}
detect_hosts(ctx);
let targets = interested_ports(ctx, envelope);
if targets.is_empty() {
return;
}
let gate = Arc::new(Gate::new(CONNECT_CONCURRENCY));
let mut pool = ProbePool::new(
CONNECT_CONCURRENCY,
ctx.clone(),
ScannerKind::Detection,
|result: Option<PortResult>, _audit| {
if let Some((key, number, protocol, findings, inconclusive)) = result {
for (id, reason) in &inconclusive {
ctx.record_failure(
ScannerKind::Detection,
format!("detection '{id}' on {} {reason}", key.endpoint(number)),
);
}
record(ctx, key, number, protocol, findings);
}
},
);
for target in targets {
if ctx.handle.should_stop() {
break;
}
if ctx.host_expired(target.address.addr()) {
continue;
}
pool.admit(detect_one(
target,
ctx.detections.clone(),
detection,
envelope,
Arc::clone(&ctx.tapes),
Arc::clone(&gate),
))
.await;
}
pool.drain().await;
}
struct PortTarget {
address: ScopedIp,
number: u16,
protocol: Protocol,
service: Option<String>,
responses: Vec<String>,
}
fn interested_ports(ctx: &ScanContext, envelope: DetectionEnvelope) -> Vec<PortTarget> {
let mut targets = Vec::new();
for host in ctx.store.iter() {
let address = host.value().scoped_ip();
for port in host.value().ports() {
if port.state() != PortState::Open {
continue;
}
if port.protocol() == Protocol::Sctp {
continue;
}
let number = port.number();
let protocol = port.protocol();
let service = port.service().map(|service| service.name().to_string());
let wanted = stage::interested(
ctx.detections.flows(),
&envelope,
service.as_deref(),
number,
protocol,
) || compute_stage::interested(
ctx.detections.modules().detections(),
&envelope,
service.as_deref(),
number,
protocol,
);
if wanted {
let responses = ctx.take_responses(&address, number, protocol);
targets.push(PortTarget {
address: address.clone(),
number,
protocol,
service,
responses,
});
}
}
}
targets
}
async fn detect_one(
target: PortTarget,
detections: crate::detect::Detections,
detection: ServiceDetection,
envelope: DetectionEnvelope,
tapes: Arc<Tapes>,
gate: Arc<Gate>,
) -> Option<PortResult> {
let PortTarget {
address,
number,
protocol,
service,
responses,
} = target;
let addr = address.to_socket_addr(number)?;
let tunnel = service.as_deref().and_then(Tunnel::from_service_label);
let host = addr.ip().to_string();
let produced = tokio::task::spawn_blocking(move || {
let flows = detections.flows();
let modules = detections.modules();
let (mut findings, flow_refusals) = stage::detect_port(
flows,
&envelope,
&host,
service.as_deref(),
number,
protocol,
|caps| {
Some(Box::new(Pooled {
inner: SocketProbe::new(addr, protocol, tunnel, &flow_budget(caps)),
_permit: gate.acquire(),
}) as Box<dyn Probe>)
},
);
let record_responses = responses.clone();
let response_bytes: Vec<Vec<u8>> = responses.into_iter().map(String::into_bytes).collect();
let response_slices: Vec<&[u8]> = response_bytes.iter().map(Vec::as_slice).collect();
let port_context = PortContext {
port: number,
protocol,
addr: Some(addr),
tunnel,
speaks_http: false,
detection,
};
let computed = compute_stage::detect_port(
modules.runtime(),
modules.detections(),
&envelope,
service.as_deref(),
&port_context,
&response_slices,
|grant| {
let permit = gate.acquire();
Some(Box::new(Permitted {
inner: LiveCapabilities::new(addr, protocol, tunnel, &grant.budget),
_permit: permit,
}) as Box<dyn Capabilities>)
},
|grant, tape| {
tapes.record(DetectionRunRecord {
host: addr.ip().to_string(),
port: number,
protocol: wire::protocol_name(protocol).to_string(),
detection: DetectionIdRecord::from(&grant.detection),
responses: record_responses.clone(),
tape: CapTapeRecord::from(&tape),
});
},
);
findings.extend(computed.findings);
let mut inconclusive: Vec<(String, String)> = computed
.inconclusive
.iter()
.map(|run| {
(
run.detection.id().to_string(),
describe_outcome(&run.outcome),
)
})
.collect();
inconclusive.extend(
flow_refusals
.into_iter()
.map(|(id, refusal)| (id, describe_refusal(refusal))),
);
(findings, inconclusive)
})
.await
.ok()?;
let (findings, inconclusive) = produced;
(!findings.is_empty() || !inconclusive.is_empty()).then_some((
address,
number,
protocol,
findings,
inconclusive,
))
}
fn describe_outcome(outcome: &RunOutcome) -> String {
match outcome {
RunOutcome::BudgetExceeded(trap) => format!("hit its {trap:?} budget"),
RunOutcome::Denied(denial) => {
format!("was denied {:?}: {}", denial.capability, denial.reason)
}
RunOutcome::Faulted(fault) => format!("faulted: {fault:?}"),
RunOutcome::HostReentered => "re-entered the runtime".to_string(),
}
}
fn describe_refusal(refusal: ProbeRefusal) -> String {
match refusal {
ProbeRefusal::Bytes => "hit its byte budget",
ProbeRefusal::Connections => "hit its connection budget",
ProbeRefusal::Deadline => "hit its time budget",
}
.to_string()
}
fn record(
ctx: &ScanContext,
key: ScopedIp,
number: u16,
protocol: Protocol,
findings: Vec<Finding>,
) {
ctx.update_host(key, |host| {
for finding in findings {
host.add_port_finding(number, protocol, finding);
}
});
}
fn detect_hosts(ctx: &ScanContext) {
let host_db = ctx.detections.hosts();
if host_db.detections().is_empty() {
return;
}
let mut per_host: Vec<(ScopedIp, BTreeSet<u16>, Vec<String>)> = Vec::new();
for host in ctx.store.iter() {
let mut open_ports = BTreeSet::new();
let mut services = Vec::new();
for port in host.value().ports() {
if port.state() == PortState::Open {
open_ports.insert(port.number());
if let Some(service) = port.service() {
services.push(service.name().to_string());
}
}
}
if !open_ports.is_empty() {
per_host.push((host.value().scoped_ip(), open_ports, services));
}
}
for (key, open_ports, service_names) in per_host {
let services: BTreeSet<&str> = service_names.iter().map(String::as_str).collect();
let findings = host_stage::detect_host(host_db.detections(), &open_ports, &services);
if findings.is_empty() {
continue;
}
ctx.update_host(key, |host| {
for finding in findings {
host.add_finding(finding);
}
});
}
}
struct Gate {
free: std::sync::Mutex<usize>,
returned: std::sync::Condvar,
}
impl Gate {
fn new(permits: usize) -> Self {
Self {
free: std::sync::Mutex::new(permits),
returned: std::sync::Condvar::new(),
}
}
fn acquire(self: &Arc<Self>) -> Permit {
let mut free = self.free.lock().unwrap_or_else(|held| held.into_inner());
while *free == 0 {
free = self
.returned
.wait(free)
.unwrap_or_else(|held| held.into_inner());
}
*free -= 1;
Permit {
gate: Arc::clone(self),
}
}
}
struct Permit {
gate: Arc<Gate>,
}
impl Drop for Permit {
fn drop(&mut self) {
let mut free = self
.gate
.free
.lock()
.unwrap_or_else(|held| held.into_inner());
*free += 1;
self.gate.returned.notify_one();
}
}
struct Permitted {
inner: LiveCapabilities,
_permit: Permit,
}
impl Capabilities for Permitted {
fn speak(&mut self, bytes: &[u8]) -> Result<Vec<u8>, CapError> {
self.inner.speak(bytes)
}
fn resolve(&mut self, name: &str) -> Result<Vec<std::net::IpAddr>, CapError> {
self.inner.resolve(name)
}
fn now(&mut self) -> ScanInstant {
self.inner.now()
}
}
struct Pooled {
inner: SocketProbe,
_permit: Permit,
}
impl Probe for Pooled {
fn speak(&mut self, bytes: &[u8]) -> Option<Vec<u8>> {
self.inner.speak(bytes)
}
fn last_refusal(&self) -> Option<ProbeRefusal> {
self.inner.last_refusal()
}
fn reply_complete(&self) -> bool {
self.inner.reply_complete()
}
}
fn flow_budget(caps: &CapabilitySpec) -> Budget {
Budget::new(
0,
Duration::from_millis(caps.max_millis.map_or(DEFAULT_MAX_MILLIS, u64::from)),
)
.with_max_bytes(caps.max_bytes.map_or(DEFAULT_MAX_BYTES, u64::from))
.with_max_connections(
caps.max_connections
.map_or(DEFAULT_MAX_CONNECTIONS, u32::from),
)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::detect::manifest::{Class, Speak};
use crate::model::host::Host;
use crate::model::port::{Port, Service};
use crate::scanner::session::ScanSession;
use std::net::IpAddr;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
fn caps(
max_bytes: Option<u32>,
max_millis: Option<u32>,
max_connections: Option<u16>,
) -> CapabilitySpec {
CapabilitySpec {
class: Class::ActiveBenign,
speak: Some(Speak::Target),
resolve: false,
max_bytes,
max_millis,
max_connections,
}
}
#[tokio::test]
async fn detect_runs_a_compute_module_over_the_gathered_response() {
let (session, ctx) = ScanSession::new();
let ip: IpAddr = "127.0.0.1".parse().unwrap();
let mut host = Host::new(ip);
host.add_port(
Port::new(80, Protocol::Tcp, PortState::Open).with_service(Service::new("http", 100)),
);
session.hosts().insert(ip, host);
ctx.record_responses(
ScopedIp::from(ip),
80,
Protocol::Tcp,
vec!["HTTP/1.1 200 OK\r\nServer: nginx\r\nContent-Type: text/html\r\n\r\n".to_string()],
);
detect(
&ctx,
ServiceDetection::default(),
DetectionEnvelope::default(),
)
.await;
let host = session.hosts().get(ip).unwrap();
let port = host.ports().find(|port| port.number() == 80).unwrap();
assert!(
port.findings()
.any(|finding| finding.detection().id() == "http-missing-security-headers"),
"the compute module did not fire over the gathered response"
);
}
#[tokio::test]
async fn a_detection_run_is_captured_as_a_tape() {
let (session, ctx) = ScanSession::new();
let ip: IpAddr = "127.0.0.1".parse().unwrap();
let mut host = Host::new(ip);
host.add_port(
Port::new(80, Protocol::Tcp, PortState::Open).with_service(Service::new("http", 100)),
);
session.hosts().insert(ip, host);
ctx.record_responses(
ScopedIp::from(ip),
80,
Protocol::Tcp,
vec!["HTTP/1.1 200 OK\r\nServer: nginx\r\nContent-Type: text/html\r\n\r\n".to_string()],
);
detect(
&ctx,
ServiceDetection::default(),
DetectionEnvelope::default(),
)
.await;
let tapes = ctx.progress().take_tapes();
let run = tapes
.iter()
.find(|run| run.detection.id == "http-missing-security-headers")
.expect("the compute detection's run was not captured as a tape");
assert_eq!(run.host, "127.0.0.1");
assert_eq!(run.port, 80);
assert!(
!run.responses.is_empty(),
"the run did not keep the responses it read"
);
}
#[tokio::test]
async fn detect_draws_a_host_finding_for_a_domain_controller() {
let (session, ctx) = ScanSession::new();
let ip: IpAddr = "127.0.0.1".parse().unwrap();
let mut host = Host::new(ip);
for number in [88u16, 389, 445] {
host.add_port(Port::new(number, Protocol::Tcp, PortState::Open));
}
session.hosts().insert(ip, host);
detect(
&ctx,
ServiceDetection::default(),
DetectionEnvelope::default(),
)
.await;
let host = session.hosts().get(ip).unwrap();
assert!(
host.findings()
.any(|finding| finding.detection().id() == "domain-controller"),
"the domain-controller host finding was not drawn"
);
}
#[tokio::test]
async fn detect_runs_a_flow_against_a_live_port_and_records_its_finding() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
if let Ok((mut sock, _)) = listener.accept().await {
let mut probe = [0u8; 64];
let _ = sock.read(&mut probe).await;
let _ = sock.write_all(b"# Server\r\nredis_version:7.2.4\r\n").await;
}
});
let (session, ctx) = ScanSession::new();
let ip = addr.ip();
let mut host = Host::new(ip);
host.add_port(
Port::new(addr.port(), Protocol::Tcp, PortState::Open)
.with_service(Service::new("redis", 100)),
);
session.hosts().insert(ip, host);
detect(
&ctx,
ServiceDetection::default(),
DetectionEnvelope::up_to(crate::model::finding::DetectionClass::ActiveBenign),
)
.await;
let host = session.hosts().get(ip).unwrap();
let port = host
.ports()
.find(|port| port.number() == addr.port())
.unwrap();
let findings: Vec<_> = port.findings().collect();
assert_eq!(
findings.len(),
1,
"the redis flow fired against the live port"
);
assert_eq!(findings[0].detection().id(), "redis-unauth-access");
assert_eq!(findings[0].detection().content_hash().len(), 64);
}
#[tokio::test]
async fn detection_off_connects_to_nothing() {
let (session, ctx) = ScanSession::new();
let unreachable: std::net::IpAddr = "192.0.2.1".parse().unwrap();
ctx.update_host(unreachable, |host| {
host.add_port(
Port::new(6379, Protocol::Tcp, PortState::Open)
.with_service(Service::new("redis", 100)),
);
});
let started = std::time::Instant::now();
detect(&ctx, ServiceDetection::Off, DetectionEnvelope::default()).await;
assert!(
started.elapsed() < crate::config::limits::CONNECT_PROBE_TIMEOUT,
"a detection turned off cannot have waited on a connection"
);
drop(session);
}
#[test]
fn a_flow_budget_takes_what_was_declared_and_defaults_the_rest() {
let declared = flow_budget(&caps(Some(20), Some(500), Some(1)));
assert_eq!(declared.max_bytes, 20);
assert_eq!(declared.deadline, Duration::from_millis(500));
assert_eq!(declared.max_connections, 1);
let open = flow_budget(&caps(None, None, None));
assert_eq!(open.max_bytes, DEFAULT_MAX_BYTES);
assert_eq!(open.deadline, Duration::from_millis(DEFAULT_MAX_MILLIS));
assert_eq!(open.max_connections, DEFAULT_MAX_CONNECTIONS);
}
}