pub mod model;
pub mod os;
mod analyzer;
pub(crate) mod authority;
mod context;
mod db;
mod extract;
mod favicon;
mod framed;
mod http;
mod jarm;
mod ldap;
mod matcher;
pub(crate) mod pattern;
mod prefilter;
mod response;
mod signature;
mod sip;
mod smb;
mod snmp;
mod ssh;
mod tls;
mod tls_cert;
mod tls_enum;
mod tls_summary;
#[cfg(test)]
mod corpus;
#[cfg(test)]
mod pattern_properties;
pub use analyzer::{Analyzer, BannerRegexAnalyzer, PortContext};
pub use db::{InvalidDefinition, SignatureDb};
pub use favicon::FaviconAnalyzer;
#[cfg(any(test, feature = "test-support"))]
pub use favicon::digest_of as favicon_digest;
pub use http::HttpHeadersAnalyzer;
pub use jarm::JarmAnalyzer;
pub use model::{Evidence, ServiceVerdict, SourceId, Tunnel};
pub use response::{Collected, ResponseSet, TlsInfo};
pub use context::{CONTEXTS, Context, Reach, context_note, reach_of};
pub use signature::{
CORPUS_ROOT, DefinitionError, MAX_COMPILED_REGEX_BYTES, MAX_UDP_PROBE_BYTES, MatchRule, Probe,
RULE_ID_SEPARATOR, RuleIdDefect, ServiceDefinition, ServiceSignature, claim_rule_id,
corpus_slug, rule_id,
};
pub(crate) use signature::unescape;
pub use ssh::SshAnalyzer;
pub use tls_cert::TlsCertAnalyzer;
pub use tls_enum::{EXCHANGE_TIMEOUT, MAX_OFFERS_PER_VERSION, enumerate_tls, enumerate_tls_named};
pub(crate) use tls_enum::enumerate_tls_while;
use std::borrow::Cow;
use std::net::SocketAddr;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
use tokio::net::TcpStream;
use tokio::time::timeout;
use crate::config::ServiceDetection;
use crate::model::port::{Port, PortState, Protocol, Service};
use crate::system::descriptors;
use crate::transport::dial::PathAllowance;
use crate::transport::dial::{Egress, Shaping};
use authority::Authority;
const BANNER_READ_TIMEOUT: Duration = Duration::from_millis(500);
const PROBE_READ_TIMEOUT: Duration = Duration::from_millis(1_000);
const CONTINUATION_GRACE: Duration = Duration::from_millis(50);
const USER_AGENT: &str = "ZondScanner/1.0";
const CONNECT_RETRY_TIMEOUT: Duration = Duration::from_millis(500);
const MAX_RESPONSE_BYTES: usize = 4096;
pub const MAX_IDENTITY_BYTES: usize = 256;
pub(crate) fn identity_field(value: &str) -> Option<&str> {
let value = value.trim();
(!value.is_empty() && value.len() <= MAX_IDENTITY_BYTES).then_some(value)
}
const MAX_CONTINUATION: Duration = Duration::from_secs(2);
const COLLECTION_BUDGET: Duration = Duration::from_secs(40);
const COLLECTION_WAITS: u32 = 20;
#[must_use]
pub fn reads_replies(port: u16, protocol: Protocol) -> bool {
extract::reads(port, protocol)
}
pub fn lookup_service_name(port: u16) -> Option<String> {
SignatureDb::global()
.service_name(port)
.map(|s| s.to_string())
}
#[non_exhaustive]
#[derive(Debug, Clone, Default)]
pub struct AboutTheHost {
pub os: Vec<crate::model::host::OsEvidence>,
pub hardware: Option<crate::model::host::HardwareInfo>,
pub names: Vec<crate::model::host::HostName>,
}
impl AboutTheHost {
pub fn is_empty(&self) -> bool {
self.os.is_empty() && self.hardware.is_none() && self.names.is_empty()
}
pub fn apply(self, host: &mut crate::model::host::Host) -> bool {
if let Some(described) = self.hardware {
match host.hardware().cloned() {
Some(mut known) => {
known.merge(described);
host.set_hardware(known);
}
None => host.set_hardware(described),
}
}
for name in self.names {
host.record_name(name);
}
os::identify(host, self.os)
}
fn of_verdict(verdict: Option<&ServiceVerdict>) -> Self {
match verdict {
Some(verdict) if !verdict.is_empty() => Self::from_evidence(&verdict.evidence),
Some(verdict) => Self {
names: names_in(&verdict.evidence),
..Self::default()
},
None => Self::default(),
}
}
fn from_evidence(evidence: &[Evidence]) -> Self {
Self {
os: evidence.iter().filter_map(|e| e.os.clone()).collect(),
hardware: evidence
.iter()
.filter_map(|e| e.hardware.as_ref())
.cloned()
.reduce(|mut best, other| {
best.merge(other);
best
}),
names: names_in(evidence),
}
}
}
fn names_in(evidence: &[Evidence]) -> Vec<crate::model::host::HostName> {
evidence
.iter()
.flat_map(|e| e.names.iter().cloned())
.collect()
}
pub fn decode_udp_reply(port: u16, datagram: &[u8]) -> Vec<String> {
extract::from_datagram(port, datagram)
}
pub fn tls_security(tls: &TlsInfo) -> crate::model::port::Security {
tls_summary::security(tls)
}
pub fn baseline_service(port: u16) -> Option<Service> {
lookup_service_name(port).map(|name| Service::new(name, 0))
}
pub(crate) async fn load_corpus() {
let _ = tokio::task::spawn_blocking(|| {
SignatureDb::global();
})
.await;
}
pub(crate) fn start_loading_corpus() {
drop(tokio::task::spawn_blocking(|| {
SignatureDb::global();
}));
}
pub fn baseline_port(port: u16, protocol: Protocol, state: PortState) -> Port {
let mut classified = Port::new(port, protocol, state);
if let Some(service) = baseline_service(port) {
classified.set_service(service);
}
classified
}
pub async fn fingerprint_tcp(stream: TcpStream, port: Port, detection: ServiceDetection) -> Port {
fingerprint_tcp_detailed(stream, port, detection).await.port
}
pub async fn fingerprint_tcp_detailed(
stream: TcpStream,
port: Port,
detection: ServiceDetection,
) -> Fingerprinted {
fingerprint_tcp_via(
stream,
port,
detection,
Egress::KERNEL,
PathAllowance::NONE,
None,
)
.await
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct Fingerprinted {
pub port: Port,
pub about_the_host: AboutTheHost,
pub responses: Vec<String>,
pub starved: bool,
pub(crate) ran_out_waiting: bool,
pub(crate) answered_late: bool,
pub(crate) waited_in_vain: Duration,
}
pub(crate) async fn fingerprint_tcp_via(
stream: TcpStream,
port: Port,
detection: ServiceDetection,
egress: Egress,
path: PathAllowance,
name: Option<Arc<str>>,
) -> Fingerprinted {
let tally = Arc::new(Tally::default());
let dialling = Dialling {
egress,
path,
tally: Arc::clone(&tally),
};
let (port, about_the_host, responses) = DIALLING
.scope(
dialling,
identify_tcp(stream, port, detection, egress, path, name),
)
.await;
Fingerprinted {
port,
about_the_host,
responses,
starved: tally.starved.load(Ordering::Relaxed),
ran_out_waiting: tally.ran_out_waiting.load(Ordering::Relaxed),
answered_late: tally.answered_late.load(Ordering::Relaxed),
waited_in_vain: *tally
.waited_in_vain
.lock()
.unwrap_or_else(|held| held.into_inner()),
}
}
async fn identify_tcp(
stream: TcpStream,
mut port: Port,
detection: ServiceDetection,
egress: Egress,
path: PathAllowance,
name: Option<Arc<str>>,
) -> (Port, AboutTheHost, Vec<String>) {
let addr = stream.peer_addr().ok().filter(|_| detection.sends());
let Ok((responses, tunnel)) = timeout(
path.over_each(COLLECTION_BUDGET, COLLECTION_WAITS),
gather(stream, port.number(), detection, egress, name.clone()),
)
.await
else {
return (port, AboutTheHost::default(), Vec::new());
};
if responses.is_empty() {
return (port, AboutTheHost::default(), Vec::new());
}
if let Some(tls) = responses.tls.as_ref() {
port.set_security(tls_summary::security(tls));
}
let fallback = first_printable(&responses.banners);
let banners = responses.banners.clone();
let stated = responses.names.clone();
let verdict = analyze(
port.number(),
Protocol::Tcp,
addr,
responses,
tunnel,
detection,
name,
)
.await;
let mut about_the_host = AboutTheHost::of_verdict(verdict.as_ref());
match verdict {
Some(verdict) if !verdict.is_empty() => {
if let Some(service) = verdict.to_service() {
port.set_service(service);
}
}
_ => {
if let Some(banner) = fallback {
let fallen_back = match port.service() {
Some(named) if named.extrainfo().is_none() => {
Some(named.clone().with_extrainfo(banner))
}
Some(_) => None,
None => Some(Service::new(format!("banner: {banner}"), 0)),
};
if let Some(service) = fallen_back {
port.set_service(service);
}
}
}
}
about_the_host.names.extend(stated);
(port, about_the_host, banners)
}
pub async fn fingerprint_udp_detailed(
addr: std::net::SocketAddr,
port: Port,
) -> Option<Fingerprinted> {
fingerprint_udp_via(addr, port, Egress::KERNEL).await
}
pub(crate) async fn fingerprint_udp_via(
addr: std::net::SocketAddr,
port: Port,
egress: Egress,
) -> Option<Fingerprinted> {
fingerprint_udp_on(addr, port, egress, PathAllowance::NONE).await
}
pub(crate) async fn fingerprint_udp_on(
addr: std::net::SocketAddr,
port: Port,
egress: Egress,
path: PathAllowance,
) -> Option<Fingerprinted> {
fingerprint_udp_within(addr, port, egress, descriptors::PATIENCE, path).await
}
async fn fingerprint_udp_within(
addr: std::net::SocketAddr,
mut port: Port,
egress: Egress,
patience: Duration,
path: PathAllowance,
) -> Option<Fingerprinted> {
let responses = match probe_udp(addr, egress, patience, path).await {
Datagram::Reply(responses) => responses,
Datagram::Silent => return None,
Datagram::Starved => {
return Some(Fingerprinted {
port,
about_the_host: AboutTheHost::default(),
responses: Vec::new(),
starved: true,
ran_out_waiting: false,
answered_late: false,
waited_in_vain: Duration::ZERO,
});
}
};
let banners = responses.banners.clone();
let stated = responses.names.clone();
let verdict = analyze(
addr.port(),
Protocol::Udp,
None,
responses,
None,
ServiceDetection::default(),
None,
)
.await;
let mut about_the_host = AboutTheHost::of_verdict(verdict.as_ref());
about_the_host.names.extend(stated);
let service = verdict.and_then(|verdict| verdict.to_service());
if service.is_none() && about_the_host.names.is_empty() {
return None;
}
if let Some(service) = service {
port.set_service(service);
}
Some(Fingerprinted {
port,
about_the_host,
responses: banners,
starved: false,
ran_out_waiting: false,
answered_late: false,
waited_in_vain: Duration::ZERO,
})
}
enum Datagram<T> {
Reply(T),
Silent,
Starved,
}
async fn probe_udp(
addr: std::net::SocketAddr,
egress: Egress,
patience: Duration,
path: PathAllowance,
) -> Datagram<ResponseSet> {
for payload in SignatureDb::global().udp_probe_payloads(addr.port()) {
match exchange_datagram(addr, payload, egress, patience, path).await {
Datagram::Reply(reply) => {
let read = ResponseSet {
banners: extract::from_datagram(addr.port(), &reply),
names: extract::names_from_datagram(addr.port(), &reply),
..ResponseSet::default()
};
if !read.banners.is_empty() || !read.names.is_empty() {
return Datagram::Reply(read);
}
}
Datagram::Silent => {}
Datagram::Starved => return Datagram::Starved,
}
}
Datagram::Silent
}
pub async fn probe_udp_with(addr: std::net::SocketAddr, payload: &[u8]) -> Vec<String> {
probe_udp_with_via(addr, payload, Egress::KERNEL).await
}
pub(crate) async fn probe_udp_with_via(
addr: std::net::SocketAddr,
payload: &[u8],
egress: Egress,
) -> Vec<String> {
match probe_udp_raw_via(addr, payload, egress).await {
Some(reply) => extract::from_datagram(addr.port(), &reply),
None => Vec::new(),
}
}
pub async fn probe_udp_raw(addr: std::net::SocketAddr, payload: &[u8]) -> Option<Vec<u8>> {
probe_udp_raw_via(addr, payload, Egress::KERNEL).await
}
pub(crate) async fn probe_udp_raw_via(
addr: std::net::SocketAddr,
payload: &[u8],
egress: Egress,
) -> Option<Vec<u8>> {
match exchange_datagram(
addr,
payload,
egress,
descriptors::PATIENCE,
PathAllowance::NONE,
)
.await
{
Datagram::Reply(reply) => Some(reply),
Datagram::Silent | Datagram::Starved => None,
}
}
async fn exchange_datagram(
addr: std::net::SocketAddr,
payload: &[u8],
egress: Egress,
patience: Duration,
path: PathAllowance,
) -> Datagram<Vec<u8>> {
let socket = match egress.udp(addr.ip(), patience).await {
Ok(socket) => socket,
Err(e) if descriptors::exhausted(&e) => return Datagram::Starved,
Err(_) => return Datagram::Silent,
};
if socket.connect(addr).await.is_err() || socket.send(payload).await.is_err() {
return Datagram::Silent;
}
let mut buffer = vec![0u8; MAX_RESPONSE_BYTES];
match timeout(path.over(PROBE_READ_TIMEOUT), socket.recv(&mut buffer)).await {
Ok(Ok(read)) => {
buffer.truncate(read);
Datagram::Reply(buffer)
}
_ => Datagram::Silent,
}
}
async fn gather(
mut stream: TcpStream,
port: u16,
detection: ServiceDetection,
egress: Egress,
name: Option<Arc<str>>,
) -> (ResponseSet, Option<Tunnel>) {
if !detection.connects() {
return (ResponseSet::default(), None);
}
if !detection.sends() {
let banner = read_response(&mut stream, BANNER_READ_TIMEOUT).await;
return (
ResponseSet::from_banners(banner.into_iter().collect()),
None,
);
}
let Ok(socket) = stream.peer_addr() else {
return (plaintext(stream, port, None, egress).await, None);
};
let peer = Authority::new(socket).named(name);
let mut opened = Some(stream);
let mut refused: Option<ResponseSet> = None;
for rung in Rung::ladder(port) {
let stream = match opened.take() {
Some(stream) => stream,
None => match redial(socket, egress).await {
Some(fresh) => fresh,
None => return (refused.unwrap_or_default(), None),
},
};
let (responses, tunnel) = rung.ask(stream, port, &peer, detection, egress).await;
if let Some(held) = refused.take() {
return match held.tls {
Some(tls) if refused_in_the_clear(&responses) => {
(responses.with_tls(tls), Some(Tunnel::Tls))
}
tls => (ResponseSet { tls, ..held }, None),
};
}
if responses.is_empty() {
continue;
}
if matches!(rung, Rung::LegacyTls) && refused_every_handshake(&responses) {
refused = Some(responses);
continue;
}
if matches!(rung, Rung::Plaintext)
&& !tls::is_tls_port(port)
&& refused_in_the_clear(&responses)
{
return asked_through_tls(responses, port, &peer, egress).await;
}
return (responses, tunnel);
}
(refused.unwrap_or_default(), None)
}
fn refused_every_handshake(responses: &ResponseSet) -> bool {
responses.banners.is_empty()
&& responses
.tls
.as_ref()
.is_some_and(|tls| tls.version == Some(tls::REFUSED))
}
fn refused_in_the_clear(responses: &ResponseSet) -> bool {
responses.banners.first().is_some_and(|reply| {
reply.starts_with("HTTP/") && reply.split_whitespace().nth(1) == Some("400")
})
}
async fn asked_through_tls(
clear: ResponseSet,
port: u16,
peer: &Authority,
egress: Egress,
) -> (ResponseSet, Option<Tunnel>) {
let Some(stream) = redial(peer.socket(), egress).await else {
return (clear, None);
};
let handshake = tls::speculative_handshake(stream, peer.server_name()).await;
let (through, tunnel) = tunneled(handshake, port, peer, egress).await;
if !through.is_empty() {
return (through, tunnel);
}
let Some(stream) = redial(peer.socket(), egress).await else {
return (clear, None);
};
match legacy_tls(stream).await.tls {
Some(tls) => (clear.with_tls(tls), Some(Tunnel::Tls)),
None => (clear, None),
}
}
async fn redial(socket: SocketAddr, egress: Egress) -> Option<TcpStream> {
dial_again(socket, egress, Some(on_path(CONNECT_RETRY_TIMEOUT)))
.await
.ok()
}
async fn dial_again(
addr: SocketAddr,
egress: Egress,
limit: Option<Duration>,
) -> std::io::Result<TcpStream> {
let refused = Refused::default();
let refused = &refused;
descriptors::patiently(descriptors::patience(), || async move {
let attempt = match limit {
Some(limit) => timeout(limit, egress.connect_shaped(addr, Shaping::default()))
.await
.unwrap_or_else(|_elapsed| Err(std::io::ErrorKind::TimedOut.into())),
None => egress.connect_shaped(addr, Shaping::default()).await,
};
refused.saw(&attempt);
attempt
})
.await
}
#[derive(Default)]
struct Refused(AtomicBool);
impl Refused {
fn saw(&self, attempt: &std::io::Result<TcpStream>) {
let refused = attempt.as_ref().is_err_and(descriptors::exhausted);
self.0.store(refused, Ordering::Relaxed);
}
}
impl Drop for Refused {
fn drop(&mut self) {
if self.0.load(Ordering::Relaxed) {
tell(|tally| tally.starved.store(true, Ordering::Relaxed));
}
}
}
#[derive(Clone, Copy)]
enum Rung {
Tls,
SpeculativeTls,
LegacyTls,
Plaintext,
LastResort,
}
impl Rung {
fn ladder(port: u16) -> &'static [Rung] {
if tls::is_tls_port(port) {
&[
Rung::Tls,
Rung::LegacyTls,
Rung::Plaintext,
Rung::LastResort,
]
} else {
&[Rung::Plaintext, Rung::SpeculativeTls, Rung::LastResort]
}
}
async fn ask(
self,
stream: TcpStream,
port: u16,
peer: &Authority,
detection: ServiceDetection,
egress: Egress,
) -> (ResponseSet, Option<Tunnel>) {
match self {
Rung::Tls => {
let handshake = tls::handshake(stream, peer.server_name()).await;
tunneled(handshake, port, peer, egress).await
}
Rung::SpeculativeTls => {
let handshake = tls::speculative_handshake(stream, peer.server_name()).await;
tunneled(handshake, port, peer, egress).await
}
Rung::LegacyTls => (legacy_tls(stream).await, None),
Rung::Plaintext => (plaintext(stream, port, Some(peer), egress).await, None),
Rung::LastResort => (
last_resort(stream, peer, port, detection, egress).await,
None,
),
}
}
}
async fn last_resort(
first: TcpStream,
peer: &Authority,
port: u16,
detection: ServiceDetection,
egress: Egress,
) -> ResponseSet {
let mut probes =
SignatureDb::global().universal_tcp_probe_payloads(port, detection.probe_intensity());
probes.truncate(guesses_worth_the_path(probes.len(), detection));
let mut replies = Vec::new();
let mut opened = Some(first);
for payload in probes {
let Some(mut stream) = (match opened.take() {
Some(stream) => Some(stream),
None => redial(peer.socket(), egress).await,
}) else {
break;
};
if stream.write_all(&peer.addressed(payload)).await.is_err() {
continue;
}
if let Some(reply) = read_document(&mut stream, PROBE_READ_TIMEOUT).await {
replies.push(reply);
}
}
ResponseSet::from_banners(replies)
}
fn guesses_worth_the_path(guesses: usize, detection: ServiceDetection) -> usize {
let slow = on_path(PROBE_READ_TIMEOUT) > PROBE_READ_TIMEOUT * 2;
match detection {
ServiceDetection::Thorough => guesses,
_ if slow => guesses.min(1),
_ => guesses,
}
}
async fn plaintext(
mut stream: TcpStream,
port: u16,
peer: Option<&Authority>,
egress: Egress,
) -> ResponseSet {
let db = SignatureDb::global();
let mut conversations = db.tcp_probe_conversations(port).peekable();
let named = db.service_name(port).is_some();
let (mut responses, mut held) = match conversations.next_if(|_| named) {
Some(first) => {
let listen = !db.asked_first(port);
let responses = collect_responses(&mut stream, port, first, peer, listen).await;
if responses
.banners
.first()
.is_some_and(|first| looks_like_tls(first.as_bytes()))
{
return ResponseSet::default();
}
(responses, Some(stream))
}
None => match ask_generically(stream, peer, egress).await {
GenericReply::Spoke(banners) => (ResponseSet::from_banners(banners), None),
GenericReply::Silent => (ResponseSet::default(), None),
GenericReply::Tls => return ResponseSet::default(),
},
};
let mut theirs = ResponseSet::default();
for probes in conversations {
if let Some(peer) = peer {
drop(held.take());
held = redial(peer.socket(), egress).await;
}
let Some(stream) = held.as_mut() else {
break;
};
theirs.extend(ask_in_turn(stream, port, probes, peer).await);
}
drop(held);
if named {
responses.extend(theirs);
let banners = std::mem::take(&mut responses.banners);
responses.banners = with_redirect_followed(banners, peer, egress).await;
} else {
theirs.banners = with_redirect_followed(theirs.banners, peer, egress).await;
responses.extend(theirs);
}
responses
}
enum GenericReply {
Spoke(Vec<String>),
Tls,
Silent,
}
async fn ask_generically(
mut stream: TcpStream,
peer: Option<&Authority>,
egress: Egress,
) -> GenericReply {
for payload in SignatureDb::global().generic_tcp_probe_payloads() {
let payload = match peer {
Some(peer) => peer.addressed(payload),
None => Cow::Borrowed(&payload[..]),
};
if stream.write_all(&payload).await.is_err() {
break;
}
}
let read = read_bytes(&mut stream, PROBE_READ_TIMEOUT, CONTINUATION_GRACE).await;
drop(stream);
let Some(bytes) = read else {
return GenericReply::Silent;
};
if looks_like_tls(&bytes) {
return GenericReply::Tls;
}
let first = extract::reply_text(&bytes);
let followed = match (peer, redirect_path(&first, peer)) {
(Some(peer), Some(path)) => follow_redirect(peer, &path, egress).await,
_ => None,
};
GenericReply::Spoke(std::iter::once(first).chain(followed).collect())
}
fn redirect_path(response: &str, peer: Option<&Authority>) -> Option<String> {
let (status, headers) = response.split_once("\r\n").or(response.split_once('\n'))?;
let code: u16 = status.split_whitespace().nth(1)?.parse().ok()?;
if !(300..400).contains(&code) {
return None;
}
let location = headers
.lines()
.take_while(|line| !line.trim_end_matches('\r').is_empty())
.find_map(|line| {
let (name, value) = line.split_once(':')?;
name.trim()
.eq_ignore_ascii_case("location")
.then(|| value.trim())
})?;
if location.is_empty() || location.chars().any(char::is_control) {
return None;
}
match location {
url if url.contains("://") || url.starts_with("//") => peer?.path_of(url),
path if path.starts_with('/') => Some(path.to_string()),
relative => Some(format!("/{relative}")),
}
}
async fn follow_redirect(peer: &Authority, path: &str, egress: Egress) -> Option<String> {
let stream = dial_again(peer.socket(), egress, Some(on_path(CONNECT_RETRY_TIMEOUT)))
.await
.ok()?;
match peer.is_tls() {
true => {
let (mut tunnel, _) = tls::handshake(stream, peer.server_name()).await?;
ask_for(&mut tunnel, peer, path).await
}
false => {
let mut stream = stream;
ask_for(&mut stream, peer, path).await
}
}
}
async fn ask_for<S>(stream: &mut S, peer: &Authority, path: &str) -> Option<String>
where
S: AsyncRead + AsyncWrite + Unpin,
{
let request = format!(
"GET {path} HTTP/1.1\r\nHost: {}\r\nUser-Agent: {USER_AGENT}\r\n\
Accept: */*\r\nConnection: close\r\n\r\n",
peer.header()
);
stream.write_all(request.as_bytes()).await.ok()?;
read_document(stream, PROBE_READ_TIMEOUT).await
}
async fn with_redirect_followed(
mut banners: Vec<String>,
peer: Option<&Authority>,
egress: Egress,
) -> Vec<String> {
let Some(peer) = peer else {
return banners;
};
let path = banners
.iter()
.find(|reply| reply.starts_with("HTTP/"))
.and_then(|reply| redirect_path(reply, Some(peer)));
if let Some(path) = path
&& let Some(page) = follow_redirect(peer, &path, egress).await
{
banners.push(page);
}
banners
}
fn looks_like_tls(bytes: &[u8]) -> bool {
matches!(bytes, [0x14..=0x17, 0x03, 0x00..=0x04, ..])
}
async fn legacy_tls(stream: TcpStream) -> ResponseSet {
match tls::legacy_version(stream).await {
Some(version) => {
ResponseSet::default().with_tls(TlsInfo::new(Vec::new()).with_version(version))
}
None => ResponseSet::default(),
}
}
async fn tunneled(
handshake: Option<(tls::TlsTunnel, TlsInfo)>,
port: u16,
peer: &Authority,
egress: Egress,
) -> (ResponseSet, Option<Tunnel>) {
let Some((mut tunnel, info)) = handshake else {
return (ResponseSet::default(), None);
};
let db = SignatureDb::global();
let peer = peer.through_tls();
let mut conversations = db.tcp_probe_conversations(port);
let mut drawn = match conversations.next() {
Some(own) => {
let listen = !db.asked_first(port);
collect_responses(&mut tunnel, port, own, Some(&peer), listen).await
}
None => {
let generic = db.generic_tcp_probe_payloads();
collect_responses(&mut tunnel, port, generic, Some(&peer), false).await
}
};
let mut held = Some(tunnel);
for probes in conversations {
drop(held.take());
held = retunneled(&peer, egress).await;
let Some(tunnel) = held.as_mut() else {
break;
};
drawn.extend(ask_in_turn(tunnel, port, probes, Some(&peer)).await);
}
drop(held);
drawn.banners = with_redirect_followed(drawn.banners, Some(&peer), egress).await;
(drawn.with_tls(info), Some(Tunnel::Tls))
}
async fn retunneled(peer: &Authority, egress: Egress) -> Option<tls::TlsTunnel> {
let stream = redial(peer.socket(), egress).await?;
let (tunnel, _) = tls::handshake(stream, peer.server_name()).await?;
Some(tunnel)
}
async fn collect_responses<S>(
stream: &mut S,
port: u16,
probes: &[Vec<u8>],
peer: Option<&Authority>,
listen: bool,
) -> ResponseSet
where
S: AsyncRead + AsyncWrite + Unpin,
{
let mut drawn = ResponseSet::default();
if listen && let Some(banner) = read_response(stream, BANNER_READ_TIMEOUT).await {
drawn.banners.push(banner);
}
drawn.extend(ask_in_turn(stream, port, probes, peer).await);
drawn
}
async fn ask_in_turn<S>(
stream: &mut S,
port: u16,
probes: &[Vec<u8>],
peer: Option<&Authority>,
) -> ResponseSet
where
S: AsyncRead + AsyncWrite + Unpin,
{
let mut drawn = ResponseSet::default();
for payload in probes {
let payload = match peer {
Some(peer) => peer.addressed(payload),
None => Cow::Borrowed(&payload[..]),
};
if stream.write_all(&payload).await.is_err() {
break;
}
let Some(bytes) = read_bytes(stream, PROBE_READ_TIMEOUT, CONTINUATION_GRACE).await else {
continue;
};
drawn.banners.extend(extract::from_stream(port, &bytes));
drawn.banners.push(extract::reply_text(&bytes));
drawn.names.extend(extract::names_from_stream(port, &bytes));
}
drawn
}
struct Dialling {
egress: Egress,
path: PathAllowance,
tally: Arc<Tally>,
}
#[derive(Debug, Default)]
struct Tally {
starved: AtomicBool,
ran_out_waiting: AtomicBool,
answered_late: AtomicBool,
waited_in_vain: std::sync::Mutex<Duration>,
}
impl Tally {
fn ran_out(&self, wait: Duration) {
self.ran_out_waiting.store(true, Ordering::Relaxed);
*self
.waited_in_vain
.lock()
.unwrap_or_else(|held| held.into_inner()) += wait;
}
}
fn tell(what: impl FnOnce(&Tally)) {
let _ = DIALLING.try_with(|dialling| what(&dialling.tally));
}
fn on_path(wait: Duration) -> Duration {
DIALLING
.try_with(|dialling| dialling.path.over(wait))
.unwrap_or(wait)
}
fn service_time(elapsed: Duration) -> Duration {
DIALLING
.try_with(|dialling| dialling.path.service_time(elapsed))
.unwrap_or(elapsed)
}
fn on_path_each(budget: Duration, waits: u32) -> Duration {
DIALLING
.try_with(|dialling| dialling.path.over_each(budget, waits))
.unwrap_or(budget)
}
tokio::task_local! {
static DIALLING: Dialling;
}
pub(crate) async fn analyzer_connect(addr: SocketAddr) -> std::io::Result<TcpStream> {
let egress = DIALLING
.try_with(|dialling| dialling.egress)
.unwrap_or(Egress::KERNEL);
dial_again(addr, egress, None).await
}
static ANALYZERS: &[&dyn Analyzer] = &[
&BannerRegexAnalyzer,
&FaviconAnalyzer,
&HttpHeadersAnalyzer,
&JarmAnalyzer,
&ldap::LdapAnalyzer,
&smb::SmbAnalyzer,
&SshAnalyzer,
&TlsCertAnalyzer,
];
#[must_use]
pub fn analyzers() -> &'static [&'static dyn Analyzer] {
ANALYZERS
}
async fn analyze(
port: u16,
protocol: Protocol,
addr: Option<std::net::SocketAddr>,
responses: ResponseSet,
tunnel: Option<Tunnel>,
detection: ServiceDetection,
name: Option<Arc<str>>,
) -> Option<ServiceVerdict> {
let speaks_http = responses
.banners
.iter()
.any(|banner| banner.starts_with("HTTP/"));
let ctx = PortContext::new(port, protocol)
.with_addr(addr)
.with_tunnel(tunnel)
.with_speaks_http(speaks_http)
.with_detection(detection)
.with_host_name(name.map(|name| name.to_string()));
analyze_with(ctx, responses, analyzers()).await
}
pub async fn analyze_with(
ctx: PortContext,
responses: ResponseSet,
analyzers: &'static [&'static dyn Analyzer],
) -> Option<ServiceVerdict> {
let mut collected = Vec::with_capacity(analyzers.len());
for analyzer in analyzers {
let interested = analyzer.interested(&ctx);
collected.push((
interested,
match interested {
true => analyzer.collect(&ctx, &responses).await,
false => Collected::default(),
},
));
}
off_the_reactor(move || {
let mut evidence = Vec::new();
for (analyzer, (interested, collected)) in analyzers.iter().zip(&collected) {
if *interested {
evidence.extend(analyzer.analyze(&ctx, &responses, collected));
}
}
(!evidence.is_empty()).then(|| ServiceVerdict::resolve(evidence))
})
.await
.flatten()
}
const ANALYSIS_THREADS: usize = 1;
fn analysis_pool() -> Option<&'static rayon::ThreadPool> {
static POOL: std::sync::OnceLock<Option<rayon::ThreadPool>> = std::sync::OnceLock::new();
POOL.get_or_init(|| {
rayon::ThreadPoolBuilder::new()
.num_threads(ANALYSIS_THREADS)
.thread_name(|index| format!("zond-identify-{index}"))
.build()
.ok()
})
.as_ref()
}
async fn off_the_reactor<T: Send + 'static>(
work: impl FnOnce() -> T + Send + 'static,
) -> Option<T> {
let Some(pool) = analysis_pool() else {
return tokio::task::spawn_blocking(work).await.ok();
};
let (sender, receiver) = tokio::sync::oneshot::channel();
pool.spawn(move || {
let done = std::panic::catch_unwind(std::panic::AssertUnwindSafe(work));
let _ = sender.send(done.ok());
});
receiver.await.ok().flatten()
}
pub(crate) fn on_the_matching_thread<T: Send>(match_work: impl FnOnce() -> T + Send) -> T {
match analysis_pool() {
Some(pool) => pool.install(match_work),
None => match_work(),
}
}
async fn read_response<S>(stream: &mut S, wait: Duration) -> Option<String>
where
S: AsyncRead + Unpin,
{
read_bytes(stream, wait, Duration::ZERO)
.await
.map(|bytes| extract::reply_text(&bytes))
}
async fn read_document<S>(stream: &mut S, wait: Duration) -> Option<String>
where
S: AsyncRead + Unpin,
{
read_bytes(stream, wait, CONTINUATION_GRACE)
.await
.map(|bytes| extract::reply_text(&bytes))
}
async fn read_bytes<S>(stream: &mut S, wait: Duration, grace: Duration) -> Option<Vec<u8>>
where
S: AsyncRead + Unpin,
{
let late = wait / 2;
let wait = on_path(wait);
let asked = tokio::time::Instant::now();
let mut collected: Vec<u8> = Vec::new();
let mut buffer = [0u8; MAX_RESPONSE_BYTES];
let mut budget = wait;
let mut deadline = None;
while collected.len() < MAX_RESPONSE_BYTES {
let first = collected.is_empty();
match timeout(budget, stream.read(&mut buffer)).await {
Ok(Ok(n)) if n > 0 => {
if first && service_time(asked.elapsed()) > late {
tell(|tally| tally.answered_late.store(true, Ordering::Relaxed));
}
let room = MAX_RESPONSE_BYTES - collected.len();
collected.extend_from_slice(&buffer[..n.min(room)]);
if grace.is_zero() {
break;
}
let deadline =
*deadline.get_or_insert_with(|| tokio::time::Instant::now() + MAX_CONTINUATION);
let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
if remaining.is_zero() {
break;
}
budget = grace.min(remaining);
}
Err(_elapsed) if first => {
tell(|tally| tally.ran_out(wait));
break;
}
_ => break,
}
}
(!collected.is_empty()).then_some(collected)
}
fn first_printable(responses: &[String]) -> Option<String> {
let printable: String = responses
.iter()
.flat_map(|response| response.chars())
.filter(|c| c.is_ascii_graphic() || *c == ' ')
.take(32)
.collect();
(!printable.is_empty()).then_some(printable)
}
#[cfg(test)]
mod tests {
use crate::testing::loopback::accept_from_this_process;
#[test]
fn a_service_describing_its_hardware_is_one_witness_and_not_two() {
let evidence = SignatureDb::global()
.identify(161, Protocol::Udp, "CSS326-24G-2S+ SwOS v2.13")
.expect("the corpus knows SwOS");
assert!(
evidence.os.is_some() && evidence.hardware.is_some(),
"test premise: the rule names both the system and the box"
);
let mut host = crate::model::host::Host::new("192.0.2.1".parse().expect("literal"));
AboutTheHost::from_evidence(&[evidence]).apply(&mut host);
let os = host
.os()
.expect("an SNMP description is a verdict on its own");
assert!(
!os.is_highly_confident(),
"one reply settled the host at {}",
os.accuracy()
);
}
#[test]
fn an_unregistered_port_is_seeded_with_nothing() {
assert!(baseline_service(22).is_some());
let unregistered = (40_000..=65_535)
.find(|port| lookup_service_name(*port).is_none())
.expect("some port in the ephemeral range is unregistered");
assert!(
baseline_service(unregistered).is_none(),
"port {unregistered} invented a service name"
);
assert!(
baseline_port(unregistered, Protocol::Tcp, PortState::Closed)
.service()
.is_none()
);
}
#[test]
fn an_sctp_port_is_named_by_the_service_that_claims_its_number() {
let port = baseline_port(2905, Protocol::Sctp, PortState::Open);
let service = port.service().expect("m3ua claims 2905");
assert_eq!(service.name(), "m3ua");
assert!(
service.is_inferred(),
"nothing asked the port what it was running"
);
}
#[test]
fn a_seeded_label_is_never_mistaken_for_an_identification() {
let seeded = baseline_service(22).expect("ssh is registered");
assert_eq!(seeded.name(), "ssh");
assert!(
seeded.is_inferred(),
"nothing asked port 22 what it was running"
);
}
#[test]
fn a_same_host_redirect_names_where_to_look_next() {
let jellyfin = "HTTP/1.1 302 Found\r\n\
Location: /web/index.html\r\n\
Server: Kestrel\r\n\r\n";
assert_eq!(
redirect_path(jellyfin, Some(&peer())).as_deref(),
Some("/web/index.html")
);
}
fn peer() -> Authority {
Authority::new("127.0.0.1:8096".parse().expect("a literal address"))
}
#[test]
fn a_relative_location_resolves_against_the_root_it_was_served_from() {
let jellyfin = "HTTP/1.1 302 Found\r\nLocation: web/\r\nServer: Kestrel\r\n\r\n";
assert_eq!(
redirect_path(jellyfin, Some(&peer())).as_deref(),
Some("/web/")
);
}
#[test]
fn an_absolute_location_naming_the_scanned_host_is_followed() {
let grafana = "HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1:8096/login\r\n\r\n";
assert_eq!(
redirect_path(grafana, Some(&peer())).as_deref(),
Some("/login")
);
let bare = "HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1:8096\r\n\r\n";
assert_eq!(redirect_path(bare, Some(&peer())).as_deref(), Some("/"));
}
#[test]
fn an_absolute_location_on_another_port_is_declined() {
let elsewhere = "HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1:9999/x\r\n\r\n";
assert_eq!(redirect_path(elsewhere, Some(&peer())), None);
}
#[test]
fn an_upgrade_to_https_is_declined_rather_than_guessed() {
let upgrade = "HTTP/1.1 301 Moved\r\nLocation: https://127.0.0.1:8096/\r\n\r\n";
assert_eq!(redirect_path(upgrade, Some(&peer())), None);
}
#[test]
fn an_absolute_location_is_declined_when_there_is_no_peer_to_check() {
let named = "HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1:8096/login\r\n\r\n";
assert_eq!(redirect_path(named, None), None);
}
#[test]
fn a_scheme_relative_location_is_declined_like_any_other_host() {
let elsewhere = "HTTP/1.1 302 Found\r\nLocation: //cdn.example/web/\r\n\r\n";
assert_eq!(redirect_path(elsewhere, Some(&peer())), None);
let here = "HTTP/1.1 302 Found\r\nLocation: //127.0.0.1:8096/web/\r\n\r\n";
assert_eq!(redirect_path(here, Some(&peer())).as_deref(), Some("/web/"));
}
#[test]
fn a_redirect_off_the_host_is_declined() {
for location in [
"https://example.com/login",
"http://somewhere.else/",
"https://127.0.0.1/web/",
"/web/\rX-Injected: 1",
"/web/\u{0}index.html",
] {
let response = format!("HTTP/1.1 302 Found\r\nLocation: {location}\r\n\r\n");
assert_eq!(
redirect_path(&response, Some(&peer())),
None,
"`{location}` is not somewhere this scan may follow"
);
}
}
#[test]
fn a_response_that_is_not_a_redirect_names_nowhere_to_go() {
assert_eq!(
redirect_path(
"HTTP/1.1 200 OK\r\nLocation: /ignored\r\n\r\n",
Some(&peer())
),
None,
"a 200 is an answer, whatever else it carries"
);
assert_eq!(
redirect_path("HTTP/1.1 302 Found\r\nServer: nginx\r\n\r\n", Some(&peer())),
None,
"and a redirect naming nowhere leads nowhere"
);
assert_eq!(
redirect_path("SSH-2.0-OpenSSH_9.2p1\r\n", Some(&peer())),
None
);
assert_eq!(redirect_path("", Some(&peer())), None);
}
#[test]
fn an_ipv6_redirect_naming_the_scanned_host_is_followed() {
let peer: SocketAddr = "[2001:db8::1]:8096".parse().expect("a literal address");
let grafana = "HTTP/1.1 302 Found\r\nLocation: http://[2001:db8::1]:8096/login\r\n\r\n";
assert_eq!(
redirect_path(grafana, Some(&Authority::new(peer))).as_deref(),
Some("/login")
);
let elsewhere = "HTTP/1.1 302 Found\r\nLocation: http://[2001:db8::2]:8096/login\r\n\r\n";
assert_eq!(redirect_path(elsewhere, Some(&Authority::new(peer))), None);
}
#[tokio::test]
async fn a_redirect_on_ipv6_is_asked_for_by_a_bracketed_host() {
let Ok(listener) = tokio::net::TcpListener::bind("[::1]:0").await else {
eprintln!("SKIP: no IPv6 loopback on this machine");
return;
};
let addr = listener.local_addr().expect("a local address");
let server = tokio::spawn(async move {
let mut first = accept_from_this_process(&listener)
.await
.expect("the first connection");
let mut buffer = [0u8; 1024];
let _ = first.read(&mut buffer).await;
let _ = first
.write_all(b"HTTP/1.1 302 Found\r\nLocation: /next\r\nContent-Length: 0\r\n\r\n")
.await;
drop(first);
let mut second = accept_from_this_process(&listener)
.await
.expect("the redirect followed");
let read = second.read(&mut buffer).await.unwrap_or(0);
let _ = second
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
.await;
String::from_utf8_lossy(&buffer[..read]).into_owned()
});
let stream = TcpStream::connect(addr).await.expect("connects");
let _ = plaintext(stream, 51987, Some(&Authority::new(addr)), Egress::KERNEL).await;
let request = server.await.expect("the listener finishes");
let host = format!("\r\nHost: [::1]:{}\r\n", addr.port());
assert!(
request.contains(&host),
"the redirect was asked for without `{}`: {request:?}",
host.trim()
);
}
#[test]
fn a_tls_alert_is_recognised_as_tls_rather_than_as_a_banner() {
assert!(looks_like_tls(&[0x15, 0x03, 0x03, 0x00, 0x02, 0x02, 0x0A]));
assert!(looks_like_tls(&[0x16, 0x03, 0x01, 0x00, 0x2A]));
assert!(!looks_like_tls(b"HTTP/1.1 200 OK"));
assert!(!looks_like_tls(b"SSH-2.0-OpenSSH_9.2p1"));
assert!(!looks_like_tls(&[0x15]), "too short to be a record");
assert!(!looks_like_tls(&[]));
}
use super::*;
struct ThreadWitness;
static WITNESSED: std::sync::Mutex<Vec<std::thread::ThreadId>> =
std::sync::Mutex::new(Vec::new());
#[async_trait::async_trait]
impl Analyzer for ThreadWitness {
fn id(&self) -> SourceId {
SourceId::BannerRegex
}
fn interested(&self, _ctx: &PortContext) -> bool {
true
}
fn analyze(&self, ctx: &PortContext, _: &ResponseSet, _: &Collected) -> Vec<Evidence> {
assert!(ctx.port != 0, "the witness was asked about port 0");
std::thread::sleep(Duration::from_millis(20));
WITNESSED
.lock()
.expect("the witness list")
.push(std::thread::current().id());
vec![
Evidence::new(
SourceId::BannerRegex,
crate::model::confidence::Confidence::Weak,
)
.with_service("witnessed"),
]
}
}
static WITNESS: [&dyn Analyzer; 1] = [&ThreadWitness];
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn identification_runs_on_one_kept_thread_and_survives_a_panic() {
let runs: Vec<_> = (1..=12u16)
.map(|port| {
tokio::spawn(analyze_with(
PortContext::new(port, Protocol::Tcp),
ResponseSet::default(),
&WITNESS,
))
})
.collect();
for run in runs {
assert!(run.await.expect("joins").is_some(), "every run is named");
}
let threads: std::collections::BTreeSet<String> = WITNESSED
.lock()
.expect("the witness list")
.iter()
.map(|id| format!("{id:?}"))
.collect();
assert_eq!(threads.len(), 1, "identification ran on {threads:?}");
let panicked = analyze_with(
PortContext::new(0, Protocol::Tcp),
ResponseSet::default(),
&WITNESS,
)
.await;
assert!(panicked.is_none(), "a panicking analyzer names nothing");
let after = analyze_with(
PortContext::new(1, Protocol::Tcp),
ResponseSet::default(),
&WITNESS,
)
.await;
assert!(after.is_some(), "and identification carries on after it");
}
#[test]
fn a_direct_match_against_the_corpus_runs_on_the_identification_thread() {
let threads: std::collections::BTreeSet<Option<String>> = (0..4)
.map(|_| {
std::thread::spawn(|| {
on_the_matching_thread(|| std::thread::current().name().map(str::to_owned))
})
})
.collect::<Vec<_>>()
.into_iter()
.map(|thread| thread.join().expect("joins"))
.collect();
assert_eq!(
threads,
[Some("zond-identify-0".to_string())].into(),
"matched on {threads:?}"
);
}
#[tokio::test]
async fn analyze_runs_both_phases_and_resolves() {
let responses = ResponseSet::from_banners(vec!["SSH-2.0-OpenSSH_9.6p1 Debian".to_string()]);
let verdict = analyze(
22,
Protocol::Tcp,
None,
responses,
None,
ServiceDetection::default(),
None,
)
.await
.expect("names a service");
assert_eq!(verdict.service.as_deref(), Some("ssh"));
assert_eq!(verdict.product.as_deref(), Some("OpenSSH"));
assert_eq!(verdict.version.as_deref(), Some("9.6p1"));
}
#[tokio::test]
async fn analyze_identifies_a_long_tail_http_server_end_to_end() {
let responses = ResponseSet::from_banners(vec![
"HTTP/1.1 200 OK\r\nServer: gunicorn/21.2.0\r\nContent-Type: text/html\r\n\r\n"
.to_string(),
]);
let verdict = analyze(
8000,
Protocol::Tcp,
None,
responses,
None,
ServiceDetection::default(),
None,
)
.await
.expect("names a service");
assert_eq!(verdict.service.as_deref(), Some("http"));
assert_eq!(verdict.product.as_deref(), Some("Gunicorn"));
assert_eq!(verdict.version.as_deref(), Some("21.2.0"));
assert_eq!(
verdict.cpe.as_deref(),
Some("cpe:/a:gunicorn:gunicorn:21.2.0"),
"the reading that won the slot is the one the correlator can use"
);
}
#[tokio::test]
async fn analyze_composes_curated_product_vendor_with_powered_by_extrainfo() {
let responses = ResponseSet::from_banners(vec![
"HTTP/1.1 200 OK\r\nServer: Apache/2.4.58\r\nX-Powered-By: PHP/8.2.1\r\n\r\n"
.to_string(),
]);
let service = analyze(
80,
Protocol::Tcp,
None,
responses,
None,
ServiceDetection::default(),
None,
)
.await
.expect("names a service")
.to_service()
.expect("projects onto a service");
assert_eq!(service.name(), "http");
assert_eq!(service.product(), Some("Apache HTTP Server"));
assert_eq!(service.vendor(), Some("Apache Software Foundation"));
assert_eq!(service.version(), Some("2.4.58"));
assert_eq!(service.extrainfo(), Some("PHP/8.2.1"));
}
#[tokio::test]
async fn analyze_resolves_a_versionless_server_to_its_name_not_generic_http() {
let responses = ResponseSet::from_banners(vec![
"HTTP/1.1 403 Forbidden\r\nServer: cloudflare\r\n\r\n".to_string(),
]);
let verdict = analyze(
8000,
Protocol::Tcp,
None,
responses,
None,
ServiceDetection::default(),
None,
)
.await
.expect("names a service");
assert_eq!(verdict.service.as_deref(), Some("http"));
assert_eq!(verdict.product.as_deref(), Some("cloudflare"));
}
#[tokio::test]
async fn a_hostile_response_cannot_fill_a_report_field() {
use tokio::net::{TcpListener, TcpStream};
let listener = TcpListener::bind("127.0.0.1:0").await.expect("a socket");
let addr = listener.local_addr().expect("its address");
let server = tokio::spawn(async move {
let Ok(mut sock) = accept_from_this_process(&listener).await else {
return;
};
let long = "A".repeat(1_500);
let response =
format!("HTTP/1.1 200 OK\r\nServer: {long}\r\nX-Powered-By: {long}\r\n\r\n");
let _ = sock.write_all(response.as_bytes()).await;
let mut buffer = [0u8; MAX_RESPONSE_BYTES];
let _ = sock.read(&mut buffer).await;
});
let stream = TcpStream::connect(addr).await.expect("connects");
let port = baseline_port(80, Protocol::Tcp, PortState::Open);
let identified = fingerprint_tcp(stream, port, ServiceDetection::Probe).await;
server.abort();
let service = identified.service().expect("the port is still named");
for (field, value) in [
("product", service.product()),
("version", service.version()),
("extrainfo", service.extrainfo()),
] {
if let Some(value) = value {
assert!(
value.len() <= MAX_IDENTITY_BYTES,
"{field} is {} bytes, past the bound",
value.len()
);
}
}
assert_eq!(
service.product(),
None,
"a fifteen-hundred-byte token is refused rather than truncated"
);
}
#[test]
fn an_ordinary_identity_field_passes_through() {
assert_eq!(identity_field("nginx/1.24.0"), Some("nginx/1.24.0"));
assert_eq!(identity_field(" PHP/8.2.1 "), Some("PHP/8.2.1"));
assert_eq!(identity_field(""), None);
assert_eq!(identity_field(" "), None);
assert_eq!(
identity_field(&"A".repeat(MAX_IDENTITY_BYTES)).map(str::len),
Some(MAX_IDENTITY_BYTES)
);
assert_eq!(identity_field(&"A".repeat(MAX_IDENTITY_BYTES + 1)), None);
}
#[tokio::test]
async fn a_server_that_speaks_only_tls_ten_is_still_recorded() {
use tokio::net::{TcpListener, TcpStream};
let listener = TcpListener::bind("127.0.0.1:0").await.expect("a socket");
let addr = listener.local_addr().expect("its address");
let server = tokio::spawn(async move {
for round in 0..2 {
let Ok(mut sock) = accept_from_this_process(&listener).await else {
return;
};
let mut buffer = [0u8; 1024];
let _ = sock.read(&mut buffer).await;
let reply: &[u8] = match round {
0 => &[0x15, 0x03, 0x01, 0x00, 0x02, 0x02, 0x46],
_ => &[
0x16, 0x03, 0x01, 0x00, 0x2a, 0x02, 0x00, 0x00, 0x26, 0x03, 0x01,
],
};
let _ = sock.write_all(reply).await;
}
});
let stream = TcpStream::connect(addr).await.expect("connects");
let port = baseline_port(443, Protocol::Tcp, PortState::Open);
let (responses, tunnel) =
gather(stream, 443, ServiceDetection::Probe, Egress::KERNEL, None).await;
server.abort();
let _ = port;
assert!(
tunnel.is_none(),
"nothing was tunnelled and nothing claims to be"
);
let tls = responses
.tls
.as_ref()
.expect("the port speaks TLS, which is the finding");
assert_eq!(tls.version, Some("TLSv1.0"));
assert_eq!(tls_security(tls).tls_version(), Some("TLSv1.0"));
}
#[tokio::test]
async fn a_trickling_port_cannot_hold_the_reader() {
use tokio::net::{TcpListener, TcpStream};
let listener = TcpListener::bind("127.0.0.1:0").await.expect("a socket");
let addr = listener.local_addr().expect("its address");
let server = tokio::spawn(async move {
let Ok(mut sock) = accept_from_this_process(&listener).await else {
return;
};
for _ in 0..2_000 {
if sock.write_all(b"A").await.is_err() {
return;
}
tokio::time::sleep(Duration::from_millis(40)).await;
}
});
let mut stream = TcpStream::connect(addr).await.expect("connects");
let started = std::time::Instant::now();
let read = read_document(&mut stream, PROBE_READ_TIMEOUT).await;
let held = started.elapsed();
server.abort();
assert!(read.is_some(), "what did arrive is still returned");
assert!(
held < PROBE_READ_TIMEOUT + MAX_CONTINUATION + Duration::from_secs(2),
"one trickling port held the reader for {held:?}"
);
}
#[test]
fn the_collection_budget_covers_every_path_through_gather() {
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
struct Walk(Duration, u32);
impl std::ops::Add for Walk {
type Output = Walk;
fn add(self, other: Walk) -> Walk {
Walk(self.0 + other.0, self.1 + other.1)
}
}
impl std::ops::Mul<u32> for Walk {
type Output = Walk;
fn mul(self, times: u32) -> Walk {
Walk(self.0 * times, self.1 * times)
}
}
let wait = |duration: Duration| Walk(duration, 1);
let continuation = Walk(MAX_CONTINUATION, 0);
let probes = SignatureDb::global()
.indexed_ports()
.map(|port| SignatureDb::global().tcp_probe_payloads(port).len())
.max()
.unwrap_or(0)
.max(1) as u32;
let spoke = |count: u32| {
wait(BANNER_READ_TIMEOUT) + (wait(PROBE_READ_TIMEOUT) + continuation) * count
};
let silent = |count: u32| wait(BANNER_READ_TIMEOUT) + wait(PROBE_READ_TIMEOUT) * count;
let read_once = wait(PROBE_READ_TIMEOUT) + continuation;
let rung = wait(CONNECT_RETRY_TIMEOUT);
let services = SignatureDb::global()
.indexed_ports()
.map(|port| SignatureDb::global().tcp_probe_conversations(port).count())
.max()
.unwrap_or(0)
.max(1) as u32;
let redials = rung * (services - 1);
let handshake = wait(tls::TLS_HANDSHAKE_TIMEOUT);
let retunnels = (rung + handshake) * (services - 1);
let legacy = wait(tls::LEGACY_PROBE_TIMEOUT);
let speculative = wait(tls::SPECULATIVE_TLS_TIMEOUT);
let followed = rung + read_once;
let followed_tls = rung + handshake + read_once;
let tls_then_redirect = handshake + spoke(probes) + retunnels + followed_tls;
let tls_all_three = handshake + rung + legacy + rung + spoke(probes) + redials + followed;
let tls_then_silence = handshake + rung + legacy + rung + silent(probes) + redials;
let claimed_then_tls = silent(probes)
+ redials
+ rung
+ speculative
+ spoke(probes)
+ retunnels
+ followed_tls;
let alert_then_tls = read_once + rung + speculative + spoke(1) + followed_tls;
let unclaimed_then_redirect = read_once + rung + read_once;
let refused_then_tls =
spoke(probes) + redials + rung + speculative + spoke(probes) + retunnels + followed_tls;
let refused_then_legacy = spoke(probes) + redials + rung + speculative + rung + legacy;
let asked = |count: u32| (wait(PROBE_READ_TIMEOUT) + continuation) * count;
let unasked = |count: u32| wait(PROBE_READ_TIMEOUT) * count;
let shared_spoke = unclaimed_then_redirect + rung * services + asked(probes) + followed;
let shared_silent = unasked(1) + rung * services + unasked(probes);
let universal = SignatureDb::global()
.universal_tcp_probe_payloads(0, ServiceDetection::Thorough.probe_intensity())
.len()
.max(1) as u32;
let last_resort = (rung + read_once) * universal;
let paths = [
tls_then_redirect,
tls_all_three + last_resort,
tls_then_silence + last_resort,
claimed_then_tls,
alert_then_tls,
unclaimed_then_redirect,
refused_then_tls,
refused_then_legacy,
silent(probes) + redials + rung + speculative + last_resort,
shared_spoke,
shared_spoke + rung + speculative + spoke(probes) + retunnels + followed_tls,
shared_spoke + rung + speculative + rung + legacy,
shared_silent + rung + speculative + spoke(probes) + retunnels + followed_tls,
shared_silent + rung + speculative + last_resort,
];
let longest = paths.iter().map(|walk| walk.0).max().expect("a path");
let most_waits = paths.iter().map(|walk| walk.1).max().expect("a path");
assert!(
longest < COLLECTION_BUDGET,
"the budget ({COLLECTION_BUDGET:?}) is below the longest honest path \
({longest:?}), so it would cut real scans short"
);
assert!(
most_waits <= COLLECTION_WAITS,
"the budget allows for the path {COLLECTION_WAITS} times and a walk \
waits on it {most_waits} times, so it would cut scans on a slow \
path short"
);
}
#[tokio::test]
async fn a_port_only_listened_to_is_not_dialled_again_by_an_analyzer() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("binds loopback");
let addr = listener.local_addr().expect("a local address");
let (closed, mut sent) = tokio::sync::mpsc::unbounded_channel::<usize>();
let server = tokio::spawn(async move {
while let Ok(mut sock) = accept_from_this_process(&listener).await {
let closed = closed.clone();
tokio::spawn(async move {
let _ = sock.write_all(b"SSH-2.0-OpenSSH_9.6p1 Debian-3\r\n").await;
let mut buffer = [0u8; 1024];
let mut received = 0;
while let Ok(n) = sock.read(&mut buffer).await {
if n == 0 {
break;
}
received += n;
}
let _ = closed.send(received);
});
}
});
let stream = TcpStream::connect(addr).await.expect("connects");
let port = baseline_port(22, Protocol::Tcp, PortState::Open);
let port = fingerprint_tcp_via(
stream,
port,
ServiceDetection::Banner,
Egress::KERNEL,
PathAllowance::NONE,
None,
)
.await
.port;
server.abort();
let mut per_connection = Vec::new();
let every_close = async {
while let Some(received) = sent.recv().await {
per_connection.push(received);
}
};
timeout(Duration::from_secs(60), every_close)
.await
.expect("every connection to the port closes");
assert_eq!(
port.service().map(Service::name),
Some("ssh"),
"the greeting alone names the service"
);
assert_eq!(per_connection.len(), 1, "the port was dialled again");
assert_eq!(
per_connection.iter().sum::<usize>(),
0,
"the port was sent a payload"
);
}
async fn asked_in_the_clear(number: u16) -> Vec<u8> {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("binds loopback");
let addr = listener.local_addr().expect("a local address");
let server = tokio::spawn(async move {
let mut sock = accept_from_this_process(&listener)
.await
.expect("one connection");
let mut received = Vec::new();
let _ = sock.read_to_end(&mut received).await;
received
});
let stream = TcpStream::connect(addr).await.expect("connects");
plaintext(stream, number, None, Egress::KERNEL).await;
server.await.expect("the listener finishes")
}
#[tokio::test]
async fn a_raw_print_port_probed_on_purpose_is_asked_what_an_unclaimed_port_is() {
let unclaimed = 51987;
assert!(
SignatureDb::global()
.tcp_probe_payloads(unclaimed)
.is_empty(),
"test assumes port {unclaimed} is unclaimed"
);
let expected: Vec<u8> = SignatureDb::global().generic_tcp_probe_payloads().concat();
assert!(
!expected.is_empty(),
"the corpus asks unclaimed ports nothing"
);
let (print, other) = tokio::join!(
asked_in_the_clear(crate::config::RAW_PRINT_PORTS[0]),
asked_in_the_clear(unclaimed)
);
assert_eq!(other, expected);
assert_eq!(
String::from_utf8_lossy(&print),
String::from_utf8_lossy(&expected),
"the raw-print port was asked something else"
);
}
async fn guesses_put(round_trip: Option<Duration>, detection: ServiceDetection) -> usize {
use crate::testing::loopback::accept_from_this_process;
use std::sync::atomic::AtomicUsize;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("binds loopback");
let addr = listener.local_addr().expect("a local address");
let taken = Arc::new(AtomicUsize::new(0));
let counting = Arc::clone(&taken);
let server = tokio::spawn(async move {
while let Ok(sock) = accept_from_this_process(&listener).await {
counting.fetch_add(1, Ordering::Relaxed);
drop(sock);
}
});
let first = TcpStream::connect(addr).await.expect("connects");
let dialling = Dialling {
egress: Egress::KERNEL,
path: PathAllowance::of_round_trips(round_trip),
tally: Arc::new(Tally::default()),
};
let peer = Authority::new(addr);
DIALLING
.scope(
dialling,
last_resort(first, &peer, 2222, detection, Egress::KERNEL),
)
.await;
server.abort();
taken.load(Ordering::Relaxed)
}
#[tokio::test]
async fn a_silent_port_across_a_slow_path_is_put_only_the_likeliest_guess() {
let slow = Some(Duration::from_millis(1_900));
let all = guesses_put(None, ServiceDetection::Probe).await;
assert!(all >= 2, "the premise needs guesses to leave out: {all}");
assert_eq!(guesses_put(slow, ServiceDetection::Probe).await, 1);
assert_eq!(
guesses_put(Some(Duration::from_millis(40)), ServiceDetection::Probe).await,
all
);
assert!(guesses_put(slow, ServiceDetection::Thorough).await >= all);
}
#[tokio::test]
async fn an_unrecognised_answer_reads_the_same_however_the_port_was_found() {
use crate::testing::loopback::accept_from_this_process;
let greeting = "WIDGET/4.2 ready";
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("binds loopback");
let addr = listener.local_addr().expect("a local address");
let server = tokio::spawn(async move {
while let Ok(mut sock) = accept_from_this_process(&listener).await {
tokio::spawn(async move {
let _ = sock.write_all(format!("{greeting}\r\n").as_bytes()).await;
let mut buffer = [0u8; 1024];
while sock.read(&mut buffer).await.is_ok_and(|read| read > 0) {}
});
}
});
let stream = TcpStream::connect(addr).await.expect("connects");
let seeded = baseline_port(6379, Protocol::Tcp, PortState::Open);
let found = fingerprint_tcp(stream, seeded.clone(), ServiceDetection::Probe).await;
server.abort();
let mut folded = seeded;
folded.merge(found.clone());
let read = |port: &Port| {
port.service().map(|service| {
(
service.name().to_owned(),
service.extrainfo().map(str::to_owned),
)
})
};
assert_eq!(
read(&found),
Some(("redis".to_owned(), Some(greeting.to_owned()))),
"recorded as identified"
);
assert_eq!(read(&folded), read(&found), "folded into the recorded port");
}
#[tokio::test]
async fn a_kdc_s_realm_reaches_its_host_masked_and_not_its_description() {
use crate::export::schema::HostDto;
use crate::export::{ExportOptions, Redaction};
use crate::model::host::Host;
use crate::testing::loopback::accept_from_this_process;
let realm = b"CORP.EXAMPLE";
let mut fields = vec![0xA6, 0x03, 0x02, 0x01, 68];
fields.extend_from_slice(&[0xA9, realm.len() as u8 + 2, 0x1B, realm.len() as u8]);
fields.extend_from_slice(realm);
let mut sequence = vec![0x30, fields.len() as u8];
sequence.extend_from_slice(&fields);
let mut error = vec![0x7E, sequence.len() as u8];
error.extend_from_slice(&sequence);
let mut reply = (error.len() as u32).to_be_bytes().to_vec();
reply.extend_from_slice(&error);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("binds loopback");
let addr = listener.local_addr().expect("a local address");
let server = tokio::spawn(async move {
while let Ok(mut sock) = accept_from_this_process(&listener).await {
let reply = reply.clone();
tokio::spawn(async move {
let mut buffer = [0u8; 1024];
if sock.read(&mut buffer).await.is_ok_and(|read| read > 0) {
let _ = sock.write_all(&reply).await;
}
while sock.read(&mut buffer).await.is_ok_and(|read| read > 0) {}
});
}
});
let stream = TcpStream::connect(addr).await.expect("connects");
let port = baseline_port(88, Protocol::Tcp, PortState::Open);
let found = fingerprint_tcp_detailed(stream, port, ServiceDetection::Probe).await;
server.abort();
let service = found.port.service().expect("the port is named");
assert_eq!(service.product(), Some("Kerberos KDC"));
assert!(
!format!("{service:?}").contains("CORP"),
"the realm reached the service: {service:?}"
);
let mut host = Host::new(addr.ip());
found.about_the_host.apply(&mut host);
let render = |options: ExportOptions| {
serde_json::to_value(HostDto::new(&host, &options)).expect("a host renders")
};
assert_eq!(
render(ExportOptions::new())["names"],
serde_json::json!([{"source": "kerberos", "kind": "domain", "name": "CORP.EXAMPLE"}])
);
let masked = render(ExportOptions::new().with_redaction(Redaction::Standard)).to_string();
assert!(
!masked.contains("CORP") && masked.contains(r#""source":"kerberos""#),
"the realm survived redaction: {masked}"
);
}
#[tokio::test]
async fn a_shared_port_asks_each_service_on_a_connection_of_its_own() {
use crate::testing::loopback::accept_from_this_process;
use std::sync::{Arc, Mutex};
let shared = 3000;
let db = SignatureDb::global();
assert!(
db.service_name(shared).is_none() && db.tcp_probe_conversations(shared).count() >= 2,
"test assumes {shared} is shared by several services and named by none"
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("binds loopback");
let addr = listener.local_addr().expect("a local address");
let heard: Arc<Mutex<Vec<Vec<u8>>>> = Arc::default();
let log = Arc::clone(&heard);
let server = tokio::spawn(async move {
while let Ok(mut sock) = accept_from_this_process(&listener).await {
let log = Arc::clone(&log);
tokio::spawn(async move {
let mut buffer = [0u8; 1024];
let read = sock.read(&mut buffer).await.unwrap_or(0);
let request = buffer[..read].to_vec();
let reply: &[u8] = if request.starts_with(b"GET /login ") {
b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n<title>Grafana</title>"
} else if request.starts_with(b"GET ") {
b"HTTP/1.1 404 Not Found\r\nConnection: close\r\n\r\n"
} else {
b"HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n"
};
log.lock().unwrap().push(request);
let _ = sock.write_all(reply).await;
});
}
});
let stream = TcpStream::connect(addr).await.expect("connects");
let port = baseline_port(shared, Protocol::Tcp, PortState::Open);
let found = fingerprint_tcp_detailed(stream, port, ServiceDetection::Probe).await;
server.abort();
let firsts = heard.lock().unwrap().clone();
let first_line = |bytes: &[u8]| -> Vec<u8> {
let end = bytes
.windows(2)
.position(|pair| pair == b"\r\n")
.unwrap_or(bytes.len());
bytes[..end].to_vec()
};
for probes in db.tcp_probe_conversations(shared) {
assert!(
firsts
.iter()
.any(|first| first_line(first) == first_line(&probes[0])),
"a service's question was never the first thing on a connection: \
{:?} not in {firsts:?}",
String::from_utf8_lossy(&probes[0])
);
}
assert_eq!(
found.port.service().map(Service::name),
Some("grafana"),
"the port was named by another service's refusal"
);
}
#[cfg(unix)]
#[tokio::test]
async fn a_connection_given_up_for_want_of_a_socket_marks_the_identification_starved() {
use crate::system::descriptors::testing::{
in_a_process_of_its_own, refuse_every_descriptor,
};
if !in_a_process_of_its_own(
module_path!(),
"a_connection_given_up_for_want_of_a_socket_marks_the_identification_starved",
) {
return;
}
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("binds loopback");
let open = listener.local_addr().expect("a local address");
let closed = crate::testing::loopback::refused_port(std::net::IpAddr::from([127, 0, 0, 1]));
let dial = |addr: SocketAddr| async move {
let tally = Arc::new(Tally::default());
let dialling = Dialling {
egress: Egress::KERNEL,
path: PathAllowance::NONE,
tally: Arc::clone(&tally),
};
let _ = DIALLING
.scope(
dialling,
timeout(Duration::from_millis(300), analyzer_connect(addr)),
)
.await;
tally.starved.load(Ordering::Relaxed)
};
let held = refuse_every_descriptor();
let refused_a_socket = dial(open).await;
drop(held);
let refused_by_the_port = dial(closed).await;
drop(listener);
assert!(
refused_a_socket,
"a connection the process had no socket for was read as the port's silence"
);
assert!(
!refused_by_the_port,
"a port that refused the connection was blamed on the file limit"
);
}
#[cfg(unix)]
#[tokio::test]
async fn a_udp_identification_refused_a_socket_is_starved_and_not_silent() {
use crate::system::descriptors::testing::{
in_a_process_of_its_own, refuse_every_descriptor,
};
if !in_a_process_of_its_own(
module_path!(),
"a_udp_identification_refused_a_socket_is_starved_and_not_silent",
) {
return;
}
let snmp: SocketAddr = "127.0.0.1:161".parse().expect("an address");
assert!(
!SignatureDb::global().udp_probe_payloads(161).is_empty(),
"test assumes the corpus asks port 161 something over UDP"
);
let port = || baseline_port(161, Protocol::Udp, PortState::Open);
let patience = Duration::from_millis(50);
let held = refuse_every_descriptor();
let unasked =
fingerprint_udp_within(snmp, port(), Egress::KERNEL, patience, PathAllowance::NONE)
.await;
drop(held);
let asked =
fingerprint_udp_within(snmp, port(), Egress::KERNEL, patience, PathAllowance::NONE)
.await;
let unasked = unasked.expect("a datagram never sent was read as the port's silence");
assert!(unasked.starved);
assert!(unasked.responses.is_empty());
assert!(
asked.is_none(),
"a port asked and silent was not read as silence: {asked:?}"
);
}
#[tokio::test]
async fn a_redirect_is_followed_after_the_connection_that_drew_it_is_closed() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("binds loopback");
let addr = listener.local_addr().expect("a local address");
let server = tokio::spawn(async move {
let mut first = accept_from_this_process(&listener)
.await
.expect("the first connection");
let mut buffer = [0u8; 1024];
let _ = first.read(&mut buffer).await;
let _ = first
.write_all(b"HTTP/1.1 302 Found\r\nLocation: /next\r\nContent-Length: 0\r\n\r\n")
.await;
let mut second = accept_from_this_process(&listener)
.await
.expect("the redirect followed");
let first_open = !matches!(
tokio::time::timeout(Duration::from_millis(500), first.read(&mut buffer)).await,
Ok(Ok(0))
);
let _ = second.read(&mut buffer).await;
let _ = second
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
.await;
first_open
});
let stream = TcpStream::connect(addr).await.expect("connects");
let banners = plaintext(stream, 51987, Some(&Authority::new(addr)), Egress::KERNEL)
.await
.banners;
let first_open = server.await.expect("the listener finishes");
assert_eq!(
banners.len(),
2,
"the redirect was not followed: {banners:?}"
);
assert!(
!first_open,
"the redirect was followed while the connection that drew it was held"
);
}
#[tokio::test]
async fn a_zabbix_agent_on_a_port_of_its_own_is_named_at_the_thorough_level_alone() {
let moved = 18801;
assert!(
SignatureDb::global().tcp_probe_payloads(moved).is_empty(),
"test assumes port {moved} is unclaimed"
);
let mut named = Vec::new();
for level in [ServiceDetection::Thorough, ServiceDetection::Probe] {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("binds loopback");
let addr = listener.local_addr().expect("a local address");
let framed = Arc::new(AtomicBool::new(false));
let heard = Arc::clone(&framed);
let agent = tokio::spawn(async move {
while let Ok(mut sock) = accept_from_this_process(&listener).await {
let mut buffer = [0u8; 1024];
let read = sock.read(&mut buffer).await.unwrap_or(0);
if buffer[..read].starts_with(b"ZBXD\x01") {
heard.store(true, Ordering::SeqCst);
let _ = sock.write_all(b"ZBXD\x01\x01\0\0\0\0\0\0\0\x31").await;
}
}
});
let stream = TcpStream::connect(addr).await.expect("connects");
let port = baseline_port(moved, Protocol::Tcp, PortState::Open);
let found = fingerprint_tcp_detailed(stream, port, level).await;
agent.abort();
named.push((
found
.port
.service()
.map(|service| service.name().to_owned()),
framed.load(Ordering::SeqCst),
));
}
assert_eq!(
named[0],
(Some("zabbix".to_owned()), true),
"a thorough identification did not name the agent (service, asked)"
);
assert_eq!(
named[1],
(None, false),
"the default level put a rare question to a stranger (service, asked)"
);
}
#[tokio::test]
async fn analyze_returns_none_when_no_evidence() {
assert!(
analyze(
1,
Protocol::Tcp,
None,
ResponseSet::default(),
None,
ServiceDetection::default(),
None,
)
.await
.is_none()
);
}
type Heard = Arc<std::sync::Mutex<Vec<String>>>;
async fn https_by_name(name: &str) -> (SocketAddr, Heard) {
let acceptor = crate::testing::loopback::tls_by_name(name);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("binds loopback");
let addr = listener.local_addr().expect("a local address");
let heard: Heard = Arc::default();
let record = Arc::clone(&heard);
tokio::spawn(async move {
while let Ok(stream) = accept_from_this_process(&listener).await {
let (acceptor, record) = (acceptor.clone(), Arc::clone(&record));
tokio::spawn(async move {
let mut first = [0u8; 1];
if stream.peek(&mut first).await.unwrap_or(0) == 0 {
return;
}
let mut buffer = [0u8; 2048];
if first[0] != 0x16 {
let mut stream = stream;
let read = stream.read(&mut buffer).await.unwrap_or(0);
record.lock().expect("not poisoned").push(format!(
"clear: {}",
String::from_utf8_lossy(&buffer[..read])
));
let _ = stream
.write_all(
b"HTTP/1.0 400 Bad Request\r\n\r\n\
Client sent an HTTP request to an HTTPS server.\n",
)
.await;
return;
}
let Ok(mut tls) = acceptor.accept(stream).await else {
return;
};
let read = tls.read(&mut buffer).await.unwrap_or(0);
let request = String::from_utf8_lossy(&buffer[..read]).into_owned();
let reply: &[u8] = match request.starts_with("GET / ") {
true => {
b"HTTP/1.1 302 Found\r\nServer: Caddy\r\nLocation: /web/\r\n\
Content-Length: 0\r\nConnection: close\r\n\r\n"
}
false => {
b"HTTP/1.1 200 OK\r\nServer: Caddy\r\n\
Content-Length: 0\r\nConnection: close\r\n\r\n"
}
};
record.lock().expect("not poisoned").push(request);
let _ = tls.write_all(reply).await;
let _ = tls.shutdown().await;
});
}
});
(addr, heard)
}
#[tokio::test]
async fn a_tls_port_on_a_named_address_is_asked_for_by_its_name() {
let (addr, heard) = https_by_name("box.example").await;
let stream = TcpStream::connect(addr).await.expect("connects");
let port = baseline_port(443, Protocol::Tcp, PortState::Open);
let found = fingerprint_tcp_via(
stream,
port,
ServiceDetection::Probe,
Egress::KERNEL,
PathAllowance::NONE,
Some(Arc::from("box.example")),
)
.await;
assert!(
found.port.security().is_some(),
"the handshake did not complete: {:?}",
heard.lock().expect("not poisoned")
);
let service = found.port.service().expect("a service was named");
assert_eq!(service.name(), "ssl/http");
let asked = heard.lock().expect("not poisoned").clone();
let host = format!("\r\nHost: box.example:{}\r\n", addr.port());
assert!(
asked.iter().any(|request| request.contains(&host)),
"nothing inside the tunnel asked for `{}`: {asked:?}",
host.trim()
);
}
#[tokio::test]
async fn each_service_sharing_a_tls_port_is_asked_through_a_handshake_of_its_own() {
let (addr, heard) = https_by_name("box.example").await;
let stream = TcpStream::connect(addr).await.expect("connects");
let port = baseline_port(8443, Protocol::Tcp, PortState::Open);
let found = fingerprint_tcp_via(
stream,
port,
ServiceDetection::Probe,
Egress::KERNEL,
PathAllowance::NONE,
Some(Arc::from("box.example")),
)
.await;
assert!(found.port.security().is_some(), "the handshake completed");
let asked = heard.lock().expect("not poisoned").clone();
for question in ["GET / ", "GET /api "] {
assert!(
asked.iter().any(|request| request.starts_with(question)),
"`{}` went unasked: {asked:?}",
question.trim()
);
}
}
struct Noting {
inner: tokio::io::DuplexStream,
seen: Vec<&'static str>,
}
impl AsyncRead for Noting {
fn poll_read(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
buf: &mut tokio::io::ReadBuf<'_>,
) -> std::task::Poll<std::io::Result<()>> {
if self.seen.last() != Some(&"read") {
self.seen.push("read");
}
std::pin::Pin::new(&mut self.inner).poll_read(cx, buf)
}
}
impl AsyncWrite for Noting {
fn poll_write(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
buf: &[u8],
) -> std::task::Poll<std::io::Result<usize>> {
if self.seen.last() != Some(&"write") {
self.seen.push("write");
}
std::pin::Pin::new(&mut self.inner).poll_write(cx, buf)
}
fn poll_flush(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<std::io::Result<()>> {
std::pin::Pin::new(&mut self.inner).poll_flush(cx)
}
fn poll_shutdown(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<std::io::Result<()>> {
std::pin::Pin::new(&mut self.inner).poll_shutdown(cx)
}
}
async fn first_move_on(number: u16) -> &'static str {
let (client, mut server) = tokio::io::duplex(8192);
let answering = tokio::spawn(async move {
let mut buffer = [0u8; 2048];
while server.read(&mut buffer).await.unwrap_or(0) > 0 {
let _ = server
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
.await;
}
});
let mut noting = Noting {
inner: client,
seen: Vec::new(),
};
let db = SignatureDb::global();
let probes = db.tcp_probe_payloads(number);
assert!(!probes.is_empty(), "port {number} registers probes");
let _ = collect_responses(&mut noting, number, probes, None, !db.asked_first(number)).await;
answering.abort();
noting.seen.first().copied().unwrap_or("nothing")
}
#[tokio::test]
async fn a_web_port_is_asked_before_it_is_listened_to() {
assert_eq!(
first_move_on(8080).await,
"write",
"8080 waited for a greeting"
);
assert_eq!(
first_move_on(9200).await,
"write",
"9200 waited for a greeting"
);
assert!(
!SignatureDb::global().asked_first(21),
"a port a greeting service claims is listened to first"
);
}
async fn https_off_the_list(name: Option<&str>) -> Fingerprinted {
let (addr, _heard) = https_by_name("box.example").await;
let number = 9443;
assert!(
!tls::is_tls_port(number),
"test assumes {number} is not numbered for TLS"
);
let stream = TcpStream::connect(addr).await.expect("connects");
fingerprint_tcp_via(
stream,
baseline_port(number, Protocol::Tcp, PortState::Open),
ServiceDetection::Probe,
Egress::KERNEL,
PathAllowance::NONE,
name.map(Arc::from),
)
.await
}
#[tokio::test]
async fn https_on_a_port_numbered_for_nothing_is_reached_through_its_handshake() {
let found = https_off_the_list(Some("box.example")).await;
let service = found.port.service().expect("a service was named");
assert_eq!(service.name(), "ssl/http");
assert_eq!(service.product(), Some("Caddy"));
let security = found.port.security().expect("the handshake is recorded");
assert!(
security.certificate().is_some(),
"the certificate was not read: {security:?}"
);
}
#[tokio::test]
async fn https_refusing_a_nameless_handshake_is_still_filed_as_https() {
let found = https_off_the_list(None).await;
let service = found.port.service().expect("a service was named");
assert_eq!(service.name(), "ssl/http");
assert!(
found.port.security().is_some(),
"the port's TLS went unrecorded"
);
}
#[tokio::test]
async fn https_on_its_own_port_refusing_a_nameless_handshake_is_filed_as_https() {
let (addr, _heard) = https_by_name("box.example").await;
let stream = TcpStream::connect(addr).await.expect("connects");
let found = fingerprint_tcp_via(
stream,
baseline_port(443, Protocol::Tcp, PortState::Open),
ServiceDetection::Probe,
Egress::KERNEL,
PathAllowance::NONE,
None,
)
.await;
let service = found.port.service().expect("a service was named");
assert_eq!(service.name(), "ssl/http");
assert!(!service.is_inferred(), "the name is still the number's");
}
#[tokio::test]
async fn a_redirect_is_followed_on_a_claimed_port_and_through_tls() {
let (addr, heard) = https_by_name("box.example").await;
let stream = TcpStream::connect(addr).await.expect("connects");
let found = fingerprint_tcp_via(
stream,
baseline_port(443, Protocol::Tcp, PortState::Open),
ServiceDetection::Probe,
Egress::KERNEL,
PathAllowance::NONE,
Some(Arc::from("box.example")),
)
.await;
let asked = heard.lock().expect("not poisoned").clone();
assert!(
asked
.iter()
.any(|request| request.starts_with("GET /web/ ")),
"the redirect through TLS was not followed: {asked:?}"
);
assert_eq!(found.responses.len(), 2, "{:?}", found.responses);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("binds loopback");
let clear = listener.local_addr().expect("a local address");
let server = tokio::spawn(async move {
let mut asked = Vec::new();
for reply in [
&b"HTTP/1.1 302 Found\r\nLocation: /web/\r\nContent-Length: 0\r\n\r\n"[..],
b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n",
] {
let Ok(mut sock) = accept_from_this_process(&listener).await else {
break;
};
let mut buffer = [0u8; 1024];
let read = sock.read(&mut buffer).await.unwrap_or(0);
asked.push(String::from_utf8_lossy(&buffer[..read]).into_owned());
let _ = sock.write_all(reply).await;
}
asked
});
let number = 8080;
assert!(!SignatureDb::global().tcp_probe_payloads(number).is_empty());
let stream = TcpStream::connect(clear).await.expect("connects");
let banners = plaintext(stream, number, Some(&Authority::new(clear)), Egress::KERNEL)
.await
.banners;
let asked = server.await.expect("the listener finishes");
assert!(
asked
.iter()
.any(|request| request.starts_with("GET /web/ ")),
"the redirect in the clear was not followed: {asked:?}"
);
assert_eq!(banners.len(), 2, "{banners:?}");
}
#[tokio::test]
async fn an_https_port_is_asked_for_its_icon_through_tls() {
let (addr, heard) = https_by_name("box.example").await;
let stream = TcpStream::connect(addr).await.expect("connects");
let _ = fingerprint_tcp_via(
stream,
baseline_port(443, Protocol::Tcp, PortState::Open),
ServiceDetection::Probe,
Egress::KERNEL,
PathAllowance::NONE,
Some(Arc::from("box.example")),
)
.await;
let asked = heard.lock().expect("not poisoned").clone();
assert!(
!asked.iter().any(|request| request.starts_with("clear:")),
"an HTTPS port was asked in the clear: {asked:?}"
);
assert!(
asked
.iter()
.any(|request| request.starts_with("GET /favicon.ico ")),
"the icon was not asked for through TLS: {asked:?}"
);
}
}