use crate::fingerprint::{MAX_COMPILED_REGEX_BYTES, pattern, unescape};
use crate::model::confidence::Confidence;
use crate::model::finding::{DetectionId, Excerpt, Finding, Version};
use crate::record::wire;
use super::schema::{FindingSpec, FlowDetection, MatchSpec, OnNoMatch, Step};
use super::schema::{MAX_FLOW_STEPS, MAX_LOOP_ITEMS, SEED_VAR_HOST, SEED_VAR_PORT};
use super::{Env, eval};
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ProbeRefusal {
Bytes,
Connections,
Deadline,
Descriptors,
}
pub trait Probe: Send {
fn speak(&mut self, bytes: &[u8]) -> Option<Vec<u8>>;
fn reply_complete(&self) -> bool {
true
}
fn plan(&mut self, exchanges: u32) {
let _ = exchanges;
}
fn last_refusal(&self) -> Option<ProbeRefusal> {
None
}
}
#[derive(PartialEq, Eq)]
enum Flow {
Continue,
Halt,
}
#[non_exhaustive]
#[derive(Debug, Clone)]
pub struct FlowSeed {
pub host: String,
pub port: u16,
}
impl FlowSeed {
pub fn new(host: impl Into<String>, port: u16) -> Self {
Self {
host: host.into(),
port,
}
}
fn seed(&self, env: &mut Env) {
env.insert(SEED_VAR_HOST.to_string(), self.host.clone());
env.insert(SEED_VAR_PORT.to_string(), self.port.to_string());
}
}
pub fn run(
flow: &FlowDetection,
content_hash: &str,
seed: &FlowSeed,
probe: &mut dyn Probe,
) -> Vec<Finding> {
let mut env = Env::new();
seed.seed(&mut env);
let mut findings = Vec::new();
probe.plan(exchanges(flow));
for step in flow.step.iter().take(MAX_FLOW_STEPS) {
match &step.for_each {
Some(for_each) => {
for item in for_each.items.iter().take(MAX_LOOP_ITEMS) {
let mut local = env.clone();
local.insert(for_each.var.clone(), item.clone());
if run_step(flow, content_hash, step, &mut local, probe, &mut findings)
== Flow::Halt
{
return findings;
}
}
}
None => {
if run_step(flow, content_hash, step, &mut env, probe, &mut findings) == Flow::Halt
{
return findings;
}
}
}
}
findings
}
pub(crate) fn exchanges(flow: &FlowDetection) -> u32 {
let sends: usize = flow
.step
.iter()
.take(MAX_FLOW_STEPS)
.filter(|step| step.send.is_some())
.map(|step| {
step.for_each
.as_ref()
.map_or(1, |for_each| for_each.items.len().min(MAX_LOOP_ITEMS))
})
.sum();
u32::try_from(sends).unwrap_or(u32::MAX)
}
fn run_step(
flow: &FlowDetection,
content_hash: &str,
step: &Step,
env: &mut Env,
probe: &mut dyn Probe,
findings: &mut Vec<Finding>,
) -> Flow {
if !eval::holds(step.when.as_deref(), env, None) {
return Flow::Continue;
}
let response = match &step.send {
Some(send) => match interpolate(send, env) {
Some(text) => probe.speak(&unescape(&text)).map(|reply| latin1(&reply)),
None => return on_no_match(step),
},
None => None,
};
if let Some(response) = &response {
for (name, spec) in &step.bind {
if let Some(value) = capture(spec, response, name) {
env.insert(name.clone(), value);
}
}
}
let matched = match &response {
Some(response) => step.expect.iter().all(|rule| matches(rule, response)),
None => step.expect.is_empty(),
};
if !matched && step.on_no_match == OnNoMatch::Halt {
return Flow::Halt;
}
for spec in &step.finding {
if eval::holds(spec.when.as_deref(), env, Some(matched))
&& let Some(finding) = build_finding(flow, content_hash, spec, env, response.as_deref())
{
findings.push(finding);
}
}
Flow::Continue
}
fn on_no_match(step: &Step) -> Flow {
match step.on_no_match {
OnNoMatch::Halt => Flow::Halt,
OnNoMatch::Continue => Flow::Continue,
}
}
fn matches(spec: &MatchSpec, text: &str) -> bool {
pattern::compile(spec.pattern(), MAX_COMPILED_REGEX_BYTES)
.is_ok_and(|compiled| compiled.identify(text, spec.version_group()).is_some())
}
pub(crate) fn check_patterns(flow: &FlowDetection) -> Result<(), String> {
for (index, step) in flow.step.iter().enumerate() {
for spec in &step.expect {
pattern::compile(spec.pattern(), MAX_COMPILED_REGEX_BYTES).map_err(|error| {
format!("step {index} `expect` has a pattern that will not compile: {error}")
})?;
}
for (var, spec) in &step.bind {
let compiled =
pattern::compile(spec.pattern(), MAX_COMPILED_REGEX_BYTES).map_err(|error| {
format!(
"step {index} bind `{var}` has a pattern that will not compile: {error}"
)
})?;
let named = compiled.capture_names().iter().any(|name| name == var);
let numbered = spec
.version_group()
.is_some_and(|group| (group as usize) < compiled.captures_len());
if !named && !numbered {
return Err(format!(
"step {index} binds `{var}`, but its pattern has no (?<{var}>…) group and \
no valid version_group, so it can never capture"
));
}
}
}
Ok(())
}
fn capture(spec: &MatchSpec, text: &str, name: &str) -> Option<String> {
let compiled = pattern::compile(spec.pattern(), MAX_COMPILED_REGEX_BYTES).ok()?;
compiled.capture(text, name).or_else(|| {
spec.version_group()
.and_then(|group| compiled.identify(text, Some(group)).and_then(|m| m.version))
})
}
fn build_finding(
flow: &FlowDetection,
content_hash: &str,
spec: &FindingSpec,
env: &Env,
response: Option<&str>,
) -> Option<Finding> {
let version = flow
.detection
.version
.parse()
.unwrap_or(Version::new(0, 0, 0));
let detection = DetectionId::new(flow.detection.id.clone(), version, content_hash).ok()?;
let title = interpolate(spec.title.as_deref().unwrap_or(spec.summary.as_str()), env)?;
let severity = spec.severity.into_model();
let confidence = spec
.confidence
.as_deref()
.and_then(wire::confidence)
.unwrap_or(Confidence::Certain);
let class = flow.detection.capabilities.class.into_model();
let mut finding = Finding::new(detection, title, severity, confidence, class).ok()?;
let excerpt = match spec.excerpt_from.as_deref() {
Some("$response") => response.map(str::to_owned),
Some(name) => env.get(name).cloned(),
None => match &spec.detail {
Some(detail) => interpolate(detail, env),
None => response.map(str::to_owned),
},
};
if let Some(excerpt) = excerpt {
finding = finding.with_excerpt(Excerpt::new(excerpt));
}
for reference in &spec.references {
if let Some(reference) = reference.to_model() {
finding = finding.with_reference(reference);
}
}
if let Some(remediation) = &spec.remediation {
finding = finding.with_remediation(remediation.clone());
}
Some(finding)
}
fn interpolate(template: &str, env: &Env) -> Option<String> {
let mut out = String::with_capacity(template.len());
let mut rest = template;
while let Some(brace) = rest.find(['{', '}']) {
out.push_str(&rest[..brace]);
let this = rest.as_bytes()[brace];
let after = &rest[brace + 1..];
if after.as_bytes().first() == Some(&this) {
out.push(this as char);
rest = &after[1..];
} else if this == b'{' {
let close = after.find('}')?;
out.push_str(env.get(&after[..close])?);
rest = &after[close + 1..];
} else {
out.push('}');
rest = after;
}
}
out.push_str(rest);
Some(out)
}
fn latin1(bytes: &[u8]) -> String {
bytes.iter().map(|&byte| byte as char).collect()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::model::finding::{Reference, Severity};
fn flow(id: &str) -> FlowDetection {
crate::detect::flow::db::shipped_flow(id)
}
fn seed() -> FlowSeed {
FlowSeed::new("192.0.2.10", 80)
}
struct Canned(Vec<u8>);
impl Probe for Canned {
fn speak(&mut self, _bytes: &[u8]) -> Option<Vec<u8>> {
Some(self.0.clone())
}
}
struct Echo {
sent: Vec<Vec<u8>>,
reply: Vec<u8>,
}
impl Probe for Echo {
fn speak(&mut self, bytes: &[u8]) -> Option<Vec<u8>> {
self.sent.push(bytes.to_vec());
Some(self.reply.clone())
}
}
#[test]
fn a_send_template_resolves_the_seeded_host_and_port() {
let toml = r#"
[detection]
id = "seed-echo"
version = "1.0.0"
title = "seed echo"
[detection.when]
service = "http"
[detection.capabilities]
class = "active-benign"
speak = "target"
[[step]]
send = "GET / HTTP/1.1\r\nHost: {host}:{port}\r\n\r\n"
expect = "200 OK"
"#;
let flow: FlowDetection = toml::from_str(toml).expect("a valid flow");
let mut probe = Echo {
sent: Vec::new(),
reply: b"HTTP/1.1 200 OK\r\n\r\n".to_vec(),
};
run(&flow, "", &FlowSeed::new("198.51.100.7", 8443), &mut probe);
assert_eq!(
probe.sent.first().map(|bytes| latin1(bytes)),
Some("GET / HTTP/1.1\r\nHost: 198.51.100.7:8443\r\n\r\n".to_string()),
"the seeded host and port replaced the template, not a fixed localhost"
);
}
#[test]
fn the_phase_one_flows_fire_on_a_confirming_reply() {
let cases: &[(&str, &[u8], Severity)] = &[
(
"memcached-unauth",
b"STAT pid 1234\r\nSTAT version 1.6.21\r\nEND\r\n",
Severity::High,
),
(
"couchdb-open",
b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n[\"_users\",\"_replicator\"]",
Severity::High,
),
(
"elasticsearch-open",
b"HTTP/1.1 200 OK\r\n\r\n{\"cluster_name\":\"prod\",\"version\":{\"number\":\"8.11.0\"}}",
Severity::High,
),
(
"http-git-exposed",
b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\nref: refs/heads/main\n",
Severity::High,
),
(
"http-server-status",
b"HTTP/1.1 200 OK\r\n\r\n<html><head><title>Apache Server Status for host</title>",
Severity::Medium,
),
(
"http-dotenv-exposed",
b"HTTP/1.1 200 OK\r\n\r\nAPP_KEY=base64:abcd\nDB_PASSWORD=hunter2\n",
Severity::High,
),
(
"mongodb-unauth",
b"\x00\x00\x00\x00...sizeOnDisk\x00...admin\x00",
Severity::High,
),
(
"http-spring-actuator",
b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{\"_links\":{\"self\":{\"href\":\"http://h/actuator\"},\"env\":{\"href\":\"http://h/actuator/env\"}}}",
Severity::High,
),
(
"http-dir-listing",
b"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<title>Index of /</title><h1>Index of /</h1>",
Severity::Low,
),
(
"anonymous-ftp",
b"220 Zond FTP\r\n331 Please specify the password.\r\n230 Login successful.\r\n221 Goodbye.\r\n",
Severity::Medium,
),
(
"ldap-anonymous-bind",
b"\x30\x0c\x02\x01\x01\x61\x07\x0a\x01\x00\x04\x00\x04\x00",
Severity::Medium,
),
(
"vnc-noauth",
b"RFB 003.008\n\x01\x01",
Severity::Critical,
),
(
"dns-version-bind",
b"\x00\x2c\x13\x37\x84\x00\x00\x01\x00\x01\x00\x00\x00\x00\x07version\x04bind\x00\x00\x10\x00\x03\xc0\x0c\x00\x10\x00\x03\x00\x00\x00\x00\x00\x0d\x0c9.16.1-Debian",
Severity::Info,
),
(
"docker-api-unauth",
b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{\"Version\":\"24.0.6\",\"ApiVersion\":\"1.43\",\"Os\":\"linux\"}",
Severity::Critical,
),
(
"k8s-api-anonymous",
b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{\"kind\":\"NamespaceList\",\"items\":[{\"metadata\":{\"name\":\"default\"}}]}",
Severity::High,
),
(
"jenkins-unauth",
b"HTTP/1.1 200 OK\r\nX-Jenkins: 2.426.1\r\nContent-Type: application/json\r\n\r\n{\"_class\":\"hudson.model.Hudson\"}",
Severity::High,
),
(
"phpmyadmin-exposed",
b"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<html><head><title>phpMyAdmin</title></head></html>",
Severity::Medium,
),
(
"etcd-unauth",
b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{\"etcdserver\":\"3.5.9\",\"etcdcluster\":\"3.5.0\"}",
Severity::High,
),
(
"consul-no-acl",
b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{\"consul\":[],\"redis\":[\"primary\"]}",
Severity::High,
),
(
"influxdb-noauth",
b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{\"results\":[{\"statement_id\":0,\"series\":[{\"name\":\"databases\",\"values\":[[\"_internal\"]]}]}]}",
Severity::High,
),
(
"nomad-no-acl",
b"HTTP/1.1 200 OK\r\nX-Nomad-Index: 42\r\nContent-Type: application/json\r\n\r\n[]",
Severity::High,
),
(
"docker-registry-catalog",
b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{\"repositories\":[\"alpine\",\"nginx\"]}",
Severity::High,
),
(
"kubelet-pods-exposed",
b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{\"kind\":\"PodList\",\"items\":[]}",
Severity::High,
),
(
"clickhouse-noauth",
b"HTTP/1.1 200 OK\r\nX-ClickHouse-Query-Id: q1\r\nContent-Type: text/tab-separated-values\r\n\r\n1\n",
Severity::High,
),
(
"prometheus-open",
b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{\"status\":\"success\",\"data\":{\"activeTargets\":[]}}",
Severity::Medium,
),
(
"kibana-open",
b"HTTP/1.1 200 OK\r\nkbn-name: kibana\r\nContent-Type: application/json\r\n\r\n{\"status\":{\"overall\":{}}}",
Severity::Medium,
),
(
"riak-open",
b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{\"riak_kv_version\":\"3.0.0\",\"ring_members\":[\"riak@127.0.0.1\"]}",
Severity::High,
),
(
"rsync-exposed",
b"@RSYNCD: 31.0\ndata\tproject files\nbackup\tnightly dumps\n@RSYNCD: EXIT\n",
Severity::Medium,
),
(
"zookeeper-4lw",
b"Zookeeper version: 3.8.0-abc, built on 2024-01-01\nClients:\n /192.0.2.9:52111[1]\nMode: standalone\n",
Severity::Medium,
),
(
"mqtt-anonymous",
b"\x20\x02\x00\x00",
Severity::High,
),
];
for (name, reply, severity) in cases {
let flow = flow(name);
let findings = run(&flow, "", &seed(), &mut Canned(reply.to_vec()));
assert_eq!(
findings.len(),
1,
"{name} drew no finding on a confirming reply"
);
assert_eq!(findings[0].severity(), *severity, "{name} graded wrong");
}
}
#[test]
fn no_flow_fires_on_a_generic_404() {
let quiet = b"HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\n<html><body>not found</body></html>";
for flow in crate::detect::flow::db::FlowDb::global().flows() {
let findings = flow.run(&seed(), &mut Canned(quiet.to_vec()));
assert!(
findings.is_empty(),
"{} fired on a bare 404",
flow.flow().detection.id
);
}
}
#[test]
fn a_panel_that_requires_authentication_does_not_fire() {
let cases: &[(&str, &[u8])] = &[
(
"jenkins-unauth",
b"HTTP/1.1 403 Forbidden\r\nX-Jenkins: 2.426.1\r\nContent-Type: text/html\r\n\r\nAuthentication required",
),
(
"k8s-api-anonymous",
b"HTTP/1.1 403 Forbidden\r\nContent-Type: application/json\r\n\r\n{\"kind\":\"Status\",\"status\":\"Failure\",\"reason\":\"Forbidden\"}",
),
];
for (name, reply) in cases {
let flow = flow(name);
let findings = run(&flow, "", &seed(), &mut Canned(reply.to_vec()));
assert!(
findings.is_empty(),
"{name} fired on a 403 that denied access"
);
}
}
#[test]
fn the_enumeration_flows_stay_quiet_when_the_service_denies_access() {
let cases: &[(&str, &[u8])] = &[
(
"anonymous-ftp",
b"220 Zond FTP\r\n331 Please specify the password.\r\n530 Login incorrect.\r\n",
),
(
"ldap-anonymous-bind",
b"\x30\x0c\x02\x01\x01\x61\x07\x0a\x01\x30\x04\x00\x04\x00",
),
("vnc-noauth", b"RFB 003.008\n\x01\x02"),
(
"dns-version-bind",
b"\x00\x1e\x13\x37\x84\x05\x00\x01\x00\x00\x00\x00\x00\x00\x07version\x04bind\x00\x00\x10\x00\x03",
),
];
for (name, reply) in cases {
let flow = flow(name);
let findings = run(&flow, "", &seed(), &mut Canned(reply.to_vec()));
assert!(
findings.is_empty(),
"{name} fired on a service that denied access"
);
}
}
#[test]
fn the_redis_flow_runs_and_produces_a_finding() {
let redis = flow("redis-unauth-access");
let mut probe = Canned(b"# Server\r\nredis_version:7.2.4\r\nrun_id:abc".to_vec());
let findings = run(&redis, "", &seed(), &mut probe);
assert_eq!(findings.len(), 1);
let finding = &findings[0];
assert_eq!(finding.detection().id(), "redis-unauth-access");
assert_eq!(finding.severity(), Severity::High);
assert_eq!(
finding.title(),
"Redis answered INFO without authentication"
);
assert_eq!(
finding.excerpt().as_str(),
"Server version 7.2.4 is reachable without a password."
);
assert!(
finding
.references()
.any(|r| matches!(r, Reference::Cwe(306)))
);
}
#[test]
fn a_gate_that_does_not_match_halts_and_emits_nothing() {
let redis = flow("redis-unauth-access");
let mut probe = Canned(b"-ERR NOAUTH Authentication required".to_vec());
assert!(run(&redis, "", &seed(), &mut probe).is_empty());
}
struct Snmp;
impl Probe for Snmp {
fn speak(&mut self, bytes: &[u8]) -> Option<Vec<u8>> {
let sys_descr_oid: &[u8] = &[0x2b, 0x06, 0x01, 0x02, 0x01, 0x01, 0x01, 0x00];
let well_formed = bytes.first() == Some(&0x30) && contains(bytes, sys_descr_oid);
let community_public = contains(bytes, b"\x04\x06public");
if well_formed && community_public {
let mut reply = vec![0xa2];
reply.extend_from_slice(b" sysDescr: Linux router 6.1");
Some(reply)
} else {
Some(b"\xa3 wrong community".to_vec())
}
}
}
#[test]
fn the_snmp_flow_probes_each_community_and_flags_the_one_that_answers() {
let snmp = flow("snmp-default-community");
let findings = run(&snmp, "", &seed(), &mut Snmp);
assert_eq!(findings.len(), 1);
let finding = &findings[0];
assert_eq!(finding.severity(), Severity::High);
assert!(
finding.title().contains("public"),
"the finding should name the community that answered, got {:?}",
finding.title()
);
assert!(finding.excerpt().as_str().contains("Linux router 6.1"));
}
fn contains(haystack: &[u8], needle: &[u8]) -> bool {
haystack
.windows(needle.len())
.any(|window| window == needle)
}
struct Grafana {
banner: &'static [u8],
leak: &'static [u8],
}
impl Probe for Grafana {
fn speak(&mut self, bytes: &[u8]) -> Option<Vec<u8>> {
if contains(bytes, b"/login") {
Some(self.banner.to_vec())
} else if contains(bytes, b"etc/passwd") {
Some(self.leak.to_vec())
} else {
None
}
}
}
#[test]
fn the_conditional_step_confirms_a_leak_on_a_vulnerable_server() {
let grafana = flow("grafana-path-traversal");
let mut probe = Grafana {
banner: b"HTTP/1.1 200 OK\r\nX-Grafana: Grafana v8.2.0\r\n\r\n",
leak: b"HTTP/1.1 200 OK\r\n\r\nroot:x:0:0:root:/root:/bin/bash\n",
};
let findings = run(&grafana, "", &seed(), &mut probe);
assert_eq!(findings.len(), 1);
let finding = &findings[0];
assert_eq!(finding.severity(), Severity::Critical);
assert!(
finding.excerpt().as_str().contains("root:x:0:0:"),
"the excerpt is the leaked passwd line, got {:?}",
finding.excerpt().as_str()
);
assert!(
finding
.references()
.any(|r| matches!(r, Reference::Cve(id) if id == "CVE-2021-43798"))
);
}
#[test]
fn an_affected_version_whose_leak_is_blocked_is_still_flagged() {
let grafana = flow("grafana-path-traversal");
let mut probe = Grafana {
banner: b"HTTP/1.1 200 OK\r\nX-Grafana: Grafana v8.2.0\r\n\r\n",
leak: b"HTTP/1.1 403 Forbidden\r\n\r\n",
};
let findings = run(&grafana, "", &seed(), &mut probe);
assert_eq!(findings.len(), 1);
assert_eq!(findings[0].severity(), Severity::Medium);
}
#[test]
fn a_patched_or_unrelated_server_never_reaches_the_exploit_step() {
let grafana = flow("grafana-path-traversal");
let mut patched = Grafana {
banner: b"HTTP/1.1 200 OK\r\nX-Grafana: Grafana v8.10.0\r\n\r\n",
leak: b"root:x:0:0:should-never-be-sent",
};
assert!(run(&grafana, "", &seed(), &mut patched).is_empty());
let mut other = Grafana {
banner: b"HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\n",
leak: b"root:x:0:0:should-never-be-sent",
};
assert!(run(&grafana, "", &seed(), &mut other).is_empty());
}
#[test]
fn run_does_not_probe_past_the_step_ceiling() {
let mut probes = 0usize;
struct Counting<'a>(&'a mut usize);
impl Probe for Counting<'_> {
fn speak(&mut self, _bytes: &[u8]) -> Option<Vec<u8>> {
*self.0 += 1;
Some(b"x".to_vec())
}
}
let mut source = String::from(
"[detection]\nid = \"x\"\nversion = \"1.0.0\"\ntitle = \"x\"\n\
[detection.when]\n[detection.capabilities]\nclass = \"active-benign\"\nspeak = \"target\"\n",
);
for _ in 0..(MAX_FLOW_STEPS + 50) {
source.push_str("[[step]]\nsend = \"p\"\non_no_match = \"continue\"\n");
}
let flow: FlowDetection = toml::from_str(&source).expect("a parseable flow");
run(&flow, "", &seed(), &mut Counting(&mut probes));
assert_eq!(
probes, MAX_FLOW_STEPS,
"run probed a flow past the {MAX_FLOW_STEPS}-step ceiling"
);
}
}