zond-engine 0.16.0

The core networking library powering Zond. Raw packet injection, protocol parsing, service and OS fingerprinting, and asynchronous network scanning.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
// Copyright (c) 2026 Erik Lening (hollowpointer) and Contributors
//
// This file is part of Zond Engine, licensed under the GNU Affero General
// Public License, version 3 or later. See the LICENSE file for details, or
// <https://www.gnu.org/licenses/agpl-3.0.html>.
//
// SPDX-License-Identifier: AGPL-3.0-or-later

//! # Fingerprinting corpus regression tests
//!
//! Locks matching behaviour against silent regression, in four layers:
//!
//! 1. **Self-consistency** ([`every_signature_matches_its_example`]): 95% of
//!    signature rules ship a recorded `example` banner they are meant to match.
//!    Every example is run through the real signature and the count of
//!    non-matching examples is pinned to a baseline.
//! 2. **Prefilter soundness** ([`prefilter_never_drops_a_matching_signature`]):
//!    for every example that matches its pattern, the global-match prefilter
//!    must select that signature as a candidate. This is what makes it safe to
//!    narrow the global set instead of scanning all of it.
//! 3. **One reply, one witness** ([`one_reply_is_one_witness`]): whatever a
//!    rule's example leaves on a host's record, it leaves under the source it
//!    was read as, so the arithmetic that combines sources never counts one
//!    reading as two witnesses.
//! 4. **Golden end-to-end** ([`golden_cases_resolve_end_to_end`],
//!    [`non_standard_port_is_identified_via_global_fallback`]): real banners
//!    driven through the whole pipeline with the exact verdict pinned.
//!
//! ## Known baseline
//!
//! 218 recorded examples do not match their own pattern, overwhelmingly case
//! mismatches (`"MIPS"` against `mips`, `"FTP server"` against `FTP Server`)
//! from imported rapid7/recog signatures whose per-pattern case flag was dropped
//! on import.
//! Restoring it takes a re-import that preserves `flags`, not a blanket
//! case-fold; the baseline is pinned so it cannot grow.

use proptest::prelude::*;
use rayon::prelude::*;

use super::db::SignatureDb;
use super::matcher::Signature;
use super::prefilter::{LiteralPrefilter, Prefilter};
use super::response::{Collected, ResponseSet, TlsInfo};
use super::{Analyzer, BannerRegexAnalyzer, PortContext, ServiceVerdict, TlsCertAnalyzer, Tunnel};
use crate::model::confidence::Confidence;

/// Recorded examples that do not match their own pattern today, from lost recog
/// case flags; see the module docs. Ratchet down as fixed, since a rise is a
/// regression.
const KNOWN_EXAMPLE_MISMATCHES: usize = 218;

/// The signature set flattened exactly as the runtime builds it, paired with
/// each signature's recorded example (if any).
fn signatures_with_examples() -> (Vec<Signature>, Vec<Option<String>>) {
    let defs = SignatureDb::embedded_definitions();
    let mut signatures = Vec::new();
    let mut examples = Vec::new();
    for def in &defs {
        for rule in &def.r#match {
            signatures.push(Signature::new(&def.service.name, rule));
            examples.push(rule.example.clone().filter(|e| !e.is_empty()));
        }
    }
    (signatures, examples)
}

#[test]
fn every_signature_matches_its_example() {
    let (signatures, examples) = signatures_with_examples();

    let mut mismatches: Vec<String> = signatures
        .par_iter()
        .zip(examples.par_iter())
        .filter_map(|(signature, example)| {
            let example = example.as_deref()?;
            signature
                .identify(example, crate::model::host::OsSource::ServiceBanner)
                .is_none()
                .then(|| format!("example={example:?} pattern={:?}", signature.pattern()))
        })
        .collect();
    mismatches.sort();

    assert_eq!(
        mismatches.len(),
        KNOWN_EXAMPLE_MISMATCHES,
        "example-match count changed (found {}, baseline {KNOWN_EXAMPLE_MISMATCHES}). If you \
         changed signatures, review the delta and update KNOWN_EXAMPLE_MISMATCHES.\nFirst:\n{}",
        mismatches.len(),
        mismatches
            .iter()
            .take(20)
            .cloned()
            .collect::<Vec<_>>()
            .join("\n"),
    );
}

#[test]
fn prefilter_never_drops_a_matching_signature() {
    let (signatures, examples) = signatures_with_examples();
    let prefilter = LiteralPrefilter::build(&signatures);

    // For every example that genuinely matches its signature, the prefilter must
    // list that signature as a candidate, or global matching would miss it.
    let violations: usize = signatures
        .par_iter()
        .zip(examples.par_iter())
        .enumerate()
        .filter(|(idx, (signature, example))| {
            let Some(example) = example.as_deref() else {
                return false;
            };
            if signature
                .identify(example, crate::model::host::OsSource::ServiceBanner)
                .is_none()
            {
                return false; // example doesn't match anyway (known baseline)
            }
            !prefilter.candidates(example).contains(idx)
        })
        .count();

    assert_eq!(
        violations,
        0,
        "prefilter dropped {}",
        crate::logging::counted(
            violations as u128,
            "matching signature",
            "matching signatures"
        )
    );
}

/// One reply is one witness, whatever the rule that read it says about the
/// machine.
///
/// Every shipped rule's example, read as each kind of text a rule is matched
/// against, and filed with a host the way a reply's reading is: what it implies
/// about the system and the hardware it describes, together, on a host with no
/// address behind it and no name. Whatever that leaves on record has to be
/// filed under the source that was read. Filed under another, the reply stands
/// in for a witness that said nothing, and beside its own reading it is counted
/// twice, which the arithmetic that combines sources takes for two witnesses
/// agreeing: a vendor a service described, read back as its address's own,
/// carries a single SNMP description past the confidence at which the active
/// OS probe is skipped.
///
/// What it sees is what one reply produces. A scanner joining two exchanges,
/// such as a name one of them learned and a record the other fetched under
/// that name, is outside any sweep of single replies, and each such join is
/// pinned where it is made.
#[test]
fn one_reply_is_one_witness() {
    use crate::model::host::{Host, OsSource};
    use std::collections::BTreeSet;

    let (signatures, examples) = signatures_with_examples();
    let read_as = [
        OsSource::ServiceBanner,
        OsSource::SnmpAgent,
        OsSource::MdnsResponder,
    ];

    let mut misfiled: Vec<String> = signatures
        .par_iter()
        .zip(examples.par_iter())
        .filter_map(|(signature, example)| Some((signature, example.as_deref()?)))
        .flat_map_iter(|(signature, example)| {
            read_as.into_iter().filter_map(move |source| {
                let matched = signature.identify(example, source)?;
                let mut host = Host::new("192.0.2.1".parse().expect("a literal address"));
                super::AboutTheHost {
                    os: matched.os.into_iter().collect(),
                    hardware: matched.hardware,
                }
                .apply(&mut host);

                let filed: BTreeSet<OsSource> =
                    host.os_evidence().map(|evidence| evidence.source).collect();
                filed
                    .iter()
                    .any(|other| *other != source)
                    .then(|| format!("{example:?} read as {source:?} left {filed:?}"))
            })
        })
        .collect();
    misfiled.sort();

    assert!(
        misfiled.is_empty(),
        "{} left a reading under a source they were not read as:\n{}",
        crate::logging::counted(misfiled.len() as u128, "example", "examples"),
        misfiled
            .iter()
            .take(20)
            .cloned()
            .collect::<Vec<_>>()
            .join("\n"),
    );
}

#[test]
fn golden_cases_resolve_end_to_end() {
    struct Case {
        port: u16,
        response: &'static str,
        service: &'static str,
        product: Option<&'static str>,
        version: Option<&'static str>,
    }

    let cases = [
        Case {
            port: 22,
            response: "SSH-2.0-OpenSSH_9.6p1 Debian-3",
            service: "ssh",
            product: Some("OpenSSH"),
            version: Some("9.6p1"),
        },
        Case {
            port: 22,
            response: "SSH-2.0-dropbear_2022.83",
            service: "ssh",
            product: Some("dropbear"),
            version: Some("2022.83"),
        },
        // Best-match, not first-match: the generic `HTTP/1.1` signature is
        // listed before the `Server: nginx` one and matches this response too,
        // but the more specific match, naming product and version, has to win.
        Case {
            port: 80,
            response: "HTTP/1.1 200 OK\r\nServer: nginx/1.25.3\r\nContent-Type: text/html\r\n\r\n",
            service: "http",
            product: Some("nginx"),
            version: Some("1.25.3"),
        },
    ];

    for case in cases {
        let responses = ResponseSet::from_banners(vec![case.response.to_string()]);
        let evidence = BannerRegexAnalyzer.analyze(
            &PortContext {
                port: case.port,
                protocol: crate::model::port::Protocol::Tcp,
                addr: None,
                tunnel: None,
                speaks_http: false,
                detection: crate::config::ServiceDetection::default(),
            },
            &responses,
            &Collected::default(),
        );
        let verdict = ServiceVerdict::resolve(evidence);

        assert_eq!(
            verdict.service.as_deref(),
            Some(case.service),
            "service for {:?}",
            case.response
        );
        if let Some(product) = case.product {
            assert_eq!(
                verdict.product.as_deref(),
                Some(product),
                "product for {:?}",
                case.response
            );
        }
        if let Some(version) = case.version {
            assert_eq!(
                verdict.version.as_deref(),
                Some(version),
                "version for {:?}",
                case.response
            );
        }
    }
}

#[test]
fn non_standard_port_is_identified_via_global_fallback() {
    // SSH on an unclaimed high port: no linked signatures, so identification
    // must come from the prefilter-narrowed global fallback.
    let port = 51987;
    assert!(
        SignatureDb::global().signatures_for_port(port).is_empty(),
        "test assumes port {port} is unclaimed"
    );

    let responses = ResponseSet::from_banners(vec!["SSH-2.0-OpenSSH_9.6p1".to_string()]);
    let evidence = BannerRegexAnalyzer.analyze(
        &PortContext {
            protocol: crate::model::port::Protocol::Tcp,
            port,
            addr: None,
            tunnel: None,
            speaks_http: false,
            detection: crate::config::ServiceDetection::default(),
        },
        &responses,
        &Collected::default(),
    );
    // A global-fallback match is not corroborated by the port.
    assert!(
        evidence.iter().all(|e| !e.port_confirmed),
        "global-fallback evidence must not be port-confirmed"
    );
    let verdict = ServiceVerdict::resolve(evidence);

    assert_eq!(verdict.service.as_deref(), Some("ssh"));
    assert_eq!(verdict.product.as_deref(), Some("OpenSSH"));
    assert_eq!(verdict.version.as_deref(), Some("9.6p1"));
}

proptest! {
    /// The headline fuzz target: adversarial banners driven through the whole
    /// banner-matching pipeline against the real signature set, so prefilter,
    /// then regex, then best-match, then resolve. It never panics and always
    /// terminates, whatever bytes arrive on the wire. Port 80 exercises the
    /// port-linked tier; unmatched banners fall through to the global prefilter,
    /// so both matching paths are covered.
    #[test]
    fn banner_pipeline_never_panics_on_adversarial_input(banner in "(?s).*") {
        let responses = ResponseSet::from_banners(vec![banner]);
        let ctx = PortContext {
            port: 80,
            protocol: crate::model::port::Protocol::Tcp,
            addr: None,
            tunnel: None,
            speaks_http: false,
            detection: crate::config::ServiceDetection::default(),
        };
        let evidence = BannerRegexAnalyzer.analyze(&ctx, &responses, &Collected::default());
        let _ = ServiceVerdict::resolve(evidence);
    }
}

#[test]
fn port_linked_match_is_tagged_port_confirmed() {
    // The same SSH banner on port 22 matches a signature registered for the
    // port, so its evidence is port-confirmed, the flag the resolver ranks on.
    let responses = ResponseSet::from_banners(vec!["SSH-2.0-OpenSSH_9.6p1 Debian-3".to_string()]);
    let evidence = BannerRegexAnalyzer.analyze(
        &PortContext {
            port: 22,
            protocol: crate::model::port::Protocol::Tcp,
            addr: None,
            tunnel: None,
            speaks_http: false,
            detection: crate::config::ServiceDetection::default(),
        },
        &responses,
        &Collected::default(),
    );
    assert!(
        evidence.iter().any(|e| e.port_confirmed),
        "a port-linked match must be tagged port-confirmed"
    );
}

/// A recorded self-signed appliance certificate (DER). Subject == issuer,
/// `O=Zond Appliance`, `CN=zond-device.local`. Serves as the parse oracle for
/// the TLS analyzer, mirroring the recorded-banner corpus above.
const SELF_SIGNED_CERT: &[u8] = include_bytes!("testdata/selfsigned.der");

#[test]
fn tls_cert_identifies_self_signed_appliance() {
    let responses = ResponseSet {
        banners: Vec::new(),
        tls: Some(TlsInfo {
            certificates: vec![SELF_SIGNED_CERT.to_vec()],
            ..TlsInfo::default()
        }),
    };

    let evidence = TlsCertAnalyzer.analyze(
        &PortContext {
            port: 8443,
            protocol: crate::model::port::Protocol::Tcp,
            addr: None,
            tunnel: Some(Tunnel::Tls),
            speaks_http: false,
            detection: crate::config::ServiceDetection::default(),
        },
        &responses,
        &Collected::default(),
    );
    let verdict = ServiceVerdict::resolve(evidence);

    // The port is identified as TLS, and the self-signed subject O= names the vendor.
    assert_eq!(verdict.service.as_deref(), Some("ssl"));
    assert_eq!(verdict.vendor.as_deref(), Some("Zond Appliance"));
    assert_eq!(verdict.confidence, Confidence::Probable);
    let service = verdict.to_service().unwrap();
    // The tunnel's own `ssl` verdict is not re-prefixed into `ssl/ssl`, even
    // under a TLS context.
    assert_eq!(service.name(), "ssl");
    // The vendor reaches the projected Service, rather than being extracted
    // here and dropped by `to_service`.
    assert_eq!(service.vendor(), Some("Zond Appliance"));
}

#[test]
fn tls_analyzer_is_silent_without_a_certificate() {
    // No TLS captured: the analyzer must produce nothing, not a bare "ssl".
    let responses = ResponseSet::from_banners(vec!["HTTP/1.1 200 OK".to_string()]);
    assert!(
        TlsCertAnalyzer
            .analyze(
                &PortContext {
                    port: 80,
                    protocol: crate::model::port::Protocol::Tcp,
                    addr: None,
                    tunnel: None,
                    speaks_http: false,
                    detection: crate::config::ServiceDetection::default(),
                },
                &responses,
                &Collected::default(),
            )
            .is_empty()
    );
}

#[test]
fn banner_matched_in_a_tunnel_is_labelled_with_scheme() {
    // A protocol identified from data read inside TLS keeps its bare service on
    // the evidence but is labelled `ssl/<proto>` for the user.
    let responses = ResponseSet::from_banners(vec!["SSH-2.0-OpenSSH_9.6p1".to_string()]);
    let ctx = PortContext {
        port: 22,
        protocol: crate::model::port::Protocol::Tcp,
        addr: None,
        tunnel: Some(Tunnel::Tls),
        speaks_http: false,
        detection: crate::config::ServiceDetection::default(),
    };
    let evidence = BannerRegexAnalyzer.analyze(&ctx, &responses, &Collected::default());
    let verdict = ServiceVerdict::resolve(evidence);

    assert_eq!(verdict.service.as_deref(), Some("ssh")); // evidence stays bare
    assert_eq!(verdict.tunnel, Some(Tunnel::Tls));
    assert_eq!(verdict.to_service().unwrap().name(), "ssl/ssh"); // label composes both
}