zond-engine 0.12.0

A lightweight, fast, and highly concurrent networking backend for packet crafting, protocol fingerprinting, and host discovery.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
// Copyright (c) 2026 Erik Lening (hollowpointer) and Contributors
//
// This file is part of Zond Engine, licensed under the GNU Affero General
// Public License, version 3 or later. See the LICENSE file for details, or
// <https://www.gnu.org/licenses/agpl-3.0.html>.
//
// SPDX-License-Identifier: AGPL-3.0-or-later

//! # Corpus regression tests for the operating-system rules
//!
//! Every shipped rule carries examples measured off real hosts. These run them.
//!
//! ## Why the second test is the important one
//!
//! [`every_example_matches_its_own_rule`] catches a rule that stopped matching
//! what it was written for. That failure is loud: detection drops to nothing and
//! somebody notices.
//!
//! [`no_example_matches_another_familys_rule`] catches the failure that is
//! *silent*. A rule with too few predicates matches hosts it was never written
//! for and names them confidently, and nothing downstream can tell that from a
//! detection that worked — the report says "Linux" either way. The build refuses
//! a rule with **no** predicates; only running each example against every other
//! family's rules catches one with merely too few.
//!
//! This is the same shape as the check in the service corpus that the prefilter
//! never drops a matching signature: the interesting property is not that the
//! thing works, but that narrowing it did not quietly break it.

use crate::model::capture::{IpObservation, Ipv4Observation};

use super::db::RuleDb;
use super::observation::StackReply;
use super::observation::{StackObservation, TcpOptionKind};
use super::rules;
use super::signature::{Example, Provenance, ReplyKind};

/// Builds the observation an example describes.
///
/// The option layout is reconstructed from its letters rather than re-parsed
/// from bytes, because an example records what was *observed* — the values a
/// host answered with — and not the frame they arrived in. The parse from bytes
/// has its own tests, against option lists recorded verbatim off the wire; this
/// tests the rules.
fn observation_from(example: &Example) -> StackReply {
    // An echo example shares only the IP header with a TCP one, so it is built
    // here rather than threaded through the TCP construction below with every
    // field left empty. A rule for one kind is never applied to the other, so
    // the two paths never meet after this point.
    if example.reply == ReplyKind::EchoReply {
        return StackReply::Echo(super::observation::EchoObservation {
            ip: IpObservation::V4(Ipv4Observation {
                ttl: example.remaining_hops,
                identification: 0,
                dont_fragment: example.dont_fragment,
                more_fragments: false,
                dscp: 0,
                ecn: 0,
            }),
            code: example.echo_code,
            // Length is not an authored field: what a rule can say about the
            // payload is whether it came back unchanged, and an example
            // recording that has nothing to say about how long it was.
            payload_len: 0,
            payload_intact: example.echo_payload_intact,
        });
    }

    tcp_observation_from(example)
}

fn tcp_observation_from(example: &Example) -> StackReply {
    let layout = example
        .option_layout
        .split(',')
        .filter(|letter| !letter.is_empty())
        .map(|letter| match letter {
            "E" => TcpOptionKind::EndOfList,
            "N" => TcpOptionKind::NoOp,
            "M" => TcpOptionKind::MaximumSegmentSize,
            "W" => TcpOptionKind::WindowScale,
            "S" => TcpOptionKind::SackPermitted,
            "K" => TcpOptionKind::Sack,
            "T" => TcpOptionKind::Timestamp,
            other => panic!("an example names an option letter nothing writes: {other:?}"),
        })
        .collect();

    let flags = match example.reply {
        ReplyKind::SynAck => crate::protocols::tcp::flags::SYN | crate::protocols::tcp::flags::ACK,
        ReplyKind::Reset => crate::protocols::tcp::flags::RST,
        // Unreachable: the caller returns before this for an echo example.
        ReplyKind::EchoReply => unreachable!("an echo example is built above"),
    };

    StackReply::Tcp(StackObservation {
        flags,
        ip: IpObservation::V4(Ipv4Observation {
            ttl: example.remaining_hops,
            identification: 0,
            dont_fragment: example.dont_fragment,
            more_fragments: false,
            dscp: 0,
            ecn: 0,
        }),
        window: example.window.unwrap_or_default(),
        option_layout: layout,
        mss: example.mss,
        window_scale: example.window_scale,
        timestamps: example
            .timestamps
            .then_some(super::observation::Timestamps { value: 1, echo: 0 }),
        sack_permitted: example.sack_permitted,
        quirks: Default::default(),
    })
}

/// A rule that no longer matches the host it was written for has stopped
/// working, and the example is the only record of what it was written for.
#[test]
fn every_example_matches_its_own_rule() {
    let db = RuleDb::global();
    let mut failures = Vec::new();

    for rule in db.rules() {
        for example in &rule.example {
            let observed = observation_from(example);
            if !rules::matches(&rule.r#match, &observed) {
                failures.push(format!(
                    "'{}' no longer matches its own example: {}",
                    rule.os.family, example.source
                ));
            }
        }
    }

    assert!(failures.is_empty(), "{}", failures.join("\n"));
}

/// The silent failure. A rule with too few predicates matches hosts it was never
/// written for and names them with the same confidence as a real match.
///
/// Rules of the *same* family are allowed to match each other's examples: two
/// Linux builds sharing a shape is a fact about Linux, not a defect. Across
/// families it is always wrong.
///
/// **Inert while the corpus holds one family**, which it does today: every pair
/// is skipped and this passes without comparing anything. That is not a reason to
/// drop it — it starts working the moment a second family is measured, which is
/// exactly when it is needed — but a test that cannot fail proves nothing while
/// it cannot, so [`the_cross_family_check_catches_a_rule_that_is_too_loose`]
/// exercises the same comparison against a rule built to fail it.
#[test]
fn no_example_matches_another_familys_rule() {
    let db = RuleDb::global();
    let mut failures = Vec::new();

    for owner in db.rules() {
        for example in &owner.example {
            let observed = observation_from(example);
            for other in db.rules() {
                if other.os.family == owner.os.family {
                    continue;
                }
                if rules::matches(&other.r#match, &observed) {
                    failures.push(format!(
                        "'{}' matches an example belonging to '{}': {}",
                        other.os.family, owner.os.family, example.source
                    ));
                }
            }
        }
    }

    assert!(failures.is_empty(), "{}", failures.join("\n"));
}

/// A rule claiming to be **measured** must ship the observation it was measured
/// from.
///
/// This is where the honesty guarantee actually lives, now that the corpus mixes
/// two kinds of rule. A published rule is allowed to have no example — there is
/// no local observation to record, which is precisely what `published` means —
/// but a rule asserting somebody saw this on real hardware has to say what they
/// saw, or the claim is unfalsifiable and scores higher than a published rule for
/// no reason anyone can check.
#[test]
fn every_measured_rule_ships_what_it_measured() {
    let mut offenders = Vec::new();
    for rule in RuleDb::global().rules() {
        if rule.provenance == Provenance::Measured && rule.example.is_empty() {
            offenders.push(rule.os.family.as_str());
        }
    }

    assert!(
        offenders.is_empty(),
        "these rules claim to be measured and record no observation: {offenders:?}. \
         Either ship the values that were seen, or mark the rule `published`."
    );
}

/// A published rule must say where its values came from.
///
/// Its whole cost is that nobody here has confirmed it, so the note is what lets
/// the next person confirm or correct it — and what stops a guess from being
/// indistinguishable from a documented default six months later.
#[test]
fn every_published_rule_says_what_it_rests_on() {
    let mut offenders = Vec::new();
    for rule in RuleDb::global().rules() {
        if rule.provenance == Provenance::Published
            && rule.notes.as_deref().unwrap_or("").trim().len() < 40
        {
            offenders.push(rule.os.family.as_str());
        }
    }

    assert!(
        offenders.is_empty(),
        "these rules are unconfirmed and do not say what they rest on: {offenders:?}"
    );
}

/// The families the corpus covers, pinned so growth is a deliberate edit.
///
/// Not a claim that each has been *verified* — most are published defaults, and
/// [`every_measured_rule_ships_what_it_measured`] is what polices that
/// distinction. This is here so that adding or losing a family is something
/// somebody chose rather than something that drifted in.
#[test]
fn the_corpus_covers_the_families_it_says_it_does() {
    let db = RuleDb::global();
    let mut families: Vec<&str> = db.rules().iter().map(|r| r.os.family.as_str()).collect();
    families.sort_unstable();
    families.dedup();

    assert_eq!(
        families,
        vec!["FreeBSD", "Linux", "Network device", "Windows", "macOS"],
        "the set of families changed; see assets/fingerprinting/os/README.md"
    );
}

/// Every shipped rule states at least one predicate, which the build also
/// enforces. Kept here as well because the build check lives in a file that does
/// not run under `cargo test`, and a rule matching every reply of its kind is the
/// worst thing this corpus can ship.
#[test]
fn every_rule_tests_something() {
    for rule in RuleDb::global().rules() {
        let r#match = &rule.r#match;
        let stated = [
            r#match.initial_hops.is_some(),
            r#match.dont_fragment.is_some(),
            r#match.option_layout.is_some(),
            r#match.window.is_some(),
            r#match.window_units.is_some(),
            r#match.window_remainder.is_some(),
            r#match.window_scale.is_some(),
            r#match.mss.is_some(),
            r#match.timestamps.is_some(),
            r#match.sack_permitted.is_some(),
            r#match.identifier_class.is_some(),
            r#match.sequence_class.is_some(),
            r#match.clock_class.is_some(),
        ]
        .into_iter()
        .filter(|stated| *stated)
        .count();

        assert!(
            stated > 0,
            "'{}' states no predicates, so it matches every reply of its kind",
            rule.os.family
        );
    }
}

/// A reset carries no TCP options at all, so a rule written for a handshake must
/// never be satisfied by one however much else agrees. Without the reply-kind
/// check being unconditional, a rule stating only IP-level predicates would match
/// both.
#[test]
fn a_handshake_rule_is_never_satisfied_by_a_reset() {
    let db = RuleDb::global();
    let example = db
        .rules()
        .iter()
        .flat_map(|rule| &rule.example)
        .find(|example| example.reply == ReplyKind::SynAck)
        .expect("the corpus ships a handshake example");

    let mut as_reset = match observation_from(example) {
        StackReply::Tcp(observed) => observed,
        StackReply::Echo(_) => unreachable!("the example was chosen as a handshake"),
    };
    as_reset.flags = crate::protocols::tcp::flags::RST;
    let as_reset = StackReply::Tcp(as_reset);

    for rule in db.rules() {
        assert!(
            !rules::matches(&rule.r#match, &as_reset),
            "'{}' matched a reset, but it is written for a handshake",
            rule.os.family
        );
    }
}

/// Proof that the check above has teeth, since the corpus cannot currently give
/// it any.
///
/// Builds the mistake it exists to catch — a rule naming a different family and
/// stating one weak predicate, so it matches almost any handshake — and asserts
/// the comparison flags it. Without this, a bug in `rules::matches` that made it
/// return `false` unconditionally would leave every corpus test passing.
#[test]
fn the_cross_family_check_catches_a_rule_that_is_too_loose() {
    use super::signature::{MatchRule, OsDefinition, OsIdentity, Predicate};

    let observed = observation_from(
        RuleDb::global()
            .rules()
            .iter()
            .flat_map(|rule| &rule.example)
            .find(|example| example.reply == ReplyKind::SynAck)
            .expect("the corpus ships a handshake example"),
    );

    let too_loose = OsDefinition {
        os: OsIdentity {
            family: "Definitely Not Linux".to_string(),
            vendor: None,
            product: None,
            version: None,
            cpe: None,
        },
        provenance: Provenance::Published,
        notes: None,
        weight: 1.0,
        r#match: MatchRule {
            reply: ReplyKind::SynAck,
            // The only predicate, and one nearly every on-link host satisfies.
            initial_hops: Some(Predicate {
                equals: Some(64),
                ..Default::default()
            }),
            ..Default::default()
        },
        example: Vec::new(),
    };

    assert!(
        rules::matches(&too_loose.r#match, &observed),
        "a rule this loose must match a Linux example, or the cross-family check \
         is incapable of catching one"
    );

    // And in the other direction: a shape no rule was written for has to be
    // declined by all of them.
    //
    // The mutation has to be one no rule can accept, which is narrower than it
    // sounds. Moving the *window* is not enough — the BSD and Darwin rules state
    // no window predicate at all, because what identifies those families is the
    // order they write their options in, and a rule is right not to test a field
    // it is not about. Nor is moving the hop counter to 255, which is precisely
    // what the network-device rule looks for. So this changes the option layout
    // to one nothing emits, and leaves the counter where no rule keys on it.
    let mut elsewhere = match observed.clone() {
        StackReply::Tcp(observed) => observed,
        StackReply::Echo(_) => unreachable!("the example was chosen as a handshake"),
    };
    elsewhere.option_layout = vec![
        TcpOptionKind::MaximumSegmentSize,
        TcpOptionKind::Sack,
        TcpOptionKind::Other(99),
    ];
    let elsewhere = StackReply::Tcp(elsewhere);
    let matched: Vec<&str> = RuleDb::global()
        .matching(&elsewhere)
        .map(|rule| rule.os.family.as_str())
        .collect();
    assert!(
        matched.is_empty(),
        "a shape nothing emits was matched by {matched:?}, so those rules are looser \
         than the families they name"
    );
}

/// **The corpus is published.** `assets/` is deliberately not excluded from the
/// packaged crate — `build.rs` compiles the rules out of it, so a package without
/// it does not build — which means every word authored there goes to crates.io
/// and stays there.
///
/// A rule's provenance has to say what *kind* of machine was measured, because
/// that is what makes the rule attributable. It must not say *whose*: an address,
/// a hostname or a cross-reference between two of them is a description of
/// somebody's network, it is of no use to anyone reading the rule, and it cannot
/// be taken back once published. This engine ships a redaction policy for exactly
/// this class of detail in its reports; its own corpus should not be the leak.
///
/// Addresses from the documentation ranges (RFC 5737 and RFC 3849) are fine and
/// are what an example should use if it needs one at all.
#[test]
fn no_rule_names_a_real_address_or_host() {
    // Deliberately crude: anything dotted-quad shaped, or with a colon-separated
    // hexadecimal run, and any obvious hostname suffix. A false positive here
    // costs one reworded line; a false negative is permanent.
    let looks_like_ipv4 = |text: &str| {
        text.split(|c: char| !(c.is_ascii_digit() || c == '.'))
            .any(|token| {
                let parts: Vec<&str> = token.split('.').collect();
                parts.len() == 4
                    && parts
                        .iter()
                        .all(|part| !part.is_empty() && part.parse::<u8>().is_ok())
            })
    };
    let documentation_range = |text: &str| {
        ["192.0.2.", "198.51.100.", "203.0.113."]
            .iter()
            .any(|prefix| text.contains(prefix))
    };

    let mut offenders = Vec::new();
    for rule in RuleDb::global().rules() {
        for example in &rule.example {
            let source = &example.source;
            if looks_like_ipv4(source) && !documentation_range(source) {
                offenders.push(format!("'{}': {source}", rule.os.family));
            }
            for suffix in [".local", ".lan", ".home", ".internal"] {
                if source.contains(suffix) {
                    offenders.push(format!("'{}': {source}", rule.os.family));
                }
            }
        }
    }

    assert!(
        offenders.is_empty(),
        "a published rule names somebody's network. Say what kind of machine was \
         measured, not which one:\n{}",
        offenders.join("\n")
    );
}

/// The echo rules reach a host the TCP rules cannot, and name it.
///
/// The point of sending a ping at all: a stock Windows firewall drops rather
/// than refuses, so a desktop with nothing listening answers no TCP probe and
/// every handshake rule in the corpus is unreachable for it.
#[test]
fn an_echo_reply_alone_can_name_a_family() {
    let windows = echo_reply(128);
    let verdict = super::verdict::classify(RuleDb::global(), &windows)
        .expect("an echo reply with a Windows hop counter names Windows");
    assert_eq!(verdict.family, "Windows");

    let device = echo_reply(255);
    let verdict = super::verdict::classify(RuleDb::global(), &device)
        .expect("an echo reply with an infrastructure hop counter names one");
    assert_eq!(verdict.family, "Network device");
}

/// A ping from a Unix-alike is deliberately **not** named, and this test is the
/// record of why.
///
/// Linux, macOS and the BSDs all start the counter at 64, so on an echo reply —
/// which carries no options, no window and no sequence number — there is nothing
/// left to tell them apart. A rule keyed on 64 alone would name every one of
/// them as whichever family it happened to claim, and would be confidently
/// wrong for most hosts it matched. Reporting nothing is the correct answer.
///
/// This fails the moment somebody adds that rule, which is the intent: the fix
/// is a second field the reply actually carries — whether a non-zero request
/// code comes back, whether the payload returns unchanged — not a looser rule.
#[test]
fn an_echo_reply_from_a_unix_hop_counter_names_nothing() {
    let unix_like = echo_reply(64);
    let matched: Vec<&str> = RuleDb::global()
        .matching(&unix_like)
        .map(|rule| rule.os.family.as_str())
        .collect();
    assert!(
        matched.is_empty(),
        "an echo reply with a hop counter of 64 was named {matched:?}, but Linux, \
         macOS and the BSDs all start there and an echo reply carries nothing \
         else to separate them"
    );
}

/// An echo-shaped reply with `hops` left, for the two tests above.
fn echo_reply(hops: u8) -> StackReply {
    StackReply::Echo(super::observation::EchoObservation {
        ip: IpObservation::V4(Ipv4Observation {
            ttl: hops,
            identification: 0,
            dont_fragment: true,
            more_fragments: false,
            dscp: 0,
            ecn: 0,
        }),
        code: crate::protocols::icmp::ECHO_PROBE_CODE,
        payload_len: 28,
        payload_intact: true,
    })
}