zond-engine 0.18.0

The core networking library powering Zond. Raw packet injection, protocol parsing, service and OS fingerprinting, and asynchronous network scanning.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
// Copyright (c) 2026 Erik Lening (hollowpointer) and Contributors
//
// This file is part of Zond Engine, licensed under the GNU Affero General
// Public License, version 3 or later. See the LICENSE file for details, or
// <https://www.gnu.org/licenses/agpl-3.0.html>.
//
// SPDX-License-Identifier: AGPL-3.0-or-later

//! # A comparison as one JSON document
//!
//! The canonical form, in the schema [`schema`](super::schema) defines. This is
//! what a pipeline ingests: a nightly comparison posted to a queue, filed as a
//! ticket, or fed to a rule that alerts on a port opening where nobody expected
//! one.

use std::io::Write;

use crate::diff::ScanDiff;
use crate::export::diff::DiffExporter;
use crate::export::diff::schema::DiffDto;
use crate::export::{ExportError, ExportOptions};

/// The format name carried in an [`ExportError::Render`].
const FORMAT: &str = "diff json";

/// Writes a comparison as one JSON document.
///
/// ```no_run
/// use std::fs::File;
/// use zond_engine::diff::ScanDiff;
/// use zond_engine::export::diff::{DiffExporter, JsonDiffExporter};
/// use zond_engine::export::{ExportOptions, Redaction};
///
/// # fn example(diff: &ScanDiff) -> Result<(), Box<dyn std::error::Error>> {
/// let options = ExportOptions::new().with_redaction(Redaction::Standard);
/// let mut file = File::create("changes.json")?;
///
/// JsonDiffExporter::new(options).export(diff, &mut file)?;
/// # Ok(())
/// # }
/// ```
#[must_use]
#[derive(Debug, Clone)]
pub struct JsonDiffExporter {
    options: ExportOptions,
    pretty: bool,
}

/// Written out rather than derived, for the reason
/// [`JsonExporter`](crate::export::JsonExporter)'s is: a derived one reads
/// `pretty` as `false` and disagrees with [`JsonDiffExporter::new`] about what
/// this exporter does by default.
impl Default for JsonDiffExporter {
    fn default() -> Self {
        Self::new(ExportOptions::default())
    }
}

impl JsonDiffExporter {
    /// An exporter that writes indented JSON.
    ///
    /// Indented for the same reason the report exporter is: the usual
    /// destination is a file somebody opens, and a document that diffs line by
    /// line is worth more than one that saves bytes.
    pub fn new(options: ExportOptions) -> Self {
        Self {
            options,
            pretty: true,
        }
    }

    /// Switches to single-line output, for a queue or an HTTP body.
    pub fn compact(mut self) -> Self {
        self.pretty = false;
        self
    }

    /// Switches back to indented output.
    pub fn pretty(mut self) -> Self {
        self.pretty = true;
        self
    }

    /// The options in force.
    pub fn options(&self) -> &ExportOptions {
        &self.options
    }
}

impl DiffExporter for JsonDiffExporter {
    fn export(&self, diff: &ScanDiff, out: &mut dyn Write) -> Result<(), ExportError> {
        let document = DiffDto::new(diff, &self.options);

        let result = if self.pretty {
            serde_json::to_writer_pretty(&mut *out, &document)
        } else {
            serde_json::to_writer(&mut *out, &document)
        };

        result.map_err(|error| crate::export::write::render_error(FORMAT, error))?;

        writeln!(out)?;
        Ok(())
    }
}

// ╔════════════════════════════════════════════╗
// ║ ████████╗███████╗███████╗████████╗███████╗ ║
// ║ ╚══██╔══╝██╔════╝██╔════╝╚══██╔══╝██╔════╝ ║
// ║    ██║   █████╗  ███████╗   ██║   ███████╗ ║
// ║    ██║   ██╔══╝  ╚════██║   ██║   ╚════██║ ║
// ║    ██║   ███████╗███████║   ██║   ███████║ ║
// ║    ╚═╝   ╚══════╝╚══════╝   ╚═╝   ╚══════╝ ║
// ╚════════════════════════════════════════════╝
#[cfg(test)]
mod tests {
    use serde_json::Value;

    use super::*;
    use crate::export::fixture;

    /// A destination that refuses every write with a nameable reason.
    struct FullDisk;

    impl Write for FullDisk {
        fn write(&mut self, _: &[u8]) -> std::io::Result<usize> {
            Err(std::io::Error::from(std::io::ErrorKind::StorageFull))
        }

        fn flush(&mut self) -> std::io::Result<()> {
            Ok(())
        }
    }

    /// A write that failed reports *why* it failed, and the report exporter and
    /// the comparison exporter say the same thing about the same disk.
    ///
    /// This exporter carried its own copy of the sorting `write::render_error`
    /// does, and the copy wrapped the error with `io::Error::other` instead of
    /// unwrapping serde_json's own. A full disk during a comparison export came
    /// back as `Other` where the same disk during a report export came back as
    /// `StorageFull`. Found by W23's byte-comparison harness.
    #[test]
    fn a_failed_write_keeps_the_kind_the_destination_reported() {
        let (before, after) = fixture::compared();
        let diff = crate::diff::ScanDiff::between(&before, &after);

        let error = JsonDiffExporter::new(ExportOptions::new())
            .export(&diff, &mut FullDisk)
            .expect_err("a destination that refuses every write");

        match error {
            ExportError::Io(io) => assert_eq!(io.kind(), std::io::ErrorKind::StorageFull),
            other => panic!("expected an I/O failure, got {other:?}"),
        }
    }

    /// The fixture pair, compared and parsed back.
    fn document() -> Value {
        let (before, after) = fixture::compared();
        let diff = ScanDiff::between(&before, &after);

        let mut bytes = Vec::new();
        JsonDiffExporter::new(ExportOptions::new())
            .export(&diff, &mut bytes)
            .expect("the export succeeds");

        serde_json::from_slice(&bytes).expect("the export parses as JSON")
    }

    /// Every change in the document, as `address port kind before after`.
    fn changes(document: &Value) -> Vec<String> {
        let mut found = Vec::new();
        for host in document["hosts"].as_array().expect("hosts") {
            let address = host["address"].as_str().expect("an address");
            for change in host["changes"].as_array().expect("changes") {
                found.push(format!(
                    "{address} - {} {} {}",
                    change["kind"].as_str().expect("a token"),
                    change["before"],
                    change["after"]
                ));
            }
            for port in host["ports"].as_array().expect("ports") {
                for change in port["changes"].as_array().expect("changes") {
                    found.push(format!(
                        "{address} {}/{} {} {} {}",
                        port["port"],
                        port["protocol"].as_str().expect("a transport"),
                        change["kind"].as_str().expect("a token"),
                        change["before"],
                        change["after"]
                    ));
                }
            }
        }
        found
    }

    /// The field a scheduled comparison is triaged by, at all three levels it is
    /// written: the endpoint, the host above it, and the document.
    ///
    /// A host is graded by the worst thing on it, so the grades have to nest.
    /// A consumer that alerts on the document and then looks for the host, and
    /// the host and then the endpoint, must not be sent somewhere the grade
    /// evaporates.
    #[test]
    fn a_grade_is_carried_at_every_level_and_nests() {
        let document = document();

        let mut worst = "routine";
        for host in document["hosts"].as_array().expect("hosts") {
            let host_grade = host["significance"].as_str().expect("a host grade");
            let address = host["address"].as_str().expect("an address");

            for port in host["ports"].as_array().expect("ports") {
                let port_grade = port["significance"].as_str().expect("an endpoint grade");
                assert!(
                    rank(port_grade) <= rank(host_grade),
                    "{address} is {host_grade} and carries a {port_grade} endpoint"
                );
            }

            if rank(host_grade) > rank(worst) {
                worst = host_grade;
            }
        }

        assert_eq!(
            document["significance"].as_str(),
            Some(worst),
            "the document is not graded by its worst host"
        );
        assert_eq!(
            worst, "urgent",
            "the fixture opens a port, so something here is urgent"
        );
    }

    /// The grades in the order the type ranks them, so a test can compare two
    /// without depending on the strings sorting that way. They do not.
    fn rank(grade: &str) -> usize {
        ["routine", "notable", "urgent"]
            .iter()
            .position(|known| *known == grade)
            .unwrap_or_else(|| panic!("'{grade}' is not a grade this document may carry"))
    }

    /// An endpoint that started accepting connections is the change the grade
    /// exists to lift out of a page of drift.
    #[test]
    fn an_endpoint_that_opened_is_the_urgent_one() {
        let document = document();

        let opened: Vec<&str> = document["hosts"]
            .as_array()
            .expect("hosts")
            .iter()
            .flat_map(|host| host["ports"].as_array().expect("ports"))
            .filter(|port| port["opened"] == Value::Bool(true))
            .map(|port| port["significance"].as_str().expect("a grade"))
            .collect();

        assert!(!opened.is_empty(), "the fixture opens a port");
        assert!(
            opened.iter().all(|grade| *grade == "urgent"),
            "an endpoint that opened was graded {opened:?}"
        );
    }

    /// The vocabulary is the contract: a rule somebody writes today keys on
    /// these strings.
    #[test]
    fn each_change_is_named_by_its_documented_token() {
        let found = changes(&document());

        for expected in [
            r#"203.0.113.1 - hostname "router.local" "gateway.local""#,
            r#"203.0.113.1 - os "Linux 5.15.0" "Linux 6.1.0""#,
            r#"203.0.113.1 22/tcp service_version "8.9p1" "9.6p1""#,
            r#"203.0.113.1 80/tcp port_state "open" "closed""#,
        ] {
            assert!(
                found.iter().any(|c| c == expected),
                "missing {expected}\nin {found:#?}"
            );
        }
    }

    /// A certificate is identified by its fingerprint, so that is what a
    /// rotation carries: two certificates are the same one exactly when they are
    /// byte for byte the same.
    #[test]
    fn a_rotation_carries_both_fingerprints() {
        let found = changes(&document());
        let rotation = found
            .iter()
            .find(|change| change.contains("certificate_rotated"))
            .expect("the fixture rotates a certificate");

        assert!(rotation.contains("aaaa"), "{rotation}");
        assert!(rotation.contains("bbbb"), "{rotation}");
    }

    /// The certificate did not move; the clock did. `after` is when it lapses,
    /// absolute, so a consumer computes whatever window it wants.
    #[test]
    fn an_expiry_crossing_carries_when_it_lapses_and_no_before() {
        let document = document();
        let host = document["hosts"]
            .as_array()
            .expect("hosts")
            .iter()
            .find(|host| host["address"] == "203.0.113.4")
            .expect("the host whose certificate is running out");

        let change = &host["ports"][0]["changes"][0];
        assert_eq!(change["kind"], "certificate_expiring");
        assert_eq!(change["before"], Value::Null);
        assert!(
            change["after"]
                .as_str()
                .is_some_and(|end| end.ends_with('Z')),
            "an absolute moment, not a window: {}",
            change["after"]
        );
    }

    /// The field a rule keys on, and the one thing this document must never get
    /// wrong.
    #[test]
    fn confirmed_says_whether_the_other_scan_looked() {
        let document = document();
        let hosts = document["hosts"].as_array().expect("hosts");

        let arrived = hosts
            .iter()
            .find(|host| host["address"] == "203.0.113.7")
            .expect("the host that arrived");
        assert_eq!(arrived["presence"], "added");
        assert_eq!(arrived["coverage"], "covered");
        assert_eq!(
            arrived["confirmed"], true,
            "the earlier scan walked this address and did not find it"
        );

        let gone = hosts
            .iter()
            .find(|host| host["address"] == "203.0.113.9")
            .expect("the host that went");
        assert_eq!(gone["presence"], "removed");
        assert_eq!(gone["confirmed"], true);

        // A port both scans walked, that only one found, is likewise a finding.
        let router = hosts
            .iter()
            .find(|host| host["address"] == "203.0.113.1")
            .expect("the gateway");
        let opened = router["ports"]
            .as_array()
            .expect("ports")
            .iter()
            .find(|port| port["port"] == 8080)
            .expect("the port that opened");
        assert_eq!(opened["presence"], "added");
        assert_eq!(
            opened["confirmed"], true,
            "both scans stated they walked 8080"
        );
        assert_eq!(opened["opened"], true);
    }

    /// A scan that never walked the ground cannot confirm what turned up on it.
    #[test]
    fn a_change_the_other_scan_never_covered_is_not_confirmed() {
        // The gateway alone, compared against a sweep that stated it walked
        // addresses and no ports at all.
        let (_, after) = fixture::compared();
        let sweep = fixture::report();

        let diff = ScanDiff::between(&sweep, &after);
        let mut bytes = Vec::new();
        JsonDiffExporter::new(ExportOptions::new())
            .export(&diff, &mut bytes)
            .expect("the export succeeds");
        let document: Value = serde_json::from_slice(&bytes).expect("JSON");

        let unconfirmed = document["hosts"]
            .as_array()
            .expect("hosts")
            .iter()
            .flat_map(|host| host["ports"].as_array().expect("ports"))
            .filter(|port| port["confirmed"] == false)
            .count();

        assert!(
            unconfirmed > 0,
            "a sweep walked no ports, so nothing found on one is a finding about the network"
        );
    }

    /// Two scans that found the same things still produce a document, because a
    /// consumer polling nightly wants the same shape either way.
    #[test]
    fn an_unchanged_comparison_still_writes_a_document() {
        let (before, _) = fixture::compared();
        let diff = ScanDiff::between(&before, &before);

        let mut bytes = Vec::new();
        JsonDiffExporter::new(ExportOptions::new())
            .export(&diff, &mut bytes)
            .expect("the export succeeds");
        let document: Value = serde_json::from_slice(&bytes).expect("JSON");

        assert_eq!(document["unchanged"], true);
        assert_eq!(document["hosts"].as_array().map(Vec::len), Some(0));
        assert_eq!(document["summary"]["hosts_changed"], 0);
    }

    /// The two ways of building one of these have to produce the same exporter,
    /// for the reason the report exporter's own test gives: a derived `Default`
    /// reads `pretty` as `false` and quietly disagrees with `new`.
    #[test]
    fn the_default_exporter_is_the_one_new_builds() {
        let (before, after) = fixture::compared();
        let diff = ScanDiff::between(&before, &after);

        // Without the stamp, which is the one field that moves between two
        // exports of one comparison.
        let render = |exporter: &JsonDiffExporter| {
            let mut bytes = Vec::new();
            exporter.export(&diff, &mut bytes).expect("exports");
            String::from_utf8(bytes)
                .expect("utf-8")
                .lines()
                .filter(|line| !line.contains("\"generated_at\""))
                .collect::<Vec<_>>()
                .join("\n")
        };

        let built = render(&JsonDiffExporter::new(ExportOptions::new()));
        assert_eq!(render(&JsonDiffExporter::default()), built);
        assert!(
            built.contains("\n  "),
            "the documented default is indented output"
        );
    }

    /// The document ends with a newline, so a file of one is a well-formed text
    /// file and a stream of them concatenates.
    #[test]
    fn the_document_ends_with_a_newline() {
        let (before, after) = fixture::compared();
        let diff = ScanDiff::between(&before, &after);

        let mut bytes = Vec::new();
        JsonDiffExporter::new(ExportOptions::new())
            .compact()
            .export(&diff, &mut bytes)
            .expect("the export succeeds");

        assert_eq!(bytes.last(), Some(&b'\n'));
    }
}