Skip to main content

zond_engine/detect/compute/
record.rs

1// Copyright (c) 2026 Erik Lening (hollowpointer) and Contributors
2//
3// This file is part of Zond Engine, licensed under the GNU Affero General
4// Public License, version 3 or later. See the LICENSE file for details, or
5// <https://www.gnu.org/licenses/agpl-3.0.html>.
6//
7// SPDX-License-Identifier: AGPL-3.0-or-later
8
9//! # The on-disk form of a capability tape
10//!
11//! A [`CapTape`] as journal data, so recorded runs can be replayed later. The
12//! tape itself has no `serde`; this module defines its wire shape. Build a
13//! record with [`From`] and read it back with
14//! [`rebuild`](CapTapeRecord::rebuild).
15//!
16//! ## Encoding
17//!
18//! Bytes are lowercase hex, as content hashes are. Each exchange records its
19//! reply or its error, so a run that branched on a refusal replays that branch.
20//! As in the model records, an unparsable value reads as the least it could
21//! mean: undecodable bytes read empty, and an unknown error kind reads as a
22//! reset.
23
24use std::net::IpAddr;
25
26use serde::{Deserialize, Serialize};
27
28use super::capability::CapError;
29use super::replay::{CapTape, ResolveExchange, SpeakExchange};
30use crate::record::DetectionIdRecord;
31
32/// One detection run, as the journal holds it: which detection ran over which
33/// subject, and the tape of what it read.
34///
35/// `#[non_exhaustive]`, so the format can gain fields.
36#[non_exhaustive]
37#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
38pub struct DetectionRunRecord {
39    /// The address the detection ran against.
40    pub host: String,
41    /// The name a target reached that address by, where it named a host: the
42    /// module's `ctx.hostname`.
43    #[serde(default, skip_serializing_if = "Option::is_none")]
44    pub host_name: Option<String>,
45    /// The port it ran over.
46    pub port: u16,
47    /// The transport, by wire name.
48    pub protocol: String,
49    /// Which detection ran, to which version, from which bytes.
50    pub detection: DetectionIdRecord,
51    /// The responses the scan had gathered and handed the detection.
52    #[serde(default)]
53    pub responses: Vec<String>,
54    /// What it read from its capabilities, kept for replay.
55    pub tape: CapTapeRecord,
56}
57
58/// The runs over one port that read the same responses, as one line of a
59/// journal: the port and its responses once, then each run's detection and
60/// tape.
61///
62/// A dozen passive detections may read one HTTP port's kilobytes of responses,
63/// so the responses are written once per port. Runs are still read back as
64/// [`DetectionRunRecord`]s.
65///
66/// An engine that only knows one run per line cannot parse this and refuses
67/// the file.
68#[cfg(feature = "journal-format")]
69#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
70pub(crate) struct PortRunsRecord {
71    host: String,
72    #[serde(default, skip_serializing_if = "Option::is_none")]
73    host_name: Option<String>,
74    port: u16,
75    protocol: String,
76    responses: Vec<String>,
77    runs: Vec<PortRunRecord>,
78}
79
80/// One run of a [`PortRunsRecord`]: what ran, and what it read from its
81/// capabilities.
82#[cfg(feature = "journal-format")]
83#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
84struct PortRunRecord {
85    detection: DetectionIdRecord,
86    tape: CapTapeRecord,
87}
88
89#[cfg(feature = "journal-format")]
90impl PortRunsRecord {
91    /// `runs` as the lines a journal writes them in: one for each port and
92    /// set of responses, in the order each first appears, holding its runs in
93    /// the order given.
94    pub(crate) fn grouping(runs: &[DetectionRunRecord]) -> Vec<Self> {
95        let mut lines: Vec<Self> = Vec::new();
96        for run in runs {
97            let line = lines.iter_mut().find(|line| {
98                (
99                    line.port,
100                    &line.host,
101                    &line.host_name,
102                    &line.protocol,
103                    &line.responses,
104                ) == (
105                    run.port,
106                    &run.host,
107                    &run.host_name,
108                    &run.protocol,
109                    &run.responses,
110                )
111            });
112            let run_record = PortRunRecord {
113                detection: run.detection.clone(),
114                tape: run.tape.clone(),
115            };
116            match line {
117                Some(line) => line.runs.push(run_record),
118                None => lines.push(Self {
119                    host: run.host.clone(),
120                    host_name: run.host_name.clone(),
121                    port: run.port,
122                    protocol: run.protocol.clone(),
123                    responses: run.responses.clone(),
124                    runs: vec![run_record],
125                }),
126            }
127        }
128        lines
129    }
130}
131
132/// A line of a journal's detection runs, in either shape it has been written
133/// in.
134///
135/// Untagged: a port's line is told apart by its runs, and a line naming a
136/// single run is read as one.
137#[cfg(feature = "journal-format")]
138#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
139#[serde(untagged)]
140pub(crate) enum DetectionLine {
141    /// A port's runs, sharing its responses.
142    Port(PortRunsRecord),
143    /// One run, carrying its own copy of what it read.
144    Run(DetectionRunRecord),
145}
146
147#[cfg(feature = "journal-format")]
148impl DetectionLine {
149    /// The runs this line holds, each with the responses it read.
150    pub(crate) fn into_runs(self) -> Vec<DetectionRunRecord> {
151        match self {
152            Self::Run(run) => vec![run],
153            Self::Port(line) => line
154                .runs
155                .into_iter()
156                .map(|run| DetectionRunRecord {
157                    host: line.host.clone(),
158                    host_name: line.host_name.clone(),
159                    port: line.port,
160                    protocol: line.protocol.clone(),
161                    detection: run.detection,
162                    responses: line.responses.clone(),
163                    tape: run.tape,
164                })
165                .collect(),
166        }
167    }
168}
169
170/// A capability tape, as a file holds it.
171#[non_exhaustive]
172#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
173pub struct CapTapeRecord {
174    /// Each recorded `speak`, in call order.
175    #[serde(default)]
176    pub speaks: Vec<SpeakExchangeRecord>,
177    /// Each recorded `resolve`, in call order.
178    #[serde(default)]
179    pub resolves: Vec<ResolveExchangeRecord>,
180    /// Each `now` tick, in call order, as milliseconds on the run's clock.
181    #[serde(default)]
182    pub nows: Vec<u64>,
183}
184
185/// One recorded `speak`: the bytes sent, and either the reply or the error.
186#[non_exhaustive]
187#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
188pub struct SpeakExchangeRecord {
189    /// The bytes the module sent, as hex.
190    pub sent: String,
191    /// The reply as hex, present when the exchange succeeded.
192    #[serde(default)]
193    pub reply: Option<String>,
194    /// The error, present when it failed.
195    #[serde(default)]
196    pub error: Option<CapErrorRecord>,
197}
198
199/// One recorded `resolve`: the name asked, and either the addresses or the error.
200#[non_exhaustive]
201#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
202pub struct ResolveExchangeRecord {
203    /// The name the module asked to resolve.
204    pub name: String,
205    /// The addresses returned, as strings, present when resolution succeeded.
206    #[serde(default)]
207    pub addresses: Vec<String>,
208    /// The error, present when it failed.
209    #[serde(default)]
210    pub error: Option<CapErrorRecord>,
211}
212
213/// A capability error, by wire name, with the reason a denial carries.
214#[non_exhaustive]
215#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
216pub struct CapErrorRecord {
217    /// The error kind, by wire name.
218    pub kind: String,
219    /// Why, for a denial. Absent for every other kind.
220    #[serde(default)]
221    pub reason: Option<String>,
222}
223
224impl From<&CapTape> for CapTapeRecord {
225    fn from(tape: &CapTape) -> Self {
226        Self {
227            speaks: tape.speaks.iter().map(SpeakExchangeRecord::from).collect(),
228            resolves: tape
229                .resolves
230                .iter()
231                .map(ResolveExchangeRecord::from)
232                .collect(),
233            nows: tape.nows.clone(),
234        }
235    }
236}
237
238impl CapTapeRecord {
239    /// Rebuilds the tape this record holds.
240    pub fn rebuild(&self) -> CapTape {
241        CapTape {
242            speaks: self
243                .speaks
244                .iter()
245                .map(SpeakExchangeRecord::rebuild)
246                .collect(),
247            resolves: self
248                .resolves
249                .iter()
250                .map(ResolveExchangeRecord::rebuild)
251                .collect(),
252            nows: self.nows.clone(),
253        }
254    }
255}
256
257impl From<&SpeakExchange> for SpeakExchangeRecord {
258    fn from(exchange: &SpeakExchange) -> Self {
259        let (reply, error) = match &exchange.reply {
260            Ok(bytes) => (Some(to_hex(bytes)), None),
261            Err(error) => (None, Some(CapErrorRecord::from(error))),
262        };
263        Self {
264            sent: to_hex(&exchange.sent),
265            reply,
266            error,
267        }
268    }
269}
270
271impl SpeakExchangeRecord {
272    fn rebuild(&self) -> SpeakExchange {
273        let reply = match &self.error {
274            Some(error) => Err(error.rebuild()),
275            None => Ok(from_hex(self.reply.as_deref().unwrap_or(""))),
276        };
277        SpeakExchange {
278            sent: from_hex(&self.sent),
279            reply,
280        }
281    }
282}
283
284impl From<&ResolveExchange> for ResolveExchangeRecord {
285    fn from(exchange: &ResolveExchange) -> Self {
286        let (addresses, error) = match &exchange.result {
287            Ok(addresses) => (addresses.iter().map(IpAddr::to_string).collect(), None),
288            Err(error) => (Vec::new(), Some(CapErrorRecord::from(error))),
289        };
290        Self {
291            name: exchange.name.clone(),
292            addresses,
293            error,
294        }
295    }
296}
297
298impl ResolveExchangeRecord {
299    fn rebuild(&self) -> ResolveExchange {
300        let result = match &self.error {
301            Some(error) => Err(error.rebuild()),
302            None => Ok(self
303                .addresses
304                .iter()
305                .filter_map(|address| address.parse().ok())
306                .collect()),
307        };
308        ResolveExchange {
309            name: self.name.clone(),
310            result,
311        }
312    }
313}
314
315impl From<&CapError> for CapErrorRecord {
316    fn from(error: &CapError) -> Self {
317        Self {
318            kind: cap_error_kind_name(error).to_owned(),
319            reason: match error {
320                CapError::Denied(reason) => Some(reason.clone()),
321                _ => None,
322            },
323        }
324    }
325}
326
327impl CapErrorRecord {
328    fn rebuild(&self) -> CapError {
329        cap_error(&self.kind, self.reason.as_deref())
330    }
331}
332
333/// The wire name of a capability error.
334fn cap_error_kind_name(error: &CapError) -> &'static str {
335    match error {
336        CapError::ByteBudgetExhausted => "byte-budget-exhausted",
337        CapError::ConnectionBudgetExhausted => "connection-budget-exhausted",
338        CapError::Denied(_) => "denied",
339        CapError::OutOfDescriptors => "out-of-descriptors",
340        CapError::Withheld => "withheld",
341        CapError::TimedOut => "timed-out",
342        CapError::ConnectionRefused => "connection-refused",
343        CapError::Reset => "reset",
344    }
345}
346
347/// The capability error a wire name and its reason name. An unknown kind reads as a
348/// reset, the most generic failure.
349fn cap_error(kind: &str, reason: Option<&str>) -> CapError {
350    match kind {
351        "byte-budget-exhausted" => CapError::ByteBudgetExhausted,
352        "connection-budget-exhausted" => CapError::ConnectionBudgetExhausted,
353        "denied" => CapError::Denied(reason.unwrap_or_default().to_owned()),
354        "out-of-descriptors" => CapError::OutOfDescriptors,
355        "withheld" => CapError::Withheld,
356        "timed-out" => CapError::TimedOut,
357        "connection-refused" => CapError::ConnectionRefused,
358        _ => CapError::Reset,
359    }
360}
361
362/// Bytes as lowercase hex, the engine's content-hash convention.
363fn to_hex(bytes: &[u8]) -> String {
364    bytes.iter().map(|byte| format!("{byte:02x}")).collect()
365}
366
367/// Bytes from lowercase hex. A malformed string, an odd length, or a non-hex or
368/// non-ASCII byte reads back empty. Indexing is over bytes, so a multi-byte
369/// character cannot split a slice.
370fn from_hex(hex: &str) -> Vec<u8> {
371    let hex = hex.as_bytes();
372    if !hex.len().is_multiple_of(2) {
373        return Vec::new();
374    }
375    (0..hex.len())
376        .step_by(2)
377        .map(|start| {
378            let hi = (hex[start] as char).to_digit(16)?;
379            let lo = (hex[start + 1] as char).to_digit(16)?;
380            Some((hi * 16 + lo) as u8)
381        })
382        .collect::<Option<Vec<u8>>>()
383        .unwrap_or_default()
384}
385
386#[cfg(test)]
387mod tests {
388    use super::*;
389    use crate::detect::compute::{
390        Budget, CapError, ComputeRuntime, Grant, LiveCapabilities, ModuleBody,
391        RecordedCapabilities, RecordingCapabilities, RhaiRuntime,
392    };
393    use crate::fingerprint::PortContext;
394    use crate::model::finding::{DetectionClass, DetectionId, Version};
395    use crate::model::port::Protocol;
396    use crate::testing::loopback::from_this_process;
397    use std::io::{Read, Write};
398    use std::net::TcpListener;
399    use std::thread;
400    use std::time::Duration;
401
402    fn budget() -> Budget {
403        Budget {
404            fuel: 1_000_000,
405            deadline: Duration::from_secs(2),
406            max_memory: 65_536,
407            max_bytes: 8_192,
408            max_connections: 4,
409        }
410    }
411
412    fn grant() -> Grant {
413        Grant {
414            group: None,
415            detection: DetectionId::new("record-test", Version::new(1, 0, 0), "hash").unwrap(),
416            class: DetectionClass::ActiveBenign,
417            budget: budget(),
418            speak: true,
419            resolve: false,
420        }
421    }
422
423    fn ctx(port: u16) -> PortContext {
424        PortContext {
425            port,
426            protocol: Protocol::Tcp,
427            addr: None,
428            tunnel: None,
429            speaks_http: false,
430            detection: crate::config::ServiceDetection::default(),
431            host_name: None,
432        }
433    }
434
435    #[test]
436    fn a_tape_survives_json_and_still_replays_identically() {
437        // A live run's tape, through JSON and back, replays to the same findings.
438        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
439        let addr = listener.local_addr().unwrap();
440        thread::spawn(move || {
441            if let Some(mut sock) = from_this_process(&listener).next() {
442                let mut probe = [0u8; 64];
443                let _ = sock.read(&mut probe);
444                // A non-ASCII byte, which a lossy encoding would corrupt.
445                let _ = sock.write_all(&[0x52, 0x45, 0x44, 0x49, 0x53, 0xff]);
446            }
447        });
448
449        let source = r#"
450            fn analyze(ctx, responses) {
451                let reply = speak(blob(4, 0x41));
452                [ #{
453                    severity: "medium",
454                    summary: "answered " + reply.len() + " bytes at " + now(),
455                } ]
456            }
457        "#;
458
459        let runtime = RhaiRuntime::new();
460        let module = runtime
461            .load(&ModuleBody::Rhai(source.to_string()))
462            .expect("the module compiles");
463
464        let mut recording =
465            RecordingCapabilities::new(LiveCapabilities::new(addr, Protocol::Tcp, None, &budget()));
466        let mut instance = runtime
467            .instantiate(&module, &grant())
468            .expect("instantiates");
469        let live = runtime
470            .run(&mut instance, &ctx(addr.port()), &[], &mut recording)
471            .expect("a clean live run");
472        let tape = recording.into_tape();
473
474        // Through JSON and back.
475        let json = serde_json::to_string(&CapTapeRecord::from(&tape)).expect("serializes");
476        let restored = serde_json::from_str::<CapTapeRecord>(&json)
477            .expect("deserializes")
478            .rebuild();
479        assert_eq!(
480            restored, tape,
481            "the tape did not survive the JSON round-trip"
482        );
483
484        let mut caps = RecordedCapabilities::from_tape(restored);
485        let mut instance = runtime
486            .instantiate(&module, &grant())
487            .expect("instantiates");
488        let replayed = runtime
489            .run(&mut instance, &ctx(addr.port()), &[], &mut caps)
490            .expect("a clean replay");
491
492        assert_eq!(
493            live, replayed,
494            "the findings did not survive the journal round-trip"
495        );
496    }
497
498    /// `ctx.hostname` survives the journal; a line without one reads as `None`.
499    #[test]
500    fn a_runs_host_name_survives_the_journal_line() {
501        let detection = DetectionId::new("d", Version::new(1, 0, 0), "h").expect("an id");
502        let run = |host_name: Option<&str>| DetectionRunRecord {
503            host: "192.0.2.1".to_string(),
504            host_name: host_name.map(str::to_string),
505            port: 443,
506            protocol: "tcp".to_string(),
507            detection: DetectionIdRecord::from(&detection),
508            responses: Vec::new(),
509            tape: CapTapeRecord::from(&CapTape::default()),
510        };
511        let runs = vec![run(Some("box.example")), run(None)];
512
513        let read: Vec<DetectionRunRecord> = PortRunsRecord::grouping(&runs)
514            .into_iter()
515            .map(|line| serde_json::to_string(&line).expect("serializes"))
516            .flat_map(|json| {
517                serde_json::from_str::<DetectionLine>(&json)
518                    .expect("deserializes")
519                    .into_runs()
520            })
521            .collect();
522        assert_eq!(read, runs);
523
524        let unnamed = serde_json::to_string(&run(None)).expect("serializes");
525        assert!(!unnamed.contains("host_name"), "{unnamed}");
526    }
527
528    #[test]
529    fn every_error_kind_is_named_and_read_back() {
530        // Every error kind's name parses back to its own variant.
531        let errors = [
532            CapError::ByteBudgetExhausted,
533            CapError::ConnectionBudgetExhausted,
534            CapError::Denied("out of scope".to_string()),
535            CapError::OutOfDescriptors,
536            CapError::Withheld,
537            CapError::TimedOut,
538            CapError::ConnectionRefused,
539            CapError::Reset,
540        ];
541        for error in errors {
542            let rebuilt = CapErrorRecord::from(&error).rebuild();
543            assert_eq!(
544                rebuilt, error,
545                "an error kind did not survive its wire name"
546            );
547        }
548    }
549
550    #[test]
551    fn a_tape_with_non_ascii_hex_reads_back_empty_rather_than_panicking() {
552        // A planted journal may hold multi-byte characters of even byte length in
553        // a hex field; reading must not slice mid-character.
554        let record = CapTapeRecord {
555            speaks: vec![SpeakExchangeRecord {
556                sent: "€€".to_string(),
557                reply: Some(" zz not hex".to_string()),
558                error: None,
559            }],
560            resolves: Vec::new(),
561            nows: Vec::new(),
562        };
563
564        let tape = record.rebuild();
565        assert_eq!(tape.speaks.len(), 1);
566        assert_eq!(tape.speaks[0].sent, Vec::<u8>::new());
567        assert_eq!(tape.speaks[0].reply, Ok(Vec::new()));
568    }
569}