nyx_agent_types/verify.rs
1//! Verifier wire types.
2//!
3//! The deterministic payload runner emits a [`VerifyResult`] per
4//! finding. The shape mirrors nyx's existing dynamic-verify schema: a
5//! tagged verdict, the oracle predicate the runner evaluated, the two
6//! per-payload runs (vuln + benign) that produced it, and the
7//! `attack_provenance` of the payload pair (Curated upstream payloads
8//! vs. LlmSynthesised pairs from PayloadSynthesis).
9//!
10//! Differential rule v1: a finding is [`VerifyVerdict::Confirmed`] iff
11//! the vuln payload trips the oracle AND the benign control does not.
12//! Any other combination is [`VerifyVerdict::NotConfirmed`]. Errors
13//! (harness failed to set up, sandbox refused to launch, both runs
14//! timed out before producing output) land as
15//! [`VerifyVerdict::Errored`] so the operator-facing UI can distinguish
16//! "we ran it and it did not exploit" from "we never got a clean
17//! signal".
18//!
19//! `replay_stable` is set by the optional second run the payload
20//! runner performs when `[run] replay_stable_check = true`. The default
21//! configuration leaves it `None` so a single run does not have to lie
22//! about determinism.
23
24use serde::{Deserialize, Serialize};
25
26use crate::payload::AttackProvenance;
27
28/// Final verdict for a single finding under differential rule v1.
29#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
30pub enum VerifyVerdict {
31 /// Vuln payload tripped the oracle AND benign control stayed clean.
32 Confirmed,
33 /// Neither payload tripped the oracle, both tripped, or only the
34 /// benign control tripped.
35 NotConfirmed,
36 /// Harness setup failed, the sandbox refused to launch, or both
37 /// runs produced no readable output before the per-run timeout.
38 Errored,
39}
40
41impl VerifyVerdict {
42 pub fn as_str(self) -> &'static str {
43 match self {
44 VerifyVerdict::Confirmed => "Confirmed",
45 VerifyVerdict::NotConfirmed => "NotConfirmed",
46 VerifyVerdict::Errored => "Errored",
47 }
48 }
49}
50
51/// Differential-rule oracle predicates the payload runner can evaluate.
52///
53/// `OutputContains` is the simplest sink probe: scan the sandboxed
54/// child's stdout + stderr for a string marker. `SinkProbe` adds a
55/// sentinel-file path the harness writes to when its instrumented sink
56/// fires; the runner checks both the file's existence and (optionally)
57/// its contents.
58#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
59#[serde(tag = "kind")]
60pub enum Oracle {
61 /// Trip when `marker` appears in captured stdout/stderr.
62 OutputContains { marker: String },
63 /// Trip when `sentinel_path` exists after the run. When
64 /// `expect_contains` is set, additionally require the file's
65 /// contents to include that substring.
66 SinkProbe {
67 sentinel_path: String,
68 #[serde(default)]
69 expect_contains: Option<String>,
70 },
71}
72
73/// Captured outcome of a single sandboxed payload run.
74#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
75pub struct VerifyRun {
76 /// The payload bytes that were spliced into the harness.
77 pub payload: Vec<u8>,
78 /// `true` when the oracle predicate fired.
79 pub oracle_fired: bool,
80 /// Exit code observed by the sandbox. Signal-killed children carry
81 /// the conventional `128 + signum`.
82 pub exit_code: i32,
83 /// `true` iff the sandbox tore the child down because the
84 /// per-run timeout fired before the child exited on its own.
85 pub timed_out: bool,
86 /// Captured stdout, capped at the sandbox's `max_output_bytes`.
87 pub stdout: Vec<u8>,
88 /// Captured stderr, capped at the sandbox's `max_output_bytes`.
89 pub stderr: Vec<u8>,
90 /// Wall-clock duration in milliseconds.
91 pub duration_ms: i64,
92}
93
94/// Verifier wire shape. The runner emits one [`VerifyResult`] per
95/// finding it confirms or rejects.
96#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
97pub struct VerifyResult {
98 pub finding_id: String,
99 pub verdict: VerifyVerdict,
100 pub oracle: Oracle,
101 /// Vuln payload run.
102 pub vuln_run: VerifyRun,
103 /// Benign-control run. Required for differential rule v1; without
104 /// it the runner refuses to emit `Confirmed`.
105 pub benign_run: VerifyRun,
106 /// Source of the payload pair the verifier consumed.
107 pub attack_provenance: AttackProvenance,
108 /// Stamped `true` when an optional second run produced an identical
109 /// verdict. Stays `None` when the replay-stable check is disabled.
110 #[serde(default)]
111 pub replay_stable: Option<bool>,
112 /// Free-form diagnostic for `Errored` verdicts. Empty on a clean
113 /// `Confirmed` / `NotConfirmed` decision.
114 #[serde(default)]
115 pub error_message: Option<String>,
116}
117
118impl VerifyResult {
119 /// Apply differential rule v1 to a fresh pair of runs.
120 pub fn from_runs(
121 finding_id: String,
122 oracle: Oracle,
123 vuln_run: VerifyRun,
124 benign_run: VerifyRun,
125 attack_provenance: AttackProvenance,
126 ) -> Self {
127 let verdict = if vuln_run.oracle_fired && !benign_run.oracle_fired {
128 VerifyVerdict::Confirmed
129 } else {
130 VerifyVerdict::NotConfirmed
131 };
132 Self {
133 finding_id,
134 verdict,
135 oracle,
136 vuln_run,
137 benign_run,
138 attack_provenance,
139 replay_stable: None,
140 error_message: None,
141 }
142 }
143
144 /// Construct an `Errored` verdict carrying `message`. Both runs are
145 /// recorded for forensics; the caller is responsible for providing
146 /// the best-effort capture they have.
147 pub fn errored(
148 finding_id: String,
149 oracle: Oracle,
150 vuln_run: VerifyRun,
151 benign_run: VerifyRun,
152 attack_provenance: AttackProvenance,
153 message: String,
154 ) -> Self {
155 Self {
156 finding_id,
157 verdict: VerifyVerdict::Errored,
158 oracle,
159 vuln_run,
160 benign_run,
161 attack_provenance,
162 replay_stable: None,
163 error_message: Some(message),
164 }
165 }
166}
167
168#[cfg(test)]
169mod tests {
170 use super::*;
171
172 fn run(oracle_fired: bool) -> VerifyRun {
173 VerifyRun {
174 payload: b"x".to_vec(),
175 oracle_fired,
176 exit_code: 0,
177 timed_out: false,
178 stdout: Vec::new(),
179 stderr: Vec::new(),
180 duration_ms: 5,
181 }
182 }
183
184 #[test]
185 fn confirmed_iff_vuln_fires_and_benign_clean() {
186 let oracle = Oracle::OutputContains { marker: "X".into() };
187 let v = VerifyResult::from_runs(
188 "f".into(),
189 oracle.clone(),
190 run(true),
191 run(false),
192 AttackProvenance::Curated,
193 );
194 assert_eq!(v.verdict, VerifyVerdict::Confirmed);
195
196 let v = VerifyResult::from_runs(
197 "f".into(),
198 oracle.clone(),
199 run(false),
200 run(false),
201 AttackProvenance::Curated,
202 );
203 assert_eq!(v.verdict, VerifyVerdict::NotConfirmed);
204
205 let v = VerifyResult::from_runs(
206 "f".into(),
207 oracle.clone(),
208 run(true),
209 run(true),
210 AttackProvenance::Curated,
211 );
212 assert_eq!(v.verdict, VerifyVerdict::NotConfirmed, "benign trip ruins the differential");
213
214 let v = VerifyResult::from_runs(
215 "f".into(),
216 oracle,
217 run(false),
218 run(true),
219 AttackProvenance::Curated,
220 );
221 assert_eq!(v.verdict, VerifyVerdict::NotConfirmed);
222 }
223
224 #[test]
225 fn errored_carries_message_and_provenance() {
226 let oracle =
227 Oracle::SinkProbe { sentinel_path: ".nyx/sentinel".into(), expect_contains: None };
228 let v = VerifyResult::errored(
229 "f".into(),
230 oracle,
231 run(false),
232 run(false),
233 AttackProvenance::LlmSynthesised,
234 "harness setup failed".into(),
235 );
236 assert_eq!(v.verdict, VerifyVerdict::Errored);
237 assert_eq!(v.error_message.as_deref(), Some("harness setup failed"));
238 assert_eq!(v.attack_provenance, AttackProvenance::LlmSynthesised);
239 }
240
241 #[test]
242 fn verify_result_roundtrips_through_serde() {
243 let oracle = Oracle::OutputContains { marker: "leak".into() };
244 let v = VerifyResult::from_runs(
245 "fid".into(),
246 oracle,
247 run(true),
248 run(false),
249 AttackProvenance::LlmSynthesised,
250 );
251 let s = serde_json::to_string(&v).unwrap();
252 let back: VerifyResult = serde_json::from_str(&s).unwrap();
253 assert_eq!(back, v);
254 }
255}