mockforge_bench/conformance/self_test.rs
1//! Positive + per-category negative request driver against a live server.
2//!
3//! Issue #79 round 13 (4) — Srikanth's (e) ask: a way to test both
4//! positive and negative compliance scenarios separately, where the
5//! positive cases should pass and the negative cases should be
6//! rejected.
7//!
8//! This module sits *alongside* the existing conformance executor
9//! (which drives k6 / native checks on a single positive call per
10//! operation). The self-test driver synthesises per-category
11//! deliberately-bad requests and asserts that the server actually
12//! rejects them with a 4xx — useful when verifying that
13//! `validate_request_with_all` is wired correctly for the user's spec
14//! (the exact gap that round-13 (3) fixed).
15//!
16//! Scope of the initial MVP: covers the highest-signal negatives —
17//! empty body when one is required, missing required query/header
18//! params, and wrong-type path params. Doesn't try to mutate every
19//! field of a JSON-Schema-validated body; that's a follow-up.
20
21use super::spec_driven::{AnnotatedOperation, ApiKeyLocation, SecuritySchemeInfo};
22use reqwest::{Client, Method};
23use std::collections::BTreeMap;
24use std::net::IpAddr;
25use std::sync::atomic::{AtomicUsize, Ordering};
26use std::sync::{Arc, Mutex};
27use std::time::Duration;
28
29/// Round 23 (c-iii) — per-direction body cap when capturing
30/// request/response payloads to `conformance-self-test-requests.jsonl`.
31/// 16 KiB keeps a 1000-case run under ~32 MB even if every payload
32/// fills the cap, while still preserving enough of a typical JSON body
33/// (or a stack-trace error response) to debug from.
34const CAPTURE_BODY_CAP_BYTES: usize = 16 * 1024;
35
36/// Round 17.2 — cap on schema-driven negatives per operation. A spec
37/// with 100 properties per body could produce hundreds of mutations
38/// for a single operation; combined with thousands of operations
39/// that's a runaway test matrix. 12 covers the highest-signal
40/// mutations (type mismatch + required-removed + a few constraint
41/// breaks) without exploding wall time on large specs.
42const SCHEMA_MUTATION_CAP: usize = 12;
43
44/// Round 25 (k) — content-type swap probes. For operations declaring a
45/// JSON request body, each entry below produces one probe that lies
46/// about Content-Type while keeping the JSON payload. A spec-compliant
47/// server should respond 415 (or 400). Order matches the order
48/// Srikanth listed in his round-23 reply: XML, YAML, multipart, and
49/// the URL-encoded variant he added in round 24.
50const CONTENT_TYPE_SWAP_VARIANTS: &[(&str, &str)] = &[
51 ("application/xml", "request-body:content-type-mismatch:xml"),
52 ("application/yaml", "request-body:content-type-mismatch:yaml"),
53 ("multipart/form-data", "request-body:content-type-mismatch:multipart"),
54 (
55 "application/x-www-form-urlencoded",
56 "request-body:content-type-mismatch:urlencoded",
57 ),
58];
59
60/// Round 27 (k variant b) — embedded content payloads. Content-Type
61/// stays `application/json` and the envelope IS valid JSON; we just
62/// stuff a non-JSON snippet into a string field's value. The test
63/// surfaces servers that try to parse string field contents (e.g.
64/// XML-EE expanders, YAML loaders, urlencoded parsers) and crash on
65/// the payload — a 5xx here is the finding. Label, payload pairs:
66const EMBEDDED_CONTENT_VARIANTS: &[(&str, &str)] = &[
67 ("request-body:embedded-content:xml", "<root><cmd>execute()</cmd></root>"),
68 ("request-body:embedded-content:yaml", "key: value\n- item1\n- item2"),
69 (
70 "request-body:embedded-content:multipart",
71 "--boundary\r\nContent-Disposition: form-data; name=\"x\"\r\n\r\nval\r\n--boundary--",
72 ),
73 ("request-body:embedded-content:urlencoded", "a=1&b=2&c=hello%20world"),
74];
75
76/// Configuration for a self-test run.
77#[derive(Debug, Clone)]
78pub struct SelfTestConfig {
79 pub target_url: String,
80 pub skip_tls_verify: bool,
81 pub timeout: Duration,
82 /// Optional extra headers to attach to every request (e.g. auth).
83 pub extra_headers: Vec<(String, String)>,
84 /// Delay between requests to avoid hammering the server.
85 pub delay_between_requests: Duration,
86 /// Round 18.1 — base path to prepend to every spec path. When the
87 /// spec declares `/users` and the deployed API is served under
88 /// `/api`, `--base-path /api` should make the self-test hit
89 /// `https://target/api/users` instead of `https://target/users`.
90 /// Pre-fix this was ignored entirely and every operation 404'd
91 /// (Srikanth's vCenter run on 0.3.152: 1275 positives, 1275 4xx).
92 pub base_path: Option<String>,
93 /// Round 18.5 — local source IPs to bind outgoing requests to.
94 /// Each IP must already be assigned to an interface on the host.
95 /// Operations round-robin through the resulting client pool.
96 pub source_ips: Vec<IpAddr>,
97 /// Round 18.5 — fake source IPs to advertise via forwarded-IP
98 /// headers (used to exercise GEODB lookup at the destination).
99 /// Rotated per operation.
100 pub geo_source_ips: Vec<IpAddr>,
101 /// Which forwarded-IP header(s) to populate when `geo_source_ips`
102 /// is non-empty. Empty → no-op; default below sets the standard
103 /// three-header set.
104 pub geo_source_headers: Vec<String>,
105 /// Round 23 (c-iii) — when `Some`, every probe captures method, URL,
106 /// request headers/body and response status/headers/body into this
107 /// sink. Caller drains it after `run_self_test` and writes
108 /// `conformance-self-test-requests.jsonl`. None → no capture (zero
109 /// extra allocations on the hot path).
110 pub capture: Option<Arc<Mutex<Vec<CaseCapture>>>>,
111 /// Round 25 — when true, validate every probe's response body
112 /// against the spec's response schema for the actual status
113 /// returned (closes round 21.3 / Srikanth's a2 / a3 ask). The
114 /// validation result lands in `CaseCapture::response_schema_error`
115 /// (None → matched, or no schema for that status). Default false:
116 /// JSON-Schema validation of large response bodies adds wall-clock
117 /// time and the user has to opt in.
118 pub validate_response_schemas: bool,
119 /// Round 33 (#823) — human-readable label for the OpenAPI spec
120 /// this run is exercising. Stamped on every `CaseCapture` so the
121 /// per-endpoint summary can attribute rows back to a spec in
122 /// multi-spec / multi-target benches. `None` when the bench didn't
123 /// track a spec path.
124 pub spec_label: Option<String>,
125 /// Round 47 (#79) — Srikanth on 0.3.191: "I did not see network
126 /// logs in the mockforge bench and conformance traffic if used
127 /// the [self-test] command". The r46 wire-level event sink only
128 /// existed on the native conformance executor; this matches it on
129 /// the self-test side. When `Some`, every `reqwest::Error` from
130 /// `send().await` is classified and pushed to this sink; caller
131 /// drains it into `conformance-network-events.json` next to the
132 /// JSONL capture. None → no extra allocations on the hot path.
133 pub network_events: Option<Arc<Mutex<Vec<NetworkEvent>>>>,
134 /// Round 49 (#79) — current iteration number (1-indexed). The
135 /// runner stamps it on every CaseCapture so the JSONL line and
136 /// violation rows carry the iteration counter. Defaults to 1
137 /// for non-looping runs.
138 pub current_iteration: u32,
139}
140
141/// Round 47 (#79) — wire-level network event captured by the self-test
142/// driver. Same shape as the native executor's `NetworkEvent` so
143/// downstream tooling can consume one file across executor variants.
144#[derive(Debug, Clone, serde::Serialize)]
145pub struct NetworkEvent {
146 pub timestamp: chrono::DateTime<chrono::Utc>,
147 pub check: String,
148 pub method: String,
149 pub url: String,
150 pub kind: String,
151 pub message: String,
152}
153
154/// Round 23 (c-iii) — one captured request/response pair, one per
155/// probe (positive or negative). Serialised as a JSON line in
156/// `conformance-self-test-requests.jsonl`. Headers are kept as
157/// `BTreeMap` for stable ordering. Bodies are truncated to
158/// `CAPTURE_BODY_CAP_BYTES`; `*_truncated` flags whether more was
159/// dropped.
160#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
161pub struct CaseCapture {
162 pub label: String,
163 pub method: String,
164 pub url: String,
165 pub request_headers: BTreeMap<String, String>,
166 pub request_body: Option<String>,
167 pub request_body_truncated: bool,
168 pub response_status: u16,
169 pub response_headers: BTreeMap<String, String>,
170 pub response_body: Option<String>,
171 pub response_body_truncated: bool,
172 pub error: Option<String>,
173 /// Round 25 — when `validate_response_schemas` is on and the spec
174 /// declares a schema for `response_status`, this carries the
175 /// validation message (or None when the body matched, or no schema
176 /// was declared for that status). Serialised verbatim in the JSONL
177 /// and rendered in the HTML viewer.
178 #[serde(default, skip_serializing_if = "Option::is_none")]
179 pub response_schema_error: Option<String>,
180 /// Round 28 — Srikanth's "Is it possible to put expected response
181 /// code status in both jsonl and jsonl report" ask. Human-readable
182 /// expected status range: `"2xx-3xx"` for positive probes,
183 /// `"4xx"` for negatives. Lets users `jq` for misses
184 /// (`.response_status as $s | .expected_status_range == "4xx"
185 /// and ($s < 400 or $s >= 500)`) and powers the HTML viewer's
186 /// "show mismatches only" filter.
187 #[serde(default)]
188 pub expected_status_range: String,
189 /// Round 33 (#823) — the spec's path template (e.g.
190 /// `/users/{id}`) before path-param substitution. Lets the
191 /// per-endpoint summary collapse `/users/X` and `/users/Y` into
192 /// one row. Empty string when the call site predates this field
193 /// (older `CaseCapture` payloads on disk also deserialise OK).
194 #[serde(default)]
195 pub path_template: String,
196 /// Round 33 (#823) — basename (or fallback to full path) of the
197 /// OpenAPI spec file this probe came from. Lets multi-spec runs
198 /// attribute rows back to the spec they came from. `None` when
199 /// the bench didn't track a spec path.
200 #[serde(default, skip_serializing_if = "Option::is_none")]
201 pub spec_label: Option<String>,
202 /// Round 36 (#876) — mockforge version that ran the probe.
203 /// Stamped from `CARGO_PKG_VERSION` at compile time. Also sent
204 /// as the `X-Mockforge-Client-Version` request header so a
205 /// matching `ServerConformanceViolation.client_mockforge_version`
206 /// can be cross-correlated. Empty string when the capture
207 /// pre-dates this field.
208 #[serde(default)]
209 pub mockforge_version: String,
210 /// Round 36 (#876) — wall-clock moment the bench driver sent the
211 /// request, as RFC3339 / ISO-8601. Also sent as the
212 /// `X-Mockforge-Client-Sent-At` request header so the server-side
213 /// `ServerConformanceViolation.client_sent_at` carries the same
214 /// value. Empty string when the capture pre-dates this field.
215 #[serde(default)]
216 pub client_sent_at: String,
217 /// Round 49 (#79) — Srikanth on 0.3.193: "Is it possible to
218 /// differentiate in the logs what is the iteration count that
219 /// way I will know how many requests are sent with that
220 /// violation." Stamped from the
221 /// `SelfTestConfig::current_iteration` field by the outer loop
222 /// in command.rs before each call to `run_self_test_with_deadline`.
223 /// 1-indexed; defaults to 1 for single-iteration runs so an older
224 /// JSONL that didn't carry the field deserialises as iteration 1.
225 #[serde(default = "default_iteration")]
226 pub iteration: u32,
227}
228
229fn default_iteration() -> u32 {
230 1
231}
232
233impl Default for SelfTestConfig {
234 fn default() -> Self {
235 Self {
236 target_url: "http://localhost:3000".into(),
237 skip_tls_verify: false,
238 timeout: Duration::from_secs(15),
239 extra_headers: Vec::new(),
240 delay_between_requests: Duration::from_millis(0),
241 base_path: None,
242 source_ips: Vec::new(),
243 geo_source_ips: Vec::new(),
244 geo_source_headers: default_geo_source_headers(),
245 capture: None,
246 validate_response_schemas: false,
247 spec_label: None,
248 network_events: None,
249 current_iteration: 1,
250 }
251 }
252}
253
254/// Truncate `body` to `CAPTURE_BODY_CAP_BYTES` on a UTF-8 boundary,
255/// returning the trimmed string and whether truncation occurred. Used
256/// for both request and response bodies in the capture sink.
257fn truncate_body_for_capture(body: &str) -> (String, bool) {
258 if body.len() <= CAPTURE_BODY_CAP_BYTES {
259 return (body.to_string(), false);
260 }
261 let mut end = CAPTURE_BODY_CAP_BYTES;
262 while end > 0 && !body.is_char_boundary(end) {
263 end -= 1;
264 }
265 (body[..end].to_string(), true)
266}
267
268/// Default forwarded-IP header set. Covers the three conventions a
269/// real GEODB front-end is likely to read in this order of
270/// preference: Cloudflare (`CF-Connecting-IP`), Akamai/CloudFront
271/// (`True-Client-IP`), then the de-facto standard
272/// `X-Forwarded-For`. Override via `--geo-source-header` to test a
273/// specific stack.
274pub fn default_geo_source_headers() -> Vec<String> {
275 vec![
276 "X-Forwarded-For".to_string(),
277 "True-Client-IP".to_string(),
278 "CF-Connecting-IP".to_string(),
279 ]
280}
281
282/// Outcome of a single test case (positive or negative).
283#[derive(Debug, Clone, serde::Serialize)]
284pub struct CaseOutcome {
285 pub label: String,
286 pub expected_4xx: bool,
287 pub actual_status: u16,
288 /// True when the response status matches expectation
289 /// (positive → 2xx-3xx, negative → 4xx).
290 pub passed: bool,
291}
292
293/// All cases run against one annotated operation.
294#[derive(Debug, Clone, serde::Serialize)]
295pub struct OperationResult {
296 pub method: String,
297 pub path: String,
298 pub positive: Option<CaseOutcome>,
299 pub negatives: Vec<CaseOutcome>,
300}
301
302/// Summary report rolled up across all operations.
303#[derive(Debug, Default, Clone, serde::Serialize)]
304pub struct SelfTestReport {
305 pub positive_pass: usize,
306 pub positive_fail: usize,
307 /// Per category: count of negative cases the server correctly
308 /// rejected with a 4xx (we caught the spec violation).
309 pub negative_caught: BTreeMap<String, usize>,
310 /// Per category: count of negative cases that should have been
311 /// rejected but came back with a non-4xx (validator gap).
312 pub negative_missed: BTreeMap<String, usize>,
313 pub operations: Vec<OperationResult>,
314}
315
316/// Round 58 (#79) — an unambiguous, no-analysis-needed problem surfaced by the
317/// self-test. Srikanth on 0.3.205: "Is it possible to give another option ...
318/// that should say for sure this is an issue. Currently both caught and missed
319/// needs manual intervention and deep analysis ... very time consuming."
320#[derive(Debug, Clone, serde::Serialize)]
321pub struct DefiniteIssue {
322 pub method: String,
323 pub path: String,
324 /// `valid_request_rejected` (target refused a spec-valid request) or
325 /// `server_error` (target returned 5xx / crashed instead of a clean reply).
326 pub kind: String,
327 pub status: u16,
328 pub detail: String,
329}
330
331/// Round 59 (#79) — per-injection-type tally of OWASP probes. Srikanth on
332/// 0.3.206 was testing a WAF (`waaptest.net` targets): "owasp: 0 caught / 8127
333/// missed" but "Definite issues: none", and asked whether real issues were
334/// being hidden. They are not hidden from the CONTRACT view (an SQLi string in
335/// a string field is spec-valid, so it is correctly not a schema violation),
336/// but for a WAF each ACCEPTED injection payload is one it did not block. This
337/// surfaces that count so a security tester sees it without eyeballing rows.
338#[derive(Debug, Clone, serde::Serialize)]
339pub struct SecurityProbeStat {
340 /// Injection family, e.g. `sqli`, `xss`, `command-injection`.
341 pub injection: String,
342 /// Probes the target let through (status < 400): for a WAF, NOT blocked.
343 pub accepted: usize,
344 /// Probes the target blocked with a `4xx`.
345 pub blocked: usize,
346 /// Probes that made the target return a `5xx` (also a Definite issue).
347 pub errored: usize,
348}
349
350/// Round 59 (#79) — a single OWASP injection probe the target ACCEPTED, for the
351/// `conformance-owasp-accepted.json` sidecar so a WAF tester can grep which URLs
352/// let which payloads through (matching their proxy's own logs).
353#[derive(Debug, Clone, serde::Serialize)]
354pub struct AcceptedOwaspProbe {
355 pub method: String,
356 pub path: String,
357 pub injection: String,
358 pub status: u16,
359}
360
361impl SelfTestReport {
362 /// All-pass means every positive case got 2xx-3xx and every
363 /// negative case got 4xx.
364 pub fn all_passed(&self) -> bool {
365 self.positive_fail == 0 && self.negative_missed.values().sum::<usize>() == 0
366 }
367
368 /// Round 58 (#79) — the subset of findings that are unambiguously wrong, so
369 /// a user does not have to eyeball every caught/missed row across every API.
370 /// Two classes need no judgement:
371 /// 1. a POSITIVE (spec-valid) request the target REJECTED (4xx) — it broke
372 /// a legitimate call;
373 /// 2. ANY probe, positive or negative, that drew a 5xx — the target
374 /// crashed instead of replying cleanly (a bad request should get a
375 /// clean 4xx, never a 500).
376 /// Deliberately EXCLUDES "missed" negatives on spec-valid probes: those are
377 /// the ones that genuinely need per-probe analysis (see the caught/missed
378 /// legend), so keeping them out is the whole point of this view.
379 pub fn definite_issues(&self) -> Vec<DefiniteIssue> {
380 let mut issues = Vec::new();
381 for op in &self.operations {
382 if let Some(pos) = &op.positive {
383 if !pos.passed {
384 let (kind, detail) = if pos.actual_status >= 500 {
385 (
386 "server_error",
387 "target returned 5xx for a spec-valid request (crashed instead of serving it)"
388 .to_string(),
389 )
390 } else {
391 (
392 "valid_request_rejected",
393 "target refused a spec-valid request (a legitimate call it should accept)"
394 .to_string(),
395 )
396 };
397 issues.push(DefiniteIssue {
398 method: op.method.clone(),
399 path: op.path.clone(),
400 kind: kind.to_string(),
401 status: pos.actual_status,
402 detail,
403 });
404 }
405 }
406 for neg in &op.negatives {
407 if neg.actual_status >= 500 {
408 issues.push(DefiniteIssue {
409 method: op.method.clone(),
410 path: op.path.clone(),
411 kind: "server_error".to_string(),
412 status: neg.actual_status,
413 detail: format!(
414 "target returned 5xx for the '{}' negative probe (should reject a bad request with a clean 4xx, not crash)",
415 neg.label
416 ),
417 });
418 }
419 }
420 }
421 issues
422 }
423
424 /// Round 59 (#79) — split the `owasp:*` injection probes by family and count
425 /// how many the target accepted (status < 400), blocked (4xx), or errored on
426 /// (5xx). Empty when the run fired no owasp probes. Labels are `owasp:<fam>`
427 /// (optionally `owasp:<fam>:<scope>`).
428 pub fn owasp_summary(&self) -> Vec<SecurityProbeStat> {
429 // (accepted, blocked, errored) per injection family, name-sorted.
430 let mut by_type: BTreeMap<String, (usize, usize, usize)> = BTreeMap::new();
431 for op in &self.operations {
432 for neg in &op.negatives {
433 let mut parts = neg.label.splitn(3, ':');
434 if parts.next() != Some("owasp") {
435 continue;
436 }
437 let injection = parts.next().unwrap_or("other").to_string();
438 let slot = by_type.entry(injection).or_insert((0, 0, 0));
439 if neg.actual_status >= 500 {
440 slot.2 += 1;
441 } else if neg.actual_status >= 400 {
442 slot.1 += 1;
443 } else {
444 slot.0 += 1;
445 }
446 }
447 }
448 by_type
449 .into_iter()
450 .map(|(injection, (accepted, blocked, errored))| SecurityProbeStat {
451 injection,
452 accepted,
453 blocked,
454 errored,
455 })
456 .collect()
457 }
458
459 /// Round 59 (#79) — the individual `owasp:*` probes the target ACCEPTED
460 /// (status < 400), for the grep-able sidecar. For a WAF, each row is an
461 /// injection payload that reached the origin unblocked.
462 pub fn owasp_accepted_probes(&self) -> Vec<AcceptedOwaspProbe> {
463 let mut out = Vec::new();
464 for op in &self.operations {
465 for neg in &op.negatives {
466 let mut parts = neg.label.splitn(3, ':');
467 if parts.next() != Some("owasp") {
468 continue;
469 }
470 if neg.actual_status < 400 {
471 out.push(AcceptedOwaspProbe {
472 method: op.method.clone(),
473 path: op.path.clone(),
474 injection: parts.next().unwrap_or("other").to_string(),
475 status: neg.actual_status,
476 });
477 }
478 }
479 }
480 out
481 }
482
483 /// Round 18.1 — detect the "self-test target is misconfigured"
484 /// case where every positive failed with the *same* status code.
485 /// The classic example: `--base-path /api` was forgotten so every
486 /// request hits a path the server doesn't know and returns 404.
487 /// Pre-warning, the user saw all-green negative buckets (because
488 /// "missing route" 404s look like "validator rejected") and no
489 /// indication that the run was meaningless. Returns Some(status)
490 /// when ≥10 positives all failed with the same status, else None.
491 pub fn detect_target_misconfiguration(&self) -> Option<u16> {
492 if self.positive_pass > 0 || self.positive_fail < 10 {
493 return None;
494 }
495 let mut seen: Option<u16> = None;
496 for op in &self.operations {
497 let Some(p) = &op.positive else {
498 continue;
499 };
500 if p.passed {
501 return None;
502 }
503 match seen {
504 None => seen = Some(p.actual_status),
505 Some(s) if s != p.actual_status => return None,
506 _ => {}
507 }
508 }
509 seen
510 }
511
512 /// Round 47 (#79) — fold a second iteration of the self-test into
513 /// this report so multi-iteration runs aggregate counters across
514 /// passes. Per-category caught / missed counters sum; positive
515 /// counters sum; the `operations` vec records every probe outcome
516 /// so the iteration-N misconfiguration detector still works. Used
517 /// by command.rs's `--conformance-self-test-iterations` /
518 /// `--conformance-self-test-duration` loop.
519 pub fn merge_iteration(&mut self, other: SelfTestReport) {
520 self.positive_pass = self.positive_pass.saturating_add(other.positive_pass);
521 self.positive_fail = self.positive_fail.saturating_add(other.positive_fail);
522 for (k, v) in other.negative_caught {
523 let slot = self.negative_caught.entry(k).or_insert(0);
524 *slot = slot.saturating_add(v);
525 }
526 for (k, v) in other.negative_missed {
527 let slot = self.negative_missed.entry(k).or_insert(0);
528 *slot = slot.saturating_add(v);
529 }
530 self.operations.extend(other.operations);
531 }
532
533 /// Human-readable summary string. One line for positives, one per
534 /// category for negatives. Designed to slot into existing
535 /// `TerminalReporter` output.
536 pub fn render_summary(&self) -> String {
537 let mut out = String::new();
538 out.push_str(&format!(
539 "Positives: {} pass / {} fail\n",
540 self.positive_pass, self.positive_fail
541 ));
542 let mut keys: Vec<&String> =
543 self.negative_caught.keys().chain(self.negative_missed.keys()).collect();
544 keys.sort();
545 keys.dedup();
546 // Round 57 (#79) — Srikanth asked what "caught" vs "missed" mean.
547 // Spell it out once, inline, so the console output is self-explanatory.
548 if !keys.is_empty() {
549 out.push_str(
550 " (negatives are deliberately-bad requests that SHOULD draw a 4xx from the TARGET: \
551 \"caught\" = the target rejected it, \"missed\" = the target accepted it. \
552 A high \"missed\" is not automatically a target bug: many probes are spec-valid by \
553 construction, e.g. a string field accepts any string or an optional query param can \
554 be dropped, so the contract permits them; those are the parameter_negative_probe / \
555 informational rows in conformance-request-violations.json.)\n",
556 );
557 }
558 for cat in keys {
559 let caught = self.negative_caught.get(cat).copied().unwrap_or(0);
560 let missed = self.negative_missed.get(cat).copied().unwrap_or(0);
561 let mark = if missed == 0 { "✓" } else { "⚠" };
562 out.push_str(&format!(
563 "Negatives [{}]: {} caught / {} missed {}\n",
564 cat, caught, missed, mark
565 ));
566 }
567 // Round 58 (#79) — surface the unambiguous problems up front so they
568 // don't have to be dug out of the caught/missed rollup. Capped in the
569 // console; the full set is in conformance-definite-issues.json.
570 let issues = self.definite_issues();
571 if issues.is_empty() {
572 out.push_str(
573 "Definite issues: none (no spec-valid request was rejected and no probe drew a 5xx)\n",
574 );
575 } else {
576 out.push_str(&format!(
577 "Definite issues ({}) — unambiguous, no per-probe analysis needed (full list in conformance-definite-issues.json):\n",
578 issues.len()
579 ));
580 const CAP: usize = 20;
581 for iss in issues.iter().take(CAP) {
582 out.push_str(&format!(
583 " {} {} -> {} [{}]: {}\n",
584 iss.method, iss.path, iss.status, iss.kind, iss.detail
585 ));
586 }
587 if issues.len() > CAP {
588 out.push_str(&format!(" ... and {} more\n", issues.len() - CAP));
589 }
590 }
591 // Round 59 (#79) — OWASP injection breakdown. For a WAF / security
592 // proxy this is the headline: how many injection payloads the target
593 // let through. It is deliberately NOT folded into "Definite issues"
594 // because an injection string is spec-valid (a string field accepts
595 // any string), so whether "accepted" is a bug depends on whether you
596 // expect a WAF to block it.
597 let owasp = self.owasp_summary();
598 if !owasp.is_empty() {
599 let total_accepted: usize = owasp.iter().map(|s| s.accepted).sum();
600 let total: usize = owasp.iter().map(|s| s.accepted + s.blocked + s.errored).sum();
601 out.push_str(&format!(
602 "Security probes (owasp injection): target ACCEPTED {}/{} payloads (status < 400). \
603 For a WAF/security proxy each accepted payload is one it did NOT block (review it); \
604 for a plain API this is expected (the schema permits arbitrary strings). \
605 Accepted payloads + URLs are in conformance-owasp-accepted.json.\n",
606 total_accepted, total
607 ));
608 for s in &owasp {
609 out.push_str(&format!(
610 " owasp:{}: {} accepted / {} blocked{}\n",
611 s.injection,
612 s.accepted,
613 s.blocked,
614 if s.errored > 0 {
615 format!(" / {} 5xx", s.errored)
616 } else {
617 String::new()
618 }
619 ));
620 }
621 }
622 out
623 }
624}
625
626/// Execute the self-test plan against `config.target_url` for every
627/// `AnnotatedOperation`. Returns the aggregated report; callers
628/// decide how to display it (e.g. via `render_summary` or by writing
629/// the JSON serialisation to disk).
630pub async fn run_self_test(
631 operations: &[AnnotatedOperation],
632 config: &SelfTestConfig,
633) -> Result<SelfTestReport, reqwest::Error> {
634 run_self_test_with_deadline(operations, config, None).await
635}
636
637/// Round 49 (#79) — Srikanth on 0.3.193: `--conformance-self-test-
638/// duration 5m` ran 5:46 because the outer iteration loop in
639/// command.rs only checks the deadline AFTER a full matrix pass
640/// completes. For long iterations this can overshoot by minutes,
641/// which breaks automation that relies on a fixed wall-clock budget.
642/// New optional `deadline` parameter lets the runner break out
643/// mid-iteration once the deadline elapses; returns the partial
644/// report with whatever operations finished before the deadline.
645pub async fn run_self_test_with_deadline(
646 operations: &[AnnotatedOperation],
647 config: &SelfTestConfig,
648 deadline: Option<std::time::Instant>,
649) -> Result<SelfTestReport, reqwest::Error> {
650 // Round 18.5 — build a client pool when `source_ips` is set,
651 // one reqwest::Client per IP, each bound to its local address.
652 // Operations round-robin through the pool. Empty pool → single
653 // default client (the pre-18.5 behaviour).
654 let clients = build_client_pool(config)?;
655 let client_cursor = AtomicUsize::new(0);
656 let geo_cursor = AtomicUsize::new(0);
657
658 let mut report = SelfTestReport::default();
659 for op in operations {
660 // Round 49 — mid-iteration deadline check. Breaks out of the
661 // per-operation loop the moment the wall-clock budget
662 // elapses, so a 5m budget never overshoots by more than one
663 // probe's round-trip.
664 if let Some(d) = deadline {
665 if std::time::Instant::now() >= d {
666 break;
667 }
668 }
669 let client_idx = client_cursor.fetch_add(1, Ordering::Relaxed) % clients.len();
670 let client = &clients[client_idx];
671 let geo_ip = if config.geo_source_ips.is_empty() {
672 None
673 } else {
674 let idx = geo_cursor.fetch_add(1, Ordering::Relaxed) % config.geo_source_ips.len();
675 Some(config.geo_source_ips[idx])
676 };
677 let result = test_operation(client, config, op, geo_ip).await;
678 if let Some(p) = &result.positive {
679 if p.passed {
680 report.positive_pass += 1;
681 } else {
682 report.positive_fail += 1;
683 }
684 }
685 for neg in &result.negatives {
686 let cat = neg.label.split(':').next().unwrap_or("other").to_string();
687 if neg.passed {
688 *report.negative_caught.entry(cat).or_insert(0) += 1;
689 } else {
690 *report.negative_missed.entry(cat).or_insert(0) += 1;
691 }
692 }
693 report.operations.push(result);
694 if !config.delay_between_requests.is_zero() {
695 tokio::time::sleep(config.delay_between_requests).await;
696 }
697 }
698 Ok(report)
699}
700
701/// Round 18.5 — append GEODB forwarded-IP headers to the
702/// operation's declared headers. Returns the original vec untouched
703/// when `geo_ip` is None or `geo_headers` is empty.
704///
705/// If the operation already declares one of the geo headers (rare
706/// but legal), we keep the operation's value — the caller's spec
707/// wins.
708fn effective_op_headers(
709 base: &[(String, String)],
710 geo_ip: Option<IpAddr>,
711 geo_headers: &[String],
712) -> Vec<(String, String)> {
713 let mut out = base.to_vec();
714 let Some(ip) = geo_ip else {
715 return out;
716 };
717 let value = ip.to_string();
718 for h in geo_headers {
719 // Case-insensitive duplicate check: don't override the
720 // spec's own declared value for the header.
721 if out.iter().any(|(k, _)| k.eq_ignore_ascii_case(h)) {
722 continue;
723 }
724 out.push((h.clone(), value.clone()));
725 }
726 out
727}
728
729/// Round 18.5 — build a pool of reqwest clients, one per declared
730/// source IP. Empty `source_ips` → a single default client.
731///
732/// The OS must already have each `source_ip` assigned to an
733/// interface; reqwest's `.local_address()` issues a `bind()` syscall
734/// at connect time, so an IP the kernel doesn't recognise surfaces
735/// as `EADDRNOTAVAIL` at request time, not at builder time.
736fn build_client_pool(config: &SelfTestConfig) -> Result<Vec<Client>, reqwest::Error> {
737 let make = |bind: Option<IpAddr>| -> Result<Client, reqwest::Error> {
738 let mut builder = Client::builder().timeout(config.timeout);
739 if config.skip_tls_verify {
740 builder = builder.danger_accept_invalid_certs(true);
741 }
742 if let Some(addr) = bind {
743 builder = builder.local_address(addr);
744 }
745 builder.build()
746 };
747 if config.source_ips.is_empty() {
748 Ok(vec![make(None)?])
749 } else {
750 config.source_ips.iter().map(|ip| make(Some(*ip))).collect()
751 }
752}
753
754async fn test_operation(
755 client: &Client,
756 config: &SelfTestConfig,
757 op: &AnnotatedOperation,
758 geo_ip: Option<IpAddr>,
759) -> OperationResult {
760 // Round 25 — track the sink length BEFORE we run any probes for
761 // this operation, so that after the probes finish we can mutate
762 // exactly the entries that belong to this op (the capture sink is
763 // shared but `run_self_test` iterates operations sequentially).
764 // Used by the response-schema validation pass below.
765 let sink_start = config.capture.as_ref().and_then(|s| s.lock().ok().map(|g| g.len()));
766
767 let url = build_url_with_base(
768 &config.target_url,
769 config.base_path.as_deref(),
770 &op.path,
771 &op.path_params,
772 );
773 let method = Method::from_bytes(op.method.to_uppercase().as_bytes()).unwrap_or(Method::GET);
774
775 // Round 34 (#828) — stamp every `CaseCapture` with the spec
776 // template PREFIXED by `--base-path`, so the per-endpoint
777 // summary's `path` column matches what the user sees in URLs
778 // and logs. Srikanth searched for `/api/appliance/access/...`
779 // and didn't find it because round 33 stored just `/appliance/
780 // access/...`. Same normalization as `build_url_with_base`:
781 // leading `/` auto-added, trailing `/` stripped, empty
782 // base_path → no prefix at all.
783 let path_template = {
784 // Round 55 — shared with build_url_with_base so `--base-path /`
785 // resolves to no prefix instead of a `//path` double slash.
786 let prefix = base_path_prefix(config.base_path.as_deref());
787 let path = if op.path.starts_with('/') {
788 op.path.clone()
789 } else {
790 format!("/{}", op.path)
791 };
792 format!("{prefix}{path}")
793 };
794
795 // Round 18.5 — pre-compute the operation's effective headers
796 // with the geo source IP baked in. Doing it once here keeps the
797 // per-case `send_case` calls below unchanged. When `geo_ip` is
798 // None the result equals `op.header_params`.
799 let op_headers = effective_op_headers(&op.header_params, geo_ip, &config.geo_source_headers);
800
801 // ── Positive case ────────────────────────────────────────────
802 let positive = send_case(
803 client,
804 config,
805 method.clone(),
806 &url,
807 "positive",
808 ExpectedOutcome::Success,
809 op.sample_body.as_deref(),
810 op.query_params.clone(),
811 op_headers.clone(),
812 &path_template,
813 )
814 .await;
815
816 // ── Negative cases ───────────────────────────────────────────
817 let mut negatives = Vec::new();
818
819 // (a) empty body when one is required.
820 //
821 // Round 16 — drop the `sample_body.is_some()` precondition. Operations
822 // whose body annotator couldn't synthesize a sample previously got
823 // zero negatives (so the self-test reported "all passing" even on
824 // POST /resource with a required body). The spec saying the operation
825 // *has* a request body is enough — an empty object is a valid
826 // negative regardless of whether we have a positive sample.
827 if op.request_body_content_type.is_some() {
828 negatives.push(
829 send_case(
830 client,
831 config,
832 method.clone(),
833 &url,
834 "request-body:empty",
835 ExpectedOutcome::ClientError,
836 Some("{}"),
837 op.query_params.clone(),
838 op_headers.clone(),
839 &path_template,
840 )
841 .await,
842 );
843
844 // (b) wrong-shaped body (array instead of object) — exercises
845 // top-level type validation independently of which fields are
846 // required.
847 negatives.push(
848 send_case(
849 client,
850 config,
851 method.clone(),
852 &url,
853 "request-body:wrong-type",
854 ExpectedOutcome::ClientError,
855 Some("[]"),
856 op.query_params.clone(),
857 op_headers.clone(),
858 &path_template,
859 )
860 .await,
861 );
862
863 // Round 25 (k) — content-type swap probes.
864 //
865 // For operations declaring `application/json` request bodies, send
866 // the SAME json payload (or a synthesised one) under four other
867 // content types: `application/xml`, `application/yaml`,
868 // `multipart/form-data`, `application/x-www-form-urlencoded`.
869 // The spec says the endpoint accepts only JSON, so a strict server
870 // should respond 415 Unsupported Media Type (or 400 if it tries
871 // to parse and fails). A 2xx means the server is accepting
872 // payloads outside its declared content negotiation, which is the
873 // failure mode behind a lot of "we crashed on a malformed XML
874 // upload" incidents.
875 //
876 // Variant (a) of Srikanth's round-23 g ask: lie about the
877 // Content-Type header. The body shape is honest JSON; only the
878 // header is swapped. Variant (b) (JSON envelope with embedded
879 // non-JSON field values) is deferred to round 26 because it
880 // requires a schema-aware field walker.
881 if op
882 .request_body_content_type
883 .as_deref()
884 .map(|ct| ct.contains("json"))
885 .unwrap_or(false)
886 {
887 let payload = op.sample_body.as_deref().unwrap_or("{}");
888 for (ct, label) in CONTENT_TYPE_SWAP_VARIANTS {
889 negatives.push(
890 send_case_with_extra(
891 client,
892 config,
893 method.clone(),
894 &url,
895 label,
896 ExpectedOutcome::ClientError,
897 Some(payload),
898 op.query_params.clone(),
899 // Strip any Content-Type already on the operation
900 // headers (the spec's positive value) so the
901 // probe's value is the only one the server sees.
902 op_headers
903 .iter()
904 .filter(|(k, _)| !k.eq_ignore_ascii_case("content-type"))
905 .cloned()
906 .collect(),
907 // The wrong Content-Type rides on `extra_headers`
908 // so it lands AFTER `send_case_with_extra`'s
909 // unconditional `application/json` insertion in
910 // request-body mode. Actually `send_case_with_extra`
911 // only sets Content-Type when a body is present
912 // AND there's no manual override; passing the
913 // override here wins because reqwest preserves
914 // the last-set header value.
915 vec![("Content-Type".to_string(), (*ct).to_string())],
916 &path_template,
917 )
918 .await,
919 );
920 }
921
922 // Round 27 (k variant b) — embedded non-JSON content
923 // inside a valid JSON envelope. Content-Type stays
924 // application/json (honest) and the body parses as JSON;
925 // only the string-valued payload changes. We expect 2xx-3xx
926 // because the envelope is spec-shape, so the probe surfaces
927 // servers that crash (5xx) trying to parse the embedded
928 // snippet as XML/YAML/etc. A 4xx is also a finding because
929 // it usually means the server's pattern/format validator
930 // tripped on the payload contents, but the user can decide
931 // from the JSONL whether that's a bug or correct narrow-
932 // string-field behaviour.
933 for (label, snippet) in EMBEDDED_CONTENT_VARIANTS {
934 let payload = op.sample_body.as_deref().unwrap_or("{}");
935 // Round 34 (#829) — skip the probe entirely when the
936 // positive sample has no string leaf we can mutate.
937 // The previous round-27 fallback `{"data": <snippet>}`
938 // produced a body that doesn't match the spec's actual
939 // schema for endpoints like vCenter's `consolecli` PUT
940 // (which wants `{enabled: bool}`), so the server
941 // correctly 400'd and the bench misreported the
942 // mismatch as an expectation failure.
943 let Some(body) = embed_payload_in_first_string_field(payload, snippet) else {
944 continue;
945 };
946 negatives.push(
947 send_case(
948 client,
949 config,
950 method.clone(),
951 &url,
952 label,
953 // expected_4xx=false: any non-2xx is a probe
954 // failure. 5xx in particular is "server panicked
955 // on the embedded content".
956 ExpectedOutcome::NotServerError,
957 Some(&body),
958 op.query_params.clone(),
959 op_headers.clone(),
960 &path_template,
961 )
962 .await,
963 );
964 }
965 }
966
967 // Round 17.2 — schema-aware negatives.
968 //
969 // When both a positive sample AND the resolved body schema are
970 // available, mutate the sample per-field (type mismatch,
971 // min/max bounds, pattern, enum out-of-range, required-field
972 // removal) and assert each is rejected with 4xx. Capped at
973 // SCHEMA_MUTATION_CAP per operation so a 100-property body
974 // doesn't explode the test matrix.
975 if let (Some(sample_str), Some(schema)) =
976 (op.sample_body.as_deref(), op.request_body_schema.as_ref())
977 {
978 if let Ok(sample) = serde_json::from_str::<serde_json::Value>(sample_str) {
979 let mutations = super::schema_mutator::mutate_body(&sample, schema);
980 for m in mutations.into_iter().take(SCHEMA_MUTATION_CAP) {
981 let body_str = serde_json::to_string(&m.body).unwrap_or_default();
982 negatives.push(
983 send_case(
984 client,
985 config,
986 method.clone(),
987 &url,
988 &m.label,
989 ExpectedOutcome::ClientError,
990 Some(&body_str),
991 op.query_params.clone(),
992 // Round 24 (f) — was `op.header_params`, which
993 // skipped the geo-IP header. Use `op_headers`
994 // so the geo IP rides with the negative probe
995 // too (positive vs negative coverage must be
996 // symmetric, otherwise a GEODB front-end sees
997 // the rotating IP only on positives).
998 op_headers.clone(),
999 &path_template,
1000 )
1001 .await,
1002 );
1003 }
1004 }
1005 }
1006 }
1007
1008 // Round 17.2 — URI-length probe. Spec-agnostic but schema-aware in
1009 // spirit: most servers cap URIs at 8 KB or so. Append a 9 KB query
1010 // string to the URL and expect 414 URI Too Long (or 400). Skipped
1011 // for operations that already have a heavy positive query.
1012 {
1013 let pad = "p=".to_string() + &"x".repeat(9_000);
1014 let bad_url = if url.contains('?') {
1015 format!("{url}&{pad}")
1016 } else {
1017 format!("{url}?{pad}")
1018 };
1019 negatives.push(
1020 send_case(
1021 client,
1022 config,
1023 method.clone(),
1024 &bad_url,
1025 "parameters:uri-too-long",
1026 ExpectedOutcome::ClientError,
1027 op.sample_body.as_deref(),
1028 op.query_params.clone(),
1029 // Round 24 (f) — see schema-mutation note above. Use
1030 // `op_headers` (carries geo IP) instead of bare
1031 // `op.header_params`.
1032 op_headers.clone(),
1033 &path_template,
1034 )
1035 .await,
1036 );
1037 }
1038
1039 // (e) Round 16 — path-param type probe. Send the first path
1040 // parameter as a literal `"self-test-invalid-id"`: a string that
1041 // contains hyphens, won't parse as an integer, won't parse as a
1042 // UUID, and won't match any typical regex pattern. Operations
1043 // whose spec types the param as `integer` or `string` with a
1044 // `format`/`pattern` will catch this (caught: server returned
1045 // 4xx); operations whose spec lets path params be free-form
1046 // strings will let it through (missed: server returned 2xx).
1047 // Either outcome is informative: a category that's all "missed"
1048 // tells the user their spec is loose on path-param types, which
1049 // is itself worth knowing. Addresses Srikanth's "always all
1050 // passing" report — operations with a path param now produce at
1051 // least one probe instead of zero.
1052 if !op.path_params.is_empty() {
1053 let mut url_with_placeholder = op.path.clone();
1054 if let Some((first_name, _)) = op.path_params.first() {
1055 // Substitute every other path-param with its sample so the
1056 // route shape stays intact and only the first param is bad.
1057 for (name, value) in op.path_params.iter().skip(1) {
1058 if !value.is_empty() {
1059 url_with_placeholder =
1060 url_with_placeholder.replace(&format!("{{{name}}}"), value);
1061 }
1062 }
1063 // Substitute the first param with a guaranteed-invalid
1064 // sentinel that's unlikely to match any reasonable schema:
1065 // contains characters disallowed in numeric IDs *and* UUIDs.
1066 url_with_placeholder =
1067 url_with_placeholder.replace(&format!("{{{first_name}}}"), "self-test-invalid-id");
1068 // Round 18.1 — honour `base_path` here too, otherwise the
1069 // probe URL differs from the positive case and the
1070 // resulting 404 is misattributed to "bad path param".
1071 let bad_url = build_url_with_base(
1072 &config.target_url,
1073 config.base_path.as_deref(),
1074 &url_with_placeholder,
1075 &[],
1076 );
1077 negatives.push(
1078 send_case(
1079 client,
1080 config,
1081 method.clone(),
1082 &bad_url,
1083 "parameters:bad-path-param",
1084 ExpectedOutcome::ClientError,
1085 op.sample_body.as_deref(),
1086 op.query_params.clone(),
1087 op_headers.clone(),
1088 &path_template,
1089 )
1090 .await,
1091 );
1092 }
1093 }
1094
1095 // (c) drop the first required query param
1096 if !op.query_params.is_empty() {
1097 let mut q = op.query_params.clone();
1098 q.remove(0);
1099 negatives.push(
1100 send_case(
1101 client,
1102 config,
1103 method.clone(),
1104 &url,
1105 "parameters:missing-query",
1106 ExpectedOutcome::ClientError,
1107 op.sample_body.as_deref(),
1108 q,
1109 op_headers.clone(),
1110 &path_template,
1111 )
1112 .await,
1113 );
1114 }
1115
1116 // (s) Round 17.3 — security probes.
1117 //
1118 // Operations whose spec declares a security requirement get a
1119 // dedicated set of negatives. The point isn't to test whether the
1120 // server's *real* auth works (the positive case already does that
1121 // via `extra_headers`) — it's to check whether deliberately-bad
1122 // credentials are still rejected, which is exactly the failure
1123 // mode that lets an attacker through a half-wired validator.
1124 //
1125 // Each probe replaces or omits the relevant auth credential and
1126 // expects 401 / 403. A 2xx here is a hard finding: "spec says
1127 // this endpoint is protected, server let unauthenticated /
1128 // wrong-credential traffic through".
1129 //
1130 // Bounded: at most one probe per declared scheme kind, so an
1131 // operation with 3 security requirements doesn't 4× the request
1132 // volume. Skips entirely when `op.security_schemes` is empty.
1133 for probe in build_security_probes(&op.security_schemes) {
1134 // Strip any pre-existing Authorization or known API-key
1135 // header from extra_headers + header_params so the probe
1136 // value is the *only* credential the server sees.
1137 let stripped_extra = strip_auth(&config.extra_headers, &op.security_schemes);
1138 let stripped_headers = strip_auth(&op.header_params, &op.security_schemes);
1139 let stripped_query = strip_auth_query(&op.query_params, &op.security_schemes);
1140 let mut req_headers = stripped_headers;
1141 for (k, v) in &probe.headers {
1142 req_headers.push((k.clone(), v.clone()));
1143 }
1144 // Round 24 (f) — security probes build req_headers from
1145 // `op.header_params` directly (we need the stripped-auth
1146 // variant), so the geo-IP header doesn't ride along
1147 // automatically. Append it here so a GEODB / WAF in front
1148 // of the auth layer still sees the rotating source IP.
1149 if let Some(ip) = geo_ip {
1150 let ip_str = ip.to_string();
1151 for h in &config.geo_source_headers {
1152 let already = req_headers.iter().any(|(k, _)| k.eq_ignore_ascii_case(h));
1153 if !already {
1154 req_headers.push((h.clone(), ip_str.clone()));
1155 }
1156 }
1157 }
1158 let mut req_query = stripped_query;
1159 for (k, v) in &probe.query {
1160 req_query.push((k.clone(), v.clone()));
1161 }
1162 negatives.push(
1163 send_case_with_extra(
1164 client,
1165 config,
1166 method.clone(),
1167 &url,
1168 &probe.label,
1169 ExpectedOutcome::ClientError,
1170 op.sample_body.as_deref(),
1171 req_query,
1172 req_headers,
1173 stripped_extra,
1174 &path_template,
1175 )
1176 .await,
1177 );
1178 }
1179
1180 // (d) drop the first required header
1181 if !op.header_params.is_empty() {
1182 // Round 24 (f) — start from `op_headers` (so the geo IP rides
1183 // along) and only strip the first OPERATION-declared header.
1184 // Slicing past `op.header_params.len()` would otherwise risk
1185 // dropping the geo header itself; `op_headers` is built as
1186 // `op.header_params ++ geo` so index 0 is always operational.
1187 let mut h = op_headers.clone();
1188 if !h.is_empty() {
1189 h.remove(0);
1190 }
1191 negatives.push(
1192 send_case(
1193 client,
1194 config,
1195 method.clone(),
1196 &url,
1197 "parameters:missing-header",
1198 ExpectedOutcome::ClientError,
1199 op.sample_body.as_deref(),
1200 op.query_params.clone(),
1201 h,
1202 &path_template,
1203 )
1204 .await,
1205 );
1206 }
1207
1208 // (w) Round 17.5 — OWASP/WAF unification.
1209 //
1210 // Pull one canonical payload per OWASP category from the existing
1211 // `SecurityPayloads` library and emit an injection probe per
1212 // category. Targets in priority order: (1) substitute the first
1213 // query param's value, (2) substitute the first string field of
1214 // the positive JSON body, (3) skip if neither is available.
1215 //
1216 // Label format `owasp:<category>`, so the existing
1217 // `negative_caught` / `negative_missed` rollup groups all OWASP
1218 // findings under one `owasp` bucket. Expected 4xx (server should
1219 // reject malicious input). A 5xx is a hard finding (server
1220 // crashed on the payload); a 2xx is a soft finding (input passed
1221 // through unfiltered — may or may not be a real vuln).
1222 //
1223 // Bounded: at most one probe per category (7 categories total).
1224 // Skips the operation entirely if no injection target is
1225 // available — open GET endpoints with no params get zero OWASP
1226 // probes, no false signal.
1227 for probe in build_owasp_probes(op) {
1228 negatives.push(
1229 send_case(
1230 client,
1231 config,
1232 method.clone(),
1233 &url,
1234 &probe.label,
1235 ExpectedOutcome::ClientError,
1236 probe.body.as_deref(),
1237 probe.query,
1238 // Round 24 (f) — OWASP injection probes must also
1239 // carry the geo IP, otherwise a WAF / GEODB rule
1240 // tuned to a specific source IP would silently let
1241 // them through.
1242 op_headers.clone(),
1243 &path_template,
1244 )
1245 .await,
1246 );
1247 }
1248
1249 // Round 25 — response-body shape validation pass. For each capture
1250 // this op pushed onto the sink, look up the spec's schema for the
1251 // actual response status and validate. Result lands in
1252 // `response_schema_error` (Some(message) on failure, None on
1253 // pass or no-schema-for-this-status). Runs only when the user
1254 // opted in AND capture is on (we need the body).
1255 if config.validate_response_schemas {
1256 if let (Some(sink), Some(start)) = (config.capture.as_ref(), sink_start) {
1257 if !op.response_schemas.is_empty() {
1258 if let Ok(mut guard) = sink.lock() {
1259 let end = guard.len();
1260 for i in start..end {
1261 let Some(entry) = guard.get_mut(i) else {
1262 continue;
1263 };
1264 let Some(body) = entry.response_body.as_deref() else {
1265 continue;
1266 };
1267 let Some(schema) = op.response_schemas.get(&entry.response_status) else {
1268 continue;
1269 };
1270 entry.response_schema_error = validate_body_against_schema(body, schema);
1271 }
1272 }
1273 }
1274 }
1275 }
1276
1277 OperationResult {
1278 method: op.method.clone(),
1279 path: op.path.clone(),
1280 positive: Some(positive),
1281 negatives,
1282 }
1283}
1284
1285/// Round 25 — validate a JSON body string against an OpenAPI response
1286/// schema (already converted to a `serde_json::Value`). Returns
1287/// `Some(message)` describing the first violation, or `None` on a
1288/// clean pass / non-JSON body / schema-build failure (in which case
1289/// the absence of an error means "we didn't have anything to compare
1290/// against", not "passed"; the caller-side semantics treat absence as
1291/// success because that's what the user sees as silence).
1292/// Round 27 (k variant b) — return a JSON body string identical to
1293/// `sample` except that the first string-valued leaf has been
1294/// replaced with `snippet`. Walks objects depth-first and stops at
1295/// the first string. Returns `None` when `sample` is not parseable
1296/// JSON or has no string field anywhere; the caller skips emitting
1297/// a probe in that case (Round 34 #829: Srikanth on 0.3.178 found
1298/// that the previous `{"data": <snippet>}` fallback envelope didn't
1299/// match real-API schemas like vCenter's `{enabled: bool}` and the
1300/// server correctly 400'd, which the bench then misreported as a
1301/// `2xx-3xx` expectation miss).
1302fn embed_payload_in_first_string_field(sample: &str, snippet: &str) -> Option<String> {
1303 let mut parsed: serde_json::Value = serde_json::from_str(sample).ok()?;
1304 if !replace_first_string(&mut parsed, snippet) {
1305 return None;
1306 }
1307 serde_json::to_string(&parsed).ok()
1308}
1309
1310/// Helper for `embed_payload_in_first_string_field`: recursively
1311/// walk the value and replace the FIRST string leaf encountered.
1312/// Returns true when a replacement happened. Honors document order
1313/// for objects (BTreeMap-backed `serde_json::Map` iterates in
1314/// insertion order) so the choice of which field to mutate is
1315/// stable across runs.
1316fn replace_first_string(v: &mut serde_json::Value, snippet: &str) -> bool {
1317 match v {
1318 serde_json::Value::String(s) => {
1319 *s = snippet.to_string();
1320 true
1321 }
1322 serde_json::Value::Object(map) => {
1323 for (_k, child) in map.iter_mut() {
1324 if replace_first_string(child, snippet) {
1325 return true;
1326 }
1327 }
1328 false
1329 }
1330 serde_json::Value::Array(arr) => {
1331 for child in arr.iter_mut() {
1332 if replace_first_string(child, snippet) {
1333 return true;
1334 }
1335 }
1336 false
1337 }
1338 _ => false,
1339 }
1340}
1341
1342fn validate_body_against_schema(body: &str, schema: &serde_json::Value) -> Option<String> {
1343 let parsed: serde_json::Value = serde_json::from_str(body).ok()?;
1344 let validator = jsonschema::validator_for(schema).ok()?;
1345 let mut errors = validator.iter_errors(&parsed);
1346 let first = errors.next()?;
1347 // Round 28 — Srikanth on 0.3.170 wanted the message to show the
1348 // actual expected schema alongside the kind label so it reads as
1349 // "expected schema {...} but got <kind>". We emit a compact JSON
1350 // serialisation of the schema as a suffix; the kind label still
1351 // names what went wrong in plain English for quick scanning.
1352 // Round 26 — Srikanth on 0.3.169: the prior `format!("{:?}", first.kind)
1353 // .split('(').next()` produced "Type { kind: Single" (broken Rust
1354 // syntax, mismatched braces). Switch to the human-readable mapping
1355 // already used in executor.rs: handle the common kinds (Type,
1356 // Required, AdditionalProperties, Enum, MinLength, MaxLength,
1357 // Minimum, Maximum, Pattern) explicitly; fall back to the
1358 // jsonschema crate's Display impl on the error (which produces
1359 // something like "{...} is not of type \"string\"") for the long
1360 // tail. Combined with `at <instance-path>` for the field location.
1361 let path = first.instance_path.to_string();
1362 let path = if path.is_empty() { "/" } else { path.as_str() };
1363 // Round 31 — Srikanth on 0.3.174 hit the vCenter case where the
1364 // error is "required field missing: comment" but the printed
1365 // schema was the WHOLE parent object schema (with descriptions of
1366 // every property), not just the missing field's sub-schema. The
1367 // jsonschema crate emits `Required` errors with
1368 // `instance_path == /` (the parent), so the round-30 sub-schema
1369 // walker had no extra info to focus the suffix. Carry the missing
1370 // property name out of the kind match so we can descend one more
1371 // step into `properties[property]` for the printed schema.
1372 let mut required_property: Option<String> = None;
1373 let kind_msg: String = match &first.kind {
1374 jsonschema::error::ValidationErrorKind::Type { kind } => {
1375 // `kind` is `TypeKind::Single(JsonType)` or
1376 // `TypeKind::Multiple(JsonTypeSet)`. `JsonType` has its
1377 // own `Display` impl ("string", "object", etc.).
1378 match kind {
1379 jsonschema::error::TypeKind::Single(t) => format!("expected type {t}"),
1380 jsonschema::error::TypeKind::Multiple(_) => "expected one of multiple types".into(),
1381 }
1382 }
1383 jsonschema::error::ValidationErrorKind::Required { property } => {
1384 // `property.to_string()` returns the Display of the JSON
1385 // value, which for a string is `"name"` (with quotes).
1386 // Strip them for the lookup; keep them in the human message.
1387 let raw = property.to_string();
1388 let unquoted = raw
1389 .strip_prefix('"')
1390 .and_then(|s| s.strip_suffix('"'))
1391 .unwrap_or(&raw)
1392 .to_string();
1393 required_property = Some(unquoted);
1394 format!("required field missing: {property}")
1395 }
1396 jsonschema::error::ValidationErrorKind::AdditionalProperties { unexpected } => {
1397 format!("unexpected additional properties: {unexpected:?}")
1398 }
1399 jsonschema::error::ValidationErrorKind::Enum { options } => {
1400 format!("value not in allowed enum: {options}")
1401 }
1402 jsonschema::error::ValidationErrorKind::MinLength { limit } => {
1403 format!("string shorter than min length ({limit})")
1404 }
1405 jsonschema::error::ValidationErrorKind::MaxLength { limit } => {
1406 format!("string longer than max length ({limit})")
1407 }
1408 jsonschema::error::ValidationErrorKind::Minimum { limit } => {
1409 format!("value below minimum ({limit})")
1410 }
1411 jsonschema::error::ValidationErrorKind::Maximum { limit } => {
1412 format!("value above maximum ({limit})")
1413 }
1414 jsonschema::error::ValidationErrorKind::Pattern { pattern } => {
1415 format!("value did not match pattern {pattern}")
1416 }
1417 // Long tail: lean on jsonschema's Display impl, which is the
1418 // built-in human-readable error message ("X is not of type Y").
1419 // Strip trailing newlines so the JSONL line stays one line.
1420 _ => first.to_string().trim().to_string(),
1421 };
1422 // Round 30 — Srikanth on 0.3.173 asked how a deeper nested mismatch
1423 // reads. The prior output printed the WHOLE top-level schema even for
1424 // a single-field mismatch, which buried the actual constraint that
1425 // failed. Walk the instance pointer through the schema's properties
1426 // chain and print the most specific sub-schema we can find. Falls
1427 // back to the full schema for paths the walker can't resolve
1428 // (additionalProperties, oneOf, allOf, $ref un-resolved, etc.).
1429 let mut focused_schema = sub_schema_at_pointer(schema, path).unwrap_or_else(|| schema.clone());
1430 // Round 31 — for Required errors, descend one more step into
1431 // `properties[<missing>]` so the printed schema is the missing
1432 // field's own constraint, not the whole parent.
1433 if let Some(prop_name) = required_property.as_ref() {
1434 if let Some(prop_schema) =
1435 focused_schema.get("properties").and_then(|p| p.get(prop_name.as_str()))
1436 {
1437 focused_schema = prop_schema.clone();
1438 }
1439 }
1440 // Round 34 (#827) — Srikanth on 0.3.178 hit the vCenter
1441 // `enabled: boolean` case where the schema's multi-paragraph
1442 // `description` (and other prose fields) ate the 300-char budget
1443 // before the actually-useful `type` keyword could appear. Strip
1444 // the noise-fields recursively before serializing so the type
1445 // signal survives truncation; constraint keywords (`type`,
1446 // `properties`, `required`, `format`, `items`, etc.) stay.
1447 let focused_schema = strip_schema_noise(&focused_schema);
1448 let schema_str = serde_json::to_string(&focused_schema).unwrap_or_else(|_| "<schema>".into());
1449 let schema_str = if schema_str.len() > 300 {
1450 format!("{}...", &schema_str[..300])
1451 } else {
1452 schema_str
1453 };
1454 // Round 29 — Srikanth on 0.3.172 was confused by `at /:` thinking
1455 // it referenced the URL path; it's actually a JSON pointer into
1456 // the RESPONSE BODY. Reword so that's unambiguous: explicit
1457 // "response body" prefix and a human label for the root case.
1458 let location = if path == "/" {
1459 "response body root".to_string()
1460 } else {
1461 format!("response body at {path}")
1462 };
1463 Some(format!("{location}: {kind_msg}; expected schema {schema_str}"))
1464}
1465
1466/// Round 34 (#827) — drop the human-readable / documentation-only
1467/// fields from a JSON Schema before printing it inside a
1468/// `response_schema_error` message. The validator only cares about
1469/// constraint keywords (`type`, `required`, `properties`, `items`,
1470/// `format`, `enum`, `min*`/`max*`, `pattern`, `oneOf`/`anyOf`/
1471/// `allOf`/`not`); the prose fields can be paragraphs long for real-
1472/// world specs (vCenter's `enabled: bool` field has a multi-paragraph
1473/// description) and were eating the 300-char truncation budget before
1474/// the actually-useful type info could appear. Stripped fields:
1475/// `description`, `example`, `examples`, `summary`, `title`,
1476/// `externalDocs`, `xml`, `discriminator.description`.
1477fn strip_schema_noise(schema: &serde_json::Value) -> serde_json::Value {
1478 const NOISE_KEYS: &[&str] = &[
1479 "description",
1480 "example",
1481 "examples",
1482 "summary",
1483 "title",
1484 "externalDocs",
1485 "xml",
1486 ];
1487 match schema {
1488 serde_json::Value::Object(map) => {
1489 let mut out = serde_json::Map::with_capacity(map.len());
1490 for (k, v) in map {
1491 if NOISE_KEYS.contains(&k.as_str()) {
1492 continue;
1493 }
1494 out.insert(k.clone(), strip_schema_noise(v));
1495 }
1496 serde_json::Value::Object(out)
1497 }
1498 serde_json::Value::Array(items) => {
1499 serde_json::Value::Array(items.iter().map(strip_schema_noise).collect())
1500 }
1501 other => other.clone(),
1502 }
1503}
1504
1505/// Round 30 — walk a JSON-Pointer-style instance path through a JSON
1506/// Schema and return the sub-schema describing the value at that
1507/// position. For path `/name/age` on
1508/// `{"properties":{"name":{"properties":{"age":{"type":"integer"}}}}}`
1509/// returns `{"type":"integer"}`. Returns `None` for paths the walker
1510/// can't follow (array indices into `items` with no per-index schema,
1511/// `additionalProperties`, `oneOf`/`allOf`, unresolved `$ref`); callers
1512/// should fall back to the full schema in that case.
1513fn sub_schema_at_pointer(schema: &serde_json::Value, pointer: &str) -> Option<serde_json::Value> {
1514 if pointer.is_empty() || pointer == "/" {
1515 return Some(schema.clone());
1516 }
1517 let mut current = schema;
1518 for seg in pointer.trim_start_matches('/').split('/') {
1519 let unescaped = seg.replace("~1", "/").replace("~0", "~");
1520 if let Some(props) = current.get("properties") {
1521 if let Some(sub) = props.get(&unescaped) {
1522 current = sub;
1523 continue;
1524 }
1525 }
1526 if let Some(items) = current.get("items") {
1527 if items.is_object() {
1528 current = items;
1529 continue;
1530 }
1531 }
1532 return None;
1533 }
1534 Some(current.clone())
1535}
1536
1537/// Round 17.5 — one OWASP injection probe to send.
1538#[derive(Debug, Clone)]
1539struct OwaspProbe {
1540 label: String,
1541 body: Option<String>,
1542 query: Vec<(String, String)>,
1543}
1544
1545/// Build one OWASP probe per `SecurityCategory` for `op`. Targets the
1546/// first query param if any, else the first string field of the
1547/// positive JSON body. Returns empty if neither target is available.
1548fn build_owasp_probes(op: &AnnotatedOperation) -> Vec<OwaspProbe> {
1549 use crate::security_payloads::{SecurityCategory, SecurityPayloads};
1550
1551 let categories = [
1552 SecurityCategory::SqlInjection,
1553 SecurityCategory::Xss,
1554 SecurityCategory::CommandInjection,
1555 SecurityCategory::PathTraversal,
1556 SecurityCategory::Ssti,
1557 SecurityCategory::LdapInjection,
1558 SecurityCategory::Xxe,
1559 ];
1560
1561 // Pick an injection target ONCE per operation; reuse it across
1562 // categories. (A single op gets up to 7 probes — one per category
1563 // — all attacking the same field.)
1564 let injection_target = pick_injection_target(op);
1565 let Some(target) = injection_target else {
1566 return Vec::new();
1567 };
1568
1569 let mut probes = Vec::new();
1570 for cat in categories {
1571 // Take the *first* payload from each category. The
1572 // collection's first entry is the canonical low-risk
1573 // representative; later entries include time-based / blind
1574 // probes that aren't useful as a one-shot rejection test.
1575 let Some(payload) = SecurityPayloads::get_by_category(cat).into_iter().next() else {
1576 continue;
1577 };
1578 let mut query = op.query_params.clone();
1579 let mut body = op.sample_body.clone();
1580 match &target {
1581 InjectionTarget::Query(idx) => {
1582 if let Some(slot) = query.get_mut(*idx) {
1583 slot.1 = payload.payload.clone();
1584 }
1585 }
1586 InjectionTarget::BodyStringField(field) => {
1587 body = inject_into_body_field(body.as_deref(), field, &payload.payload);
1588 }
1589 }
1590 probes.push(OwaspProbe {
1591 label: format!("owasp:{}", cat),
1592 body,
1593 query,
1594 });
1595 }
1596 probes
1597}
1598
1599#[derive(Debug, Clone)]
1600enum InjectionTarget {
1601 Query(usize),
1602 BodyStringField(String),
1603}
1604
1605fn pick_injection_target(op: &AnnotatedOperation) -> Option<InjectionTarget> {
1606 if !op.query_params.is_empty() {
1607 return Some(InjectionTarget::Query(0));
1608 }
1609 let sample = op.sample_body.as_deref()?;
1610 let parsed: serde_json::Value = serde_json::from_str(sample).ok()?;
1611 let obj = parsed.as_object()?;
1612 for (k, v) in obj {
1613 if v.is_string() {
1614 return Some(InjectionTarget::BodyStringField(k.clone()));
1615 }
1616 }
1617 None
1618}
1619
1620/// Replace the value of `field` in a JSON-object body with `payload`.
1621/// Returns the mutated body as a JSON string. Returns `None` if the
1622/// body doesn't parse as a JSON object.
1623fn inject_into_body_field(body: Option<&str>, field: &str, payload: &str) -> Option<String> {
1624 let raw = body?;
1625 let mut parsed: serde_json::Value = serde_json::from_str(raw).ok()?;
1626 let obj = parsed.as_object_mut()?;
1627 obj.insert(field.to_string(), serde_json::json!(payload));
1628 serde_json::to_string(&parsed).ok()
1629}
1630
1631#[allow(clippy::too_many_arguments)]
1632/// Round 17.3 — one synthesised bad credential to send.
1633#[derive(Debug, Clone)]
1634struct SecurityProbe {
1635 /// Self-test label, e.g. `security:bad-bearer`.
1636 label: String,
1637 /// Headers to attach to the probe request.
1638 headers: Vec<(String, String)>,
1639 /// Query parameters to attach (API key in query case).
1640 query: Vec<(String, String)>,
1641}
1642
1643/// For each declared security scheme, produce one bad-credential
1644/// probe plus a single "no auth at all" probe that exercises the
1645/// missing-credential code path. Deduplicates by scheme kind so an
1646/// operation declaring `[bearer, bearer]` only yields one Bearer
1647/// probe.
1648fn build_security_probes(schemes: &[SecuritySchemeInfo]) -> Vec<SecurityProbe> {
1649 if schemes.is_empty() {
1650 return Vec::new();
1651 }
1652 let mut probes: Vec<SecurityProbe> = Vec::new();
1653 let mut seen_bearer = false;
1654 let mut seen_basic = false;
1655 // `(loc_tag, name)` — ApiKeyLocation doesn't implement Ord, so
1656 // we tag it with a short discriminant string for dedup.
1657 let mut seen_apikey: std::collections::BTreeSet<(&'static str, String)> = Default::default();
1658 for s in schemes {
1659 match s {
1660 SecuritySchemeInfo::Bearer if !seen_bearer => {
1661 seen_bearer = true;
1662 probes.push(SecurityProbe {
1663 label: "security:bad-bearer".into(),
1664 headers: vec![(
1665 "Authorization".into(),
1666 "Bearer self-test-invalid-token".into(),
1667 )],
1668 query: Vec::new(),
1669 });
1670 }
1671 SecuritySchemeInfo::Basic if !seen_basic => {
1672 seen_basic = true;
1673 // base64("self-test:invalid") — valid base64, wrong creds.
1674 probes.push(SecurityProbe {
1675 label: "security:bad-basic".into(),
1676 headers: vec![(
1677 "Authorization".into(),
1678 "Basic c2VsZi10ZXN0OmludmFsaWQ=".into(),
1679 )],
1680 query: Vec::new(),
1681 });
1682 }
1683 SecuritySchemeInfo::ApiKey { location, name } => {
1684 let loc_tag = match location {
1685 ApiKeyLocation::Header => "header",
1686 ApiKeyLocation::Query => "query",
1687 ApiKeyLocation::Cookie => "cookie",
1688 };
1689 if seen_apikey.contains(&(loc_tag, name.clone())) {
1690 continue;
1691 }
1692 seen_apikey.insert((loc_tag, name.clone()));
1693 let label = format!("security:bad-apikey:{}", name);
1694 let bad = "self-test-invalid-key".to_string();
1695 match location {
1696 ApiKeyLocation::Header => probes.push(SecurityProbe {
1697 label,
1698 headers: vec![(name.clone(), bad)],
1699 query: Vec::new(),
1700 }),
1701 ApiKeyLocation::Query => probes.push(SecurityProbe {
1702 label,
1703 headers: Vec::new(),
1704 query: vec![(name.clone(), bad)],
1705 }),
1706 ApiKeyLocation::Cookie => probes.push(SecurityProbe {
1707 label,
1708 headers: vec![("Cookie".into(), format!("{}={}", name, bad))],
1709 query: Vec::new(),
1710 }),
1711 }
1712 }
1713 _ => {}
1714 }
1715 }
1716 // Always add a "no auth at all" probe when *any* security scheme
1717 // is declared — useful even if all schemes failed to resolve to a
1718 // testable kind, because it surfaces validators that aren't
1719 // checking auth presence at all.
1720 probes.push(SecurityProbe {
1721 label: "security:no-auth".into(),
1722 headers: Vec::new(),
1723 query: Vec::new(),
1724 });
1725 probes
1726}
1727
1728/// Remove Authorization and any API-key headers declared by the
1729/// operation's security schemes from `headers`, so a security probe
1730/// can supply its own credential (or none) cleanly.
1731fn strip_auth(
1732 headers: &[(String, String)],
1733 schemes: &[SecuritySchemeInfo],
1734) -> Vec<(String, String)> {
1735 let mut apikey_headers: std::collections::BTreeSet<String> = Default::default();
1736 for s in schemes {
1737 if let SecuritySchemeInfo::ApiKey {
1738 location: ApiKeyLocation::Header,
1739 name,
1740 } = s
1741 {
1742 apikey_headers.insert(name.to_lowercase());
1743 }
1744 if let SecuritySchemeInfo::ApiKey {
1745 location: ApiKeyLocation::Cookie,
1746 ..
1747 } = s
1748 {
1749 apikey_headers.insert("cookie".into());
1750 }
1751 }
1752 headers
1753 .iter()
1754 .filter(|(k, _)| {
1755 let lk = k.to_lowercase();
1756 lk != "authorization" && !apikey_headers.contains(&lk)
1757 })
1758 .cloned()
1759 .collect()
1760}
1761
1762/// Remove API-key query parameters declared by the operation's
1763/// security schemes from `query`, so a probe can supply its own.
1764fn strip_auth_query(
1765 query: &[(String, String)],
1766 schemes: &[SecuritySchemeInfo],
1767) -> Vec<(String, String)> {
1768 let mut apikey_query: std::collections::BTreeSet<String> = Default::default();
1769 for s in schemes {
1770 if let SecuritySchemeInfo::ApiKey {
1771 location: ApiKeyLocation::Query,
1772 name,
1773 } = s
1774 {
1775 apikey_query.insert(name.clone());
1776 }
1777 }
1778 query.iter().filter(|(k, _)| !apikey_query.contains(k)).cloned().collect()
1779}
1780
1781/// Round 35 (#859) — Srikanth on 0.3.179: embedded-content variant-b
1782/// probes were flagging well-behaved 4xx responses as mismatches when
1783/// in reality only a 5xx (server CRASHED trying to parse the embedded
1784/// XML/YAML/multipart/urlencoded payload) is the bug the probe was
1785/// designed to find. Tristate replaces the older `expected_4xx: bool`
1786/// so variant-b probes can opt into "anything but 5xx is fine".
1787#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1788pub(crate) enum ExpectedOutcome {
1789 /// Positive probe: spec-compliant request, expect 2xx or 3xx.
1790 Success,
1791 /// Negative probe: invalid request, expect 4xx.
1792 ClientError,
1793 /// Embedded-content variant-b probe: spec-shape envelope with a
1794 /// non-JSON payload embedded in the first string field. Any
1795 /// response that isn't a 5xx is fine; the probe is here to catch
1796 /// server crashes on the embedded payload.
1797 NotServerError,
1798}
1799
1800impl ExpectedOutcome {
1801 /// Whether `actual_status` counts as a pass for this outcome.
1802 fn passes(self, actual_status: u16) -> bool {
1803 match self {
1804 ExpectedOutcome::Success => (200..400).contains(&actual_status),
1805 ExpectedOutcome::ClientError => (400..500).contains(&actual_status),
1806 ExpectedOutcome::NotServerError => {
1807 actual_status >= 200 && !(500..600).contains(&actual_status)
1808 }
1809 }
1810 }
1811
1812 /// Human-readable hint persisted in the JSONL capture + HTML
1813 /// viewer's "show mismatches only" filter; also what users `jq`
1814 /// against.
1815 fn as_str(self) -> &'static str {
1816 match self {
1817 ExpectedOutcome::Success => "2xx-3xx",
1818 ExpectedOutcome::ClientError => "4xx",
1819 ExpectedOutcome::NotServerError => "2xx-4xx",
1820 }
1821 }
1822}
1823
1824/// Variant of `send_case` that takes an explicit `extra_headers`
1825/// (rather than reading them from `config`). Used by security probes
1826/// to substitute or strip the configured Authorization header.
1827#[allow(clippy::too_many_arguments)]
1828async fn send_case_with_extra(
1829 client: &Client,
1830 config: &SelfTestConfig,
1831 method: Method,
1832 url: &str,
1833 label: &str,
1834 expected: ExpectedOutcome,
1835 body: Option<&str>,
1836 query: Vec<(String, String)>,
1837 headers: Vec<(String, String)>,
1838 extra_headers: Vec<(String, String)>,
1839 // Round 33 (#823) — spec path template (e.g. `/users/{id}`)
1840 // for the operation this probe belongs to. Stamped on the
1841 // capture so the per-endpoint summary can group by template.
1842 path_template: &str,
1843) -> CaseOutcome {
1844 let mut req = client.request(method.clone(), url);
1845 let mut capture_headers: BTreeMap<String, String> = BTreeMap::new();
1846 for (k, v) in &query {
1847 req = req.query(&[(k.as_str(), v.as_str())]);
1848 }
1849 // Round 36 (#876) — stamp the client side first so the same
1850 // `client_sent_at` string flows into both the request headers
1851 // (so the server-side `ServerConformanceViolation` records it
1852 // verbatim) and the on-disk `CaseCapture` JSONL line. Don't
1853 // re-call `Utc::now()` after `req.send()` — that would record
1854 // a different timestamp than the server sees.
1855 let mockforge_version = env!("CARGO_PKG_VERSION").to_string();
1856 let client_sent_at = chrono::Utc::now().to_rfc3339();
1857 // Round 28 — reqwest's `.header(k, v)` APPENDS rather than replaces
1858 // (.headers().insert() would replace but isn't on the builder).
1859 // The previous round-25 fix relied on "last-write-wins" semantics
1860 // that don't exist; for content-type-swap probes the request went
1861 // out with BOTH `Content-Type: application/json` AND `Content-Type:
1862 // application/xml`, and axum's `Json<>` extractor picked the JSON
1863 // one and accepted, so the server-side validator never saw the
1864 // mismatch. Build a `HeaderMap` ourselves so the override
1865 // replaces the body-block default exactly once.
1866 let mut final_headers: reqwest::header::HeaderMap = reqwest::header::HeaderMap::new();
1867 if let Some(_b) = body {
1868 if let Ok(v) = reqwest::header::HeaderValue::from_str("application/json") {
1869 final_headers.insert(reqwest::header::CONTENT_TYPE, v);
1870 }
1871 capture_headers.insert("Content-Type".to_string(), "application/json".to_string());
1872 }
1873 for (k, v) in &headers {
1874 if let (Ok(hn), Ok(hv)) = (
1875 reqwest::header::HeaderName::from_bytes(k.as_bytes()),
1876 reqwest::header::HeaderValue::from_str(v),
1877 ) {
1878 final_headers.insert(hn, hv);
1879 }
1880 capture_headers.insert(k.clone(), v.clone());
1881 }
1882 for (k, v) in &extra_headers {
1883 if let (Ok(hn), Ok(hv)) = (
1884 reqwest::header::HeaderName::from_bytes(k.as_bytes()),
1885 reqwest::header::HeaderValue::from_str(v),
1886 ) {
1887 final_headers.insert(hn, hv);
1888 }
1889 capture_headers.insert(k.clone(), v.clone());
1890 }
1891 // Round 36 (#876) — outbound client stamps. Inserted last so
1892 // they can't be clobbered by user-supplied extra-headers, and
1893 // recorded in `capture_headers` so the JSONL line shows the
1894 // exact bytes that went on the wire.
1895 {
1896 let v_header = mockforge_foundation::conformance_violations::CLIENT_VERSION_HEADER;
1897 let s_header = mockforge_foundation::conformance_violations::CLIENT_SENT_AT_HEADER;
1898 if let (Ok(hn), Ok(hv)) = (
1899 reqwest::header::HeaderName::from_bytes(v_header.as_bytes()),
1900 reqwest::header::HeaderValue::from_str(&mockforge_version),
1901 ) {
1902 final_headers.insert(hn, hv);
1903 }
1904 if let (Ok(hn), Ok(hv)) = (
1905 reqwest::header::HeaderName::from_bytes(s_header.as_bytes()),
1906 reqwest::header::HeaderValue::from_str(&client_sent_at),
1907 ) {
1908 final_headers.insert(hn, hv);
1909 }
1910 capture_headers.insert(v_header.to_string(), mockforge_version.clone());
1911 capture_headers.insert(s_header.to_string(), client_sent_at.clone());
1912 }
1913 if let Some(b) = body {
1914 req = req.body(b.to_string());
1915 }
1916 req = req.headers(final_headers);
1917 let (actual_status, response_capture) = match req.send().await {
1918 Ok(resp) => {
1919 let status = resp.status().as_u16();
1920 if let Some(sink) = &config.capture {
1921 let resp_headers: BTreeMap<String, String> = resp
1922 .headers()
1923 .iter()
1924 .map(|(k, v)| (k.as_str().to_string(), v.to_str().unwrap_or("").to_string()))
1925 .collect();
1926 let text = resp.text().await.unwrap_or_default();
1927 let (rb, truncated) = truncate_body_for_capture(&text);
1928 (status, Some((Some((rb, truncated)), resp_headers, None, sink.clone())))
1929 } else {
1930 (status, None)
1931 }
1932 }
1933 Err(e) => {
1934 let err_str = e.to_string();
1935 // Round 47 (#79) — classify + push to the wire-level
1936 // network-events sink (when present) so the user has a
1937 // grep-able log of connect/timeout/tls failures during
1938 // self-test, matching the r46 native-executor behaviour.
1939 if let Some(sink) = &config.network_events {
1940 let kind = if e.is_connect() {
1941 "connect"
1942 } else if e.is_timeout() {
1943 "timeout"
1944 } else if e.is_request() {
1945 "request"
1946 } else if e.is_body() {
1947 "body"
1948 } else if e.is_decode() {
1949 "decode"
1950 } else if err_str.to_ascii_lowercase().contains("tls") {
1951 "tls"
1952 } else {
1953 "other"
1954 };
1955 if let Ok(mut guard) = sink.lock() {
1956 guard.push(NetworkEvent {
1957 timestamp: chrono::Utc::now(),
1958 check: label.to_string(),
1959 method: method.to_string(),
1960 url: build_query_url(url, &query),
1961 kind: kind.to_string(),
1962 message: err_str.clone(),
1963 });
1964 }
1965 }
1966 if let Some(sink) = &config.capture {
1967 (0, Some((None, BTreeMap::new(), Some(err_str), sink.clone())))
1968 } else {
1969 (0, None)
1970 }
1971 }
1972 };
1973 let passed = expected.passes(actual_status);
1974 if let Some((resp_body, resp_headers, error, sink)) = response_capture {
1975 let (request_body, request_body_truncated) = match body {
1976 Some(b) => {
1977 let (rb, t) = truncate_body_for_capture(b);
1978 (Some(rb), t)
1979 }
1980 None => (None, false),
1981 };
1982 let (response_body, response_body_truncated) = match resp_body {
1983 Some((rb, t)) => (Some(rb), t),
1984 None => (None, false),
1985 };
1986 let entry = CaseCapture {
1987 label: label.to_string(),
1988 method: method.to_string(),
1989 url: build_query_url(url, &query),
1990 request_headers: capture_headers,
1991 request_body,
1992 request_body_truncated,
1993 response_status: actual_status,
1994 response_headers: resp_headers,
1995 response_body,
1996 response_body_truncated,
1997 error,
1998 // Filled in by the per-operation validation pass after
1999 // every probe finishes; the capture itself is unaware of
2000 // the schema map.
2001 response_schema_error: None,
2002 // Round 28 — derive the expected range from the probe's
2003 // outcome shape so the JSONL line and HTML viewer can
2004 // filter mismatches without re-deriving on the read side.
2005 // Round 35 (#859) — add a third value `"2xx-4xx"` for
2006 // embedded-content variant-b probes whose only failure
2007 // mode is a 5xx server crash.
2008 expected_status_range: expected.as_str().to_string(),
2009 // Round 33 (#823) — path_template carries the spec's
2010 // pre-substitution path so the per-endpoint summary can
2011 // collapse `/users/X` and `/users/Y` into one row.
2012 // spec_label is constant per run, read from the config.
2013 path_template: path_template.to_string(),
2014 spec_label: config.spec_label.clone(),
2015 // Round 36 (#876) — same values that went on the wire as
2016 // request headers, so a server-side
2017 // `ServerConformanceViolation` recorded with
2018 // `client_mockforge_version` + `client_sent_at` matches
2019 // the JSONL line byte-for-byte.
2020 mockforge_version: mockforge_version.clone(),
2021 client_sent_at: client_sent_at.clone(),
2022 iteration: config.current_iteration.max(1),
2023 };
2024 if let Ok(mut guard) = sink.lock() {
2025 guard.push(entry);
2026 }
2027 }
2028 // Round 35 (#859) — keep the `expected_4xx` field on `CaseOutcome`
2029 // semantically tied to "negative probe expecting 400-class", so
2030 // downstream code in `report_html.rs` doesn't have to learn about
2031 // the new tristate. `NotServerError` reports as `expected_4xx:
2032 // false` (it's a positive probe in spirit) and instead carries
2033 // its outcome through the per-capture `expected_status_range`.
2034 let expected_4xx = matches!(expected, ExpectedOutcome::ClientError);
2035 CaseOutcome {
2036 label: label.to_string(),
2037 expected_4xx,
2038 actual_status,
2039 passed,
2040 }
2041}
2042
2043// HTTP request shape needs all of these: client, config (for capture
2044// sink + extra headers), method, url, label (probe id), expected_4xx
2045// (pass/fail decision), body, query, headers. A struct wrapper would
2046// just move the arity from positional to field access without making
2047// the call sites clearer.
2048#[allow(clippy::too_many_arguments)]
2049async fn send_case(
2050 client: &Client,
2051 config: &SelfTestConfig,
2052 method: Method,
2053 url: &str,
2054 label: &str,
2055 expected: ExpectedOutcome,
2056 body: Option<&str>,
2057 query: Vec<(String, String)>,
2058 headers: Vec<(String, String)>,
2059 path_template: &str,
2060) -> CaseOutcome {
2061 // Forwarding to `send_case_with_extra` keeps the capture logic in
2062 // one place so request/response tracing can't drift between the
2063 // two entrypoints.
2064 send_case_with_extra(
2065 client,
2066 config,
2067 method,
2068 url,
2069 label,
2070 expected,
2071 body,
2072 query,
2073 headers,
2074 config.extra_headers.clone(),
2075 path_template,
2076 )
2077 .await
2078}
2079
2080/// Round 23 (c-iii) — rebuild the query-stringified URL for capture so
2081/// the JSONL trace shows the URL that actually went over the wire
2082/// (reqwest applies `.query(..)` after the request URL string is
2083/// rendered, so capturing the raw `url` argument alone loses the
2084/// query params).
2085fn build_query_url(base: &str, query: &[(String, String)]) -> String {
2086 if query.is_empty() {
2087 return base.to_string();
2088 }
2089 let qs: String = query
2090 .iter()
2091 .map(|(k, v)| format!("{}={}", urlencoding::encode(k), urlencoding::encode(v)))
2092 .collect::<Vec<_>>()
2093 .join("&");
2094 if base.contains('?') {
2095 format!("{base}&{qs}")
2096 } else {
2097 format!("{base}?{qs}")
2098 }
2099}
2100
2101/// Substitute `{param}` placeholders in the spec path with their
2102/// sample values from `path_params`, then prepend `target_url`. Empty
2103/// values are kept as `{param}` so an upstream router still matches
2104/// the template — useful when `path_params` is empty and we want to
2105/// hit the same route the spec defines.
2106///
2107/// All current call sites went through `build_url_with_base` after
2108/// round 18.1, so this no-base-path helper is unused; keep it as the
2109/// documented shim for future external callers (one-arg simplification).
2110#[allow(dead_code)]
2111fn build_url(target: &str, path_template: &str, path_params: &[(String, String)]) -> String {
2112 build_url_with_base(target, None, path_template, path_params)
2113}
2114
2115/// Round 55 (#79) — normalise a `--base-path` into a URL path prefix.
2116///
2117/// `Some("/api")` / `Some("api")` / `Some("/api/")` all become `/api`;
2118/// `Some("/")`, `Some("")`, and `None` all become `""` (root, no prefix).
2119///
2120/// Srikanth on 0.3.202 ran with `--base-path /` and got ZERO violations even
2121/// though the console reported thousands of misses. The old logic did
2122/// `"/".trim_end_matches('/')` = `""`, decided `"".starts_with('/')` was
2123/// false, and produced the prefix `"/"`. Combined with the leading `/` on the
2124/// spec path that yielded `<target>//v1/organizations` (double slash), which
2125/// never matched the spec's `/v1/organizations`, so every emitted request was
2126/// skipped by the validator and no violations were logged. Trimming slashes
2127/// from BOTH ends and treating the empty result as "no prefix" fixes it.
2128fn base_path_prefix(base_path: Option<&str>) -> String {
2129 match base_path {
2130 Some(bp) => {
2131 let trimmed = bp.trim_matches('/');
2132 if trimmed.is_empty() {
2133 String::new()
2134 } else {
2135 format!("/{trimmed}")
2136 }
2137 }
2138 None => String::new(),
2139 }
2140}
2141
2142/// Round 18.1 — variant of `build_url` that takes a `base_path`
2143/// (e.g. `Some("/api")`). When set, prepends it to the spec path so a
2144/// spec declaring `/users` against a target served behind `/api`
2145/// resolves to `<target>/api/users`. `base_path` is normalised via
2146/// [`base_path_prefix`] (both ends trimmed; `/` means root/no prefix).
2147fn build_url_with_base(
2148 target: &str,
2149 base_path: Option<&str>,
2150 path_template: &str,
2151 path_params: &[(String, String)],
2152) -> String {
2153 let mut url = path_template.to_string();
2154 for (name, value) in path_params {
2155 let placeholder = format!("{{{}}}", name);
2156 if !value.is_empty() {
2157 url = url.replace(&placeholder, value);
2158 }
2159 }
2160 let target = target.trim_end_matches('/');
2161 let prefix = base_path_prefix(base_path);
2162 let path = if url.starts_with('/') {
2163 url
2164 } else {
2165 format!("/{url}")
2166 };
2167 format!("{target}{prefix}{path}")
2168}
2169
2170#[cfg(test)]
2171mod tests {
2172 use super::*;
2173
2174 fn op(
2175 method: &str,
2176 path: &str,
2177 body: Option<&str>,
2178 query: Vec<(&str, &str)>,
2179 headers: Vec<(&str, &str)>,
2180 path_params: Vec<(&str, &str)>,
2181 ) -> AnnotatedOperation {
2182 AnnotatedOperation {
2183 method: method.into(),
2184 path: path.into(),
2185 features: Vec::new(),
2186 request_body_content_type: body.map(|_| "application/json".into()),
2187 sample_body: body.map(|s| s.to_string()),
2188 query_params: query.into_iter().map(|(a, b)| (a.into(), b.into())).collect(),
2189 header_params: headers.into_iter().map(|(a, b)| (a.into(), b.into())).collect(),
2190 path_params: path_params.into_iter().map(|(a, b)| (a.into(), b.into())).collect(),
2191 response_schema: None,
2192 response_schemas: std::collections::BTreeMap::new(),
2193 request_body_schema: None,
2194 security_schemes: Vec::new(),
2195 }
2196 }
2197
2198 /// Round 36 (#876) — older JSONL lines (written before the stamp
2199 /// fields existed) must still deserialise without error and
2200 /// default to empty strings. Prevents a back-compat regression
2201 /// the next time we extend `CaseCapture`.
2202 #[test]
2203 fn case_capture_back_compat_when_stamp_fields_missing() {
2204 let pre_r36 = serde_json::json!({
2205 "label": "positive",
2206 "method": "GET",
2207 "url": "http://api/users",
2208 "request_headers": {},
2209 "request_body_truncated": false,
2210 "response_status": 200,
2211 "response_headers": {},
2212 "response_body_truncated": false,
2213 });
2214 let capture: CaseCapture =
2215 serde_json::from_value(pre_r36).expect("pre-r36 payload must deserialise");
2216 assert!(capture.mockforge_version.is_empty(), "default to empty");
2217 assert!(capture.client_sent_at.is_empty(), "default to empty");
2218 }
2219
2220 /// Round 36 (#876) — when the bench stamps fields itself (the
2221 /// happy path), they round-trip through serde unchanged. Pins
2222 /// the on-wire shape so tooling that grep's `mockforge_version`
2223 /// out of the JSONL stays valid.
2224 #[test]
2225 fn case_capture_stamps_round_trip_through_serde() {
2226 let stamped = CaseCapture {
2227 label: "positive".into(),
2228 method: "GET".into(),
2229 url: "http://api/users".into(),
2230 request_headers: BTreeMap::new(),
2231 request_body: None,
2232 request_body_truncated: false,
2233 response_status: 200,
2234 response_headers: BTreeMap::new(),
2235 response_body: None,
2236 response_body_truncated: false,
2237 error: None,
2238 response_schema_error: None,
2239 expected_status_range: "2xx-3xx".into(),
2240 path_template: "/users".into(),
2241 spec_label: None,
2242 mockforge_version: "0.3.183".into(),
2243 client_sent_at: "2026-06-17T12:34:56+00:00".into(),
2244 iteration: 1,
2245 };
2246 let json = serde_json::to_string(&stamped).unwrap();
2247 assert!(json.contains("\"mockforge_version\":\"0.3.183\""));
2248 assert!(json.contains("\"client_sent_at\":\"2026-06-17T12:34:56+00:00\""));
2249 let back: CaseCapture = serde_json::from_str(&json).unwrap();
2250 assert_eq!(back.mockforge_version, "0.3.183");
2251 assert_eq!(back.client_sent_at, "2026-06-17T12:34:56+00:00");
2252 }
2253
2254 #[test]
2255 fn build_url_substitutes_path_params() {
2256 let url = build_url(
2257 "https://api.test/",
2258 "/users/{id}/posts/{pid}",
2259 &[("id".into(), "42".into()), ("pid".into(), "7".into())],
2260 );
2261 assert_eq!(url, "https://api.test/users/42/posts/7");
2262 }
2263
2264 /// Round 18.1 — a run where every positive 404s should be flagged
2265 /// as a likely target misconfiguration, not silently treated as a
2266 /// successful conformance run.
2267 #[test]
2268 fn detect_target_misconfiguration_when_all_positives_share_status() {
2269 let mut report = SelfTestReport {
2270 positive_pass: 0,
2271 positive_fail: 50,
2272 ..Default::default()
2273 };
2274 for i in 0..50 {
2275 report.operations.push(OperationResult {
2276 method: "GET".into(),
2277 path: format!("/r/{i}"),
2278 positive: Some(CaseOutcome {
2279 label: "positive".into(),
2280 expected_4xx: false,
2281 actual_status: 404,
2282 passed: false,
2283 }),
2284 negatives: Vec::new(),
2285 });
2286 }
2287 assert_eq!(report.detect_target_misconfiguration(), Some(404));
2288 }
2289
2290 #[test]
2291 fn detect_target_misconfiguration_returns_none_when_some_pass() {
2292 let mut report = SelfTestReport {
2293 positive_pass: 5,
2294 positive_fail: 50,
2295 ..Default::default()
2296 };
2297 for i in 0..55 {
2298 report.operations.push(OperationResult {
2299 method: "GET".into(),
2300 path: format!("/r/{i}"),
2301 positive: Some(CaseOutcome {
2302 label: "positive".into(),
2303 expected_4xx: false,
2304 actual_status: if i < 5 { 200 } else { 404 },
2305 passed: i < 5,
2306 }),
2307 negatives: Vec::new(),
2308 });
2309 }
2310 assert_eq!(report.detect_target_misconfiguration(), None);
2311 }
2312
2313 /// Round 18.1 — `--base-path /api` should prepend `/api` to
2314 /// every spec path. Pre-fix, the self-test ignored base_path and
2315 /// 404'd every positive when the deployed API was behind a path
2316 /// prefix.
2317 #[test]
2318 fn build_url_applies_base_path_when_present() {
2319 let url = build_url_with_base(
2320 "https://api.example.com",
2321 Some("/api"),
2322 "/users/{id}",
2323 &[("id".into(), "42".into())],
2324 );
2325 assert_eq!(url, "https://api.example.com/api/users/42");
2326 }
2327
2328 /// Round 18.1 — base_path is normalised: missing leading slash
2329 /// gets one added, trailing slash is stripped, empty string is
2330 /// the same as None.
2331 #[test]
2332 fn build_url_normalises_base_path() {
2333 let no_slash = build_url_with_base("https://t", Some("api"), "/x", &[]);
2334 assert_eq!(no_slash, "https://t/api/x");
2335 let trailing = build_url_with_base("https://t", Some("/api/"), "/x", &[]);
2336 assert_eq!(trailing, "https://t/api/x");
2337 let empty = build_url_with_base("https://t", Some(""), "/x", &[]);
2338 assert_eq!(empty, "https://t/x");
2339 let none = build_url_with_base("https://t", None, "/x", &[]);
2340 assert_eq!(none, "https://t/x");
2341 }
2342
2343 /// Round 55 (#79) — Srikanth on 0.3.202 passed `--base-path /` and got a
2344 /// `//v1/organizations` double slash, which broke all validation. `/`
2345 /// (and `//`) must be treated as root, i.e. no prefix.
2346 #[test]
2347 fn base_path_slash_is_root_no_double_slash() {
2348 assert_eq!(base_path_prefix(Some("/")), "");
2349 assert_eq!(base_path_prefix(Some("//")), "");
2350 assert_eq!(base_path_prefix(Some("")), "");
2351 assert_eq!(base_path_prefix(None), "");
2352 assert_eq!(base_path_prefix(Some("/api")), "/api");
2353 assert_eq!(base_path_prefix(Some("api/")), "/api");
2354 assert_eq!(base_path_prefix(Some("/api/v1")), "/api/v1");
2355 // The end-to-end URL no longer double-slashes.
2356 assert_eq!(
2357 build_url_with_base("https://t", Some("/"), "/v1/organizations", &[]),
2358 "https://t/v1/organizations"
2359 );
2360 }
2361
2362 #[test]
2363 fn build_url_keeps_placeholders_when_no_sample() {
2364 let url = build_url("https://api.test", "/users/{id}", &[]);
2365 assert_eq!(url, "https://api.test/users/{id}");
2366 }
2367
2368 #[test]
2369 fn report_summary_calls_out_misses() {
2370 let r = SelfTestReport {
2371 positive_pass: 3,
2372 positive_fail: 0,
2373 negative_caught: BTreeMap::from([("request-body".into(), 2)]),
2374 negative_missed: BTreeMap::from([("request-body".into(), 1)]),
2375 operations: Vec::new(),
2376 };
2377 let summary = r.render_summary();
2378 assert!(summary.contains("Positives: 3 pass / 0 fail"));
2379 assert!(summary.contains("Negatives [request-body]: 2 caught / 1 missed"));
2380 assert!(summary.contains("⚠"));
2381 // Round 57 (#79) — the caught/missed legend is printed once when there
2382 // are negatives, so the console explains the terms inline.
2383 assert!(summary.contains("\"caught\" = the target rejected it"));
2384 assert!(summary.contains("spec-valid by construction"));
2385 assert!(!r.all_passed());
2386 }
2387
2388 #[test]
2389 fn report_summary_omits_legend_when_no_negatives() {
2390 // Round 57 (#79) — a positives-only report has no Negatives lines, so
2391 // the legend would be noise; it must not appear.
2392 let r = SelfTestReport {
2393 positive_pass: 1,
2394 positive_fail: 0,
2395 negative_caught: BTreeMap::new(),
2396 negative_missed: BTreeMap::new(),
2397 operations: Vec::new(),
2398 };
2399 let summary = r.render_summary();
2400 assert!(!summary.contains("deliberately-bad requests"));
2401 }
2402
2403 #[test]
2404 fn definite_issues_flags_rejected_positives_and_5xx_but_not_spec_valid_misses() {
2405 // Round 58 (#79) — the "for sure this is an issue" view.
2406 let mut r = SelfTestReport::default();
2407 // (1) valid request the target rejected with 4xx -> definite issue.
2408 r.operations.push(OperationResult {
2409 method: "POST".into(),
2410 path: "/v1/organizations".into(),
2411 positive: Some(CaseOutcome {
2412 label: "positive".into(),
2413 expected_4xx: false,
2414 actual_status: 400,
2415 passed: false,
2416 }),
2417 negatives: vec![],
2418 });
2419 // (2) negative probe that crashed the target with 5xx -> definite issue.
2420 r.operations.push(OperationResult {
2421 method: "GET".into(),
2422 path: "/v1/items".into(),
2423 positive: Some(CaseOutcome {
2424 label: "positive".into(),
2425 expected_4xx: false,
2426 actual_status: 200,
2427 passed: true,
2428 }),
2429 negatives: vec![CaseOutcome {
2430 label: "owasp:sqli".into(),
2431 expected_4xx: true,
2432 actual_status: 500,
2433 passed: false,
2434 }],
2435 });
2436 // (3) spec-valid negative the target accepted (2xx) -> NOT a definite
2437 // issue (this is a "missed" that needs judgement, must be excluded).
2438 r.operations.push(OperationResult {
2439 method: "GET".into(),
2440 path: "/v1/ok".into(),
2441 positive: Some(CaseOutcome {
2442 label: "positive".into(),
2443 expected_4xx: false,
2444 actual_status: 200,
2445 passed: true,
2446 }),
2447 negatives: vec![CaseOutcome {
2448 label: "parameters:missing-query".into(),
2449 expected_4xx: true,
2450 actual_status: 200,
2451 passed: false,
2452 }],
2453 });
2454
2455 let issues = r.definite_issues();
2456 assert_eq!(issues.len(), 2, "only the rejected-positive and the 5xx");
2457 assert!(issues.iter().any(|i| i.kind == "valid_request_rejected"
2458 && i.path == "/v1/organizations"
2459 && i.status == 400));
2460 assert!(issues
2461 .iter()
2462 .any(|i| i.kind == "server_error" && i.path == "/v1/items" && i.status == 500));
2463 // The spec-valid missed negative is not present.
2464 assert!(!issues.iter().any(|i| i.path == "/v1/ok"));
2465
2466 let summary = r.render_summary();
2467 assert!(summary.contains("Definite issues (2)"));
2468 }
2469
2470 #[test]
2471 fn definite_issues_empty_renders_none_line() {
2472 let r = SelfTestReport {
2473 positive_pass: 1,
2474 positive_fail: 0,
2475 negative_caught: BTreeMap::from([("owasp".into(), 3)]),
2476 negative_missed: BTreeMap::new(),
2477 operations: vec![OperationResult {
2478 method: "GET".into(),
2479 path: "/v1/ok".into(),
2480 positive: Some(CaseOutcome {
2481 label: "positive".into(),
2482 expected_4xx: false,
2483 actual_status: 200,
2484 passed: true,
2485 }),
2486 negatives: vec![],
2487 }],
2488 };
2489 assert!(r.definite_issues().is_empty());
2490 assert!(r.render_summary().contains("Definite issues: none"));
2491 }
2492
2493 #[test]
2494 fn owasp_summary_splits_by_injection_and_counts_accepted_vs_blocked() {
2495 // Round 59 (#79) — a WAF that lets SQLi through (200) but blocks XSS (403).
2496 let neg = |label: &str, status: u16| CaseOutcome {
2497 label: label.into(),
2498 expected_4xx: true,
2499 actual_status: status,
2500 passed: status >= 400 && status < 500,
2501 };
2502 let mut r = SelfTestReport::default();
2503 r.operations.push(OperationResult {
2504 method: "POST".into(),
2505 path: "/v1/orgs".into(),
2506 positive: None,
2507 negatives: vec![
2508 neg("owasp:sqli", 200), // accepted (not blocked)
2509 neg("owasp:xss", 403), // blocked
2510 neg("owasp:command-injection", 500), // errored (5xx)
2511 neg("parameters:missing-query", 200), // not owasp -> ignored here
2512 ],
2513 });
2514 r.operations.push(OperationResult {
2515 method: "GET".into(),
2516 path: "/v1/items".into(),
2517 positive: None,
2518 negatives: vec![neg("owasp:sqli", 200)], // second accepted sqli
2519 });
2520
2521 let s = r.owasp_summary();
2522 let sqli = s.iter().find(|x| x.injection == "sqli").unwrap();
2523 assert_eq!((sqli.accepted, sqli.blocked, sqli.errored), (2, 0, 0));
2524 let xss = s.iter().find(|x| x.injection == "xss").unwrap();
2525 assert_eq!((xss.accepted, xss.blocked, xss.errored), (0, 1, 0));
2526 let cmd = s.iter().find(|x| x.injection == "command-injection").unwrap();
2527 assert_eq!((cmd.accepted, cmd.blocked, cmd.errored), (0, 0, 1));
2528 // parameters:* must not appear in the owasp summary.
2529 assert!(!s.iter().any(|x| x.injection.contains("query")));
2530
2531 // Accepted-probes sidecar lists only the two accepted sqli (URLs).
2532 let accepted = r.owasp_accepted_probes();
2533 assert_eq!(accepted.len(), 2);
2534 assert!(accepted.iter().all(|p| p.injection == "sqli" && p.status == 200));
2535
2536 // Console section renders with the WAF framing.
2537 let summary = r.render_summary();
2538 assert!(summary.contains("Security probes (owasp injection)"));
2539 assert!(summary.contains("target ACCEPTED 2/4 payloads"));
2540 assert!(summary.contains("owasp:sqli: 2 accepted / 0 blocked"));
2541 }
2542
2543 #[test]
2544 fn owasp_summary_empty_when_no_owasp_probes() {
2545 let r = SelfTestReport {
2546 positive_pass: 1,
2547 positive_fail: 0,
2548 negative_caught: BTreeMap::new(),
2549 negative_missed: BTreeMap::from([("parameters".into(), 2)]),
2550 operations: vec![OperationResult {
2551 method: "GET".into(),
2552 path: "/x".into(),
2553 positive: None,
2554 negatives: vec![CaseOutcome {
2555 label: "parameters:missing-query".into(),
2556 expected_4xx: true,
2557 actual_status: 200,
2558 passed: false,
2559 }],
2560 }],
2561 };
2562 assert!(r.owasp_summary().is_empty());
2563 assert!(!r.render_summary().contains("Security probes (owasp"));
2564 }
2565
2566 #[test]
2567 fn report_all_passed_when_no_miss() {
2568 let r = SelfTestReport {
2569 positive_pass: 5,
2570 positive_fail: 0,
2571 negative_caught: BTreeMap::from([("parameters".into(), 3)]),
2572 negative_missed: BTreeMap::new(),
2573 operations: Vec::new(),
2574 };
2575 assert!(r.all_passed());
2576 assert!(r.render_summary().contains("✓"));
2577 }
2578
2579 #[tokio::test]
2580 async fn run_self_test_against_unreachable_target_marks_all_failed() {
2581 // Use an obviously-dead port so we exercise the timeout/error
2582 // path without needing a live server in tests.
2583 let cfg = SelfTestConfig {
2584 target_url: "http://127.0.0.1:1".into(),
2585 timeout: Duration::from_millis(200),
2586 ..Default::default()
2587 };
2588 let ops = vec![op(
2589 "POST",
2590 "/users",
2591 Some("{\"name\":\"a\"}"),
2592 vec![],
2593 vec![],
2594 vec![],
2595 )];
2596 let report = run_self_test(&ops, &cfg).await.expect("client builds");
2597 // All cases hit the connect-error path → actual_status=0.
2598 // Positive expects 2xx-3xx → 0 is fail. Negatives expect 4xx
2599 // → 0 is also fail (we missed catching).
2600 assert_eq!(report.positive_fail, 1);
2601 assert!(report.negative_missed.values().sum::<usize>() >= 1);
2602 assert!(!report.all_passed());
2603 }
2604
2605 /// Round 17.2 — operations with both a positive sample AND a
2606 /// resolved request-body schema produce schema-driven negatives
2607 /// in addition to the spec-agnostic empty/wrong-type ones. The
2608 /// labels carry the field path so a per-category report can tell
2609 /// you exactly which field caught.
2610 #[tokio::test]
2611 async fn schema_driven_negatives_fire_when_schema_present() {
2612 use openapiv3::{ObjectType, ReferenceOr, Schema, SchemaData, SchemaKind, Type};
2613 let cfg = SelfTestConfig {
2614 target_url: "http://127.0.0.1:1".into(),
2615 timeout: Duration::from_millis(200),
2616 ..Default::default()
2617 };
2618 // Build an operation whose schema has a required `name` string
2619 // and an `age` integer. The mutator should produce, at
2620 // minimum: required-removed:name, required-removed:age,
2621 // type-mismatch:name, type-mismatch:age, integer-as-float:age,
2622 // plus the root-level type-mismatch.
2623 let mut obj = ObjectType::default();
2624 obj.properties.insert(
2625 "name".to_string(),
2626 ReferenceOr::Item(Box::new(Schema {
2627 schema_data: SchemaData::default(),
2628 schema_kind: SchemaKind::Type(Type::String(Default::default())),
2629 })),
2630 );
2631 obj.properties.insert(
2632 "age".to_string(),
2633 ReferenceOr::Item(Box::new(Schema {
2634 schema_data: SchemaData::default(),
2635 schema_kind: SchemaKind::Type(Type::Integer(Default::default())),
2636 })),
2637 );
2638 obj.required = vec!["name".into(), "age".into()];
2639 let schema = Schema {
2640 schema_data: SchemaData::default(),
2641 schema_kind: SchemaKind::Type(Type::Object(obj)),
2642 };
2643
2644 let mut o =
2645 op("POST", "/users", Some(r#"{"name":"Ada","age":30}"#), vec![], vec![], vec![]);
2646 o.request_body_schema = Some(schema);
2647 let report = run_self_test(&[o], &cfg).await.expect("client builds");
2648 // Bucket labels from the operation result.
2649 let labels: std::collections::BTreeSet<String> = report
2650 .operations
2651 .iter()
2652 .flat_map(|op| op.negatives.iter().map(|n| n.label.clone()))
2653 .collect();
2654 assert!(
2655 labels.iter().any(|l| l.starts_with("request-body:type-mismatch:")),
2656 "missing type-mismatch negative; got {labels:?}"
2657 );
2658 assert!(
2659 labels.iter().any(|l| l.starts_with("request-body:required-removed:")),
2660 "missing required-removed negative; got {labels:?}"
2661 );
2662 assert!(
2663 labels.iter().any(|l| l == "parameters:uri-too-long"),
2664 "missing URI-length negative; got {labels:?}"
2665 );
2666 }
2667
2668 /// Round 16 — operations with a body OR a path-param now produce
2669 /// negatives even without a sample body. Previously a POST whose
2670 /// body annotator failed produced *zero* negatives, so the self-test
2671 /// always reported "all passing" for that endpoint.
2672 #[tokio::test]
2673 async fn no_sample_body_still_produces_request_body_negatives() {
2674 let cfg = SelfTestConfig {
2675 target_url: "http://127.0.0.1:1".into(),
2676 timeout: Duration::from_millis(200),
2677 ..Default::default()
2678 };
2679 // POST with a body content type but no sample (annotator gap).
2680 let ops = vec![op("POST", "/x", None, vec![], vec![], vec![])];
2681 // No sample_body but request_body_content_type set:
2682 let mut ops_fixed = ops;
2683 ops_fixed[0].request_body_content_type = Some("application/json".into());
2684 let report = run_self_test(&ops_fixed, &cfg).await.expect("client builds");
2685 // Both request-body negatives (empty + wrong-type) should fire,
2686 // landing in `negative_missed` because the unreachable target
2687 // returns no 4xx. The point: count > 0.
2688 assert!(
2689 report.negative_missed.values().sum::<usize>() >= 2,
2690 "expected ≥2 request-body negatives, got {:?}",
2691 report.negative_missed
2692 );
2693 }
2694
2695 /// Round 16 — operations with a path-param now get a probe even
2696 /// when there's no body / required query / required header.
2697 /// Previously `/teams/{team-id}` with no other required fields
2698 /// produced zero negatives → always "all passing".
2699 #[tokio::test]
2700 async fn path_param_only_endpoint_produces_a_probe() {
2701 let cfg = SelfTestConfig {
2702 target_url: "http://127.0.0.1:1".into(),
2703 timeout: Duration::from_millis(200),
2704 ..Default::default()
2705 };
2706 let ops = vec![op(
2707 "GET",
2708 "/teams/{team-id}",
2709 None,
2710 vec![],
2711 vec![],
2712 vec![("team-id", "1")],
2713 )];
2714 let report = run_self_test(&ops, &cfg).await.expect("client builds");
2715 let total: usize = report.negative_caught.values().sum::<usize>()
2716 + report.negative_missed.values().sum::<usize>();
2717 assert!(total >= 1, "expected ≥1 path-param probe, got {:?}", report);
2718 }
2719
2720 /// Round 18.5 — when `geo_ip` is set, every default forwarded-
2721 /// IP header gets the IP appended (X-Forwarded-For,
2722 /// True-Client-IP, CF-Connecting-IP).
2723 #[test]
2724 fn effective_op_headers_appends_geo_ip_to_default_headers() {
2725 let ip: IpAddr = "203.0.113.42".parse().unwrap();
2726 let headers = effective_op_headers(
2727 &[("Accept".into(), "application/json".into())],
2728 Some(ip),
2729 &default_geo_source_headers(),
2730 );
2731 let names: Vec<&str> = headers.iter().map(|(k, _)| k.as_str()).collect();
2732 assert!(names.contains(&"Accept"));
2733 assert!(names.contains(&"X-Forwarded-For"));
2734 assert!(names.contains(&"True-Client-IP"));
2735 assert!(names.contains(&"CF-Connecting-IP"));
2736 // Every geo header carries the same IP value.
2737 let geo_values: Vec<&str> =
2738 headers.iter().filter(|(k, _)| k != "Accept").map(|(_, v)| v.as_str()).collect();
2739 for v in geo_values {
2740 assert_eq!(v, "203.0.113.42");
2741 }
2742 }
2743
2744 /// Round 18.5 — operations that already declare a forwarded-IP
2745 /// header (rare but legal — some specs hard-code one) keep their
2746 /// declared value; we don't clobber the spec.
2747 #[test]
2748 fn effective_op_headers_respects_spec_declared_header() {
2749 let ip: IpAddr = "203.0.113.99".parse().unwrap();
2750 let headers = effective_op_headers(
2751 &[("x-forwarded-for".into(), "10.0.0.1".into())],
2752 Some(ip),
2753 &["X-Forwarded-For".to_string()],
2754 );
2755 // The spec's lower-case value wins; we shouldn't add a
2756 // second X-Forwarded-For row that overrides it.
2757 let xff: Vec<&str> = headers
2758 .iter()
2759 .filter(|(k, _)| k.eq_ignore_ascii_case("x-forwarded-for"))
2760 .map(|(_, v)| v.as_str())
2761 .collect();
2762 assert_eq!(xff, vec!["10.0.0.1"]);
2763 }
2764
2765 /// Round 18.5 — None geo_ip and/or empty header list is a no-op.
2766 #[test]
2767 fn effective_op_headers_is_a_noop_without_geo_ip() {
2768 let base = vec![("Accept".into(), "json".into())];
2769 let h1 = effective_op_headers(&base, None, &default_geo_source_headers());
2770 assert_eq!(h1, base);
2771 let ip: IpAddr = "10.0.0.1".parse().unwrap();
2772 let h2 = effective_op_headers(&base, Some(ip), &[]);
2773 assert_eq!(h2, base);
2774 }
2775
2776 /// Round 18.5 — empty `source_ips` builds a single default
2777 /// client; a non-empty list builds N clients each attempting to
2778 /// bind. We can't reliably test the actual bind on CI (no
2779 /// loopback aliases), but a loopback IP is always bind-able.
2780 #[test]
2781 fn build_client_pool_one_per_source_ip() {
2782 let mut cfg = SelfTestConfig {
2783 target_url: "http://127.0.0.1:1".into(),
2784 timeout: Duration::from_millis(200),
2785 ..Default::default()
2786 };
2787 // Empty → one default client.
2788 assert_eq!(build_client_pool(&cfg).expect("default builds").len(), 1);
2789 // Non-empty → one per IP. Loopback bind is portable.
2790 cfg.source_ips = vec!["127.0.0.1".parse().unwrap()];
2791 assert_eq!(build_client_pool(&cfg).expect("bind loopback").len(), 1);
2792 }
2793
2794 /// Round 18.5 — geo IPs round-robin across operations. Hits an
2795 /// unreachable target so we can inspect the case outcomes; the
2796 /// point is to confirm `op_headers` carried the geo IP through
2797 /// (CaseOutcome doesn't surface headers directly, so we just
2798 /// verify the run completes without panicking and the result
2799 /// shape is correct when source_ips is non-empty too).
2800 #[tokio::test]
2801 async fn run_self_test_with_geo_source_completes() {
2802 let cfg = SelfTestConfig {
2803 target_url: "http://127.0.0.1:1".into(),
2804 timeout: Duration::from_millis(200),
2805 geo_source_ips: vec![
2806 "203.0.113.1".parse().unwrap(),
2807 "203.0.113.2".parse().unwrap(),
2808 ],
2809 ..Default::default()
2810 };
2811 let ops = vec![
2812 op("GET", "/a", None, vec![], vec![], vec![]),
2813 op("GET", "/b", None, vec![], vec![], vec![]),
2814 op("GET", "/c", None, vec![], vec![], vec![]),
2815 ];
2816 let report = run_self_test(&ops, &cfg).await.expect("client builds");
2817 assert_eq!(report.operations.len(), 3);
2818 }
2819
2820 /// Round 24 (f) — Srikanth saw the geo header on positive probes
2821 /// only; the four negative-probe call sites were passing
2822 /// `op.header_params` directly instead of `op_headers`, so the
2823 /// geo IP got dropped. This test runs a self-test that includes
2824 /// negative probes (uri-too-long, missing-query, etc.) under
2825 /// `--conformance-self-test-capture`, then asserts that EVERY
2826 /// captured probe (positive AND negative) carries one of the
2827 /// configured forwarded-IP headers.
2828 #[tokio::test]
2829 async fn geo_headers_present_on_every_probe_with_capture() {
2830 let sink: Arc<Mutex<Vec<CaseCapture>>> = Arc::new(Mutex::new(Vec::new()));
2831 let cfg = SelfTestConfig {
2832 target_url: "http://127.0.0.1:1".into(),
2833 timeout: Duration::from_millis(50),
2834 geo_source_ips: vec!["203.0.113.5".parse().unwrap()],
2835 capture: Some(sink.clone()),
2836 ..Default::default()
2837 };
2838 // An operation rich enough to trip several negative-probe
2839 // branches: header param (→ missing-header), query param
2840 // (→ missing-query), and a sample body (→ schema mutations
2841 // wouldn't fire without a schema, but uri-too-long always
2842 // does).
2843 let ops = vec![op(
2844 "GET",
2845 "/items",
2846 Some("{}"),
2847 vec![("id", "1")],
2848 vec![("X-Trace", "x")],
2849 vec![],
2850 )];
2851 let _ = run_self_test(&ops, &cfg).await.expect("client builds");
2852 let captures = sink.lock().unwrap();
2853 assert!(!captures.is_empty(), "self-test should record probes");
2854 // For every captured probe, at least one of the default geo
2855 // headers must be present and equal to the configured IP.
2856 let geo_headers: std::collections::HashSet<&str> =
2857 ["X-Forwarded-For", "True-Client-IP", "CF-Connecting-IP"].into_iter().collect();
2858 for c in captures.iter() {
2859 let has_geo = c
2860 .request_headers
2861 .iter()
2862 .any(|(k, v)| geo_headers.contains(k.as_str()) && v == "203.0.113.5");
2863 assert!(
2864 has_geo,
2865 "probe `{}` is missing the geo IP header; got headers: {:?}",
2866 c.label, c.request_headers
2867 );
2868 }
2869 }
2870
2871 /// Round 25 (k) — operations with a JSON request body now get four
2872 /// content-type-swap probes (xml / yaml / multipart / urlencoded).
2873 /// Verify they:
2874 /// 1. fire only when the operation declares a JSON body
2875 /// 2. carry the wrong Content-Type the probe is testing for
2876 /// 3. don't fire on body-less operations
2877 #[tokio::test]
2878 async fn content_type_swap_probes_fire_for_json_bodies() {
2879 let sink: Arc<Mutex<Vec<CaseCapture>>> = Arc::new(Mutex::new(Vec::new()));
2880 let cfg = SelfTestConfig {
2881 target_url: "http://127.0.0.1:1".into(),
2882 timeout: Duration::from_millis(50),
2883 capture: Some(sink.clone()),
2884 ..Default::default()
2885 };
2886 let ops = vec![
2887 op("POST", "/users", Some("{\"name\":\"a\"}"), vec![], vec![], vec![]),
2888 op("GET", "/ping", None, vec![], vec![], vec![]),
2889 ];
2890 let _ = run_self_test(&ops, &cfg).await.expect("client builds");
2891 let captures = sink.lock().unwrap();
2892
2893 let swap_labels: Vec<&str> = captures
2894 .iter()
2895 .filter(|c| c.label.starts_with("request-body:content-type-mismatch:"))
2896 .map(|c| c.label.as_str())
2897 .collect();
2898 assert_eq!(
2899 swap_labels.len(),
2900 4,
2901 "expected 4 content-type-swap probes (one per variant), got: {swap_labels:?}"
2902 );
2903 let expected_labels = [
2904 "request-body:content-type-mismatch:xml",
2905 "request-body:content-type-mismatch:yaml",
2906 "request-body:content-type-mismatch:multipart",
2907 "request-body:content-type-mismatch:urlencoded",
2908 ];
2909 for want in expected_labels {
2910 assert!(swap_labels.contains(&want), "missing swap probe `{want}`");
2911 }
2912
2913 // Each swap probe must carry the wrong Content-Type it's
2914 // testing for — that's the whole point.
2915 for c in captures.iter() {
2916 let Some(suffix) = c.label.strip_prefix("request-body:content-type-mismatch:") else {
2917 continue;
2918 };
2919 let want_ct = match suffix {
2920 "xml" => "application/xml",
2921 "yaml" => "application/yaml",
2922 "multipart" => "multipart/form-data",
2923 "urlencoded" => "application/x-www-form-urlencoded",
2924 _ => continue,
2925 };
2926 let got_ct = c
2927 .request_headers
2928 .iter()
2929 .find(|(k, _)| k.eq_ignore_ascii_case("content-type"))
2930 .map(|(_, v)| v.as_str())
2931 .unwrap_or("");
2932 assert_eq!(got_ct, want_ct, "swap probe `{}` sent wrong CT", c.label);
2933 }
2934
2935 // The body-less operation must NOT produce content-type-swap
2936 // probes (no body → no content type to lie about).
2937 let body_less_swaps = captures
2938 .iter()
2939 .filter(|c| {
2940 c.label.starts_with("request-body:content-type-mismatch:")
2941 && c.url.ends_with("/ping")
2942 })
2943 .count();
2944 assert_eq!(
2945 body_less_swaps, 0,
2946 "GET /ping has no request body; should not produce content-type-swap probes"
2947 );
2948 }
2949
2950 /// Round 27 (k variant b) — Srikanth's round-23 follow-up on (k):
2951 /// JSON envelope with embedded non-JSON field values. For each
2952 /// JSON-body operation, four extra probes fire that send valid
2953 /// JSON with an XML/YAML/multipart/urlencoded snippet stuffed
2954 /// into a string field. Content-Type stays `application/json`;
2955 /// expected is 2xx-3xx (the body parses); a 5xx flags a server
2956 /// that crashed on the embedded content.
2957 #[tokio::test]
2958 async fn embedded_content_probes_fire_with_honest_content_type() {
2959 let sink: Arc<Mutex<Vec<CaseCapture>>> = Arc::new(Mutex::new(Vec::new()));
2960 let cfg = SelfTestConfig {
2961 target_url: "http://127.0.0.1:1".into(),
2962 timeout: Duration::from_millis(50),
2963 capture: Some(sink.clone()),
2964 ..Default::default()
2965 };
2966 let ops = vec![op(
2967 "POST",
2968 "/users",
2969 Some("{\"name\":\"alice\",\"age\":30}"),
2970 vec![],
2971 vec![],
2972 vec![],
2973 )];
2974 let _ = run_self_test(&ops, &cfg).await.expect("client builds");
2975 let captures = sink.lock().unwrap();
2976 let embedded: Vec<&CaseCapture> = captures
2977 .iter()
2978 .filter(|c| c.label.starts_with("request-body:embedded-content:"))
2979 .collect();
2980 assert_eq!(
2981 embedded.len(),
2982 4,
2983 "expected 4 embedded-content probes, got: {:?}",
2984 embedded.iter().map(|c| &c.label).collect::<Vec<_>>()
2985 );
2986 // Every embedded probe must carry the honest application/json
2987 // Content-Type (NOT lie like the variant-a content-type-swap
2988 // probes do) and a request body that still parses as JSON.
2989 for c in &embedded {
2990 let ct = c
2991 .request_headers
2992 .iter()
2993 .find(|(k, _)| k.eq_ignore_ascii_case("content-type"))
2994 .map(|(_, v)| v.as_str())
2995 .unwrap_or("");
2996 assert!(
2997 ct.contains("application/json"),
2998 "embedded probe `{}` should keep Content-Type honest, got {ct}",
2999 c.label
3000 );
3001 let body = c.request_body.as_deref().unwrap_or("");
3002 assert!(
3003 serde_json::from_str::<serde_json::Value>(body).is_ok(),
3004 "embedded probe `{}` body should still be valid JSON, got: {body}",
3005 c.label
3006 );
3007 }
3008 }
3009
3010 /// `embed_payload_in_first_string_field` walks objects depth-first
3011 /// and replaces only the FIRST string-valued leaf, leaving the
3012 /// surrounding structure intact.
3013 #[test]
3014 fn embed_payload_replaces_first_string_only() {
3015 let sample = r#"{"name":"alice","age":30,"tags":["admin","user"]}"#;
3016 let mutated = embed_payload_in_first_string_field(sample, "<x/>")
3017 .expect("string field present so probe constructed");
3018 let v: serde_json::Value = serde_json::from_str(&mutated).unwrap();
3019 assert_eq!(v["name"], serde_json::json!("<x/>"));
3020 // age stays an integer (not stringified by the mutation).
3021 assert_eq!(v["age"], serde_json::json!(30));
3022 // tags array's strings stay untouched (we only replace the
3023 // first encountered string leaf, depth-first).
3024 assert_eq!(v["tags"][0], serde_json::json!("admin"));
3025 assert_eq!(v["tags"][1], serde_json::json!("user"));
3026 }
3027
3028 /// Round 34 (#829) — Srikanth on 0.3.178: when the positive
3029 /// sample has NO string field, the previous `{"data": <snippet>}`
3030 /// fallback produced an envelope that doesn't match real-API
3031 /// schemas (e.g. vCenter's `consolecli` PUT wants
3032 /// `{enabled: bool}`), so the server correctly 400'd and the
3033 /// bench misreported the 2xx-3xx expectation. Now we return None
3034 /// and the caller skips the probe.
3035 #[test]
3036 fn embed_payload_returns_none_when_no_string_field() {
3037 let no_strings = r#"{"a":1,"b":[2,3]}"#;
3038 assert!(embed_payload_in_first_string_field(no_strings, "<x><y></y></x>").is_none());
3039 // The exact vCenter-style case Srikanth hit.
3040 let bool_only = r#"{"enabled":true}"#;
3041 assert!(embed_payload_in_first_string_field(bool_only, "<x/>").is_none());
3042 }
3043
3044 #[test]
3045 fn embed_payload_returns_none_for_invalid_json_sample() {
3046 assert!(embed_payload_in_first_string_field("garbage", "a=1&b=2").is_none());
3047 }
3048
3049 /// Round 35 (#859) — Srikanth on 0.3.179 saw variant-b probes flag
3050 /// every 4xx as a mismatch when the spec field had a `pattern` /
3051 /// `format` validator that correctly rejected the embedded
3052 /// payload. The probe was only ever meant to catch 5xx (server
3053 /// crashed parsing the embedded content); 4xx is the well-behaved
3054 /// outcome. Tristate `ExpectedOutcome::NotServerError` lets a
3055 /// variant-b probe pass on 2xx-4xx and fail only on 5xx.
3056 #[test]
3057 fn expected_outcome_pass_rules() {
3058 // Success (positive): 2xx-3xx pass, 4xx + 5xx fail.
3059 assert!(ExpectedOutcome::Success.passes(200));
3060 assert!(ExpectedOutcome::Success.passes(201));
3061 assert!(ExpectedOutcome::Success.passes(204));
3062 assert!(ExpectedOutcome::Success.passes(301));
3063 assert!(!ExpectedOutcome::Success.passes(400));
3064 assert!(!ExpectedOutcome::Success.passes(415));
3065 assert!(!ExpectedOutcome::Success.passes(500));
3066 assert!(!ExpectedOutcome::Success.passes(0));
3067
3068 // ClientError (negative): only 4xx pass.
3069 assert!(!ExpectedOutcome::ClientError.passes(200));
3070 assert!(ExpectedOutcome::ClientError.passes(400));
3071 assert!(ExpectedOutcome::ClientError.passes(404));
3072 assert!(ExpectedOutcome::ClientError.passes(422));
3073 assert!(!ExpectedOutcome::ClientError.passes(500));
3074
3075 // NotServerError (variant-b): 2xx-4xx pass, 5xx fails.
3076 assert!(ExpectedOutcome::NotServerError.passes(200));
3077 assert!(ExpectedOutcome::NotServerError.passes(204));
3078 assert!(ExpectedOutcome::NotServerError.passes(400), "Srikanth's vCenter consolecli case: 400 from a pattern validator should NOT be a probe failure");
3079 assert!(ExpectedOutcome::NotServerError.passes(415));
3080 assert!(ExpectedOutcome::NotServerError.passes(422));
3081 assert!(
3082 !ExpectedOutcome::NotServerError.passes(500),
3083 "Server CRASH on embedded content is the only real failure"
3084 );
3085 assert!(!ExpectedOutcome::NotServerError.passes(502));
3086 assert!(!ExpectedOutcome::NotServerError.passes(503));
3087 // status 0 (network error / probe never reached the server) does not pass either
3088 assert!(!ExpectedOutcome::NotServerError.passes(0));
3089 }
3090
3091 /// Round 35 (#859) — the per-capture `expected_status_range`
3092 /// string is what the HTML viewer's "show mismatches only"
3093 /// filter and Srikanth's `jq` pipelines key off, so the new
3094 /// tristate must surface a third distinct value.
3095 #[test]
3096 fn expected_outcome_string_labels() {
3097 assert_eq!(ExpectedOutcome::Success.as_str(), "2xx-3xx");
3098 assert_eq!(ExpectedOutcome::ClientError.as_str(), "4xx");
3099 assert_eq!(ExpectedOutcome::NotServerError.as_str(), "2xx-4xx");
3100 }
3101
3102 /// Round 26 — Srikanth saw `at /: Type { kind: Single` in his
3103 /// 0.3.169 capture for the vCenter `infraprofile/configs` 202
3104 /// response (spec promised `type: string`, server returned a
3105 /// JSON object). The output was a broken-syntax debug string.
3106 /// This test reproduces his exact spec+body and asserts the
3107 /// message is readable.
3108 #[test]
3109 fn response_schema_error_message_is_readable() {
3110 let schema = serde_json::json!({"type": "string"});
3111 let body = r#"{"data":{},"id":"generated_id","status":"created"}"#;
3112 let err = validate_body_against_schema(body, &schema).expect("type-mismatch fires");
3113 // The message must NOT contain Rust debug syntax leftovers
3114 // ("Type { kind:", trailing "{" or "(" tokens). It SHOULD say
3115 // what type was expected.
3116 assert!(!err.contains("Type { kind"), "stale debug output: {err}");
3117 assert!(!err.contains("{ kind:"), "stale debug output: {err}");
3118 assert!(err.contains("string"), "should name expected type: {err}");
3119 // Round 29 — Srikanth on 0.3.172 was confused by `at /:`,
3120 // thinking it pointed to the URL path. The new format
3121 // explicitly says "response body root" for the root case
3122 // (and "response body at /<pointer>" for nested fields).
3123 assert!(
3124 err.contains("response body root"),
3125 "should label root explicitly so reader knows it's not the URL: {err}"
3126 );
3127 // Round 28 — Srikanth wanted the expected schema embedded
3128 // in the message so it reads as 'expected schema {"type":"string"}'.
3129 assert!(
3130 err.contains("expected schema") && err.contains("\"type\":\"string\""),
3131 "should include expected schema JSON: {err}"
3132 );
3133 }
3134
3135 /// Round 29 — for non-root paths the format reads
3136 /// "response body at /name: ...". Catches the case where the
3137 /// root rewording accidentally dropped the JSON-pointer for
3138 /// nested fields.
3139 #[test]
3140 fn response_schema_error_uses_response_body_prefix_for_nested_paths() {
3141 let schema = serde_json::json!({
3142 "type": "object",
3143 "required": ["name"],
3144 "properties": {"name": {"type": "string"}}
3145 });
3146 let body = r#"{"name": 123}"#;
3147 let err = validate_body_against_schema(body, &schema).expect("type-mismatch fires");
3148 assert!(
3149 err.contains("response body at /name"),
3150 "nested path should read 'response body at /name': {err}"
3151 );
3152 assert!(!err.contains("response body root"), "wrong label for nested: {err}");
3153 // Round 30 — the "expected schema" suffix should be the
3154 // sub-schema at /name, not the entire object schema. Reader
3155 // shouldn't have to scan a 300-char object to find the
3156 // constraint that failed.
3157 assert!(
3158 err.contains(r#"expected schema {"type":"string"}"#),
3159 "should show only the /name sub-schema, not the full object: {err}"
3160 );
3161 }
3162
3163 /// Round 30 — Srikanth asked how a deeper nested mismatch reads.
3164 /// Schema: `name.type` should be a string; body has it as a number.
3165 /// JSON pointer is `/name/type`.
3166 #[test]
3167 fn response_schema_error_uses_response_body_prefix_for_deep_nested_paths() {
3168 let schema = serde_json::json!({
3169 "type": "object",
3170 "properties": {
3171 "name": {
3172 "type": "object",
3173 "properties": {"type": {"type": "string"}}
3174 }
3175 }
3176 });
3177 let body = r#"{"name": {"type": 123}}"#;
3178 let err = validate_body_against_schema(body, &schema).expect("type-mismatch fires");
3179 assert!(
3180 err.contains("response body at /name/type"),
3181 "deep nested path should read 'response body at /name/type': {err}"
3182 );
3183 // Round 30 — for deep paths the sub-schema is the leaf
3184 // {"type":"string"}, not the wrapping object schemas.
3185 assert!(
3186 err.contains(r#"expected schema {"type":"string"}"#),
3187 "should show only the /name/type leaf sub-schema: {err}"
3188 );
3189 }
3190
3191 /// Round 30 — when the instance pointer can't be resolved through
3192 /// the schema's `properties` chain (e.g. additionalProperties hit),
3193 /// `sub_schema_at_pointer` returns None and the message falls back
3194 /// to the full schema. Verifies the fallback path is wired.
3195 #[test]
3196 fn sub_schema_at_pointer_falls_back_for_unresolvable_paths() {
3197 let schema = serde_json::json!({"type":"object","additionalProperties":true});
3198 // Walker can't resolve /unknown, so we get the full schema back.
3199 assert_eq!(
3200 sub_schema_at_pointer(&schema, "/unknown"),
3201 None,
3202 "unresolvable path should return None to trigger fallback"
3203 );
3204 // Root path returns the whole schema.
3205 assert_eq!(sub_schema_at_pointer(&schema, "/"), Some(schema.clone()));
3206 assert_eq!(sub_schema_at_pointer(&schema, ""), Some(schema));
3207 }
3208
3209 #[test]
3210 fn response_schema_error_required_field_is_readable() {
3211 let schema = serde_json::json!({
3212 "type": "object",
3213 "required": ["id"],
3214 "properties": {"id": {"type": "integer"}}
3215 });
3216 let body = r#"{"other": 1}"#;
3217 let err = validate_body_against_schema(body, &schema).expect("required-missing fires");
3218 assert!(err.contains("required field missing"), "{err}");
3219 assert!(err.contains("id"), "{err}");
3220 }
3221
3222 /// Round 31 — Srikanth's vCenter case on 0.3.174: the
3223 /// `Appliance.Recovery.Backup.SystemName.Archive.Info` schema has
3224 /// a multi-paragraph description and ~6 required fields, of which
3225 /// `comment` was missing in the response. Before this fix the
3226 /// printed schema was the WHOLE parent object schema (parent's
3227 /// description bleeding in, all sibling property schemas dumped)
3228 /// truncated to 300 chars; after the fix it's the missing field's
3229 /// own schema. Verifies (a) parent description is gone and
3230 /// (b) sibling property names don't appear in the message.
3231 #[test]
3232 fn response_schema_error_required_focuses_on_missing_field_only() {
3233 let schema = serde_json::json!({
3234 "description": "The Appliance.Recovery.Backup.SystemName.Archive.Info schema represents backup archive information.\n\nThis schema was added in vSphere API 6.7.",
3235 "type": "object",
3236 "required": ["comment", "location", "parts", "system_name", "timestamp", "version"],
3237 "properties": {
3238 "comment": {
3239 "type": "string",
3240 "description": "Custom comment added by the user for this backup."
3241 },
3242 "location": {"type": "string", "description": "Backup location URL."},
3243 "parts": {"type": "array", "items": {"type": "string"}},
3244 "system_name": {"type": "string"},
3245 "timestamp": {"type": "string", "format": "date-time"},
3246 "version": {"type": "string"}
3247 }
3248 });
3249 let body = r#"{"location":"x","parts":[],"system_name":"y","timestamp":"z","version":"v"}"#;
3250 let err = validate_body_against_schema(body, &schema).expect("required-missing fires");
3251 assert!(err.contains("required field missing: \"comment\""), "{err}");
3252 // Parent's description should not appear; only the `comment`
3253 // field's own description (if any) may.
3254 assert!(
3255 !err.contains("Appliance.Recovery.Backup"),
3256 "parent description should not bleed into focused schema: {err}"
3257 );
3258 // No sibling property names should appear in the focused schema
3259 // suffix.
3260 for sibling in ["location", "parts", "system_name", "timestamp", "version"] {
3261 assert!(
3262 !err.contains(&format!("\"{sibling}\"")),
3263 "sibling field {sibling} should not appear in focused schema: {err}"
3264 );
3265 }
3266 }
3267
3268 #[test]
3269 fn response_schema_error_none_on_match() {
3270 let schema = serde_json::json!({"type": "string"});
3271 assert_eq!(validate_body_against_schema("\"hello\"", &schema), None);
3272 }
3273
3274 /// Round 34 (#827) — Srikanth on 0.3.178 hit the vCenter
3275 /// `consolecli` PUT where the `enabled: boolean` property has a
3276 /// multi-paragraph description. The schema printout truncated
3277 /// mid-description, hiding `type: boolean` past the 300-char cap.
3278 /// Stripping `description` (and friends) before serializing must
3279 /// keep the type info visible.
3280 #[test]
3281 fn response_schema_error_strips_description_so_type_survives_truncation() {
3282 // Schema crafted so without stripping, `description` would
3283 // push `type` past the 300-char truncation cap. The
3284 // description we use here is intentionally close to the
3285 // vCenter-spec wording Srikanth quoted.
3286 let big_desc = "In the result of the #get and #list operations this property indicates whether proxying is enabled for a particular protocol. In the input to the test and set operations this property specifies whether proxying should be enabled for a particular protocol. This property was added in vSphere API 6.7. Defaults to enabled if both this field and the value field are unset.";
3287 let schema = serde_json::json!({
3288 "type": "object",
3289 "required": ["enabled"],
3290 "properties": {
3291 "enabled": {
3292 "type": "boolean",
3293 "description": big_desc,
3294 "example": true,
3295 }
3296 }
3297 });
3298 let body = r#"{}"#;
3299 let err = validate_body_against_schema(body, &schema).expect("required-missing fires");
3300 assert!(err.contains("required field missing: \"enabled\""), "{err}");
3301 assert!(
3302 err.contains(r#""type":"boolean""#),
3303 "the `type: boolean` keyword must survive truncation: {err}"
3304 );
3305 // Description should NOT appear (we stripped it) so the
3306 // suffix is type-focused, not prose.
3307 assert!(
3308 !err.contains("proxying is enabled"),
3309 "description should be stripped from the printed schema: {err}"
3310 );
3311 assert!(
3312 !err.contains("\"example\""),
3313 "`example` field should be stripped from the printed schema: {err}"
3314 );
3315 }
3316
3317 /// Round 34 (#827) — strip_schema_noise should keep all
3318 /// constraint keywords intact; only the prose noise goes.
3319 #[test]
3320 fn strip_schema_noise_preserves_constraint_keywords() {
3321 let schema = serde_json::json!({
3322 "type": "object",
3323 "required": ["a", "b"],
3324 "description": "should be stripped",
3325 "title": "should be stripped",
3326 "example": {"a": 1, "b": 2},
3327 "properties": {
3328 "a": {"type": "string", "format": "uri", "minLength": 1, "description": "drop"},
3329 "b": {"type": "integer", "minimum": 0, "maximum": 100, "summary": "drop"},
3330 },
3331 });
3332 let stripped = strip_schema_noise(&schema);
3333 let s = serde_json::to_string(&stripped).unwrap();
3334 // Constraint keywords survive.
3335 for keep in [
3336 "\"type\"",
3337 "\"required\"",
3338 "\"properties\"",
3339 "\"format\"",
3340 "\"minLength\"",
3341 "\"minimum\"",
3342 "\"maximum\"",
3343 ] {
3344 assert!(s.contains(keep), "should keep {keep}: {s}");
3345 }
3346 // Noise fields are gone.
3347 for drop in ["description", "title", "example", "summary"] {
3348 assert!(!s.contains(&format!("\"{drop}\"")), "should strip {drop}: {s}");
3349 }
3350 }
3351
3352 #[test]
3353 fn json_serialises_report() {
3354 let r = SelfTestReport {
3355 positive_pass: 1,
3356 positive_fail: 0,
3357 negative_caught: BTreeMap::new(),
3358 negative_missed: BTreeMap::new(),
3359 operations: vec![OperationResult {
3360 method: "GET".into(),
3361 path: "/x".into(),
3362 positive: Some(CaseOutcome {
3363 label: "positive".into(),
3364 expected_4xx: false,
3365 actual_status: 200,
3366 passed: true,
3367 }),
3368 negatives: Vec::new(),
3369 }],
3370 };
3371 let json = serde_json::to_value(&r).expect("serialises");
3372 assert_eq!(json["positive_pass"], serde_json::json!(1));
3373 assert_eq!(json["operations"][0]["positive"]["actual_status"], serde_json::json!(200));
3374 }
3375}