mockforge_bench/conformance/
report_html.rs

1//! HTML report renderer for the conformance self-test.
2//!
3//! Issue #79 round 17.6 — Srikanth's (17.6) ask: a human-readable
4//! report grouped by category, severity, and OWASP class. The JSON
5//! reports from rounds 17.x are precise but hard to skim under a
6//! deadline; this module renders a self-contained HTML file that
7//! drops into a browser without any external assets.
8//!
9//! Sections:
10//! 1. Header: target URL, timestamp, headline counts.
11//! 2. Self-test summary cards: positives, negatives caught / missed
12//!    per category.
13//! 3. Negative detail table (rolled up by category + label) so a
14//!    user can drill from "owasp had 12 misses" → which routes.
15//! 4. Optional spec-audit section (if a round-17.4 audit JSON is
16//!    passed in alongside).
17//!
18//! Output is one self-contained HTML string: inline CSS, no external
19//! fonts or scripts, safe to email or commit to a CI artefact bucket.
20
21use super::self_test::{CaseOutcome, OperationResult, SelfTestReport};
22use std::collections::BTreeMap;
23
24/// Render a complete HTML report for the given self-test report.
25/// `audit` is an optional `SpecAuditReport`-shaped JSON value — when
26/// present, an audit section is appended. We accept `&serde_json::Value`
27/// rather than the strongly-typed `SpecAuditReport` to keep this
28/// module decoupled from `spec_audit` (which lives on a separate
29/// in-flight branch).
30pub fn render_html(report: &SelfTestReport, audit: Option<&serde_json::Value>) -> String {
31    render_html_with_options(report, audit, &RenderOptions::default())
32}
33
34/// Round 21.1 — render options surfaced via CLI flags. Currently:
35/// - `missed_cap`: max rows in the missed-negative drill-down table.
36///   `Some(N)` caps at N (default 200); `None` shows all rows. Set
37///   via `--report-missed-cap` (with `--report-missed-cap 0` mapping
38///   to None for "no cap").
39#[derive(Debug, Clone)]
40pub struct RenderOptions {
41    pub missed_cap: Option<usize>,
42}
43
44impl Default for RenderOptions {
45    fn default() -> Self {
46        Self {
47            missed_cap: Some(200),
48        }
49    }
50}
51
52/// Round 21.1 — like `render_html` but lets the caller override the
53/// drill-down cap. Used by the CLI when `--report-missed-cap` is set.
54pub fn render_html_with_options(
55    report: &SelfTestReport,
56    audit: Option<&serde_json::Value>,
57    opts: &RenderOptions,
58) -> String {
59    let mut html = String::new();
60    html.push_str(HEAD);
61    push_header(&mut html, report);
62    push_summary_cards(&mut html, report);
63    push_category_table(&mut html, report);
64    push_operations_table(&mut html, report, opts);
65    if let Some(a) = audit {
66        push_spec_audit(&mut html, a);
67    }
68    html.push_str(FOOT);
69    html
70}
71
72/// Inline-CSS opening — no external assets, prints fine.
73const HEAD: &str = r#"<!doctype html>
74<html lang="en">
75<head>
76<meta charset="utf-8">
77<title>MockForge Conformance Report</title>
78<style>
79  body { font-family: -apple-system, system-ui, sans-serif; max-width: 1100px;
80         margin: 2rem auto; padding: 0 1rem; color: #1f2933; line-height: 1.5; }
81  h1 { font-size: 1.8rem; margin: 0 0 0.5rem; }
82  h2 { font-size: 1.3rem; margin: 2rem 0 0.5rem; border-bottom: 1px solid #d1d5db; padding-bottom: 0.3rem; }
83  .meta { color: #6b7280; font-size: 0.9rem; }
84  .cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 0.75rem; margin: 1rem 0; }
85  .card { padding: 0.75rem 1rem; border-radius: 6px; background: #f3f4f6; }
86  .card .label { font-size: 0.75rem; text-transform: uppercase; letter-spacing: 0.05em; color: #6b7280; }
87  .card .value { font-size: 1.6rem; font-weight: 600; color: #1f2933; }
88  .card.ok { background: #ecfdf5; } .card.ok .value { color: #047857; }
89  .card.warn { background: #fffbeb; } .card.warn .value { color: #b45309; }
90  .card.err { background: #fef2f2; } .card.err .value { color: #b91c1c; }
91  table { width: 100%; border-collapse: collapse; margin: 0.5rem 0 1.5rem; font-size: 0.9rem; }
92  th, td { text-align: left; padding: 0.5rem 0.75rem; border-bottom: 1px solid #e5e7eb; }
93  th { background: #f9fafb; font-weight: 600; color: #374151; }
94  tr:hover { background: #f9fafb; }
95  .badge { display: inline-block; padding: 0.1rem 0.5rem; border-radius: 999px; font-size: 0.75rem; font-weight: 500; }
96  .badge.pass { background: #d1fae5; color: #047857; }
97  .badge.fail { background: #fee2e2; color: #b91c1c; }
98  .badge.info { background: #dbeafe; color: #1d4ed8; }
99  .badge.warn { background: #fef3c7; color: #92400e; }
100  .badge.err  { background: #fee2e2; color: #b91c1c; }
101  .small { color: #6b7280; font-size: 0.85rem; }
102  code { background: #f3f4f6; padding: 0.05rem 0.3rem; border-radius: 3px; font-size: 0.9em; }
103</style>
104</head>
105<body>
106"#;
107
108const FOOT: &str = "\n</body>\n</html>\n";
109
110fn push_header(out: &mut String, _report: &SelfTestReport) {
111    out.push_str("<h1>MockForge Conformance Report</h1>\n");
112    // Round 22.6 — link the probe-label reference next to the
113    // generator credit so users can decode labels like
114    // `request-body:type-mismatch:user.email` without leaving
115    // the report. The book is generated separately, so we link
116    // to the canonical hosted location.
117    out.push_str(
118        "<p class=\"meta\">Generated by <code>mockforge bench --conformance-self-test</code>. \
119         Probe-label reference: \
120         <a href=\"https://docs.mockforge.dev/reference/conformance-self-test-probes.html\">\
121         docs.mockforge.dev/reference/conformance-self-test-probes</a>.</p>\n",
122    );
123}
124
125fn push_summary_cards(out: &mut String, report: &SelfTestReport) {
126    let positives = report.positive_pass + report.positive_fail;
127    let neg_caught: usize = report.negative_caught.values().sum();
128    let neg_missed: usize = report.negative_missed.values().sum();
129    let pos_class = if report.positive_fail == 0 {
130        "ok"
131    } else {
132        "err"
133    };
134    let miss_class = if neg_missed == 0 { "ok" } else { "warn" };
135    out.push_str("<div class=\"cards\">\n");
136    push_card(out, "Positive cases", positives, pos_class);
137    push_card(out, "Positive failures", report.positive_fail, pos_class);
138    push_card(out, "Negatives matched (4xx)", neg_caught, "ok");
139    push_card(out, "Negatives mismatched (non-4xx)", neg_missed, miss_class);
140    push_card(out, "Operations", report.operations.len(), "");
141    out.push_str("</div>\n");
142}
143
144fn push_card(out: &mut String, label: &str, value: usize, class: &str) {
145    let class_attr = if class.is_empty() {
146        String::new()
147    } else {
148        format!(" {}", class)
149    };
150    out.push_str(&format!(
151        "  <div class=\"card{class_attr}\"><div class=\"label\">{}</div><div class=\"value\">{}</div></div>\n",
152        html_escape(label),
153        value
154    ));
155}
156
157fn push_category_table(out: &mut String, report: &SelfTestReport) {
158    out.push_str("<h2>Negatives by category</h2>\n");
159    let mut keys: Vec<&String> =
160        report.negative_caught.keys().chain(report.negative_missed.keys()).collect();
161    keys.sort();
162    keys.dedup();
163    if keys.is_empty() {
164        out.push_str("<p class=\"small\">No negative probes ran — typically means no operations had any injectable surface.</p>\n");
165        return;
166    }
167    out.push_str("<table>\n<thead><tr><th>Category</th><th>Matched (4xx)</th><th>Mismatched (non-4xx)</th><th>Status</th></tr></thead>\n<tbody>\n");
168    for cat in keys {
169        let caught = report.negative_caught.get(cat).copied().unwrap_or(0);
170        let missed = report.negative_missed.get(cat).copied().unwrap_or(0);
171        // Round 23 (d) — Srikanth: "rejection gaps" was still too soft,
172        // and missed/caught wasn't intuitive. Switch the column headers
173        // to Matched/Mismatched (server's 4xx response matched the
174        // probe's expectation, or didn't) and reduce the status badge
175        // to a plain PASS/FAIL since the count column already conveys
176        // the magnitude.
177        let (badge_class, badge_text) = if missed == 0 {
178            ("pass", "PASS")
179        } else {
180            ("fail", "FAIL")
181        };
182        // Round 23 (d) — clickable count: link the Mismatched count to
183        // the per-row anchor in the drill-down table below, so a reader
184        // can jump from "this category has 3 fails" → "here are the 3
185        // probes". Empty → no link.
186        let missed_cell = if missed > 0 {
187            format!("<a href=\"#miss-cat-{}\">{}</a>", html_escape(cat), missed)
188        } else {
189            missed.to_string()
190        };
191        out.push_str(&format!(
192            "<tr><td><code>{}</code></td><td>{}</td><td>{}</td><td><span class=\"badge {}\">{}</span></td></tr>\n",
193            html_escape(cat),
194            caught,
195            missed_cell,
196            badge_class,
197            badge_text
198        ));
199    }
200    out.push_str("</tbody></table>\n");
201}
202
203fn push_operations_table(out: &mut String, report: &SelfTestReport, opts: &RenderOptions) {
204    out.push_str("<h2>Per-operation results</h2>\n");
205    if report.operations.is_empty() {
206        out.push_str("<p class=\"small\">No operations.</p>\n");
207        return;
208    }
209    out.push_str("<table>\n<thead><tr><th>Method</th><th>Path</th><th>Positive</th><th>Matched / Mismatched</th></tr></thead>\n<tbody>\n");
210    for op in &report.operations {
211        let pos_badge = match &op.positive {
212            Some(p) if p.passed => "<span class=\"badge pass\">2xx ✓</span>".to_string(),
213            Some(p) => format!("<span class=\"badge fail\">{} ✗</span>", p.actual_status),
214            None => "<span class=\"badge info\">none</span>".into(),
215        };
216        let (caught, missed) = op.negatives.iter().partition::<Vec<&CaseOutcome>, _>(|n| n.passed);
217        // Round 23 (d) — clickable count: link the Mismatched count to
218        // the operation's anchor in the drill-down table below.
219        let op_slug = op_anchor_slug(&op.method, &op.path);
220        let missed_cell = if missed.is_empty() {
221            "0".to_string()
222        } else {
223            format!("<a href=\"#miss-op-{}\">{}</a>", op_slug, missed.len())
224        };
225        out.push_str(&format!(
226            "<tr><td><code>{}</code></td><td><code>{}</code></td><td>{}</td><td>{} / {}</td></tr>\n",
227            html_escape(&op.method),
228            html_escape(&op.path),
229            pos_badge,
230            caught.len(),
231            missed_cell
232        ));
233    }
234    out.push_str("</tbody></table>\n");
235    push_missed_detail(out, report, opts);
236}
237
238/// Round 23 (d) — stable slug for the per-operation anchor in the
239/// missed-negative drill-down table. Lowercase, [a-z0-9_] only so the
240/// resulting `id` is HTML-valid and the `#miss-op-...` link from the
241/// Per-operation table resolves predictably. Collisions across very
242/// similar paths are acceptable: clicking lands on the first matching
243/// row and the table is short enough to scan from there.
244fn op_anchor_slug(method: &str, path: &str) -> String {
245    let mut s = format!("{method}_{path}");
246    s = s.to_ascii_lowercase();
247    s = s.chars().map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }).collect();
248    s
249}
250
251/// Round 21.1 — human-readable expected status range derived from the
252/// probe's `expected_4xx` flag, so the missed-negative table tells you
253/// what status it WAS expecting alongside what it ACTUALLY saw.
254fn expected_status_label(case: &CaseOutcome) -> &'static str {
255    if case.expected_4xx {
256        "4xx (reject)"
257    } else {
258        "2xx-3xx (accept)"
259    }
260}
261
262fn push_missed_detail(out: &mut String, report: &SelfTestReport, opts: &RenderOptions) {
263    // List every individual missed negative for drill-down. By default
264    // capped at 200 rows to keep the HTML file under a reasonable size
265    // on huge specs; raise or remove via `--report-missed-cap`. The
266    // JSON report always has the full set.
267    let mut missed: Vec<(&OperationResult, &CaseOutcome)> = Vec::new();
268    for op in &report.operations {
269        for neg in &op.negatives {
270            if !neg.passed {
271                missed.push((op, neg));
272            }
273        }
274    }
275    if missed.is_empty() {
276        return;
277    }
278    out.push_str(
279        "<h2>Mismatched negatives (server returned non-4xx to a probe expecting 4xx)</h2>\n",
280    );
281    // Cap message: surface the cap explicitly so the user knows
282    // whether the table is truncated or complete.
283    let total = missed.len();
284    let cap_msg = match opts.missed_cap {
285        Some(cap) if total > cap => format!(
286            "{} mismatched negative(s). Showing first {} (raise with <code>--report-missed-cap N</code>, or <code>0</code> for no cap); full set in <code>conformance-self-test.json</code>.",
287            total, cap
288        ),
289        Some(_) => format!("{} mismatched negative(s). All shown.", total),
290        None => format!("{} mismatched negative(s). All shown (no cap).", total),
291    };
292    out.push_str(&format!("<p class=\"small\">{cap_msg}</p>\n"));
293    out.push_str("<table>\n<thead><tr><th>Method</th><th>Path</th><th>Label</th><th>Expected</th><th>Actual</th></tr></thead>\n<tbody>\n");
294    let take = opts.missed_cap.unwrap_or(usize::MAX);
295    // Round 23 (d) — emit anchor ids so the count-cells in the
296    // Negatives-by-category and Per-operation tables can link straight
297    // to the first matching drill-down row:
298    //   `miss-cat-<category>` on the <tr>
299    //   `miss-op-<slug>`      on a zero-size <span> in the first cell
300    // (a <tr> can only carry one `id`, so the per-op anchor rides on
301    // the span; HTML treats both as valid jump targets). First-seen
302    // wins to avoid duplicate IDs across the truncated table.
303    let mut seen_cat: std::collections::HashSet<String> = std::collections::HashSet::new();
304    let mut seen_op: std::collections::HashSet<String> = std::collections::HashSet::new();
305    for (op, neg) in missed.iter().take(take) {
306        let cat = neg.label.split(':').next().unwrap_or("other").to_string();
307        let op_slug = op_anchor_slug(&op.method, &op.path);
308        let tr_id = if seen_cat.insert(cat.clone()) {
309            format!(" id=\"miss-cat-{}\"", html_escape(&cat))
310        } else {
311            String::new()
312        };
313        let op_anchor = if seen_op.insert(op_slug.clone()) {
314            format!("<span id=\"miss-op-{op_slug}\"></span>")
315        } else {
316            String::new()
317        };
318        out.push_str(&format!(
319            "<tr{}><td>{}<code>{}</code></td><td><code>{}</code></td><td><code>{}</code></td><td><span class=\"badge info\">{}</span></td><td>{}</td></tr>\n",
320            tr_id,
321            op_anchor,
322            html_escape(&op.method),
323            html_escape(&op.path),
324            html_escape(&neg.label),
325            expected_status_label(neg),
326            neg.actual_status
327        ));
328    }
329    out.push_str("</tbody></table>\n");
330}
331
332fn push_spec_audit(out: &mut String, audit: &serde_json::Value) {
333    out.push_str("<h2>Spec audit</h2>\n");
334    let findings = audit.get("findings").and_then(|v| v.as_array());
335    let coverage = audit.get("datatype_coverage").and_then(|v| v.as_object());
336    let ops = audit.get("operations_audited").and_then(|v| v.as_u64()).unwrap_or(0);
337    out.push_str(&format!(
338        "<p class=\"small\">Audited {ops} operation(s). Coverage map: {} datatype kind(s).</p>\n",
339        coverage.map(|c| c.len()).unwrap_or(0)
340    ));
341    if let Some(findings) = findings {
342        if findings.is_empty() {
343            out.push_str("<p class=\"small\">No findings.</p>\n");
344        } else {
345            // Group findings by severity for an easy scan.
346            let mut by_sev: BTreeMap<String, Vec<&serde_json::Value>> = BTreeMap::new();
347            for f in findings {
348                let sev = f.get("severity").and_then(|v| v.as_str()).unwrap_or("info").to_string();
349                by_sev.entry(sev).or_default().push(f);
350            }
351            out.push_str("<table>\n<thead><tr><th>Severity</th><th>Category</th><th>Location</th><th>Message</th></tr></thead>\n<tbody>\n");
352            for (sev, items) in by_sev {
353                let badge_class = match sev.as_str() {
354                    "error" => "err",
355                    "warning" => "warn",
356                    _ => "info",
357                };
358                for item in items {
359                    let cat = item.get("category").and_then(|v| v.as_str()).unwrap_or("");
360                    let loc = item.get("location").and_then(|v| v.as_str()).unwrap_or("");
361                    let msg = item.get("message").and_then(|v| v.as_str()).unwrap_or("");
362                    out.push_str(&format!(
363                        "<tr><td><span class=\"badge {}\">{}</span></td><td><code>{}</code></td><td><code>{}</code></td><td>{}</td></tr>\n",
364                        badge_class,
365                        html_escape(&sev),
366                        html_escape(cat),
367                        html_escape(loc),
368                        html_escape(msg)
369                    ));
370                }
371            }
372            out.push_str("</tbody></table>\n");
373        }
374    }
375    if let Some(coverage) = coverage {
376        let mut entries: Vec<(&String, u64)> =
377            coverage.iter().filter_map(|(k, v)| v.as_u64().map(|c| (k, c))).collect();
378        entries.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(b.0)));
379        if !entries.is_empty() {
380            out.push_str("<h2>Datatype coverage</h2>\n");
381            out.push_str("<table>\n<thead><tr><th>Type</th><th>Count</th></tr></thead>\n<tbody>\n");
382            for (kind, count) in entries.iter().take(40) {
383                out.push_str(&format!(
384                    "<tr><td><code>{}</code></td><td>{}</td></tr>\n",
385                    html_escape(kind),
386                    count
387                ));
388            }
389            out.push_str("</tbody></table>\n");
390        }
391    }
392}
393
394fn html_escape(s: &str) -> String {
395    let mut out = String::with_capacity(s.len());
396    for c in s.chars() {
397        match c {
398            '&' => out.push_str("&amp;"),
399            '<' => out.push_str("&lt;"),
400            '>' => out.push_str("&gt;"),
401            '"' => out.push_str("&quot;"),
402            '\'' => out.push_str("&#39;"),
403            _ => out.push(c),
404        }
405    }
406    out
407}
408
409#[cfg(test)]
410mod tests {
411    use super::*;
412    use crate::conformance::self_test::{CaseOutcome, OperationResult, SelfTestReport};
413
414    fn sample_report() -> SelfTestReport {
415        SelfTestReport {
416            positive_pass: 3,
417            positive_fail: 1,
418            negative_caught: BTreeMap::from([("request-body".into(), 4), ("parameters".into(), 2)]),
419            negative_missed: BTreeMap::from([("owasp".into(), 1)]),
420            operations: vec![OperationResult {
421                method: "POST".into(),
422                path: "/users".into(),
423                positive: Some(CaseOutcome {
424                    label: "positive".into(),
425                    expected_4xx: false,
426                    actual_status: 201,
427                    passed: true,
428                }),
429                negatives: vec![CaseOutcome {
430                    label: "owasp:sqli".into(),
431                    expected_4xx: true,
432                    actual_status: 200,
433                    passed: false,
434                }],
435            }],
436        }
437    }
438
439    #[test]
440    fn html_contains_expected_sections() {
441        let html = render_html(&sample_report(), None);
442        assert!(html.contains("<title>MockForge Conformance Report</title>"));
443        assert!(html.contains("Positive cases"));
444        assert!(html.contains("Negatives by category"));
445        assert!(html.contains("Per-operation results"));
446        // Round 23 wording polish: "Missed negatives" → "Mismatched negatives".
447        assert!(html.contains("Mismatched negatives"));
448        // Specific data points from the sample report:
449        assert!(html.contains("request-body"));
450        assert!(html.contains("owasp:sqli"));
451        assert!(html.contains("/users"));
452    }
453
454    #[test]
455    fn html_renders_audit_section_when_present() {
456        let audit = serde_json::json!({
457            "findings": [
458                {"category": "servers", "severity": "warning",
459                 "location": "#/servers", "message": "no servers declared"}
460            ],
461            "datatype_coverage": {"string": 5, "integer": 3},
462            "operations_audited": 7
463        });
464        let html = render_html(&sample_report(), Some(&audit));
465        assert!(html.contains("Spec audit"));
466        assert!(html.contains("no servers declared"));
467        assert!(html.contains("Datatype coverage"));
468        assert!(html.contains("string"));
469        assert!(html.contains("Audited 7 operation"));
470    }
471
472    #[test]
473    fn html_escapes_special_chars_in_labels() {
474        let mut report = sample_report();
475        report.operations[0].path = "/items/<script>".into();
476        report.operations[0].negatives[0].label = "owasp:xss:<>\"&".into();
477        let html = render_html(&report, None);
478        // The literal special chars should be escaped, not rendered raw.
479        assert!(!html.contains("/items/<script>"));
480        assert!(html.contains("&lt;script&gt;"));
481        assert!(html.contains("&quot;"));
482    }
483
484    #[test]
485    fn html_handles_empty_report() {
486        let html = render_html(&SelfTestReport::default(), None);
487        assert!(html.contains("No negative probes ran"));
488        assert!(html.contains("No operations."));
489    }
490
491    #[test]
492    fn html_caps_missed_detail_at_default_200_rows() {
493        let mut report = SelfTestReport::default();
494        for i in 0..250 {
495            report.operations.push(OperationResult {
496                method: "GET".into(),
497                path: format!("/r/{i}"),
498                positive: None,
499                negatives: vec![CaseOutcome {
500                    label: "parameters:missing-query".into(),
501                    expected_4xx: true,
502                    actual_status: 200,
503                    passed: false,
504                }],
505            });
506        }
507        report.negative_missed.insert("parameters".into(), 250);
508        let html = render_html(&report, None);
509        // Cap message visible and references the new flag (round 23: "missed" → "mismatched"):
510        assert!(html.contains("250 mismatched negative"));
511        assert!(html.contains("Showing first 200"));
512        assert!(html.contains("--report-missed-cap"));
513    }
514
515    /// Round 21.1 — when `missed_cap` is `None` (set via
516    /// `--report-missed-cap 0`), all rows are shown and the message
517    /// says so explicitly.
518    #[test]
519    fn html_no_cap_shows_all_rows() {
520        let mut report = SelfTestReport::default();
521        for i in 0..50 {
522            report.operations.push(OperationResult {
523                method: "GET".into(),
524                path: format!("/r/{i}"),
525                positive: None,
526                negatives: vec![CaseOutcome {
527                    label: "parameters:missing-query".into(),
528                    expected_4xx: true,
529                    actual_status: 200,
530                    passed: false,
531                }],
532            });
533        }
534        let opts = RenderOptions { missed_cap: None };
535        let html = render_html_with_options(&report, None, &opts);
536        assert!(html.contains("50 mismatched negative"));
537        assert!(html.contains("All shown (no cap)"));
538        assert!(!html.contains("Showing first"));
539    }
540
541    /// Round 21.1 — the missed-negative table now has an Expected
542    /// column derived from the probe's `expected_4xx` flag.
543    #[test]
544    fn html_missed_table_has_expected_column() {
545        let mut report = sample_report();
546        // The sample's single missed negative is `owasp:sqli` with
547        // expected_4xx: true.
548        report.operations[0].negatives = vec![CaseOutcome {
549            label: "security:bad-bearer".into(),
550            expected_4xx: true,
551            actual_status: 200,
552            passed: false,
553        }];
554        let html = render_html(&report, None);
555        assert!(html.contains("Expected"), "Expected column header missing");
556        assert!(
557            html.contains("4xx (reject)"),
558            "expected-status badge for negative probe missing"
559        );
560    }
561}
mockforge_bench/conformance/report_html.rs

mockforge_bench/conformance/
report_html.rs
