mockforge_bench/conformance/
per_endpoint_summary.rs

1//! Per-endpoint send/received summary derived from the
2//! `CaseCapture` JSONL sink.
3//!
4//! Issue #79 round 32 — Srikanth on 0.3.176: "HTML(Manual
5//! Verification)/JSON(for Automation Verification) would help where we
6//! show API Endpoint details. Something like:
7//! `[GET/POST/PUT/...]: <send_request_count>, 2xx or 3xx or 4xx or 5xx
8//! count separately Per end Point` and per-(method, path) request
9//! body / response body length p95."
10//!
11//! The bench already records every request/response in
12//! `conformance-self-test-requests.jsonl`. This module rolls them up
13//! per (method, resolved-path) so a human (or `jq`) doesn't have to
14//! re-aggregate from scratch. v1 groups by the resolved URL path
15//! (everything after the host, minus the query string); a future
16//! round can collapse to the spec's path template once we surface the
17//! `op.path` template on each `CaseCapture` entry.
18//!
19//! Output:
20//! - `conformance-per-endpoint.json` next to the existing
21//!   `conformance-self-test.json` for automation.
22//! - HTML section spliced into `conformance-report.html` for humans.
23
24use std::collections::BTreeMap;
25
26use serde::{Deserialize, Serialize};
27
28use super::self_test::CaseCapture;
29
30#[derive(Debug, Clone, Serialize, Deserialize)]
31pub struct PerEndpointSummary {
32    /// HTTP method, uppercase.
33    pub method: String,
34    /// Round 33 (#823) — spec path template (e.g. `/users/{id}`)
35    /// pre path-param substitution. Falls back to the resolved URL
36    /// path when the capture predates the template field.
37    pub path: String,
38    /// Round 33 (#823) — basename of the OpenAPI spec the probes for
39    /// this endpoint came from. `None` for single-spec runs that didn't
40    /// stamp a label, or for legacy captures.
41    #[serde(default, skip_serializing_if = "Option::is_none")]
42    pub spec: Option<String>,
43    pub sent: usize,
44    pub status_2xx: usize,
45    pub status_3xx: usize,
46    pub status_4xx: usize,
47    pub status_5xx: usize,
48    /// Network errors (`response_status == 0`).
49    pub errors: usize,
50    /// Length stats on the captured REQUEST body (bytes). `None` when
51    /// no request body was sent on any probe for this endpoint.
52    pub request_body_len: Option<LenStats>,
53    /// Length stats on the captured RESPONSE body (bytes). `None`
54    /// when no captured response body had content.
55    pub response_body_len: Option<LenStats>,
56    /// Length stats on the resolved query string (raw bytes after
57    /// `?`). `None` when no probe carried a query string for this
58    /// endpoint.
59    pub query_len: Option<LenStats>,
60}
61
62#[derive(Debug, Clone, Serialize, Deserialize)]
63pub struct LenStats {
64    pub samples: usize,
65    pub avg: f64,
66    pub p50: u64,
67    pub p95: u64,
68    pub max: u64,
69}
70
71impl LenStats {
72    fn from_samples(mut samples: Vec<u64>) -> Option<Self> {
73        if samples.is_empty() {
74            return None;
75        }
76        samples.sort_unstable();
77        let n = samples.len();
78        let sum: u64 = samples.iter().sum();
79        let avg = sum as f64 / n as f64;
80        let pick = |q: f64| -> u64 {
81            // Nearest-rank percentile, 1-indexed. Matches k6's
82            // `http_req_duration{tag:p95}` calculation closely enough
83            // for spot checks.
84            let idx = (q * n as f64).ceil() as usize;
85            let idx = idx.clamp(1, n) - 1;
86            samples[idx]
87        };
88        Some(LenStats {
89            samples: n,
90            avg,
91            p50: pick(0.50),
92            p95: pick(0.95),
93            max: *samples.last().unwrap(),
94        })
95    }
96}
97
98/// Build the per-endpoint summary. Pass the captured slice as
99/// produced by the conformance self-test sink.
100///
101/// Round 33 (#823) — grouping key is `(method, path_template, spec)`
102/// when the capture carries a non-empty `path_template`, and falls
103/// back to `(method, resolved-path)` otherwise so this stays
104/// compatible with older capture files that don't have the field.
105pub fn build_summary(captures: &[CaseCapture]) -> Vec<PerEndpointSummary> {
106    let mut by_key: BTreeMap<(String, String, Option<String>), EndpointAccumulator> =
107        BTreeMap::new();
108
109    for c in captures {
110        let (resolved_path, query) = split_url(&c.url);
111        // Prefer the spec template; resolved URL path is the fallback
112        // only for legacy captures that predate `path_template`.
113        let path = if c.path_template.is_empty() {
114            resolved_path
115        } else {
116            c.path_template.clone()
117        };
118        let key = (c.method.to_ascii_uppercase(), path, c.spec_label.clone());
119        let entry = by_key.entry(key).or_default();
120        entry.sent += 1;
121        match c.response_status {
122            0 => entry.errors += 1,
123            s if (200..300).contains(&s) => entry.status_2xx += 1,
124            s if (300..400).contains(&s) => entry.status_3xx += 1,
125            s if (400..500).contains(&s) => entry.status_4xx += 1,
126            s if (500..600).contains(&s) => entry.status_5xx += 1,
127            _ => {}
128        }
129        if let Some(body) = &c.request_body {
130            entry.request_lens.push(body.len() as u64);
131        }
132        if let Some(body) = &c.response_body {
133            entry.response_lens.push(body.len() as u64);
134        }
135        if let Some(q) = query {
136            if !q.is_empty() {
137                entry.query_lens.push(q.len() as u64);
138            }
139        }
140    }
141
142    let mut out: Vec<PerEndpointSummary> = by_key
143        .into_iter()
144        .map(|((method, path, spec), acc)| PerEndpointSummary {
145            spec,
146            method,
147            path,
148            sent: acc.sent,
149            status_2xx: acc.status_2xx,
150            status_3xx: acc.status_3xx,
151            status_4xx: acc.status_4xx,
152            status_5xx: acc.status_5xx,
153            errors: acc.errors,
154            request_body_len: LenStats::from_samples(acc.request_lens),
155            response_body_len: LenStats::from_samples(acc.response_lens),
156            query_len: LenStats::from_samples(acc.query_lens),
157        })
158        .collect();
159    // Sort by sent count desc, then by (method, path) for stable order.
160    out.sort_by(|a, b| b.sent.cmp(&a.sent).then(a.method.cmp(&b.method)).then(a.path.cmp(&b.path)));
161    out
162}
163
164#[derive(Default)]
165struct EndpointAccumulator {
166    sent: usize,
167    status_2xx: usize,
168    status_3xx: usize,
169    status_4xx: usize,
170    status_5xx: usize,
171    errors: usize,
172    request_lens: Vec<u64>,
173    response_lens: Vec<u64>,
174    query_lens: Vec<u64>,
175}
176
177/// Return `(path, query)` from a fully-qualified URL. Falls back to
178/// returning the input unchanged as the path when parsing fails (so
179/// the summary still groups, just without query metrics).
180fn split_url(url: &str) -> (String, Option<String>) {
181    // Strip scheme + host. URLs the bench produces always start with
182    // a scheme; defensive against the rare relative-URL case.
183    let after_host = if let Some(idx) = url.find("://") {
184        let rest = &url[idx + 3..];
185        match rest.find('/') {
186            Some(i) => &rest[i..],
187            None => "/",
188        }
189    } else {
190        url
191    };
192    match after_host.find('?') {
193        Some(i) => (after_host[..i].to_string(), Some(after_host[i + 1..].to_string())),
194        None => (after_host.to_string(), None),
195    }
196}
197
198/// Render the per-endpoint summary as a self-contained HTML
199/// `<section>` block suitable for splicing into
200/// `conformance-report.html`. Uses the same `<table>` styling the
201/// rest of the report already has.
202pub fn render_html_section(summaries: &[PerEndpointSummary]) -> String {
203    if summaries.is_empty() {
204        return String::new();
205    }
206    // Round 33 (#823) — show the Spec column only when at least one row
207    // carries a spec label, so single-spec runs don't get an empty
208    // column and multi-spec runs can attribute rows.
209    let show_spec = summaries.iter().any(|s| s.spec.is_some());
210    let mut out = String::from("<h2 id=\"per-endpoint\">Per-endpoint traffic summary</h2>\n");
211    out.push_str(
212        "<p class=\"small\">Aggregated from the JSONL capture sink. Path is the spec template; lengths are byte counts on the captured (truncated) bodies.</p>\n",
213    );
214    out.push_str("<table>\n<thead><tr>");
215    if show_spec {
216        out.push_str("<th>Spec</th>");
217    }
218    out.push_str(
219        "<th>Method</th><th>Path</th>\
220         <th>Sent</th><th>2xx</th><th>3xx</th><th>4xx</th><th>5xx</th><th>Err</th>\
221         <th>Req p95 (B)</th><th>Resp p95 (B)</th><th>Query p95 (B)</th>\
222         </tr></thead>\n<tbody>\n",
223    );
224    for s in summaries {
225        let req = s
226            .request_body_len
227            .as_ref()
228            .map(|l| l.p95.to_string())
229            .unwrap_or_else(|| "-".to_string());
230        let resp = s
231            .response_body_len
232            .as_ref()
233            .map(|l| l.p95.to_string())
234            .unwrap_or_else(|| "-".to_string());
235        let query = s
236            .query_len
237            .as_ref()
238            .map(|l| l.p95.to_string())
239            .unwrap_or_else(|| "-".to_string());
240        out.push_str("<tr>");
241        if show_spec {
242            let spec_cell = s.spec.as_deref().unwrap_or("-");
243            out.push_str(&format!("<td><code>{}</code></td>", html_escape(spec_cell)));
244        }
245        out.push_str(&format!(
246            "<td><code>{}</code></td><td><code>{}</code></td>\
247             <td>{}</td><td>{}</td><td>{}</td><td>{}</td><td>{}</td><td>{}</td>\
248             <td>{}</td><td>{}</td><td>{}</td></tr>\n",
249            html_escape(&s.method),
250            html_escape(&s.path),
251            s.sent,
252            s.status_2xx,
253            s.status_3xx,
254            s.status_4xx,
255            s.status_5xx,
256            s.errors,
257            req,
258            resp,
259            query,
260        ));
261    }
262    out.push_str("</tbody></table>\n");
263    out
264}
265
266fn html_escape(s: &str) -> String {
267    s.replace('&', "&amp;").replace('<', "&lt;").replace('>', "&gt;")
268}
269
270#[cfg(test)]
271mod tests {
272    use super::*;
273
274    fn cap(
275        method: &str,
276        url: &str,
277        status: u16,
278        req: Option<&str>,
279        resp: Option<&str>,
280    ) -> CaseCapture {
281        cap_with_template(method, url, status, req, resp, "")
282    }
283
284    fn cap_with_template(
285        method: &str,
286        url: &str,
287        status: u16,
288        req: Option<&str>,
289        resp: Option<&str>,
290        path_template: &str,
291    ) -> CaseCapture {
292        CaseCapture {
293            label: "x".to_string(),
294            method: method.to_string(),
295            url: url.to_string(),
296            request_headers: BTreeMap::new(),
297            request_body: req.map(|s| s.to_string()),
298            request_body_truncated: false,
299            response_status: status,
300            response_headers: BTreeMap::new(),
301            response_body: resp.map(|s| s.to_string()),
302            response_body_truncated: false,
303            error: None,
304            response_schema_error: None,
305            expected_status_range: "2xx-3xx".to_string(),
306            path_template: path_template.to_string(),
307            spec_label: None,
308            mockforge_version: String::new(),
309            client_sent_at: String::new(),
310            iteration: 1,
311        }
312    }
313
314    #[test]
315    fn groups_by_method_and_resolved_path() {
316        let caps = vec![
317            cap("GET", "https://host/api/foo", 200, None, Some("hello")),
318            cap("GET", "https://host/api/foo", 404, None, Some("not found")),
319            cap("POST", "https://host/api/bar", 201, Some(r#"{"x":1}"#), Some(r#"{"id":7}"#)),
320        ];
321        let s = build_summary(&caps);
322        assert_eq!(s.len(), 2, "two distinct (method, path) groups");
323        let foo = s.iter().find(|x| x.path == "/api/foo").unwrap();
324        assert_eq!(foo.sent, 2);
325        assert_eq!(foo.status_2xx, 1);
326        assert_eq!(foo.status_4xx, 1);
327        assert!(foo.request_body_len.is_none(), "no request bodies on GET probes");
328        assert!(foo.response_body_len.is_some());
329        let bar = s.iter().find(|x| x.path == "/api/bar").unwrap();
330        assert!(bar.request_body_len.is_some());
331        assert_eq!(bar.request_body_len.as_ref().unwrap().samples, 1);
332    }
333
334    #[test]
335    fn strips_query_string_into_separate_metric() {
336        let caps = vec![
337            cap("GET", "https://host/api/x?a=1&b=2", 200, None, Some("ok")),
338            cap("GET", "https://host/api/x?c=3", 200, None, Some("ok")),
339        ];
340        let s = build_summary(&caps);
341        assert_eq!(s.len(), 1, "query string strip must collapse into one group");
342        let row = &s[0];
343        assert_eq!(row.path, "/api/x");
344        assert_eq!(row.sent, 2);
345        let qlen = row.query_len.as_ref().expect("query stats present");
346        assert_eq!(qlen.samples, 2);
347        assert_eq!(qlen.max, 7); // "a=1&b=2" is 7 bytes
348    }
349
350    #[test]
351    fn p95_is_nearest_rank() {
352        let stats = LenStats::from_samples(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]).unwrap();
353        assert_eq!(stats.p50, 5);
354        assert_eq!(stats.p95, 10);
355        assert_eq!(stats.max, 10);
356    }
357
358    #[test]
359    fn empty_input_renders_to_empty_html() {
360        assert_eq!(render_html_section(&[]), "");
361    }
362
363    /// Round 33 (#823) — Srikanth's vCenter spec resolves the same
364    /// path template to many distinct URLs (`/users/{id}` →
365    /// `/users/test-value`, `/users/abc`, etc). Without template
366    /// grouping the report blows up to one row per VU. With it, every
367    /// hit on the same `(method, path_template)` collapses into one row.
368    #[test]
369    fn template_grouping_collapses_distinct_resolved_urls() {
370        let caps = vec![
371            cap_with_template(
372                "GET",
373                "https://host/api/users/test-value",
374                200,
375                None,
376                Some("ok"),
377                "/users/{id}",
378            ),
379            cap_with_template(
380                "GET",
381                "https://host/api/users/abc",
382                404,
383                None,
384                Some("nf"),
385                "/users/{id}",
386            ),
387            cap_with_template(
388                "GET",
389                "https://host/api/users/zzz",
390                200,
391                None,
392                Some("ok"),
393                "/users/{id}",
394            ),
395        ];
396        let s = build_summary(&caps);
397        assert_eq!(s.len(), 1, "all three URLs collapse into one template-grouped row");
398        let row = &s[0];
399        assert_eq!(row.path, "/users/{id}", "path field carries the spec template");
400        assert_eq!(row.sent, 3);
401        assert_eq!(row.status_2xx, 2);
402        assert_eq!(row.status_4xx, 1);
403    }
404
405    /// Round 33 (#823) — when probes from two different specs share
406    /// the same `(method, path_template)` they stay separate rows, so
407    /// a multi-spec run keeps the attribution.
408    #[test]
409    fn spec_label_keeps_same_template_rows_separate() {
410        let mut a = cap_with_template(
411            "POST",
412            "https://host/api/foo",
413            201,
414            Some("body"),
415            Some("ok"),
416            "/foo",
417        );
418        a.spec_label = Some("specA.yaml".to_string());
419        let mut b = cap_with_template(
420            "POST",
421            "https://host/api/foo",
422            201,
423            Some("body"),
424            Some("ok"),
425            "/foo",
426        );
427        b.spec_label = Some("specB.yaml".to_string());
428        let caps = vec![a, b];
429        let s = build_summary(&caps);
430        assert_eq!(s.len(), 2, "different specs must not collapse same-template rows");
431        let labels: Vec<Option<&str>> = s.iter().map(|x| x.spec.as_deref()).collect();
432        assert!(labels.contains(&Some("specA.yaml")));
433        assert!(labels.contains(&Some("specB.yaml")));
434    }
435
436    /// Round 33 (#823) — the HTML section only emits a Spec column
437    /// when at least one row carries a spec label. Keeps the
438    /// single-spec single-target run from showing a useless column.
439    #[test]
440    fn html_spec_column_only_appears_with_labels() {
441        let no_label = vec![cap_with_template(
442            "GET",
443            "https://h/a",
444            200,
445            None,
446            Some("x"),
447            "/a",
448        )];
449        let html_no = render_html_section(&build_summary(&no_label));
450        assert!(!html_no.contains("<th>Spec</th>"), "single-spec runs hide the column");
451
452        let mut labelled = cap_with_template("GET", "https://h/b", 200, None, Some("x"), "/b");
453        labelled.spec_label = Some("spec.yaml".to_string());
454        let html_yes = render_html_section(&build_summary(&[labelled]));
455        assert!(html_yes.contains("<th>Spec</th>"), "labelled runs surface the column");
456        assert!(html_yes.contains("spec.yaml"), "spec label rendered in the row");
457    }
458
459    /// Round 33 (#823) — captures with empty `path_template` (e.g.
460    /// legacy JSONL on disk) still group by resolved path, so we
461    /// don't break backward compatibility.
462    #[test]
463    fn empty_template_falls_back_to_resolved_path() {
464        let caps = vec![
465            cap("GET", "https://host/api/foo", 200, None, Some("ok")),
466            cap("GET", "https://host/api/foo", 200, None, Some("ok")),
467        ];
468        let s = build_summary(&caps);
469        assert_eq!(s.len(), 1);
470        assert_eq!(s[0].path, "/api/foo");
471        assert_eq!(s[0].sent, 2);
472    }
473
474    /// Round 34 (#828) — Srikanth on 0.3.178 searched for
475    /// `/api/appliance/access/consolecli` in the per-endpoint JSON
476    /// and didn't find it; r33 stored just the spec path
477    /// `/appliance/access/consolecli` because the bench bench
478    /// threaded `&op.path` (no base_path prefix). r34 prefixes
479    /// `--base-path` so the stored `path_template` matches the URL
480    /// the user actually sent. Captures stamped with the prefixed
481    /// template here simulate the new behavior and the summary
482    /// surfaces the user-facing URL path.
483    #[test]
484    fn base_path_prefixed_template_appears_in_path_column() {
485        let caps = vec![
486            cap_with_template(
487                "PUT",
488                "https://host/api/appliance/access/consolecli",
489                204,
490                Some(r#"{"enabled":true}"#),
491                None,
492                "/api/appliance/access/consolecli",
493            ),
494            cap_with_template(
495                "PUT",
496                "https://host/api/appliance/access/consolecli",
497                400,
498                Some(r#"{"data":"x"}"#),
499                Some("bad"),
500                "/api/appliance/access/consolecli",
501            ),
502        ];
503        let s = build_summary(&caps);
504        assert_eq!(s.len(), 1, "both probes collapse into one row");
505        assert_eq!(
506            s[0].path, "/api/appliance/access/consolecli",
507            "stored path includes the --base-path prefix so it matches the URL the user sent"
508        );
509        assert_eq!(s[0].sent, 2);
510        assert_eq!(s[0].status_2xx, 1);
511        assert_eq!(s[0].status_4xx, 1);
512    }
513}
mockforge_bench/conformance/per_endpoint_summary.rs

mockforge_bench/conformance/
per_endpoint_summary.rs
