mockforge_bench/conformance/
spec_audit.rs

1//! Spec-level audits for the conformance self-test.
2//!
3//! Issue #79 round 17.4 — Srikanth's (17.4) ask: in addition to driving
4//! a server with positive + negative *requests*, also audit the
5//! OpenAPI document itself for things that will silently degrade
6//! validator quality at runtime. These are not server-rejection
7//! findings — they're spec-quality findings, surfaced before any
8//! request is sent.
9//!
10//! Categories: `servers` (missing / localhost-only / relative-only),
11//! `callbacks` (unsecured webhook operations), `polymorphism`
12//! (`oneOf` / `anyOf` without a `discriminator`), `datatypes`
13//! (coverage of every `(type, format)` combination in the spec).
14//!
15//! The audit is a pure function of `&openapiv3::OpenAPI`; no network
16//! traffic, no server side-effects. Output ships alongside the
17//! self-test JSON report.
18
19use openapiv3::{
20    OpenAPI, ReferenceOr, Schema, SchemaKind, StringFormat, Type, VariantOrUnknownOrEmpty,
21};
22use std::collections::BTreeMap;
23
24/// Severity of a finding. Maps to traffic-light colours in the report.
25#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
26#[serde(rename_all = "lowercase")]
27pub enum Severity {
28    /// Informational — not a defect, but useful coverage data.
29    Info,
30    /// Likely-degraded validator behaviour, but not a hard bug.
31    Warning,
32    /// Clear finding that the user should address.
33    Error,
34}
35
36/// One audit finding.
37#[derive(Debug, Clone, serde::Serialize)]
38pub struct SpecFinding {
39    pub category: String,
40    pub severity: Severity,
41    /// JSON-pointer-ish location, e.g. `#/paths/~1users/post/callbacks/onCreated`.
42    pub location: String,
43    pub message: String,
44}
45
46/// Roll-up of all findings + the datatype coverage map.
47#[derive(Debug, Default, Clone, serde::Serialize)]
48pub struct SpecAuditReport {
49    pub findings: Vec<SpecFinding>,
50    /// Per `(type, format)` count of how many schemas in the spec use it.
51    /// Format `""` is used when no format is declared.
52    pub datatype_coverage: BTreeMap<String, usize>,
53    pub operations_audited: usize,
54}
55
56impl SpecAuditReport {
57    /// Count of findings by severity. Useful for one-line summaries.
58    pub fn counts_by_severity(&self) -> (usize, usize, usize) {
59        let mut info = 0;
60        let mut warn = 0;
61        let mut err = 0;
62        for f in &self.findings {
63            match f.severity {
64                Severity::Info => info += 1,
65                Severity::Warning => warn += 1,
66                Severity::Error => err += 1,
67            }
68        }
69        (info, warn, err)
70    }
71
72    /// Human-readable single-paragraph summary.
73    pub fn render_summary(&self) -> String {
74        let (info, warn, err) = self.counts_by_severity();
75        let coverage_kinds = self.datatype_coverage.len();
76        format!(
77            "Spec audit: {err} error(s), {warn} warning(s), {info} info; covered {coverage_kinds} datatype kind(s) across {} operation(s)",
78            self.operations_audited
79        )
80    }
81}
82
83/// Walk the OpenAPI document and produce all findings + coverage.
84/// Pure; no I/O.
85pub fn audit_spec(spec: &OpenAPI) -> SpecAuditReport {
86    let mut report = SpecAuditReport::default();
87    audit_servers(spec, &mut report);
88    audit_callbacks(spec, &mut report);
89    audit_polymorphism_and_datatypes(spec, &mut report);
90    report
91}
92
93fn audit_servers(spec: &OpenAPI, report: &mut SpecAuditReport) {
94    if spec.servers.is_empty() {
95        report.findings.push(SpecFinding {
96            category: "servers".into(),
97            severity: Severity::Warning,
98            location: "#/servers".into(),
99            message: "No `servers` declared — clients have to guess the base URL".into(),
100        });
101        return;
102    }
103    let mut all_localhost = true;
104    let mut all_relative = true;
105    for s in &spec.servers {
106        let url = s.url.as_str();
107        let is_local = url.contains("localhost") || url.contains("127.0.0.1");
108        let is_rel = !url.starts_with("http://") && !url.starts_with("https://");
109        if !is_local {
110            all_localhost = false;
111        }
112        if !is_rel {
113            all_relative = false;
114        }
115    }
116    if all_localhost && !spec.servers.is_empty() {
117        report.findings.push(SpecFinding {
118            category: "servers".into(),
119            severity: Severity::Warning,
120            location: "#/servers".into(),
121            message: format!(
122                "All {} declared server(s) are localhost — production base URL missing",
123                spec.servers.len()
124            ),
125        });
126    }
127    if all_relative && !spec.servers.is_empty() {
128        report.findings.push(SpecFinding {
129            category: "servers".into(),
130            severity: Severity::Warning,
131            location: "#/servers".into(),
132            message: "All declared servers use relative URLs — clients must resolve against the spec's host".into(),
133        });
134    }
135}
136
137fn audit_callbacks(spec: &OpenAPI, report: &mut SpecAuditReport) {
138    for (path, path_item_ref) in &spec.paths.paths {
139        let path_item = match path_item_ref {
140            ReferenceOr::Item(p) => p,
141            ReferenceOr::Reference { .. } => continue,
142        };
143        for (method, op) in operations_of(path_item) {
144            for (cb_name, cb) in &op.callbacks {
145                // `Callback = IndexMap<String, PathItem>` — no ReferenceOr
146                // on the value, so we walk directly.
147                for (cb_path, cb_path_item) in cb {
148                    for (cb_method, cb_op) in operations_of(cb_path_item) {
149                        if cb_op.security.as_ref().is_none_or(|s| s.is_empty()) {
150                            report.findings.push(SpecFinding {
151                                category: "callbacks".into(),
152                                severity: Severity::Warning,
153                                location: format!(
154                                    "#/paths/{}/{}/callbacks/{}/{}/{}",
155                                    path, method, cb_name, cb_path, cb_method
156                                ),
157                                message: format!(
158                                    "Callback `{}` on `{} {}` has no security requirement — webhook deliveries are unauthenticated",
159                                    cb_name, method.to_uppercase(), path
160                                ),
161                            });
162                        }
163                    }
164                }
165            }
166        }
167    }
168}
169
170fn audit_polymorphism_and_datatypes(spec: &OpenAPI, report: &mut SpecAuditReport) {
171    // Walk all schemas in components + every inline request/response
172    // body schema. Single pass — we count datatype coverage and find
173    // polymorphism findings at the same time.
174    if let Some(components) = &spec.components {
175        for (name, schema_ref) in &components.schemas {
176            if let ReferenceOr::Item(schema) = schema_ref {
177                walk_schema(schema, &format!("#/components/schemas/{}", name), report);
178            }
179        }
180    }
181    for (path, path_item_ref) in &spec.paths.paths {
182        let path_item = match path_item_ref {
183            ReferenceOr::Item(p) => p,
184            ReferenceOr::Reference { .. } => continue,
185        };
186        report.operations_audited += operations_of(path_item).len();
187        for (method, op) in operations_of(path_item) {
188            if let Some(ReferenceOr::Item(rb)) = &op.request_body {
189                for (ct, mt) in &rb.content {
190                    if let Some(ReferenceOr::Item(schema)) = &mt.schema {
191                        walk_schema(
192                            schema,
193                            &format!("#/paths/{}/{}/requestBody/{}", path, method, ct),
194                            report,
195                        );
196                    }
197                }
198            }
199            for (status, resp_ref) in &op.responses.responses {
200                if let ReferenceOr::Item(resp) = resp_ref {
201                    for (ct, mt) in &resp.content {
202                        if let Some(ReferenceOr::Item(schema)) = &mt.schema {
203                            walk_schema(
204                                schema,
205                                &format!(
206                                    "#/paths/{}/{}/responses/{:?}/{}",
207                                    path, method, status, ct
208                                ),
209                                report,
210                            );
211                        }
212                    }
213                }
214            }
215        }
216    }
217}
218
219fn walk_schema(schema: &Schema, location: &str, report: &mut SpecAuditReport) {
220    match &schema.schema_kind {
221        SchemaKind::Type(t) => {
222            count_datatype(t, &mut report.datatype_coverage);
223            // Recurse into object properties + array items.
224            match t {
225                Type::Object(obj) => {
226                    for (k, v) in &obj.properties {
227                        if let ReferenceOr::Item(inner) = v {
228                            walk_schema(inner, &format!("{}.{}", location, k), report);
229                        }
230                    }
231                }
232                Type::Array(arr) => {
233                    if let Some(ReferenceOr::Item(inner)) = &arr.items {
234                        walk_schema(inner, &format!("{}[]", location), report);
235                    }
236                }
237                _ => {}
238            }
239        }
240        SchemaKind::OneOf { one_of } | SchemaKind::AnyOf { any_of: one_of } => {
241            let kind = if matches!(schema.schema_kind, SchemaKind::OneOf { .. }) {
242                "oneOf"
243            } else {
244                "anyOf"
245            };
246            if schema.schema_data.discriminator.is_none() {
247                report.findings.push(SpecFinding {
248                    category: "polymorphism".into(),
249                    severity: Severity::Warning,
250                    location: location.to_string(),
251                    message: format!(
252                        "{} composition has no `discriminator` — validator cannot pick the variant deterministically",
253                        kind
254                    ),
255                });
256            }
257            for (i, variant) in one_of.iter().enumerate() {
258                if let ReferenceOr::Item(inner) = variant {
259                    walk_schema(inner, &format!("{}/{}/{}", location, kind, i), report);
260                }
261            }
262        }
263        SchemaKind::AllOf { all_of } => {
264            for (i, variant) in all_of.iter().enumerate() {
265                if let ReferenceOr::Item(inner) = variant {
266                    walk_schema(inner, &format!("{}/allOf/{}", location, i), report);
267                }
268            }
269        }
270        _ => {}
271    }
272}
273
274fn count_datatype(t: &Type, coverage: &mut BTreeMap<String, usize>) {
275    let key = match t {
276        Type::String(s) => match &s.format {
277            VariantOrUnknownOrEmpty::Item(StringFormat::Date) => "string:date".to_string(),
278            VariantOrUnknownOrEmpty::Item(StringFormat::DateTime) => "string:date-time".to_string(),
279            VariantOrUnknownOrEmpty::Item(StringFormat::Password) => "string:password".to_string(),
280            VariantOrUnknownOrEmpty::Item(StringFormat::Byte) => "string:byte".to_string(),
281            VariantOrUnknownOrEmpty::Item(StringFormat::Binary) => "string:binary".to_string(),
282            VariantOrUnknownOrEmpty::Unknown(f) => format!("string:{}", f),
283            VariantOrUnknownOrEmpty::Empty => "string".to_string(),
284        },
285        Type::Number(_) => "number".to_string(),
286        Type::Integer(_) => "integer".to_string(),
287        Type::Boolean(_) => "boolean".to_string(),
288        Type::Object(_) => "object".to_string(),
289        Type::Array(_) => "array".to_string(),
290    };
291    *coverage.entry(key).or_insert(0) += 1;
292}
293
294/// `(method_name, &Operation)` for each declared HTTP method on a path
295/// item. Mirrors what `openapiv3::PathItem` exposes individually.
296fn operations_of(p: &openapiv3::PathItem) -> Vec<(&'static str, &openapiv3::Operation)> {
297    let mut out = Vec::new();
298    if let Some(o) = &p.get {
299        out.push(("get", o));
300    }
301    if let Some(o) = &p.post {
302        out.push(("post", o));
303    }
304    if let Some(o) = &p.put {
305        out.push(("put", o));
306    }
307    if let Some(o) = &p.patch {
308        out.push(("patch", o));
309    }
310    if let Some(o) = &p.delete {
311        out.push(("delete", o));
312    }
313    if let Some(o) = &p.head {
314        out.push(("head", o));
315    }
316    if let Some(o) = &p.options {
317        out.push(("options", o));
318    }
319    if let Some(o) = &p.trace {
320        out.push(("trace", o));
321    }
322    out
323}
324
325#[cfg(test)]
326mod tests {
327    use super::*;
328    use openapiv3::{ObjectType, SchemaData, Server};
329
330    fn empty_spec() -> OpenAPI {
331        OpenAPI {
332            openapi: "3.0.0".into(),
333            info: Default::default(),
334            ..Default::default()
335        }
336    }
337
338    #[test]
339    fn no_servers_yields_servers_warning() {
340        let spec = empty_spec();
341        let report = audit_spec(&spec);
342        assert!(report
343            .findings
344            .iter()
345            .any(|f| f.category == "servers" && f.severity == Severity::Warning));
346    }
347
348    #[test]
349    fn localhost_only_servers_warn() {
350        let mut spec = empty_spec();
351        spec.servers = vec![
352            Server {
353                url: "http://localhost:3000".into(),
354                ..Default::default()
355            },
356            Server {
357                url: "http://127.0.0.1:8080".into(),
358                ..Default::default()
359            },
360        ];
361        let report = audit_spec(&spec);
362        assert!(report
363            .findings
364            .iter()
365            .any(|f| f.category == "servers" && f.message.contains("localhost")));
366    }
367
368    #[test]
369    fn relative_only_servers_warn() {
370        let mut spec = empty_spec();
371        spec.servers = vec![Server {
372            url: "/v1".into(),
373            ..Default::default()
374        }];
375        let report = audit_spec(&spec);
376        assert!(report
377            .findings
378            .iter()
379            .any(|f| f.category == "servers" && f.message.contains("relative URLs")));
380    }
381
382    #[test]
383    fn production_servers_no_warning() {
384        let mut spec = empty_spec();
385        spec.servers = vec![Server {
386            url: "https://api.example.com".into(),
387            ..Default::default()
388        }];
389        let report = audit_spec(&spec);
390        assert!(!report.findings.iter().any(|f| f.category == "servers"));
391    }
392
393    #[test]
394    fn datatype_coverage_records_string_format() {
395        use openapiv3::{Components, StringType};
396        let mut spec = empty_spec();
397        let mut components = Components::default();
398        let mut email_schema = Schema {
399            schema_data: SchemaData::default(),
400            schema_kind: SchemaKind::Type(Type::String(StringType {
401                format: VariantOrUnknownOrEmpty::Unknown("email".into()),
402                ..Default::default()
403            })),
404        };
405        // Reuse the same shape with no format for a second schema.
406        components
407            .schemas
408            .insert("Email".into(), ReferenceOr::Item(email_schema.clone()));
409        email_schema.schema_kind = SchemaKind::Type(Type::String(Default::default()));
410        components.schemas.insert("Plain".into(), ReferenceOr::Item(email_schema));
411        spec.components = Some(components);
412        let report = audit_spec(&spec);
413        assert_eq!(report.datatype_coverage.get("string:email"), Some(&1));
414        assert_eq!(report.datatype_coverage.get("string"), Some(&1));
415    }
416
417    #[test]
418    fn oneof_without_discriminator_flags_polymorphism() {
419        use openapiv3::Components;
420        let mut spec = empty_spec();
421        let mut components = Components::default();
422        let one_of_schema = Schema {
423            schema_data: SchemaData::default(),
424            schema_kind: SchemaKind::OneOf {
425                one_of: vec![
426                    ReferenceOr::Item(Schema {
427                        schema_data: SchemaData::default(),
428                        schema_kind: SchemaKind::Type(Type::Object(ObjectType::default())),
429                    }),
430                    ReferenceOr::Item(Schema {
431                        schema_data: SchemaData::default(),
432                        schema_kind: SchemaKind::Type(Type::Object(ObjectType::default())),
433                    }),
434                ],
435            },
436        };
437        components.schemas.insert("Shape".into(), ReferenceOr::Item(one_of_schema));
438        spec.components = Some(components);
439        let report = audit_spec(&spec);
440        assert!(report
441            .findings
442            .iter()
443            .any(|f| f.category == "polymorphism" && f.message.contains("oneOf")));
444    }
445
446    #[test]
447    fn summary_counts_severities() {
448        let report = SpecAuditReport {
449            findings: vec![
450                SpecFinding {
451                    category: "servers".into(),
452                    severity: Severity::Warning,
453                    location: "#/servers".into(),
454                    message: "x".into(),
455                },
456                SpecFinding {
457                    category: "callbacks".into(),
458                    severity: Severity::Error,
459                    location: "#/x".into(),
460                    message: "y".into(),
461                },
462            ],
463            datatype_coverage: BTreeMap::from([("string".into(), 5)]),
464            operations_audited: 3,
465        };
466        let (info, warn, err) = report.counts_by_severity();
467        assert_eq!((info, warn, err), (0, 1, 1));
468        let s = report.render_summary();
469        assert!(s.contains("1 error"));
470        assert!(s.contains("1 warning"));
471        assert!(s.contains("3 operation"));
472    }
473}
mockforge_bench/conformance/spec_audit.rs

mockforge_bench/conformance/
spec_audit.rs
