mcp_tester/
post_deploy_report.rs

1//! Phase 79 Wave 0: machine-readable test-command output contract.
2//!
3//! `PostDeployReport` is the structured report emitted by
4//! `cargo pmcp test {check, conformance, apps} --format=json`. The Phase 79
5//! post-deploy verifier (`cargo-pmcp/src/deployment/post_deploy_tests.rs`,
6//! shipped in Plan 79-03) consumes this as
7//! `serde_json::from_str::<PostDeployReport>(stdout)?`, eliminating the need
8//! to regex-parse pretty terminal output.
9//!
10//! ## Contract stability
11//!
12//! `schema_version` is the wire-format guard. Phase 79 ships `"1"`. Future
13//! breaking changes MUST bump this and downstream consumers MUST check it
14//! before deserializing. Additive field changes (new optional fields with
15//! `#[serde(default)]`) do NOT bump the version.
16//!
17//! ## Why a new struct (vs. extending `TestReport`)
18//!
19//! `mcp_tester::TestReport` (in `report.rs`) is the existing per-test-suite
20//! report. `PostDeployReport` wraps it with metadata the verifier needs:
21//! - `command` discriminator (which subcommand emitted this)
22//! - `url` for traceability
23//! - `mode` for the `apps` subcommand variant
24//! - `outcome` enum for the trinary verdict (Passed / `TestFailed` / `InfraError`)
25//! - `failures: Vec<FailureDetail>` with pre-formatted `reproduce` strings
26//! - `schema_version` for forward-compat
27//!
28//! Re-using `TestReport` directly would mix concerns; this wrapper keeps the
29//! per-test-suite reporter (`TestReport`) and the per-subcommand verifier
30//! contract (`PostDeployReport`) cleanly separated.
31
32use crate::TestSummary;
33use serde::{Deserialize, Serialize};
34
35/// Top-level machine-readable report emitted by `cargo pmcp test {check,
36/// conformance, apps} --format=json`. The canonical contract for Phase 79's
37/// post-deploy verifier (Plan 79-03 consumes this via
38/// `serde_json::from_str::<PostDeployReport>(stdout)`).
39///
40/// ## Schema version
41/// - `"1"` (Phase 79) — initial release.
42///
43/// ## Outcome semantics
44/// - `Passed` — subcommand exit code 0; `summary` populated where applicable.
45/// - `TestFailed` — subcommand exit code 1 (verdict on the new code);
46///   `failures` populated.
47/// - `InfraError` — subcommand exit code 2 (network / spawn failure / timeout);
48///   `failures` may be empty; `summary` may be `None`.
49///
50/// ## URL hygiene (Threat T-79-15)
51/// `cargo pmcp` URL parsing strips embedded credentials before display per the
52/// existing convention; `PostDeployReport.url` inherits that hygiene — callers
53/// MUST NOT inject a URL with embedded credentials.
54#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
55pub struct PostDeployReport {
56    /// Which subcommand emitted this report.
57    pub command: TestCommand,
58
59    /// URL the subcommand probed.
60    pub url: String,
61
62    /// For `apps`: validation mode (`"claude-desktop"`, `"chatgpt"`, `"standard"`).
63    /// For `check` and `conformance`: always `None`.
64    #[serde(default, skip_serializing_if = "Option::is_none")]
65    pub mode: Option<String>,
66
67    /// Trinary outcome — see struct doc.
68    pub outcome: TestOutcome,
69
70    /// Pass/fail counts. `None` when not applicable (e.g., `check` connectivity
71    /// is binary so `summary` is `None` for that subcommand).
72    #[serde(default, skip_serializing_if = "Option::is_none")]
73    pub summary: Option<TestSummary>,
74
75    /// Per-failure detail with pre-formatted reproduce commands. Empty `Vec`
76    /// when no failures (e.g., `Passed` outcome).
77    #[serde(default)]
78    pub failures: Vec<FailureDetail>,
79
80    /// Wall-clock duration of the subcommand invocation (milliseconds).
81    pub duration_ms: u64,
82
83    /// Wire-format version. Currently `"1"`. Future breaking changes bump.
84    pub schema_version: String,
85}
86
87impl Default for PostDeployReport {
88    fn default() -> Self {
89        Self {
90            command: TestCommand::Check,
91            url: String::new(),
92            mode: None,
93            outcome: TestOutcome::Passed,
94            summary: None,
95            failures: Vec::new(),
96            duration_ms: 0,
97            schema_version: "1".to_string(),
98        }
99    }
100}
101
102/// Discriminator for which subcommand emitted the report.
103#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
104#[serde(rename_all = "kebab-case")]
105pub enum TestCommand {
106    /// `cargo pmcp test check`
107    Check,
108    /// `cargo pmcp test conformance`
109    Conformance,
110    /// `cargo pmcp test apps`
111    Apps,
112}
113
114/// Trinary outcome for the verifier consumer. Maps to subcommand exit codes:
115/// `Passed=0`, `TestFailed=1`, `InfraError=2`.
116#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
117#[serde(rename_all = "kebab-case")]
118pub enum TestOutcome {
119    /// All tests passed; exit code 0.
120    Passed,
121    /// One or more tests failed; exit code 1.
122    TestFailed,
123    /// Network / spawn / timeout failure prevented the test from running; exit code 2.
124    InfraError,
125}
126
127/// Per-failure detail with verbatim message and pre-formatted reproduce command.
128///
129/// Wave 3's banner formatter consumes `reproduce` directly as a copy-paste-able
130/// command line — it MUST include `--mode <mode>` and `--tool <name>` for the
131/// `apps` subcommand. For `conformance` failures, the `reproduce` line includes
132/// `--domain <name>` when the failing domain is identifiable.
133///
134/// ## Threat note (T-79-14)
135/// The `reproduce` field is documentation-only — never `eval`'d, never
136/// `Command::spawn`'d by the producer. Phase 79 Wave 3 verifier likewise
137/// treats it as a copy-paste-able UX hint, not an executable. Consumers MUST
138/// preserve this rule: never auto-execute the string.
139#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
140pub struct FailureDetail {
141    /// Tool name for `apps` failures; domain name for `conformance` failures;
142    /// `None` for connectivity / framework-level failures.
143    #[serde(default, skip_serializing_if = "Option::is_none")]
144    pub tool: Option<String>,
145
146    /// Verbatim failure message from the existing `TestResult.error` /
147    /// `TestResult.details`. NOT mutated for the verifier — preserves the same
148    /// detail the human terminal output shows.
149    pub message: String,
150
151    /// Pre-formatted `cargo pmcp test ...` command line that reproduces this
152    /// single failure in isolation. Includes `--mode` and `--tool` for `apps`,
153    /// `--domain` for `conformance` (when domain identifiable), or the bare
154    /// subcommand for connectivity failures.
155    pub reproduce: String,
156}
157
158#[cfg(test)]
159mod tests {
160    //! Round-trip and serialization-format tests that lock the wire-format
161    //! contract Phase 79 Wave 3 (Plan 79-03) depends on.
162    use super::*;
163    use crate::TestSummary;
164
165    fn sample_summary() -> TestSummary {
166        TestSummary {
167            total: 9,
168            passed: 7,
169            failed: 1,
170            warnings: 1,
171            skipped: 0,
172        }
173    }
174
175    #[test]
176    fn post_deploy_report_round_trips_via_serde_json() {
177        // Test 1.1: lock the wire-format contract Wave 3 depends on.
178        let original = PostDeployReport {
179            command: TestCommand::Check,
180            url: "http://x".to_string(),
181            mode: None,
182            outcome: TestOutcome::Passed,
183            summary: None,
184            failures: vec![],
185            duration_ms: 200,
186            schema_version: "1".to_string(),
187        };
188        let json = serde_json::to_string_pretty(&original)
189            .expect("PostDeployReport must serialize cleanly");
190        let round_tripped: PostDeployReport =
191            serde_json::from_str(&json).expect("PostDeployReport must deserialize cleanly");
192        assert_eq!(original, round_tripped);
193    }
194
195    #[test]
196    fn test_command_serializes_kebab_case() {
197        // Test 1.2: kebab-case lock — Wave 3 deserializer relies on it.
198        assert_eq!(
199            serde_json::to_string(&TestCommand::Check).unwrap(),
200            "\"check\""
201        );
202        assert_eq!(
203            serde_json::to_string(&TestCommand::Conformance).unwrap(),
204            "\"conformance\""
205        );
206        assert_eq!(
207            serde_json::to_string(&TestCommand::Apps).unwrap(),
208            "\"apps\""
209        );
210    }
211
212    #[test]
213    fn test_outcome_serializes_kebab_case() {
214        // Test 1.3: trinary verdict serialization lock.
215        assert_eq!(
216            serde_json::to_string(&TestOutcome::Passed).unwrap(),
217            "\"passed\""
218        );
219        assert_eq!(
220            serde_json::to_string(&TestOutcome::TestFailed).unwrap(),
221            "\"test-failed\""
222        );
223        assert_eq!(
224            serde_json::to_string(&TestOutcome::InfraError).unwrap(),
225            "\"infra-error\""
226        );
227    }
228
229    #[test]
230    fn failure_detail_construction() {
231        // Test 1.4: --mode and --tool MUST appear in the reproduce string for
232        // apps failures so Wave 3 can split / display them.
233        let detail = FailureDetail {
234            tool: Some("get_spend_summary".to_string()),
235            message: "widget cost-summary.html missing onteardown handler".to_string(),
236            reproduce:
237                "cargo pmcp test apps --url http://x --mode claude-desktop --tool get_spend_summary"
238                    .to_string(),
239        };
240        let json = serde_json::to_string(&detail).unwrap();
241        let round_tripped: FailureDetail = serde_json::from_str(&json).unwrap();
242        assert_eq!(detail, round_tripped);
243        assert!(detail.reproduce.contains("--mode"));
244        assert!(detail.reproduce.contains("--tool"));
245    }
246
247    #[test]
248    fn schema_version_field_is_serialized() {
249        // Test 1.5: forward-compat — consumers MUST be able to find this field.
250        let report = PostDeployReport::default();
251        let json = serde_json::to_string(&report).unwrap();
252        assert!(
253            json.contains("\"schema_version\":\"1\""),
254            "schema_version field missing or wrong; got: {json}"
255        );
256    }
257
258    #[test]
259    fn test_summary_reuses_mcp_tester_test_summary() {
260        // Test 1.6: the embedded TestSummary preserves all 5 buckets through
261        // serde round-trip (passed/total/failed/warnings/skipped).
262        let report = PostDeployReport {
263            command: TestCommand::Conformance,
264            url: "http://x".to_string(),
265            mode: None,
266            outcome: TestOutcome::TestFailed,
267            summary: Some(sample_summary()),
268            failures: vec![],
269            duration_ms: 50,
270            schema_version: "1".to_string(),
271        };
272        let json = serde_json::to_string(&report).unwrap();
273        let round_tripped: PostDeployReport = serde_json::from_str(&json).unwrap();
274        let summary = round_tripped.summary.expect("summary preserved");
275        assert_eq!(summary.total, 9);
276        assert_eq!(summary.passed, 7);
277        assert_eq!(summary.failed, 1);
278        assert_eq!(summary.warnings, 1);
279        assert_eq!(summary.skipped, 0);
280    }
281
282    #[test]
283    fn default_schema_version_is_1() {
284        // Test 1.7: cannot accidentally ship a report with an empty version.
285        let report = PostDeployReport::default();
286        assert_eq!(report.schema_version, "1");
287    }
288}
mcp_tester/post_deploy_report.rs

mcp_tester/
post_deploy_report.rs
