mcpsec 1.1.0

MCP Security Benchmark Framework — vendor-neutral security evaluation for MCP gateways
Documentation
// Copyright 2026 Paolo Vella
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0

//! JSON and Markdown report generation.

use crate::BenchmarkResult;

/// Generate a JSON report string.
pub fn to_json(result: &BenchmarkResult) -> String {
    match serde_json::to_string_pretty(result) {
        Ok(json) => json,
        Err(e) => {
            let escaped = e.to_string().replace('\\', "\\\\").replace('"', "\\\"");
            format!(
                "{{\"error\":\"failed to serialize benchmark result\",\"details\":\"{}\"}}",
                escaped
            )
        }
    }
}

/// Generate a Markdown report string.
pub fn to_markdown(result: &BenchmarkResult) -> String {
    let mut md = String::new();

    // Header
    md.push_str("# MCPSEC Benchmark Report\n\n");
    md.push_str(&format!(
        "**Framework:** {} v{}\n\n",
        result.framework, result.version
    ));
    md.push_str(&format!("**Gateway:** {}\n\n", result.gateway));
    md.push_str(&format!("**Timestamp:** {}\n\n", result.timestamp));

    // Overall score
    md.push_str("## Overall Score\n\n");
    md.push_str(&format!(
        "| Score | Tier | Tier Name |\n|-------|------|-----------|\n| {:.1}% | {} | {} |\n\n",
        result.overall_score, result.tier, result.tier_name
    ));

    // Summary
    md.push_str("## Summary\n\n");
    md.push_str(&format!(
        "| Total Tests | Passed | Failed | Skipped |\n|-------------|--------|--------|---------|\n| {} | {} | {} | {} |\n\n",
        result.summary.total_tests,
        result.summary.passed,
        result.summary.failed,
        result.summary.skipped
    ));

    // Property scores
    md.push_str("## Property Scores\n\n");
    md.push_str("| Property | Name | Score | Passed | Total |\n");
    md.push_str("|----------|------|-------|--------|-------|\n");
    for prop in &result.properties {
        md.push_str(&format!(
            "| {} | {} | {:.0}% | {} | {} |\n",
            prop.property_id, prop.name, prop.score, prop.tests_passed, prop.tests_total
        ));
    }
    md.push('\n');

    // Attack results by class
    md.push_str("## Attack Results\n\n");

    let mut current_class = String::new();
    for attack in &result.attacks {
        if attack.class != current_class {
            current_class = attack.class.clone();
            md.push_str(&format!("### {}\n\n", current_class));
            md.push_str("| ID | Name | Result | Latency |\n");
            md.push_str("|----|------|--------|---------|\n");
        }
        let status = if attack.passed { "PASS" } else { "**FAIL**" };
        let latency = format_latency(attack.latency_ns);
        md.push_str(&format!(
            "| {} | {} | {} | {} |\n",
            attack.attack_id, attack.name, status, latency
        ));
    }
    md.push('\n');

    // Failed tests detail
    let failed: Vec<_> = result.attacks.iter().filter(|a| !a.passed).collect();
    if !failed.is_empty() {
        md.push_str("## Failed Tests\n\n");
        for attack in &failed {
            md.push_str(&format!(
                "- **{}** ({}): {}\n",
                attack.attack_id, attack.name, attack.details
            ));
        }
        md.push('\n');
    }

    md.push_str("---\n\n");
    md.push_str("Generated by [MCPSEC](https://github.com/vellaveto/vellaveto/tree/main/mcpsec)\n");

    md
}

fn format_latency(ns: u64) -> String {
    if ns < 1_000 {
        format!("{ns}ns")
    } else if ns < 1_000_000 {
        format!("{:.1}us", ns as f64 / 1_000.0)
    } else {
        format!("{:.1}ms", ns as f64 / 1_000_000.0)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::{AttackResult, BenchmarkResult, BenchmarkSummary, PropertyScore};

    fn sample_result() -> BenchmarkResult {
        BenchmarkResult {
            framework: "MCPSEC".to_string(),
            version: "1.0.0".to_string(),
            timestamp: "2026-02-15T12:00:00Z".to_string(),
            gateway: "test-gateway".to_string(),
            gateway_version: "1.0.0".to_string(),
            overall_score: 75.0,
            tier: 3,
            tier_name: "Strong".to_string(),
            properties: vec![PropertyScore {
                property_id: "P1".to_string(),
                name: "Tool-Level Access Control".to_string(),
                score: 100.0,
                tests_passed: 6,
                tests_total: 6,
            }],
            attacks: vec![
                AttackResult {
                    attack_id: "A1.1".to_string(),
                    name: "Classic injection phrase".to_string(),
                    class: "Prompt Injection Evasion".to_string(),
                    passed: true,
                    latency_ns: 28_000,
                    details: "Detected".to_string(),
                },
                AttackResult {
                    attack_id: "A1.2".to_string(),
                    name: "Zero-width evasion".to_string(),
                    class: "Prompt Injection Evasion".to_string(),
                    passed: false,
                    latency_ns: 150_000,
                    details: "Not detected".to_string(),
                },
            ],
            summary: BenchmarkSummary {
                total_tests: 2,
                passed: 1,
                failed: 1,
                skipped: 0,
            },
        }
    }

    #[test]
    fn test_json_roundtrip() {
        let result = sample_result();
        let json = to_json(&result);
        let parsed: BenchmarkResult = serde_json::from_str(&json).unwrap();
        assert_eq!(parsed.framework, "MCPSEC");
        assert_eq!(parsed.overall_score, 75.0);
    }

    #[test]
    fn test_markdown_contains_key_sections() {
        let result = sample_result();
        let md = to_markdown(&result);
        assert!(md.contains("# MCPSEC Benchmark Report"));
        assert!(md.contains("## Overall Score"));
        assert!(md.contains("## Property Scores"));
        assert!(md.contains("## Attack Results"));
        assert!(md.contains("## Failed Tests"));
        assert!(md.contains("PASS"));
        assert!(md.contains("**FAIL**"));
    }

    #[test]
    fn test_format_latency() {
        assert_eq!(format_latency(500), "500ns");
        assert_eq!(format_latency(28_000), "28.0us");
        assert_eq!(format_latency(1_500_000), "1.5ms");
    }
}