mcp_tester/
scenario.rs

1use anyhow::{Context, Result};
2use serde::{Deserialize, Serialize};
3use serde_json::Value;
4use std::collections::HashMap;
5use std::fs;
6use std::path::Path;
7use std::time::Duration;
8
9/// Represents a test scenario file that defines a sequence of MCP operations
10#[derive(Debug, Clone, Serialize, Deserialize)]
11pub struct TestScenario {
12    /// Name of the test scenario
13    pub name: String,
14
15    /// Description of what the scenario tests
16    pub description: Option<String>,
17
18    /// Timeout for the entire scenario (in seconds)
19    #[serde(default = "default_timeout")]
20    pub timeout: u64,
21
22    /// Whether to stop on first failure
23    #[serde(default = "default_stop_on_failure")]
24    pub stop_on_failure: bool,
25
26    /// Variables that can be used in the steps
27    #[serde(default)]
28    pub variables: HashMap<String, Value>,
29
30    /// Setup steps to run before the test
31    #[serde(default)]
32    pub setup: Vec<TestStep>,
33
34    /// The main test steps
35    pub steps: Vec<TestStep>,
36
37    /// Cleanup steps to run after the test
38    #[serde(default)]
39    pub cleanup: Vec<TestStep>,
40}
41
42/// Represents a single test step in a scenario
43#[derive(Debug, Clone, Serialize, Deserialize)]
44pub struct TestStep {
45    /// Name/description of this step
46    pub name: String,
47
48    /// The type of operation
49    pub operation: Operation,
50
51    /// Optional timeout for this specific step (in seconds)
52    pub timeout: Option<u64>,
53
54    /// Whether to continue if this step fails
55    #[serde(default)]
56    pub continue_on_failure: bool,
57
58    /// Store the result in a variable for later use
59    pub store_result: Option<String>,
60
61    /// Assertions to validate the response
62    #[serde(default)]
63    pub assertions: Vec<Assertion>,
64}
65
66/// Types of operations that can be performed
67#[derive(Debug, Clone, Serialize, Deserialize)]
68#[serde(tag = "type")]
69pub enum Operation {
70    /// Call a tool with arguments
71    #[serde(rename = "tool_call")]
72    ToolCall {
73        tool: String,
74        #[serde(default)]
75        arguments: Value,
76    },
77
78    /// List available tools
79    #[serde(rename = "list_tools")]
80    ListTools,
81
82    /// List available resources
83    #[serde(rename = "list_resources")]
84    ListResources,
85
86    /// Read a resource
87    #[serde(rename = "read_resource")]
88    ReadResource { uri: String },
89
90    /// List available prompts
91    #[serde(rename = "list_prompts")]
92    ListPrompts,
93
94    /// Get a prompt
95    #[serde(rename = "get_prompt")]
96    GetPrompt {
97        name: String,
98        #[serde(default)]
99        arguments: Value,
100    },
101
102    /// Send a custom JSON-RPC request
103    #[serde(rename = "custom")]
104    Custom {
105        method: String,
106        #[serde(default)]
107        params: Value,
108    },
109
110    /// Wait for a specified duration
111    #[serde(rename = "wait")]
112    Wait { seconds: f64 },
113
114    /// Set a variable
115    #[serde(rename = "set_variable")]
116    SetVariable { name: String, value: Value },
117}
118
119/// Types of assertions that can be made on responses
120#[derive(Debug, Clone, Serialize, Deserialize)]
121#[serde(tag = "type")]
122pub enum Assertion {
123    /// Check if a field equals a specific value
124    #[serde(rename = "equals")]
125    Equals {
126        path: String,
127        value: Value,
128        #[serde(default)]
129        ignore_case: bool,
130    },
131
132    /// Check if a field contains a substring
133    #[serde(rename = "contains")]
134    Contains {
135        path: String,
136        value: String,
137        #[serde(default)]
138        ignore_case: bool,
139    },
140
141    /// Check if a field matches a regex pattern
142    #[serde(rename = "matches")]
143    Matches { path: String, pattern: String },
144
145    /// Check if a field exists (is not null/undefined)
146    #[serde(rename = "exists")]
147    Exists { path: String },
148
149    /// Check if a field does not exist (is null/undefined)
150    #[serde(rename = "not_exists")]
151    NotExists { path: String },
152
153    /// Check if response indicates success (no error field)
154    #[serde(rename = "success")]
155    Success,
156
157    /// Check if response indicates failure (has error field)
158    #[serde(rename = "failure")]
159    Failure,
160
161    /// Check array length
162    #[serde(rename = "array_length")]
163    ArrayLength {
164        path: String,
165        #[serde(flatten)]
166        comparison: Comparison,
167    },
168
169    /// Check numeric value
170    #[serde(rename = "numeric")]
171    Numeric {
172        path: String,
173        #[serde(flatten)]
174        comparison: Comparison,
175    },
176
177    /// Custom JSONPath assertion
178    #[serde(rename = "jsonpath")]
179    JsonPath {
180        expression: String,
181        expected: Option<Value>,
182    },
183}
184
185/// Numeric comparison operators
186#[derive(Debug, Clone, Serialize, Deserialize)]
187#[serde(rename_all = "snake_case")]
188pub enum Comparison {
189    Equals(f64),
190    NotEquals(f64),
191    GreaterThan(f64),
192    GreaterThanOrEqual(f64),
193    LessThan(f64),
194    LessThanOrEqual(f64),
195    Between { min: f64, max: f64 },
196}
197
198/// Result of running a test scenario
199#[derive(Debug, Clone, Serialize, Deserialize)]
200pub struct ScenarioResult {
201    pub scenario_name: String,
202    pub success: bool,
203    pub duration: Duration,
204    pub steps_completed: usize,
205    pub steps_total: usize,
206    pub step_results: Vec<StepResult>,
207    pub error: Option<String>,
208}
209
210/// Result of running a single test step
211#[derive(Debug, Clone, Serialize, Deserialize)]
212pub struct StepResult {
213    pub step_name: String,
214    pub success: bool,
215    pub duration: Duration,
216    pub response: Option<Value>,
217    pub assertion_results: Vec<AssertionResult>,
218    pub error: Option<String>,
219}
220
221/// Result of evaluating an assertion
222#[derive(Debug, Clone, Serialize, Deserialize)]
223pub struct AssertionResult {
224    pub assertion: String,
225    pub passed: bool,
226    pub actual_value: Option<Value>,
227    pub expected_value: Option<Value>,
228    pub message: Option<String>,
229}
230
231impl TestScenario {
232    /// Load a test scenario from a YAML file
233    pub fn from_yaml_file<P: AsRef<Path>>(path: P) -> Result<Self> {
234        let content = fs::read_to_string(path.as_ref())
235            .with_context(|| format!("Failed to read scenario file: {:?}", path.as_ref()))?;
236        serde_yaml::from_str(&content)
237            .with_context(|| format!("Failed to parse YAML scenario: {:?}", path.as_ref()))
238    }
239
240    /// Load a test scenario from a JSON file
241    pub fn from_json_file<P: AsRef<Path>>(path: P) -> Result<Self> {
242        let content = fs::read_to_string(path.as_ref())
243            .with_context(|| format!("Failed to read scenario file: {:?}", path.as_ref()))?;
244        serde_json::from_str(&content)
245            .with_context(|| format!("Failed to parse JSON scenario: {:?}", path.as_ref()))
246    }
247
248    /// Load a test scenario from a file (auto-detect format)
249    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self> {
250        let path_ref = path.as_ref();
251        match path_ref.extension().and_then(|s| s.to_str()) {
252            Some("yaml") | Some("yml") => Self::from_yaml_file(path),
253            Some("json") => Self::from_json_file(path),
254            _ => {
255                // Try YAML first, then JSON
256                Self::from_yaml_file(path_ref)
257                    .or_else(|_| Self::from_json_file(path_ref))
258                    .context("Failed to parse scenario file as YAML or JSON")
259            },
260        }
261    }
262
263    /// Validate the scenario structure
264    pub fn validate(&self) -> Result<()> {
265        if self.name.is_empty() {
266            anyhow::bail!("Scenario name cannot be empty");
267        }
268
269        if self.steps.is_empty() {
270            anyhow::bail!("Scenario must have at least one step");
271        }
272
273        // Validate variable references
274        for step in &self.steps {
275            if let Some(var_name) = &step.store_result {
276                if var_name.is_empty() {
277                    anyhow::bail!("Variable name for storing result cannot be empty");
278                }
279            }
280        }
281
282        Ok(())
283    }
284}
285
286fn default_timeout() -> u64 {
287    60
288}
289
290fn default_stop_on_failure() -> bool {
291    true
292}
293
294#[cfg(test)]
295mod tests {
296    use super::*;
297
298    #[test]
299    fn test_parse_simple_scenario() {
300        let yaml = r#"
301name: Simple Tool Test
302description: Test basic tool functionality
303steps:
304  - name: List available tools
305    operation:
306      type: list_tools
307    assertions:
308      - type: success
309      - type: exists
310        path: tools
311        
312  - name: Call echo tool
313    operation:
314      type: tool_call
315      tool: echo
316      arguments:
317        message: "Hello, World!"
318    assertions:
319      - type: success
320      - type: contains
321        path: result
322        value: "Hello, World!"
323"#;
324
325        let scenario: TestScenario = serde_yaml::from_str(yaml).unwrap();
326        assert_eq!(scenario.name, "Simple Tool Test");
327        assert_eq!(scenario.steps.len(), 2);
328        scenario.validate().unwrap();
329    }
330
331    #[test]
332    fn test_parse_complex_scenario() {
333        let yaml = r#"
334name: Complex Scenario
335timeout: 120
336variables:
337  test_message: "Test message"
338  expected_count: 5
339
340setup:
341  - name: Initialize test data
342    operation:
343      type: set_variable
344      name: test_id
345      value: "test_123"
346
347steps:
348  - name: Test with variable
349    operation:
350      type: tool_call
351      tool: process
352      arguments:
353        id: "${test_id}"
354        message: "${test_message}"
355    store_result: process_result
356    assertions:
357      - type: success
358      - type: numeric
359        path: count
360        greater_than_or_equal: 5
361
362cleanup:
363  - name: Clean up test data
364    operation:
365      type: tool_call
366      tool: cleanup
367      arguments:
368        id: "${test_id}"
369"#;
370
371        let scenario: TestScenario = serde_yaml::from_str(yaml).unwrap();
372        assert_eq!(scenario.name, "Complex Scenario");
373        assert_eq!(scenario.timeout, 120);
374        assert_eq!(scenario.setup.len(), 1);
375        assert_eq!(scenario.steps.len(), 1);
376        assert_eq!(scenario.cleanup.len(), 1);
377        scenario.validate().unwrap();
378    }
379}
mcp_tester/scenario.rs

mcp_tester/
scenario.rs
