pipeline_service/testing/
mod.rs

1// Testing Framework Module
2// Provides pipeline test definitions, execution, assertions, and reporting
3
4pub mod assertions;
5pub mod parser;
6pub mod reporter;
7pub mod runner;
8
9// Re-export key types
10pub use assertions::{Assertion, AssertionResult};
11pub use parser::TestFileParser;
12pub use reporter::{ReportFormat, TestReporter};
13pub use runner::{TestResult, TestRunner, TestSuiteResult};
14
15use crate::parser::models::Value;
16
17use std::collections::HashMap;
18use std::path::PathBuf;
19
20use serde::de::{self, MapAccess, Visitor};
21use serde::{Deserialize, Deserializer, Serialize};
22
23// =============================================================================
24// Test Definition Models
25// =============================================================================
26
27/// A complete test suite loaded from a roxid-test.yml file
28#[derive(Debug, Clone, Serialize, Deserialize)]
29pub struct TestSuite {
30    /// Optional suite name
31    #[serde(default)]
32    pub name: Option<String>,
33    /// Test definitions
34    pub tests: Vec<PipelineTest>,
35    /// Default variables applied to all tests
36    #[serde(default)]
37    pub defaults: Option<TestDefaults>,
38}
39
40/// Default values applied to all tests in a suite
41#[derive(Debug, Clone, Serialize, Deserialize)]
42pub struct TestDefaults {
43    /// Default variables
44    #[serde(default)]
45    pub variables: HashMap<String, String>,
46    /// Default parameters
47    #[serde(default)]
48    pub parameters: HashMap<String, serde_yaml::Value>,
49    /// Default working directory
50    #[serde(default)]
51    pub working_dir: Option<String>,
52}
53
54/// A single pipeline test definition
55#[derive(Debug, Clone, Serialize, Deserialize)]
56pub struct PipelineTest {
57    /// Test name (used in reporting)
58    pub name: String,
59    /// Path to the pipeline YAML file (relative to test file)
60    pub pipeline: PathBuf,
61    /// Variables to set for this test run
62    #[serde(default)]
63    pub variables: HashMap<String, String>,
64    /// Parameters to pass for this test run
65    #[serde(default)]
66    pub parameters: HashMap<String, serde_yaml::Value>,
67    /// Working directory for execution
68    #[serde(default)]
69    pub working_dir: Option<String>,
70    /// Assertions to evaluate after execution
71    #[serde(default)]
72    pub assertions: Vec<AssertionDef>,
73}
74
75/// An assertion definition as parsed from YAML
76///
77/// Each variant maps to a YAML key in the assertions list.
78/// This is the serializable form; it gets converted to `Assertion`
79/// for evaluation.
80///
81/// Supports YAML formats:
82/// - Bare string: `pipeline_succeeded`
83/// - Key-value: `step_succeeded: Build`
84/// - Key-struct: `step_output_contains: { step: Build, pattern: "..." }`
85#[derive(Debug, Clone, Serialize)]
86pub enum AssertionDef {
87    /// Assert a step succeeded
88    StepSucceeded(String),
89
90    /// Assert a step failed
91    StepFailed(String),
92
93    /// Assert a step was skipped
94    StepSkipped(String),
95
96    /// Assert a job succeeded
97    JobSucceeded(String),
98
99    /// Assert a job failed
100    JobFailed(String),
101
102    /// Assert a job was skipped
103    JobSkipped(String),
104
105    /// Assert a stage succeeded
106    StageSucceeded(String),
107
108    /// Assert a stage failed
109    StageFailed(String),
110
111    /// Assert a stage was skipped
112    StageSkipped(String),
113
114    /// Assert step output equals a value
115    StepOutputEquals(StepOutputAssertion),
116
117    /// Assert step output contains a pattern
118    StepOutputContains(StepOutputPatternAssertion),
119
120    /// Assert a step ran before another step
121    StepRanBefore(OrderAssertion),
122
123    /// Assert steps ran in parallel (within the same stage/job level)
124    StepsRanInParallel(ParallelAssertion),
125
126    /// Assert a variable has a specific value after execution
127    VariableEquals(VariableAssertion),
128
129    /// Assert a variable contains a pattern
130    VariableContains(VariablePatternAssertion),
131
132    /// Assert the pipeline succeeded overall
133    PipelineSucceeded,
134
135    /// Assert the pipeline failed overall
136    PipelineFailed,
137}
138
139impl<'de> Deserialize<'de> for AssertionDef {
140    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
141    where
142        D: Deserializer<'de>,
143    {
144        struct AssertionDefVisitor;
145
146        impl<'de> Visitor<'de> for AssertionDefVisitor {
147            type Value = AssertionDef;
148
149            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
150                formatter.write_str(
151                    "a string like 'pipeline_succeeded' or a mapping like 'step_succeeded: Build'",
152                )
153            }
154
155            // Handle bare strings: `- pipeline_succeeded`
156            fn visit_str<E>(self, value: &str) -> Result<AssertionDef, E>
157            where
158                E: de::Error,
159            {
160                match value {
161                    "pipeline_succeeded" => Ok(AssertionDef::PipelineSucceeded),
162                    "pipeline_failed" => Ok(AssertionDef::PipelineFailed),
163                    _ => Err(de::Error::unknown_variant(
164                        value,
165                        &["pipeline_succeeded", "pipeline_failed"],
166                    )),
167                }
168            }
169
170            // Handle mappings: `- step_succeeded: Build` or `- step_output_contains: { ... }`
171            fn visit_map<M>(self, mut map: M) -> Result<AssertionDef, M::Error>
172            where
173                M: MapAccess<'de>,
174            {
175                let key: String = map
176                    .next_key()?
177                    .ok_or_else(|| de::Error::custom("expected assertion key"))?;
178
179                let result = match key.as_str() {
180                    "step_succeeded" => {
181                        let val: String = map.next_value()?;
182                        Ok(AssertionDef::StepSucceeded(val))
183                    }
184                    "step_failed" => {
185                        let val: String = map.next_value()?;
186                        Ok(AssertionDef::StepFailed(val))
187                    }
188                    "step_skipped" => {
189                        let val: String = map.next_value()?;
190                        Ok(AssertionDef::StepSkipped(val))
191                    }
192                    "job_succeeded" => {
193                        let val: String = map.next_value()?;
194                        Ok(AssertionDef::JobSucceeded(val))
195                    }
196                    "job_failed" => {
197                        let val: String = map.next_value()?;
198                        Ok(AssertionDef::JobFailed(val))
199                    }
200                    "job_skipped" => {
201                        let val: String = map.next_value()?;
202                        Ok(AssertionDef::JobSkipped(val))
203                    }
204                    "stage_succeeded" => {
205                        let val: String = map.next_value()?;
206                        Ok(AssertionDef::StageSucceeded(val))
207                    }
208                    "stage_failed" => {
209                        let val: String = map.next_value()?;
210                        Ok(AssertionDef::StageFailed(val))
211                    }
212                    "stage_skipped" => {
213                        let val: String = map.next_value()?;
214                        Ok(AssertionDef::StageSkipped(val))
215                    }
216                    "step_output_equals" => {
217                        let val: StepOutputAssertion = map.next_value()?;
218                        Ok(AssertionDef::StepOutputEquals(val))
219                    }
220                    "step_output_contains" => {
221                        let val: StepOutputPatternAssertion = map.next_value()?;
222                        Ok(AssertionDef::StepOutputContains(val))
223                    }
224                    "step_ran_before" => {
225                        let val: OrderAssertion = map.next_value()?;
226                        Ok(AssertionDef::StepRanBefore(val))
227                    }
228                    "steps_ran_in_parallel" => {
229                        let val: ParallelAssertion = map.next_value()?;
230                        Ok(AssertionDef::StepsRanInParallel(val))
231                    }
232                    "variable_equals" => {
233                        let val: VariableAssertion = map.next_value()?;
234                        Ok(AssertionDef::VariableEquals(val))
235                    }
236                    "variable_contains" => {
237                        let val: VariablePatternAssertion = map.next_value()?;
238                        Ok(AssertionDef::VariableContains(val))
239                    }
240                    "pipeline_succeeded" => {
241                        // Allow `pipeline_succeeded:` with null/empty value in mapping form
242                        let _: serde_yaml::Value = map.next_value()?;
243                        Ok(AssertionDef::PipelineSucceeded)
244                    }
245                    "pipeline_failed" => {
246                        let _: serde_yaml::Value = map.next_value()?;
247                        Ok(AssertionDef::PipelineFailed)
248                    }
249                    _ => Err(de::Error::unknown_field(
250                        &key,
251                        &[
252                            "step_succeeded",
253                            "step_failed",
254                            "step_skipped",
255                            "job_succeeded",
256                            "job_failed",
257                            "job_skipped",
258                            "stage_succeeded",
259                            "stage_failed",
260                            "stage_skipped",
261                            "step_output_equals",
262                            "step_output_contains",
263                            "step_ran_before",
264                            "steps_ran_in_parallel",
265                            "variable_equals",
266                            "variable_contains",
267                            "pipeline_succeeded",
268                            "pipeline_failed",
269                        ],
270                    )),
271                };
272
273                result
274            }
275        }
276
277        deserializer.deserialize_any(AssertionDefVisitor)
278    }
279}
280
281/// Assertion for step output equality
282#[derive(Debug, Clone, Serialize, Deserialize)]
283pub struct StepOutputAssertion {
284    /// Step name (the `name:` field of the step)
285    pub step: String,
286    /// Output variable name
287    pub output: String,
288    /// Expected value
289    pub expected: serde_yaml::Value,
290}
291
292/// Assertion for step output pattern matching
293#[derive(Debug, Clone, Serialize, Deserialize)]
294pub struct StepOutputPatternAssertion {
295    /// Step name
296    pub step: String,
297    /// Substring or pattern to search for in stdout
298    pub pattern: String,
299    /// Optional: which output to check ("stdout", "stderr", or specific output variable)
300    #[serde(default)]
301    pub output: Option<String>,
302}
303
304/// Assertion for execution ordering
305#[derive(Debug, Clone, Serialize, Deserialize)]
306pub struct OrderAssertion {
307    /// The step that should run first
308    pub step: String,
309    /// The step that should run after
310    pub before: String,
311}
312
313/// Assertion for parallel execution
314#[derive(Debug, Clone, Serialize, Deserialize)]
315pub struct ParallelAssertion {
316    /// Steps that should have run in parallel
317    pub steps: Vec<String>,
318}
319
320/// Assertion for variable values
321#[derive(Debug, Clone, Serialize, Deserialize)]
322pub struct VariableAssertion {
323    /// Variable name
324    pub name: String,
325    /// Expected value
326    pub expected: serde_yaml::Value,
327}
328
329/// Assertion for variable pattern matching
330#[derive(Debug, Clone, Serialize, Deserialize)]
331pub struct VariablePatternAssertion {
332    /// Variable name
333    pub name: String,
334    /// Pattern to match
335    pub pattern: String,
336}
337
338// =============================================================================
339// Conversion helpers
340// =============================================================================
341
342impl AssertionDef {
343    /// Convert this YAML assertion definition into an evaluable `Assertion`
344    pub fn to_assertion(&self) -> Assertion {
345        match self {
346            AssertionDef::StepSucceeded(name) => Assertion::StepSucceeded { step: name.clone() },
347            AssertionDef::StepFailed(name) => Assertion::StepFailed { step: name.clone() },
348            AssertionDef::StepSkipped(name) => Assertion::StepSkipped { step: name.clone() },
349            AssertionDef::JobSucceeded(name) => Assertion::JobSucceeded { job: name.clone() },
350            AssertionDef::JobFailed(name) => Assertion::JobFailed { job: name.clone() },
351            AssertionDef::JobSkipped(name) => Assertion::JobSkipped { job: name.clone() },
352            AssertionDef::StageSucceeded(name) => Assertion::StageSucceeded {
353                stage: name.clone(),
354            },
355            AssertionDef::StageFailed(name) => Assertion::StageFailed {
356                stage: name.clone(),
357            },
358            AssertionDef::StageSkipped(name) => Assertion::StageSkipped {
359                stage: name.clone(),
360            },
361            AssertionDef::StepOutputEquals(a) => Assertion::StepOutputEquals {
362                step: a.step.clone(),
363                output: a.output.clone(),
364                expected: yaml_to_value(&a.expected),
365            },
366            AssertionDef::StepOutputContains(a) => Assertion::StepOutputContains {
367                step: a.step.clone(),
368                pattern: a.pattern.clone(),
369                output: a.output.clone(),
370            },
371            AssertionDef::StepRanBefore(a) => Assertion::StepRanBefore {
372                step: a.step.clone(),
373                before: a.before.clone(),
374            },
375            AssertionDef::StepsRanInParallel(a) => Assertion::StepsRanInParallel {
376                steps: a.steps.clone(),
377            },
378            AssertionDef::VariableEquals(a) => Assertion::VariableEquals {
379                name: a.name.clone(),
380                expected: yaml_to_value(&a.expected),
381            },
382            AssertionDef::VariableContains(a) => Assertion::VariableContains {
383                name: a.name.clone(),
384                pattern: a.pattern.clone(),
385            },
386            AssertionDef::PipelineSucceeded => Assertion::PipelineSucceeded,
387            AssertionDef::PipelineFailed => Assertion::PipelineFailed,
388        }
389    }
390}
391
392/// Convert a serde_yaml::Value to our internal Value type
393fn yaml_to_value(v: &serde_yaml::Value) -> Value {
394    match v {
395        serde_yaml::Value::Null => Value::Null,
396        serde_yaml::Value::Bool(b) => Value::Bool(*b),
397        serde_yaml::Value::Number(n) => {
398            if let Some(i) = n.as_i64() {
399                Value::Number(i as f64)
400            } else if let Some(f) = n.as_f64() {
401                Value::Number(f)
402            } else {
403                Value::Null
404            }
405        }
406        serde_yaml::Value::String(s) => Value::String(s.clone()),
407        serde_yaml::Value::Sequence(seq) => Value::Array(seq.iter().map(yaml_to_value).collect()),
408        serde_yaml::Value::Mapping(map) => {
409            let mut obj = HashMap::new();
410            for (k, v) in map {
411                if let serde_yaml::Value::String(key) = k {
412                    obj.insert(key.clone(), yaml_to_value(v));
413                }
414            }
415            Value::Object(obj)
416        }
417        serde_yaml::Value::Tagged(tagged) => yaml_to_value(&tagged.value),
418    }
419}
420
421#[cfg(test)]
422mod tests {
423    use super::*;
424
425    #[test]
426    fn test_assertion_def_to_assertion_step_succeeded() {
427        let def = AssertionDef::StepSucceeded("Build".to_string());
428        let assertion = def.to_assertion();
429        assert!(matches!(
430            assertion,
431            Assertion::StepSucceeded { step } if step == "Build"
432        ));
433    }
434
435    #[test]
436    fn test_assertion_def_to_assertion_variable_equals() {
437        let def = AssertionDef::VariableEquals(VariableAssertion {
438            name: "BUILD_CONFIG".to_string(),
439            expected: serde_yaml::Value::String("Release".to_string()),
440        });
441        let assertion = def.to_assertion();
442        assert!(matches!(
443            assertion,
444            Assertion::VariableEquals { name, expected }
445                if name == "BUILD_CONFIG" && expected == Value::String("Release".to_string())
446        ));
447    }
448
449    #[test]
450    fn test_yaml_to_value_primitives() {
451        assert_eq!(yaml_to_value(&serde_yaml::Value::Null), Value::Null);
452        assert_eq!(
453            yaml_to_value(&serde_yaml::Value::Bool(true)),
454            Value::Bool(true)
455        );
456        assert_eq!(
457            yaml_to_value(&serde_yaml::Value::String("hello".to_string())),
458            Value::String("hello".to_string())
459        );
460    }
461
462    #[test]
463    fn test_pipeline_test_deserialize() {
464        let yaml = r#"
465name: "Build test"
466pipeline: azure-pipelines.yml
467variables:
468  BUILD_CONFIG: Release
469assertions:
470  - step_succeeded: Build
471  - pipeline_succeeded
472"#;
473        let test: PipelineTest = serde_yaml::from_str(yaml).unwrap();
474        assert_eq!(test.name, "Build test");
475        assert_eq!(test.pipeline, PathBuf::from("azure-pipelines.yml"));
476        assert_eq!(test.variables.get("BUILD_CONFIG").unwrap(), "Release");
477        assert_eq!(test.assertions.len(), 2);
478    }
479
480    #[test]
481    fn test_test_suite_deserialize() {
482        let yaml = r#"
483tests:
484  - name: "Build stage runs correctly"
485    pipeline: azure-pipelines.yml
486    variables:
487      BUILD_CONFIG: Release
488    assertions:
489      - step_succeeded: Build
490      - step_output_contains:
491          step: Build
492          pattern: "Build succeeded"
493      - step_ran_before:
494          step: Test
495          before: Deploy
496
497  - name: "Deploy is skipped on PR"
498    pipeline: azure-pipelines.yml
499    variables:
500      BUILD_REASON: PullRequest
501    assertions:
502      - step_skipped: Deploy
503"#;
504        let suite: TestSuite = serde_yaml::from_str(yaml).unwrap();
505        assert_eq!(suite.tests.len(), 2);
506        assert_eq!(suite.tests[0].name, "Build stage runs correctly");
507        assert_eq!(suite.tests[0].assertions.len(), 3);
508        assert_eq!(suite.tests[1].name, "Deploy is skipped on PR");
509        assert_eq!(suite.tests[1].assertions.len(), 1);
510    }
511}
pipeline_service/testing/mod.rs

pipeline_service/testing/
mod.rs
