nmbrs_workload/
spectest.rs

1// Copyright 2024-2026 Jonathan Shook
2// SPDX-License-Identifier: Apache-2.0
3
4//! SpecTest: parse and validate workload specification examples.
5//!
6//! The workload_definition markdown files contain test triples:
7//! 1. `*yaml:*` — YAML source
8//! 2. `*json:*` — JSON equivalent (validated against YAML parse)
9//! 3. `*ops:*`  — ParsedOp API view (validated against normalization)
10//!
11//! This module parses those triples from markdown and validates them.
12
13/// A single spectest triple extracted from a markdown file.
14#[derive(Debug, Clone)]
15pub struct SpecTestCase {
16    pub title: String,
17    pub yaml_source: String,
18    pub json_expected: String,
19    pub ops_expected: String,
20    pub line_number: usize,
21}
22
23/// Extract all spectest triples from a markdown file.
24pub fn extract_spec_tests(markdown: &str) -> Vec<SpecTestCase> {
25    let mut tests = Vec::new();
26    let lines: Vec<&str> = markdown.lines().collect();
27    let mut i = 0;
28    let mut current_title = String::new();
29
30    while i < lines.len() {
31        let line = lines[i].trim();
32
33        // Track headings for test titles
34        if line.starts_with('#') {
35            current_title = line.trim_start_matches('#').trim().to_string();
36        }
37
38        // Look for *yaml:* marker
39        if line == "*yaml:*" {
40            let yaml_line = i + 1;
41            let yaml = extract_code_block(&lines, &mut i);
42            let json = if i < lines.len() && lines[i].trim() == "*json:*" {
43                extract_code_block(&lines, &mut i)
44            } else {
45                String::new()
46            };
47            let ops = if i < lines.len() && lines[i].trim() == "*ops:*" {
48                extract_code_block(&lines, &mut i)
49            } else {
50                String::new()
51            };
52
53            if !yaml.is_empty() {
54                tests.push(SpecTestCase {
55                    title: current_title.clone(),
56                    yaml_source: yaml,
57                    json_expected: json,
58                    ops_expected: ops,
59                    line_number: yaml_line,
60                });
61            }
62        } else {
63            i += 1;
64        }
65    }
66
67    tests
68}
69
70/// Extract the content of a fenced code block starting after the current line.
71fn extract_code_block(lines: &[&str], pos: &mut usize) -> String {
72    *pos += 1; // skip the marker line
73
74    // Find opening fence
75    while *pos < lines.len() {
76        let trimmed = lines[*pos].trim();
77        if trimmed.starts_with("```") {
78            *pos += 1;
79            break;
80        }
81        *pos += 1;
82    }
83
84    // Collect content until closing fence
85    let mut content = Vec::new();
86    while *pos < lines.len() {
87        let trimmed = lines[*pos].trim();
88        if trimmed.starts_with("```") {
89            *pos += 1;
90            break;
91        }
92        content.push(lines[*pos]);
93        *pos += 1;
94    }
95
96    content.join("\n").trim().to_string()
97}
98
99/// Validate a single spectest case.
100///
101/// Returns `Ok(())` if the YAML parses correctly and the normalized
102/// ops match the expected ops JSON. Returns `Err(message)` on failure.
103pub fn validate_spec_test(test: &SpecTestCase) -> Result<(), String> {
104    // Stage 1: Parse YAML
105    let yaml_value: serde_json::Value = serde_yaml::from_str(&test.yaml_source)
106        .map_err(|e| format!("[line {}] YAML parse error: {e}", test.line_number))?;
107
108    // Stage 2: Validate YAML↔JSON equivalence (if json provided)
109    if !test.json_expected.is_empty() {
110        let json_value: serde_json::Value =
111            serde_json::from_str(&test.json_expected).map_err(|e| {
112                format!(
113                    "[line {}] JSON parse error in expected: {e}",
114                    test.line_number
115                )
116            })?;
117
118        if !json_values_equivalent(&yaml_value, &json_value) {
119            return Err(format!(
120                "[line {}] YAML↔JSON mismatch in '{}'\n  YAML parsed as: {}\n  JSON expected:  {}",
121                test.line_number,
122                test.title,
123                serde_json::to_string(&yaml_value).unwrap_or_default(),
124                serde_json::to_string(&json_value).unwrap_or_default(),
125            ));
126        }
127    }
128
129    // Stage 3: Validate normalized ops (if ops provided)
130    if !test.ops_expected.is_empty() && test.ops_expected != "[]" {
131        let expected_ops: serde_json::Value = serde_json::from_str(&test.ops_expected)
132            .map_err(|e| format!("[line {}] Ops JSON parse error: {e}", test.line_number))?;
133
134        // Parse through our workload parser
135        let parsed = crate::parse::parse_ops(&test.yaml_source)
136            .map_err(|e| format!("[line {}] Workload parse error: {e}", test.line_number))?;
137
138        // Convert our ParsedOps to JSON for comparison
139        let our_ops: serde_json::Value = serde_json::to_value(&parsed)
140            .map_err(|e| format!("[line {}] Serialization error: {e}", test.line_number))?;
141
142        // Compare each expected op against our output
143        if let serde_json::Value::Array(expected_arr) = &expected_ops
144            && let serde_json::Value::Array(our_arr) = &our_ops
145        {
146            for (idx, expected_op) in expected_arr.iter().enumerate() {
147                // Find matching op by name
148                let expected_name = expected_op
149                    .get("name")
150                    .and_then(|v| v.as_str())
151                    .unwrap_or("");
152
153                let our_op = our_arr
154                    .iter()
155                    .find(|o| o.get("name").and_then(|v| v.as_str()) == Some(expected_name));
156
157                if let Some(our_op) = our_op {
158                    // Compare key fields
159                    validate_op_fields(expected_op, our_op, &test.title, test.line_number, idx)?;
160                } else {
161                    return Err(format!(
162                        "[line {}] Op '{}' expected but not found in output for '{}'",
163                        test.line_number, expected_name, test.title,
164                    ));
165                }
166            }
167        }
168    }
169
170    Ok(())
171}
172
173/// Compare two JSON values for structural equivalence.
174/// Order-insensitive for objects, order-sensitive for arrays.
175fn json_values_equivalent(a: &serde_json::Value, b: &serde_json::Value) -> bool {
176    match (a, b) {
177        (serde_json::Value::Object(am), serde_json::Value::Object(bm)) => {
178            // All keys in a must be in b with equivalent values
179            am.iter()
180                .all(|(k, v)| bm.get(k).is_some_and(|bv| json_values_equivalent(v, bv)))
181                && bm.iter().all(|(k, _)| am.contains_key(k))
182        }
183        (serde_json::Value::Array(aa), serde_json::Value::Array(ba)) => {
184            aa.len() == ba.len()
185                && aa
186                    .iter()
187                    .zip(ba.iter())
188                    .all(|(a, b)| json_values_equivalent(a, b))
189        }
190        (serde_json::Value::String(a), serde_json::Value::String(b)) => a == b,
191        (serde_json::Value::Number(a), serde_json::Value::Number(b)) => a == b,
192        (serde_json::Value::Bool(a), serde_json::Value::Bool(b)) => a == b,
193        (serde_json::Value::Null, serde_json::Value::Null) => true,
194        _ => false,
195    }
196}
197
198/// Validate key fields of a parsed op against expected.
199fn validate_op_fields(
200    expected: &serde_json::Value,
201    actual: &serde_json::Value,
202    title: &str,
203    line: usize,
204    idx: usize,
205) -> Result<(), String> {
206    // Check name
207    let exp_name = expected.get("name").and_then(|v| v.as_str()).unwrap_or("");
208    let act_name = actual.get("name").and_then(|v| v.as_str()).unwrap_or("");
209    if exp_name != act_name {
210        return Err(format!(
211            "[line {line}] Op {idx} name mismatch in '{title}': expected '{exp_name}', got '{act_name}'"
212        ));
213    }
214
215    // Check op fields
216    if let Some(exp_op) = expected.get("op")
217        && let Some(act_op) = actual.get("op")
218        && !json_values_equivalent(exp_op, act_op)
219    {
220        return Err(format!(
221            "[line {line}] Op '{exp_name}' op fields mismatch in '{title}'\n  expected: {}\n  actual:   {}",
222            serde_json::to_string(exp_op).unwrap_or_default(),
223            serde_json::to_string(act_op).unwrap_or_default(),
224        ));
225    }
226
227    // Check tags (if expected has them)
228    if let Some(exp_tags) = expected.get("tags")
229        && let Some(act_tags) = actual.get("tags")
230        && let (Some(exp_map), Some(act_map)) = (exp_tags.as_object(), act_tags.as_object())
231    {
232        for (key, exp_val) in exp_map {
233            if let Some(act_val) = act_map.get(key) {
234                if exp_val != act_val {
235                    return Err(format!(
236                        "[line {line}] Op '{exp_name}' tag '{key}' mismatch in '{title}': expected {exp_val}, got {act_val}"
237                    ));
238                }
239            } else {
240                return Err(format!(
241                    "[line {line}] Op '{exp_name}' missing tag '{key}' in '{title}'"
242                ));
243            }
244        }
245    }
246
247    // Check bindings (if expected has them)
248    if let Some(exp_bindings) = expected.get("bindings")
249        && let Some(act_bindings) = actual.get("bindings")
250        && !json_values_equivalent(exp_bindings, act_bindings)
251    {
252        return Err(format!(
253            "[line {line}] Op '{exp_name}' bindings mismatch in '{title}'"
254        ));
255    }
256
257    // Check params (if expected has them)
258    if let Some(exp_params) = expected.get("params")
259        && let Some(act_params) = actual.get("params")
260        && !json_values_equivalent(exp_params, act_params)
261    {
262        return Err(format!(
263            "[line {line}] Op '{exp_name}' params mismatch in '{title}'"
264        ));
265    }
266
267    Ok(())
268}
269
270/// Run all spectest cases from a markdown file.
271/// Returns (passed, failed, errors).
272pub fn run_spec_tests(markdown: &str) -> (usize, usize, Vec<String>) {
273    let tests = extract_spec_tests(markdown);
274    let mut passed = 0;
275    let mut failed = 0;
276    let mut errors = Vec::new();
277
278    for test in &tests {
279        match validate_spec_test(test) {
280            Ok(()) => passed += 1,
281            Err(e) => {
282                failed += 1;
283                errors.push(e);
284            }
285        }
286    }
287
288    (passed, failed, errors)
289}
nmbrs_workload/spectest.rs

nmbrs_workload/
spectest.rs
