Skip to main content

sprocket_test_types/
lib.rs

1//! Facilities for unit testing WDL documents.
2
3use std::collections::BTreeMap;
4use std::collections::HashSet;
5
6use anyhow::Result;
7use indexmap::IndexMap;
8use schemars::JsonSchema;
9use serde::Deserialize;
10use serde_json::Value;
11mod assertions;
12pub use assertions::*;
13mod inputs;
14pub mod yaml;
15
16use wdl_analysis::Diagnostics;
17use wdl_analysis::Document;
18use wdl_ast::Diagnostic;
19use wdl_ast::Span;
20
21use crate::assertions::RawAssertions;
22use crate::inputs::InputMatrix;
23use crate::yaml::MaybeMap;
24use crate::yaml::Spanned;
25use crate::yaml::SpannedField;
26use crate::yaml::spanned_fields;
27
28/// Convert a [`serde_saphyr::Span`] to our [`Span`] type.
29pub(crate) fn convert_yaml_span(span: serde_saphyr::Span) -> Span {
30    // SAFETY: `serde-saphyr` guarantees that byte-level information is
31    // available         when parsing from a string, which we always do.
32    Span::new(
33        span.byte_offset().expect("byte info should be available") as usize,
34        span.byte_len().expect("byte info should be available") as usize,
35    )
36}
37
38/// The parser encountered an unknown field.
39fn unknown_field(span: serde_saphyr::Span, field: &str) -> Diagnostic {
40    Diagnostic::warning(format!("unknown field `{field}`")).with_highlight(convert_yaml_span(span))
41}
42
43/// A target has multiple test definitions with the same name.
44fn duplicate_test_definition(first: &Spanned<Value>, duplicate: &Spanned<Value>) -> Diagnostic {
45    let Value::String(name) = &first.0.value else {
46        unreachable!("should be validated beforehand")
47    };
48    Diagnostic::error(format!("the name `{name}` is defined multiple times"))
49        .with_label(
50            "first definition here",
51            convert_yaml_span(first.0.defined.span()),
52        )
53        .with_label(
54            format!("`{name}` redefined here",),
55            convert_yaml_span(duplicate.0.defined.span()),
56        )
57}
58
59/// The parser expected a YAML mapping.
60fn expected_mapping(field: &Spanned<String>) -> Diagnostic {
61    Diagnostic::error(format!(
62        "expected a mapping of values in field `{}`",
63        field.0.value
64    ))
65    .with_highlight(convert_yaml_span(field.0.defined.span()))
66}
67
68/// Collection of tests for an entire WDL document.
69#[derive(Clone, Debug, JsonSchema)]
70#[schemars(transparent)]
71pub struct DocumentTests {
72    /// Tasks or Workflows with test definitions.
73    ///
74    /// Each task or workflow may have one or more test definitions.
75    pub targets: IndexMap<Spanned<String>, Vec<TestDefinition>>,
76}
77
78impl DocumentTests {
79    /// Attempt to parse a [`DocumentTests`] from a YAML source string.
80    pub fn parse(source: &str) -> Result<(Self, Diagnostics), Diagnostics> {
81        #[derive(Debug, Deserialize)]
82        #[serde(transparent)]
83        struct RawDocumentTests {
84            pub targets: IndexMap<Spanned<String>, Vec<Spanned<RawTestDefinition>>>,
85        }
86
87        let mut diagnostics = Diagnostics::default();
88        let raw: RawDocumentTests = match serde_saphyr::from_str(source) {
89            Ok(map) => map,
90            Err(_e) => {
91                // TODO(serial): create nice diagnostics from serde-saphyr
92                // errors
93                diagnostics.add(Diagnostic::error(
94                    "expected test document to be a YAML mapping from target names to test \
95                     definitions",
96                ));
97                return Err(diagnostics);
98            }
99        };
100
101        let mut document_tests = DocumentTests {
102            targets: IndexMap::with_capacity(raw.targets.len()),
103        };
104
105        for (target, definitions) in raw.targets {
106            let mut definition_names = HashSet::new();
107            let mut parsed_definitions = Vec::with_capacity(definitions.len());
108            for definition_val in definitions {
109                let name_span = definition_val
110                    .0
111                    .value
112                    .name
113                    .as_ref()
114                    .map(|name| name.value.clone());
115                match TestDefinition::parse(definition_val) {
116                    Ok(test_definition) => {
117                        // SAFETY: The parse would fail if no name was provided
118                        let name_span = name_span.unwrap();
119                        if definition_names.insert(name_span.clone()) {
120                            parsed_definitions.push(test_definition);
121                        } else {
122                            diagnostics.add(duplicate_test_definition(
123                                definition_names
124                                    .get(&name_span)
125                                    .expect("just verified it exists"),
126                                &name_span,
127                            ));
128                        }
129                    }
130                    Err(e) => diagnostics.extend(e),
131                }
132            }
133
134            document_tests.targets.insert(target, parsed_definitions);
135        }
136
137        if diagnostics.has_errors() {
138            Err(diagnostics)
139        } else {
140            Ok((document_tests, diagnostics))
141        }
142    }
143
144    /// Validate the test definition against the associated WDL document.
145    pub fn validate(&self, associated_wdl: &Document) -> Result<(), Diagnostics> {
146        let mut diagnostics = Diagnostics::default();
147
148        for (target, definitions) in &self.targets {
149            let target_callable = match associated_wdl.callable_by_name(&target.0.value) {
150                Some(callable) => callable,
151                None => {
152                    diagnostics.add(
153                        Diagnostic::error(format!(
154                            "no target named `{name}` in `{path}`",
155                            name = target.0.value,
156                            path = associated_wdl.path()
157                        ))
158                        .with_highlight(convert_yaml_span(target.0.defined.span())),
159                    );
160                    continue;
161                }
162            };
163
164            for definition in definitions {
165                definition
166                    .assertions
167                    .validate(target_callable, &mut diagnostics);
168            }
169        }
170
171        if diagnostics.is_empty() {
172            Ok(())
173        } else {
174            Err(diagnostics)
175        }
176    }
177}
178
179/// A test definition. Defines at least a single execution, but may define many
180/// executions.
181#[derive(Clone, Debug, JsonSchema)]
182pub struct TestDefinition {
183    /// Name for the test.
184    pub name: Spanned<String>,
185    /// Any tags associated with the test.
186    #[schemars(default)]
187    pub tags: HashSet<String>,
188    /// Matrix of inputs to combinatorially execute.
189    #[schemars(default)]
190    pub inputs: InputMatrix,
191    /// Assertions (shared for all executions).
192    ///
193    /// If no assertions defined, it is assumed that failing execution for any
194    /// reason is considered a test fail.
195    #[schemars(default)]
196    pub assertions: Assertions,
197}
198
199spanned_fields! {
200    #[derive(Debug)]
201    struct RawTestDefinition {
202        name: Option<SpannedField<Spanned<Value>>>,
203        tags: Option<SpannedField<Value>>,
204        inputs: Option<SpannedField<MaybeMap<MaybeMap<Value>>>>,
205        assertions: Option<SpannedField<RawAssertions>>,
206    }
207}
208
209impl TestDefinition {
210    /// Attempt to parse a [`TestDefinition`] within the given callable
211    /// `target`.
212    fn parse(raw: Spanned<RawTestDefinition>) -> Result<Self, Diagnostics> {
213        let mut diagnostics = Diagnostics::default();
214
215        let parsed_name = match raw.0.value.name {
216            Some(raw_name) => {
217                let Value::String(name) = raw_name.value.0.value else {
218                    diagnostics.add(
219                        Diagnostic::error("`name` must be a string")
220                            .with_highlight(convert_yaml_span(raw_name.value.0.defined.span())),
221                    );
222                    return Err(diagnostics);
223                };
224                Some(Spanned(serde_saphyr::Spanned {
225                    value: name,
226                    defined: raw_name.value.0.defined,
227                    referenced: raw_name.value.0.referenced,
228                }))
229            }
230            None => None,
231        };
232
233        let parsed_tags = match raw.0.value.tags {
234            Some(tags) => match serde_json::from_value::<HashSet<String>>(tags.value) {
235                Ok(tags) => tags,
236                Err(e) => {
237                    diagnostics.add(Diagnostic::error("invalid tags").with_help(e.to_string()));
238                    HashSet::new()
239                }
240            },
241            None => HashSet::new(),
242        };
243        let parsed_inputs = match raw.0.value.inputs {
244            Some(inputs) => match InputMatrix::parse(inputs) {
245                Ok(inputs) => inputs,
246                Err(d) => {
247                    diagnostics.extend(d);
248                    InputMatrix::default()
249                }
250            },
251            None => InputMatrix::default(),
252        };
253        let parsed_assertions = {
254            match raw.0.value.assertions {
255                Some(assertions) => match Assertions::parse(assertions.value) {
256                    Ok(assertions) => assertions,
257                    Err(d) => {
258                        diagnostics.extend(d);
259                        Assertions::default()
260                    }
261                },
262                _ => Assertions::default(),
263            }
264        };
265
266        for (unknown, _) in raw.0.value.unknown_fields {
267            diagnostics.add(unknown_field(unknown.0.defined.span(), &unknown.0.value));
268        }
269
270        let Some(name) = parsed_name else {
271            diagnostics.add(
272                Diagnostic::error("missing required field `name`")
273                    .with_highlight(convert_yaml_span(raw.0.defined.span())),
274            );
275            return Err(diagnostics);
276        };
277
278        if diagnostics.is_empty() {
279            Ok(Self {
280                name,
281                tags: parsed_tags,
282                inputs: parsed_inputs,
283                assertions: parsed_assertions,
284            })
285        } else {
286            Err(diagnostics)
287        }
288    }
289}
290
291#[cfg(test)]
292mod tests {
293    use std::path::Path;
294    use std::path::PathBuf;
295
296    use schemars::schema_for;
297
298    use super::*;
299
300    fn json_schema_path() -> PathBuf {
301        Path::new(env!("CARGO_MANIFEST_DIR"))
302            .join("..")
303            .join("..")
304            .join("jsonschemas")
305            .join("sprocket-test.json")
306    }
307
308    fn json_schema() -> String {
309        std::fs::read_to_string(json_schema_path()).unwrap()
310    }
311
312    fn full_test_path() -> PathBuf {
313        Path::new(env!("CARGO_MANIFEST_DIR"))
314            .join("assets")
315            .join("full-test.yaml")
316    }
317
318    #[test]
319    fn public_schema_up_to_date() {
320        let current_schema = schema_for!(DocumentTests);
321        let current_schema_pretty = serde_json::to_string_pretty(&current_schema).unwrap();
322
323        pretty_assertions::assert_eq!(
324            current_schema_pretty,
325            json_schema().trim(),
326            "The `sprocket test` schema at `{}` is out of date! Update it with the output of \
327             `sprocket test schema`.",
328            json_schema_path().display()
329        );
330    }
331
332    #[test]
333    fn schema_matches_config() {
334        let config_str = std::fs::read_to_string(full_test_path()).unwrap();
335
336        let schema = serde_yaml_ng::from_str::<Value>(&json_schema()).unwrap();
337        let config = serde_yaml_ng::from_str::<Value>(&config_str).unwrap();
338
339        let mut failed = false;
340        let validator = jsonschema::draft202012::new(&schema).unwrap();
341        for error in validator.iter_errors(&config) {
342            failed = true;
343            eprintln!("{error}");
344        }
345        assert!(
346            !failed,
347            "the generated schema does not match the current `TestDefinition` spec!"
348        );
349    }
350}