cfn_guard/commands/
test.rs

1use crate::commands::reporters::test::generic::GenericReporter;
2use crate::commands::reporters::test::structured::{
3    ContextAwareRule, Err, StructuredTestReporter, TestResult,
4};
5use crate::commands::reporters::JunitReport;
6use crate::commands::{
7    Executable, SUCCESS_STATUS_CODE, TEST_ERROR_STATUS_CODE, TEST_FAILURE_STATUS_CODE,
8};
9use clap::Args;
10use serde::{Deserialize, Serialize};
11use std::collections::{BTreeMap, HashMap};
12use std::fs::File;
13use std::io::Write;
14use std::path::{Path, PathBuf};
15use std::time::Instant;
16use walkdir::DirEntry;
17
18use validate::validate_path;
19
20use crate::commands::files::{
21    alphabetical, get_files_with_filter, last_modified, read_file_content, regular_ordering,
22};
23use crate::commands::validate::{OutputFormatType, OUTPUT_FORMAT_HELP};
24use crate::commands::{
25    validate, ALPHABETICAL, DIRECTORY, DIRECTORY_ONLY, LAST_MODIFIED, RULES_AND_TEST_FILE,
26    RULES_FILE, TEST_DATA,
27};
28use crate::rules::errors::Error;
29use crate::rules::Result;
30use crate::utils::reader::Reader;
31use crate::utils::writer::Writer;
32
33const ABOUT: &str = r#"Built in unit testing capability to validate a Guard rules file against
34unit tests specified in YAML format to determine each individual rule's success
35or failure testing.
36"#;
37const RULES_HELP: &str = "Provide a rules file";
38const TEST_DATA_HELP: &str = "Provide a file or dir for data files in JSON or YAML";
39const DIRECTORY_HELP: &str = "Provide the root directory for rules";
40const ALPHABETICAL_HELP: &str = "Sort alphabetically inside a directory";
41const LAST_MODIFIED_HELP: &str = "Sort by last modified times within a directory";
42const VERBOSE_HELP: &str = "Verbose logging";
43
44#[derive(Debug, Clone, Eq, PartialEq, Args)]
45#[clap(about=ABOUT)]
46#[clap(
47    group=clap::ArgGroup::new(RULES_AND_TEST_FILE)
48    .requires_all([RULES_FILE.0, TEST_DATA.0])
49    .conflicts_with(DIRECTORY_ONLY))
50]
51#[clap(
52    group=clap::ArgGroup::new(DIRECTORY_ONLY).args([DIRECTORY.0])
53    .requires_all([DIRECTORY.0])
54    .conflicts_with(RULES_AND_TEST_FILE))
55]
56#[clap(arg_required_else_help = true)]
57/// .
58/// The test command evaluates rules against data files to determine success or failure based on
59/// pre-defined expected outcomes
60pub struct Test {
61    /// the path to a rules file that a data file will have access to
62    /// default None
63    /// conflicts with directory attribute
64    #[arg(name="rules-file", short, long, help=RULES_HELP)]
65    pub(crate) rules: Option<String>,
66    /// the path to the test-data file
67    /// default None
68    /// conflicts with directory attribute
69    #[arg(name="test-data", short, long, help=TEST_DATA_HELP)]
70    pub(crate) test_data: Option<String>,
71    /// the path to the directory that includes rule files, and a subdirectory labeled tests that
72    /// includes test-data files
73    /// default None
74    /// conflicts with rules, and test_data attributes
75    #[arg(name=DIRECTORY.0, short, long=DIRECTORY.0, help=DIRECTORY_HELP)]
76    pub(crate) directory: Option<String>,
77    /// Sort alphabetically inside a directory
78    /// default false
79    /// conflicts with last_modified attribute
80    #[arg(short, long, help=ALPHABETICAL_HELP, conflicts_with=LAST_MODIFIED.0)]
81    pub(crate) alphabetical: bool,
82    /// Sort by last modified times within a directory
83    /// default false
84    /// conflicts with last_modified attribute
85    #[arg(name="last-modified", short=LAST_MODIFIED.1, long=LAST_MODIFIED.0, help=LAST_MODIFIED_HELP, conflicts_with=ALPHABETICAL.0)]
86    pub(crate) last_modified: bool,
87    /// Output verbose logging, conflicts with output_format when not using single-line-summary
88    /// when set to true
89    /// default is false
90    #[arg(short, long, help=VERBOSE_HELP)]
91    pub(crate) verbose: bool,
92    /// Specify the format in which the output should be displayed
93    /// default is single-line-summary
94    /// if junit, json or yaml are chosen, will conflict with verbose logging if set to true
95    #[arg(short, long, help=OUTPUT_FORMAT_HELP, value_enum, default_value_t=OutputFormatType::SingleLineSummary)]
96    pub(crate) output_format: OutputFormatType,
97}
98
99#[derive(Debug)]
100pub(crate) struct GuardFile {
101    prefix: String,
102    file: DirEntry,
103    test_files: Vec<DirEntry>,
104}
105
106impl GuardFile {
107    fn get_test_files(&self) -> Vec<PathBuf> {
108        self.test_files
109            .iter()
110            .map(|de| de.path().to_path_buf())
111            .collect::<Vec<PathBuf>>()
112    }
113}
114
115impl Executable for Test {
116    /// .
117    /// test rules against provided data inputs, comparing expected outcomes to what's evaluated
118    ///
119    /// This function will return an error if
120    /// - conflicting attributes have been set
121    /// - any of the specified paths do not exist
122    /// - parse errors occur in the rule file
123    /// - illegal json or yaml syntax present in any of the data input files
124    fn execute(&self, writer: &mut Writer, _: &mut Reader) -> Result<i32> {
125        let mut exit_code = SUCCESS_STATUS_CODE;
126        let cmp = if self.alphabetical {
127            alphabetical
128        } else if self.last_modified {
129            last_modified
130        } else {
131            regular_ordering
132        };
133
134        if self.output_format.is_structured() && self.verbose {
135            return Err(Error::IllegalArguments(String::from("Cannot provide an output_type of JSON, YAML, or JUnit while the verbose flag is set")));
136        } else if matches!(self.output_format, OutputFormatType::Sarif) {
137            return Err(Error::IllegalArguments(String::from(
138                "Cannot provide an output_type of SARIF, SARIF reporter is unsupported.",
139            )));
140        }
141
142        if let Some(dir) = &self.directory {
143            validate_path(dir)?;
144            let walk = walkdir::WalkDir::new(dir);
145            let ordered_directory = OrderedTestDirectory::from(walk);
146
147            match self.output_format {
148                OutputFormatType::SingleLineSummary => {
149                    handle_plaintext_directory(ordered_directory, writer, self.verbose)
150                }
151                OutputFormatType::JSON | OutputFormatType::YAML | OutputFormatType::Junit => {
152                    let test_exit_code = handle_structured_directory_report(
153                        ordered_directory,
154                        writer,
155                        self.output_format,
156                    )?;
157                    exit_code = if exit_code == SUCCESS_STATUS_CODE {
158                        test_exit_code
159                    } else {
160                        exit_code
161                    };
162
163                    Ok(exit_code)
164                }
165                OutputFormatType::Sarif => unreachable!(),
166            }
167        } else {
168            let file = self.rules.as_ref().unwrap();
169            let data = self.test_data.as_ref().unwrap();
170
171            validate_path(file)?;
172            validate_path(data)?;
173
174            let data_test_files = get_files_with_filter(data, cmp, |entry| {
175                entry
176                    .file_name()
177                    .to_str()
178                    .map(|name| {
179                        name.ends_with(".json")
180                            || name.ends_with(".yaml")
181                            || name.ends_with(".JSON")
182                            || name.ends_with(".YAML")
183                            || name.ends_with(".yml")
184                            || name.ends_with(".jsn")
185                    })
186                    .unwrap_or(false)
187            })?;
188
189            let path = PathBuf::from(file);
190
191            let rule_file = File::open(&path)?;
192            if !rule_file.metadata()?.is_file() {
193                return Err(Error::IoError(std::io::Error::from(
194                    std::io::ErrorKind::InvalidInput,
195                )));
196            }
197
198            match self.output_format {
199                OutputFormatType::SingleLineSummary => handle_plaintext_single_file(
200                    rule_file,
201                    path.as_path(),
202                    writer,
203                    &data_test_files,
204                    self.verbose,
205                ),
206                OutputFormatType::Sarif => unreachable!(),
207                OutputFormatType::YAML | OutputFormatType::JSON | OutputFormatType::Junit => {
208                    handle_structured_single_report(
209                        rule_file,
210                        path.as_path(),
211                        writer,
212                        &data_test_files,
213                        self.output_format,
214                    )
215                }
216            }
217        }
218    }
219}
220
221fn handle_plaintext_directory(
222    directory: OrderedTestDirectory,
223    writer: &mut Writer,
224    verbose: bool,
225) -> Result<i32> {
226    let mut exit_code = SUCCESS_STATUS_CODE;
227
228    for (_, guard_files) in directory {
229        for each_rule_file in guard_files {
230            if each_rule_file.test_files.is_empty() {
231                writeln!(
232                    writer,
233                    "Guard File {} did not have any tests associated, skipping.",
234                    each_rule_file.file.path().display()
235                )?;
236                writeln!(writer, "---")?;
237                continue;
238            }
239
240            writeln!(
241                writer,
242                "Testing Guard File {}",
243                each_rule_file.file.path().display()
244            )?;
245
246            let path = each_rule_file.file.path();
247            let content = get_rule_content(path)?;
248            let span = crate::rules::parser::Span::new_extra(&content, &each_rule_file.prefix);
249
250            match crate::rules::parser::rules_file(span) {
251                Err(e) => {
252                    writeln!(writer, "Parse Error on ruleset file {e}",)?;
253                    exit_code = TEST_FAILURE_STATUS_CODE;
254                }
255                Ok(Some(rules)) => {
256                    let data_test_files = each_rule_file
257                        .test_files
258                        .iter()
259                        .map(|de| de.path().to_path_buf())
260                        .collect::<Vec<PathBuf>>();
261
262                    let mut reporter = GenericReporter {
263                        test_data: &data_test_files,
264                        rules,
265                        verbose,
266                        writer,
267                    };
268
269                    let test_exit_code = reporter.report()?;
270
271                    exit_code = if exit_code == SUCCESS_STATUS_CODE {
272                        test_exit_code
273                    } else {
274                        exit_code
275                    };
276                }
277                Ok(None) => {}
278            }
279            writeln!(writer, "---")?;
280        }
281    }
282
283    Ok(exit_code)
284}
285
286fn handle_plaintext_single_file(
287    rule_file: File,
288    path: &Path,
289    writer: &mut Writer,
290    data_test_files: &[PathBuf],
291    verbose: bool,
292) -> Result<i32> {
293    match read_file_content(rule_file) {
294        Err(e) => {
295            write!(writer, "Unable to read rule file content {e}")?;
296            Ok(TEST_ERROR_STATUS_CODE)
297        }
298        Ok(content) => {
299            let span = crate::rules::parser::Span::new_extra(&content, path.to_str().unwrap_or(""));
300            match crate::rules::parser::rules_file(span) {
301                Err(e) => {
302                    writeln!(writer, "Parse Error on ruleset file {e}")?;
303                    Ok(TEST_ERROR_STATUS_CODE)
304                }
305
306                Ok(Some(rules)) => {
307                    let mut reporter = GenericReporter {
308                        test_data: data_test_files,
309                        writer,
310                        verbose,
311                        rules,
312                    };
313
314                    reporter.report()
315                }
316                Ok(None) => Ok(SUCCESS_STATUS_CODE),
317            }
318        }
319    }
320}
321fn get_rule_content(path: &Path) -> Result<String> {
322    let rule_file = File::open(path)?;
323    read_file_content(rule_file)
324}
325
326pub(crate) fn handle_structured_single_report(
327    rule_file: File,
328    path: &Path,
329    writer: &mut Writer,
330    data_test_files: &[PathBuf],
331    output: OutputFormatType,
332) -> Result<i32> {
333    let mut exit_code = SUCCESS_STATUS_CODE;
334    let now = Instant::now();
335
336    let result = match read_file_content(rule_file) {
337        Err(e) => TestResult::Err(Err {
338            rule_file: path.to_str().unwrap_or("").to_string(),
339            error: e.to_string(),
340            time: now.elapsed().as_millis(),
341        }),
342
343        Ok(content) => {
344            let span = crate::rules::parser::Span::new_extra(&content, path.to_str().unwrap_or(""));
345            match crate::rules::parser::rules_file(span) {
346                Err(e) => TestResult::Err(Err {
347                    rule_file: path.to_str().unwrap_or("").to_string(),
348                    error: e.to_string(),
349                    time: now.elapsed().as_millis(),
350                }),
351                Ok(Some(rule)) => {
352                    let mut reporter = StructuredTestReporter {
353                        data_test_files,
354                        output,
355                        rules: ContextAwareRule {
356                            rule,
357                            name: path.to_str().unwrap_or("").to_string(),
358                        },
359                    };
360
361                    let test = reporter.evaluate()?;
362                    let test_code = test.get_exit_code();
363                    exit_code = get_exit_code(exit_code, test_code);
364
365                    test
366                }
367                Ok(None) => return Ok(exit_code),
368            }
369        }
370    };
371
372    match output {
373        OutputFormatType::YAML => serde_yaml::to_writer(writer, &result)?,
374        OutputFormatType::JSON => serde_json::to_writer_pretty(writer, &result)?,
375        OutputFormatType::Junit => JunitReport::from(&vec![result]).serialize(writer)?,
376        OutputFormatType::SingleLineSummary => unreachable!(),
377        OutputFormatType::Sarif => unreachable!(),
378    }
379
380    Ok(exit_code)
381}
382
383fn handle_structured_directory_report(
384    directory: OrderedTestDirectory,
385    writer: &mut Writer,
386    output: OutputFormatType,
387) -> Result<i32> {
388    let mut test_results = vec![];
389    let mut exit_code = SUCCESS_STATUS_CODE;
390
391    for (_, guard_files) in directory {
392        for each_rule_file in guard_files {
393            let now = Instant::now();
394
395            if each_rule_file.test_files.is_empty() {
396                continue;
397            }
398
399            let path = each_rule_file.file.path();
400            let content = match get_rule_content(path) {
401                Ok(content) => content,
402                Err(e) => {
403                    exit_code = TEST_ERROR_STATUS_CODE;
404                    test_results.push(TestResult::Err(Err {
405                        rule_file: path.to_str().unwrap().to_string(),
406                        error: e.to_string(),
407                        time: now.elapsed().as_millis(),
408                    }));
409                    continue;
410                }
411            };
412
413            let span = crate::rules::parser::Span::new_extra(&content, &each_rule_file.prefix);
414
415            match crate::rules::parser::rules_file(span) {
416                Err(e) => {
417                    exit_code = TEST_ERROR_STATUS_CODE;
418                    test_results.push(TestResult::Err(Err {
419                        rule_file: path.to_str().unwrap().to_string(),
420                        error: e.to_string(),
421                        time: now.elapsed().as_millis(),
422                    }))
423                }
424                Ok(Some(rules)) => {
425                    let data_test_files = each_rule_file.get_test_files();
426
427                    let mut reporter = StructuredTestReporter {
428                        data_test_files: &data_test_files,
429                        output,
430                        rules: ContextAwareRule {
431                            rule: rules,
432                            name: path.to_str().unwrap().to_string(),
433                        },
434                    };
435
436                    let test = reporter.evaluate()?;
437                    let test_code = test.get_exit_code();
438                    exit_code = get_exit_code(exit_code, test_code);
439
440                    test_results.push(test);
441                }
442                Ok(None) => {}
443            }
444        }
445    }
446
447    match output {
448        OutputFormatType::YAML => serde_yaml::to_writer(writer, &test_results)?,
449        OutputFormatType::JSON => serde_json::to_writer_pretty(writer, &test_results)?,
450        OutputFormatType::Junit => JunitReport::from(&test_results).serialize(writer)?,
451        // NOTE: safe since output type is checked prior to calling this function
452        OutputFormatType::Sarif => unreachable!(),
453        OutputFormatType::SingleLineSummary => unreachable!(),
454    }
455
456    Ok(exit_code)
457}
458
459fn get_exit_code(exit_code: i32, test_code: i32) -> i32 {
460    match exit_code {
461        SUCCESS_STATUS_CODE => test_code,
462        TEST_ERROR_STATUS_CODE => exit_code,
463        TEST_FAILURE_STATUS_CODE => {
464            if test_code == TEST_ERROR_STATUS_CODE {
465                TEST_ERROR_STATUS_CODE
466            } else {
467                TEST_FAILURE_STATUS_CODE
468            }
469        }
470        _ => unreachable!(),
471    }
472}
473
474#[derive(Serialize, Deserialize, Debug)]
475pub struct TestExpectations {
476    pub rules: HashMap<String, String>,
477}
478
479#[derive(Serialize, Deserialize, Debug)]
480pub struct TestSpec {
481    pub name: Option<String>,
482    pub input: serde_yaml::Value,
483    pub expectations: TestExpectations,
484}
485
486struct OrderedTestDirectory(BTreeMap<String, Vec<GuardFile>>);
487
488impl IntoIterator for OrderedTestDirectory {
489    fn into_iter(self) -> Self::IntoIter {
490        self.0.into_iter()
491    }
492
493    type IntoIter = std::collections::btree_map::IntoIter<String, Vec<GuardFile>>;
494    type Item = (String, Vec<GuardFile>);
495}
496
497impl From<walkdir::WalkDir> for OrderedTestDirectory {
498    fn from(walk: walkdir::WalkDir) -> Self {
499        let mut non_guard: Vec<DirEntry> = vec![];
500        let mut files: BTreeMap<String, Vec<GuardFile>> = BTreeMap::new();
501        for file in walk
502            .follow_links(true)
503            .sort_by_file_name()
504            .into_iter()
505            .flatten()
506        {
507            if file.path().is_file() {
508                let name = file
509                    .file_name()
510                    .to_str()
511                    .map_or("".to_string(), |s| s.to_string());
512
513                if name.ends_with(".guard") || name.ends_with(".ruleset") {
514                    let prefix = name
515                        .strip_suffix(".guard")
516                        .or_else(|| name.strip_suffix(".ruleset"))
517                        .unwrap()
518                        .to_string();
519
520                    files
521                        .entry(
522                            file.path()
523                                .parent()
524                                .map_or("".to_string(), |p| format!("{}", p.display())),
525                        )
526                        .or_default()
527                        .push(GuardFile {
528                            prefix,
529                            file,
530                            test_files: vec![],
531                        });
532                    continue;
533                } else {
534                    non_guard.push(file);
535                }
536            }
537        }
538
539        for file in non_guard {
540            let name = file
541                .file_name()
542                .to_str()
543                .map_or("".to_string(), |s| s.to_string());
544
545            if name.ends_with(".yaml")
546                || name.ends_with(".yml")
547                || name.ends_with(".json")
548                || name.ends_with(".jsn")
549            {
550                let parent = file.path().parent();
551
552                if parent.map_or(false, |p| p.ends_with("tests")) {
553                    if let Some(candidates) = parent.unwrap().parent().and_then(|grand| {
554                        let grand = format!("{}", grand.display());
555                        files.get_mut(&grand)
556                    }) {
557                        for guard_file in candidates {
558                            if name.starts_with(&guard_file.prefix) {
559                                guard_file.test_files.push(file);
560                                break;
561                            }
562                        }
563                    }
564                }
565            }
566        }
567
568        OrderedTestDirectory(files)
569    }
570}
cfn_guard/commands/test.rs

cfn_guard/commands/
test.rs
