cargo_regression/
regression.rs

1use core::fmt;
2use std::{
3  io,
4  path::PathBuf,
5  process::{ExitCode, Termination},
6  sync::Arc,
7  time::Instant,
8};
9
10use itertools::{Either, Itertools};
11use tokio::{fs::remove_dir_all, sync::Semaphore};
12
13use crate::{
14  args::match_extension,
15  assert::{AssertError, DisplayErrs},
16  config::FullConfig,
17  Args,
18};
19
20#[derive(Debug, thiserror::Error)]
21pub enum BuildError {
22  #[error("file \"{0}\": {1}")]
23  Toml(PathBuf, toml::de::Error),
24  #[error("task \"{0}\": its permit = {1}, exceed total permits = {2}")]
25  PermitEcxceed(PathBuf, u32, u32),
26  #[error("task \"{0}\": need to specify '{1}'")]
27  MissConfig(PathBuf, &'static str),
28  #[error("file \"{0}\": {1}")]
29  UnableToRead(PathBuf, io::Error),
30  #[error("read dir \"{0}\": {1}")]
31  ReadDir(PathBuf, io::Error),
32  #[error("clean dir \"{0}\": {1}")]
33  CleanDir(PathBuf, io::Error),
34  #[error("input extensions can not contains 'toml'")]
35  InputExtToml,
36  #[error("This is regolden mode, all test case will pass!")]
37  Regolden,
38}
39
40pub(crate) enum FailedState {
41  ReportSaved(PathBuf),
42  NoReport(PathBuf, Vec<AssertError>),
43}
44pub(crate) enum State {
45  Ok,
46  Failed(Option<FailedState>),
47  Ignored,
48  FilteredOut,
49}
50
51impl fmt::Display for FailedState {
52  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
53    match self {
54      Self::ReportSaved(report) => {
55        write!(f, "\n     report: {}", report.display())
56      }
57      Self::NoReport(input, errs) => {
58        write!(f, "\n----------- {} -----------\n{}", input.display(), DisplayErrs(errs))
59      }
60    }
61  }
62}
63impl fmt::Display for State {
64  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
65    match self {
66      Self::Ok => write!(f, "\x1B[32mok\x1B[0m"),
67      Self::Failed(_) => write!(f, "\x1B[31mFAILED\x1B[0m"),
68      Self::Ignored => write!(f, "\x1B[33mignored\x1B[0m"),
69      Self::FilteredOut => write!(f, "\x1B[2mfiltered out\x1B[0m"),
70    }
71  }
72}
73
74pub(crate) struct TestResult {
75  count_ok: usize,
76  count_ignored: usize,
77  count_filtered: usize,
78  faileds: Vec<FailedState>,
79}
80
81pub struct TestExitCode(Result<TestResult, Vec<BuildError>>, Instant);
82
83impl Termination for TestExitCode {
84  fn report(self) -> ExitCode {
85    let time = self.1.elapsed().as_secs_f32();
86    match self.0 {
87      Ok(TestResult { count_ok, count_ignored, count_filtered, faileds }) => {
88        if faileds.is_empty() {
89          println!("test result: {}. {count_ok} passed; 0 failed; {count_ignored} ignored; {count_filtered} filtered out; finished in {time:.2}s", State::Ok);
90          ExitCode::SUCCESS
91        } else {
92          print!("\nfailures:");
93          for failed in &faileds {
94            print!("{failed}");
95          }
96          println!("\n\ntest result: {}. {count_ok} passed; {} failed; {count_ignored} ignored; {count_filtered} filtered out; finished in {time:.2}s", State::Failed(None), faileds.len());
97          ExitCode::FAILURE
98        }
99      }
100      Err(build_errs) => {
101        println!("Fail to build test:");
102        for err in &build_errs {
103          println!("{err}");
104        }
105        ExitCode::FAILURE
106      }
107    }
108  }
109}
110
111impl Args {
112  pub async fn test(self) -> TestExitCode {
113    let now = Instant::now();
114    TestExitCode(
115      match self.rebuild() {
116        Ok(args) => _test(args).await,
117        Err(e) => Err(vec![e]),
118      },
119      now,
120    )
121  }
122}
123async fn _test(args: Args) -> Result<TestResult, Vec<BuildError>> {
124  let f1 = async move {
125    let work_dir = PathBuf::from(args.work_dir);
126    if work_dir.exists() {
127      remove_dir_all(&work_dir)
128        .await
129        .map_err(|e| BuildError::CleanDir(work_dir, e))
130    } else {
131      Ok(())
132    }
133  };
134  let f2 =
135    async move { walk(FullConfig::new(args), PathBuf::from(args.root_dir), args).await };
136  // walkthrough all config
137  let (clean_dir, file_configs) = tokio::join!(f1, f2);
138  if let Err(e) = clean_dir {
139    return Err(vec![e]);
140  }
141  let scheduler = Arc::new(Semaphore::new(args.permits as usize));
142  let handles: Vec<_> = file_configs?
143    .into_iter()
144    .map(|(path, config)| {
145      let scheduler = scheduler.clone();
146      tokio::spawn(async move {
147        let _permit = scheduler
148          .acquire_many(*config.permit)
149          .await
150          .expect("Semaphore closed");
151        let state = config.test(&path, args).await;
152        println!("test {} ... {}", path.display(), state);
153        state
154      })
155    })
156    .collect();
157
158  let mut count_ok = 0;
159  let mut count_ignored = 0;
160  let mut count_filtered = 0;
161  let mut faileds = Vec::with_capacity(handles.len());
162  // TODO: iter
163  for handle in handles {
164    match handle.await.unwrap() {
165      State::Ok => count_ok += 1,
166      State::Failed(Some(failed)) => faileds.push(failed),
167      State::Failed(None) => unreachable!(),
168      State::Ignored => count_ignored += 1,
169      State::FilteredOut => count_filtered += 1,
170    }
171  }
172  scheduler.close();
173  if args.regolden {
174    return Err(vec![BuildError::Regolden]);
175  }
176  Ok(TestResult { count_ok, count_ignored, count_filtered, faileds })
177}
178
179#[async_recursion::async_recursion]
180async fn walk(
181  mut current_config: FullConfig,
182  current_path: PathBuf,
183  args: Args,
184) -> Result<Vec<(PathBuf, FullConfig)>, Vec<BuildError>> {
185  let all_path = current_path.join("__all__.toml");
186  if all_path.exists() {
187    match current_config.update(&all_path, args.debug) {
188      Ok(_config) => current_config = _config,
189      Err(e) => return Err(vec![e]),
190    }
191  }
192  let read_dir = match current_path.read_dir() {
193    Ok(read_dir) => read_dir,
194    Err(e) => return Err(vec![BuildError::ReadDir(current_path, e)]),
195  };
196  let (sub_dir_futures, files): (Vec<_>, Vec<_>) =
197    read_dir.into_iter().partition_map(|entry| {
198      let path = entry.unwrap().path();
199      if path.is_dir() {
200        if path.file_name().unwrap() == "__golden__" {
201          Either::Left(None)
202        } else {
203          let current_config = current_config.clone();
204          Either::Left(Some(tokio::spawn(async move {
205            walk(current_config, path, args).await
206          })))
207        }
208      } else {
209        Either::Right(path)
210      }
211    });
212  let mut errs = Vec::new();
213  let mut file_configs = files
214    .into_iter()
215    .filter_map(|file| {
216      if match_extension(&file, current_config.extensions.iter()) {
217        match args.filtered(&file) {
218          Ok(filtered) => {
219            if filtered {
220              Some((file, FullConfig::new_filtered()))
221            } else {
222              let config_file = file.with_extension("toml");
223              let current_config = current_config.clone();
224              if config_file.is_file() {
225                match current_config.update(&config_file, args.debug) {
226                  Ok(config) => Some((file, config)),
227                  Err(e) => {
228                    errs.push(e);
229                    None
230                  }
231                }
232              } else {
233                Some((file, current_config))
234              }
235              .and_then(|(file, config)| match config.eval(&file, args) {
236                Ok(config) => Some((file, config)),
237                Err(e) => {
238                  errs.push(e);
239                  None
240                }
241              })
242            }
243          }
244          Err(e) => {
245            errs.push(e);
246            None
247          }
248        }
249      } else {
250        None
251      }
252    })
253    .collect::<Vec<_>>();
254  for f in sub_dir_futures.into_iter().flatten() {
255    match f.await.expect("join handle") {
256      Ok(res) => file_configs.extend(res),
257      Err(e) => errs.extend(e),
258    }
259  }
260  if errs.is_empty() {
261    Ok(file_configs)
262  } else {
263    Err(errs)
264  }
265}
cargo_regression/regression.rs

cargo_regression/
regression.rs
