test_r_core/
args.rs

1use crate::execution::TestSuiteExecution;
2use crate::output::TestRunnerOutput;
3use clap::{Parser, ValueEnum};
4use std::ffi::OsString;
5use std::num::NonZero;
6use std::str::FromStr;
7use std::sync::Arc;
8use std::time::Duration;
9
10/// Command line arguments.
11///
12/// This type represents everything the user can specify via CLI args. The main
13/// method is [`from_args`][Arguments::from_args] which reads the global
14/// `std::env::args()` and parses them into this type.
15#[derive(Parser, Debug, Clone, Default)]
16#[command(
17    help_template = "USAGE: [OPTIONS] [FILTERS...]\n\n{all-args}\n",
18    disable_version_flag = true
19)]
20pub struct Arguments {
21    /// Run ignored and not ignored tests
22    #[arg(long = "include-ignored")]
23    pub include_ignored: bool,
24
25    /// Run only ignored tests
26    #[arg(long = "ignored")]
27    pub ignored: bool,
28
29    /// Excludes tests marked as should_panic
30    #[arg(long = "exclude-should-panic")]
31    pub exclude_should_panic: bool,
32
33    /// Run tests and not benchmarks
34    #[arg(long = "test", conflicts_with = "bench")]
35    pub test: bool,
36
37    /// Run benchmarks instead of tests
38    #[arg(long = "bench")]
39    pub bench: bool,
40
41    /// List all tests and benchmarks
42    #[arg(long = "list")]
43    pub list: bool,
44
45    /// Write logs to the specified file
46    #[arg(long = "logfile", value_name = "PATH")]
47    pub logfile: Option<String>,
48
49    /// don't capture stdout/stderr of each task, allow printing directly
50    #[arg(long = "nocapture")]
51    pub nocapture: bool,
52
53    /// Number of threads used for running tests in parallel
54    #[arg(long = "test-threads")]
55    pub test_threads: Option<usize>,
56
57    /// Skip tests whose names contains FILTER (this flag can be used multiple times)
58    #[arg(long = "skip", value_name = "FILTER")]
59    pub skip: Vec<String>,
60
61    /// Display one character per test instead of one line.
62    /// Alias to `--format=terse`
63    #[arg(short = 'q', long = "quiet", conflicts_with = "format")]
64    pub quiet: bool,
65
66    /// Exactly match filters rather than by substring
67    #[arg(long = "exact")]
68    pub exact: bool,
69
70    /// Configure coloring of output
71    #[arg(long = "color", value_enum, value_name = "auto|always|never")]
72    pub color: Option<ColorSetting>,
73
74    /// Configure formatting of output
75    #[arg(long = "format", value_enum, value_name = "pretty|terse|json|junit")]
76    pub format: Option<FormatSetting>,
77
78    /// Show captured stdout of successful tests
79    #[arg(long = "show-output")]
80    pub show_output: bool,
81
82    /// Enable nightly-only flags
83    #[arg(short = 'Z')]
84    pub unstable_flags: Option<UnstableFlags>,
85
86    /// Show execution time of each test.
87    /// Threshold values for colorized output can be configured via `RUST_TEST_TIME_UNIT`,
88    /// `RUST_TEST_TIME_INTEGRATION` and `RUST_TEST_TIME_DOCTEST` environment variables.
89    /// Expected format of the environment variables is `VARIABLE=WARN_TIME,CRITICAL_TIME`.
90    /// Durations must be specified in milliseconds, e.g. `500,2000` means that the warn time is 0.5
91    /// seconds, and the critical time is 2 seconds.
92    /// Not available for `--format=terse`.
93    #[arg(long = "report-time")]
94    pub report_time: bool,
95
96    /// Treat excess of the test execution time limit as error.
97    /// Threshold values for this option can be configured via `RUST_TEST_TIME_UNIT`,
98    /// `RUST_TEST_TIME_INTEGRATION` and `RUST_TEST_TIME_DOCTEST` environment variables.
99    /// Expected format of the environment variables is `VARIABLE=WARN_TIME,CRITICAL_TIME`.
100    /// `CRITICAL_TIME` here means the limit that should not be exceeded by test.
101    #[arg(long = "ensure-time")]
102    pub ensure_time: bool,
103
104    /// Run tests in random order
105    #[arg(long = "shuffle", conflicts_with = "shuffle_seed")]
106    pub shuffle: bool,
107
108    /// Run tests in random order; seed the random number generator with SEED
109    #[arg(long = "shuffle-seed", value_name = "SEED", conflicts_with = "shuffle")]
110    pub shuffle_seed: Option<u64>,
111
112    /// Show detailed benchmark statistics for each benchmark
113    #[arg(long = "show-stats")]
114    pub show_stats: bool,
115
116    /// The FILTER string is tested against the name of all tests, and only those
117    /// tests whose names contain the filter are run. Multiple filter strings may
118    /// be passed, which will run all tests matching any of the filters.
119    #[arg(value_name = "FILTER")]
120    pub filter: Option<String>,
121
122    /// Marks the whole run (all the selected tests) as expected to be flaky, to
123    /// be retried on top level a given number of times if needed.
124    #[arg(long = "flaky-run", value_name = "COUNT")]
125    pub flaky_run: Option<usize>,
126
127    /// Run the test suite in worker IPC mode - listening on the given local socket waiting
128    /// for the test runner to connect and send test execution requests. The only stdout/stderr
129    /// output will be the one emitted by the actual test runs so the test runner can capture them.
130    #[arg(long = "ipc", hide = true)]
131    pub ipc: Option<String>,
132
133    /// If true, spawn worker processes in IPC mode and run the tests on those
134    #[arg(long = "spawn-workers", hide = true)]
135    pub spawn_workers: bool,
136}
137
138impl Arguments {
139    /// Parses the global CLI arguments given to the application.
140    ///
141    /// If the parsing fails (due to incorrect CLI args), an error is shown and
142    /// the application exits. If help is requested (`-h` or `--help`), a help
143    /// message is shown and the application exits, too.
144    pub fn from_args() -> Self {
145        let mut result: Self = Parser::parse();
146        if result.shuffle && result.shuffle_seed.is_none() {
147            // Setting a specific shuffle seed so all spawned workers use the same
148            result.shuffle_seed = Some(rand::random());
149            result.shuffle = false;
150        }
151        result
152    }
153
154    /// Renders the arguments as a list of strings that can be passed to a subprocess
155    pub fn to_args(&self) -> Vec<OsString> {
156        let mut result = Vec::new();
157
158        if self.include_ignored {
159            result.push(OsString::from("--include-ignored"));
160        }
161
162        if self.ignored {
163            result.push(OsString::from("--ignored"));
164        }
165
166        if self.exclude_should_panic {
167            result.push(OsString::from("--exclude-should-panic"));
168        }
169
170        if self.test {
171            result.push(OsString::from("--test"));
172        }
173
174        if self.bench {
175            result.push(OsString::from("--bench"));
176        }
177
178        if self.list {
179            result.push(OsString::from("--list"));
180        }
181
182        if let Some(logfile) = &self.logfile {
183            result.push(OsString::from("--logfile"));
184            result.push(OsString::from(logfile));
185        }
186
187        if self.nocapture {
188            result.push(OsString::from("--nocapture"));
189        }
190
191        if let Some(test_threads) = self.test_threads {
192            result.push(OsString::from("--test-threads"));
193            result.push(OsString::from(test_threads.to_string()));
194        }
195
196        for skip in &self.skip {
197            result.push(OsString::from("--skip"));
198            result.push(OsString::from(skip));
199        }
200
201        if self.quiet {
202            result.push(OsString::from("--quiet"));
203        }
204
205        if self.exact {
206            result.push(OsString::from("--exact"));
207        }
208
209        if let Some(color) = self.color {
210            result.push(OsString::from("--color"));
211            match color {
212                ColorSetting::Auto => result.push(OsString::from("auto")),
213                ColorSetting::Always => result.push(OsString::from("always")),
214                ColorSetting::Never => result.push(OsString::from("never")),
215            }
216        }
217
218        if let Some(format) = self.format {
219            result.push(OsString::from("--format"));
220            match format {
221                FormatSetting::Pretty => result.push(OsString::from("pretty")),
222                FormatSetting::Terse => result.push(OsString::from("terse")),
223                FormatSetting::Json => result.push(OsString::from("json")),
224                FormatSetting::Junit => result.push(OsString::from("junit")),
225                FormatSetting::Ctrf => result.push(OsString::from("ctrf")),
226            }
227        }
228
229        if self.show_output {
230            result.push(OsString::from("--show-output"));
231        }
232
233        if let Some(unstable_flags) = &self.unstable_flags {
234            result.push(OsString::from("-Z"));
235            match unstable_flags {
236                UnstableFlags::UnstableOptions => result.push(OsString::from("unstable-options")),
237            }
238        }
239
240        if self.report_time {
241            result.push(OsString::from("--report-time"));
242        }
243
244        if self.ensure_time {
245            result.push(OsString::from("--ensure-time"));
246        }
247
248        if self.shuffle {
249            result.push(OsString::from("--shuffle"));
250        }
251
252        if let Some(shuffle_seed) = &self.shuffle_seed {
253            result.push(OsString::from("--shuffle-seed"));
254            result.push(OsString::from(shuffle_seed.to_string()));
255        }
256
257        if self.show_stats {
258            result.push(OsString::from("--show-stats"));
259        }
260
261        if let Some(filter) = &self.filter {
262            result.push(OsString::from(filter));
263        }
264
265        if let Some(flaky_run) = &self.flaky_run {
266            result.push(OsString::from("--flaky-run"));
267            result.push(OsString::from(flaky_run.to_string()));
268        }
269
270        if let Some(ipc) = &self.ipc {
271            result.push(OsString::from("--ipc"));
272            result.push(OsString::from(ipc));
273        }
274
275        if self.spawn_workers {
276            result.push(OsString::from("--spawn-workers"));
277        }
278
279        result
280    }
281
282    pub fn unit_test_threshold(&self) -> TimeThreshold {
283        TimeThreshold::from_env_var("RUST_TEST_TIME_UNIT").unwrap_or(TimeThreshold::new(
284            Duration::from_millis(50),
285            Duration::from_millis(100),
286        ))
287    }
288
289    pub fn integration_test_threshold(&self) -> TimeThreshold {
290        TimeThreshold::from_env_var("RUST_TEST_TIME_INTEGRATION").unwrap_or(TimeThreshold::new(
291            Duration::from_millis(500),
292            Duration::from_millis(1000),
293        ))
294    }
295
296    pub(crate) fn test_threads(&self) -> NonZero<usize> {
297        if self.ipc.is_some() {
298            // When running as an IPC-controlled worker, always use a single thread
299            NonZero::new(1).unwrap()
300        } else {
301            self.test_threads
302                .and_then(NonZero::new)
303                .or_else(|| std::thread::available_parallelism().ok())
304                .unwrap_or(NonZero::new(1).unwrap())
305        }
306    }
307
308    /// Make necessary adjustments to the configuration if needed based on the final execution plan
309    pub(crate) fn finalize_for_execution(
310        &mut self,
311        execution: &TestSuiteExecution,
312        output: Arc<dyn TestRunnerOutput>,
313    ) {
314        let requires_capturing = execution.requires_capturing(!self.nocapture);
315
316        if !requires_capturing || self.ipc.is_some() {
317            // If there is no need to capture the output, there are no restrictions to check and apply
318            // If this is an IPC worker, we don't need to do anything either, as the top level test runner already sets the proper arguments
319        } else {
320            // If capture is enabled, we need to spawn at least one worker process
321            self.spawn_workers = true;
322
323            if self.test_threads().get() > 1 {
324                // If tests are executed in parallel, and output needs to be captured, there cannot be any dependencies
325                // because it can only be done through spawned workers
326
327                if execution.has_dependencies() {
328                    if execution.remaining() > 1 {
329                        // If there is only one test, the warning does not make sense
330                        output.warning("Cannot run tests in parallel when tests have shared dependencies and output capturing is on. Using a single thread.");
331                    }
332                    self.test_threads = Some(1); // Falling back to single-threaded execution
333                }
334            }
335        }
336    }
337}
338
339impl<A: Into<OsString> + Clone> FromIterator<A> for Arguments {
340    fn from_iter<T: IntoIterator<Item = A>>(iter: T) -> Self {
341        Parser::parse_from(iter)
342    }
343}
344
345/// Possible values for the `--color` option.
346#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum, Default)]
347pub enum ColorSetting {
348    /// Colorize if stdout is a tty and tests are run on serially (default)
349    #[default]
350    Auto,
351
352    /// Always colorize output
353    Always,
354
355    /// Never colorize output
356    Never,
357}
358
359/// Possible values for the `-Z` option
360#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
361pub enum UnstableFlags {
362    /// Allow use of experimental features
363    UnstableOptions,
364}
365
366/// Possible values for the `--format` option.
367#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum, Default)]
368pub enum FormatSetting {
369    /// Print verbose output
370    #[default]
371    Pretty,
372
373    /// Display one character per test
374    Terse,
375
376    /// Output a json document
377    Json,
378
379    /// Output a JUnit document
380    Junit,
381
382    /// Output to Common Test Report Format
383    Ctrf,
384}
385
386/// Structure denoting time limits for test execution.
387///
388/// From https://github.com/rust-lang/rust/blob/master/library/test/src/time.rs
389#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)]
390pub struct TimeThreshold {
391    pub warn: Duration,
392    pub critical: Duration,
393}
394
395impl TimeThreshold {
396    /// Creates a new `TimeThreshold` instance with provided durations.
397    pub fn new(warn: Duration, critical: Duration) -> Self {
398        Self { warn, critical }
399    }
400
401    /// Attempts to create a `TimeThreshold` instance with values obtained
402    /// from the environment variable, and returns `None` if the variable
403    /// is not set.
404    /// Environment variable format is expected to match `\d+,\d+`.
405    ///
406    /// # Panics
407    ///
408    /// Panics if variable with provided name is set but contains inappropriate
409    /// value.
410    pub fn from_env_var(env_var_name: &str) -> Option<Self> {
411        let durations_str = std::env::var(env_var_name).ok()?;
412        let (warn_str, critical_str) = durations_str.split_once(',').unwrap_or_else(|| {
413            panic!(
414                "Duration variable {env_var_name} expected to have 2 numbers separated by comma, but got {durations_str}"
415            )
416        });
417
418        let parse_u64 = |v| {
419            u64::from_str(v).unwrap_or_else(|_| {
420                panic!(
421                    "Duration value in variable {env_var_name} is expected to be a number, but got {v}"
422                )
423            })
424        };
425
426        let warn = parse_u64(warn_str);
427        let critical = parse_u64(critical_str);
428        if warn > critical {
429            panic!("Test execution warn time should be less or equal to the critical time");
430        }
431
432        Some(Self::new(
433            Duration::from_millis(warn),
434            Duration::from_millis(critical),
435        ))
436    }
437
438    pub fn is_critical(&self, duration: &Duration) -> bool {
439        *duration >= self.critical
440    }
441
442    pub fn is_warn(&self, duration: &Duration) -> bool {
443        *duration >= self.warn
444    }
445}
446
447#[cfg(test)]
448mod tests {
449    use super::*;
450
451    #[test]
452    fn verify_cli() {
453        use clap::CommandFactory;
454        Arguments::command().debug_assert();
455    }
456}
test_r_core/args.rs

test_r_core/
args.rs
