qcheck/
tester.rs

1use std::cmp;
2use std::env;
3use std::fmt::Debug;
4use std::panic;
5
6use crate::{
7    tester::Status::{Discard, Fail, Pass},
8    Arbitrary, Gen,
9};
10
11/// The main QuickCheck type for setting configuration and running QuickCheck.
12pub struct QuickCheck {
13    tests: u64,
14    max_tests: u64,
15    min_tests_passed: u64,
16    gen: Gen,
17}
18
19fn qc_tests() -> u64 {
20    let default = 100;
21    match env::var("QUICKCHECK_TESTS") {
22        Ok(val) => val.parse().unwrap_or(default),
23        Err(_) => default,
24    }
25}
26
27fn qc_max_tests() -> u64 {
28    let default = 10_000;
29    match env::var("QUICKCHECK_MAX_TESTS") {
30        Ok(val) => val.parse().unwrap_or(default),
31        Err(_) => default,
32    }
33}
34
35fn qc_gen_size() -> usize {
36    let default = Gen::DEFAULT_SIZE;
37    match env::var("QUICKCHECK_GENERATOR_SIZE") {
38        Ok(val) => val.parse().unwrap_or(default),
39        Err(_) => default,
40    }
41}
42
43fn qc_min_tests_passed() -> u64 {
44    let default = 0;
45    match env::var("QUICKCHECK_MIN_TESTS_PASSED") {
46        Ok(val) => val.parse().unwrap_or(default),
47        Err(_) => default,
48    }
49}
50
51impl QuickCheck {
52    /// Creates a new QuickCheck value.
53    ///
54    /// This can be used to run QuickCheck on things that implement `Testable`.
55    /// You may also adjust the configuration, such as the number of tests to
56    /// run.
57    ///
58    /// By default, the maximum number of passed tests is set to `100`, the max
59    /// number of overall tests is set to `10000` and the generator is created
60    /// with a size of `100`.
61    pub fn new() -> QuickCheck {
62        let gen = Gen::new(qc_gen_size());
63        let tests = qc_tests();
64        let max_tests = cmp::max(tests, qc_max_tests());
65        let min_tests_passed = qc_min_tests_passed();
66
67        QuickCheck {
68            tests,
69            max_tests,
70            min_tests_passed,
71            gen,
72        }
73    }
74
75    /// Set the random number generator to be used by QuickCheck.
76    pub fn gen(self, gen: Gen) -> QuickCheck {
77        QuickCheck { gen, ..self }
78    }
79
80    /// Set the number of tests to run.
81    ///
82    /// This actually refers to the maximum number of *passed* tests that
83    /// can occur. Namely, if a test causes a failure, future testing on that
84    /// property stops. Additionally, if tests are discarded, there may be
85    /// fewer than `tests` passed.
86    pub fn tests(mut self, tests: u64) -> QuickCheck {
87        self.tests = tests;
88        self
89    }
90
91    /// Set the maximum number of tests to run.
92    ///
93    /// The number of invocations of a property will never exceed this number.
94    /// This is necessary to cap the number of tests because QuickCheck
95    /// properties can discard tests.
96    pub fn max_tests(mut self, max_tests: u64) -> QuickCheck {
97        self.max_tests = max_tests;
98        self
99    }
100
101    /// Set the minimum number of tests that needs to pass.
102    ///
103    /// This actually refers to the minimum number of *valid* *passed* tests
104    /// that needs to pass for the property to be considered successful.
105    pub fn min_tests_passed(mut self, min_tests_passed: u64) -> QuickCheck {
106        self.min_tests_passed = min_tests_passed;
107        self
108    }
109
110    /// Tests a property and returns the result.
111    ///
112    /// The result returned is either the number of tests passed or a witness
113    /// of failure.
114    ///
115    /// (If you're using Rust's unit testing infrastructure, then you'll
116    /// want to use the `quickcheck` method, which will `panic!` on failure.)
117    pub fn quicktest<A>(&mut self, f: A) -> Result<u64, TestResult>
118    where
119        A: Testable,
120    {
121        let mut n_tests_passed = 0;
122        for _ in 0..self.max_tests {
123            if n_tests_passed >= self.tests {
124                break;
125            }
126            match f.result(&mut self.gen) {
127                TestResult { status: Pass, .. } => n_tests_passed += 1,
128                TestResult {
129                    status: Discard, ..
130                } => continue,
131                r @ TestResult { status: Fail, .. } => return Err(r),
132            }
133        }
134        Ok(n_tests_passed)
135    }
136
137    /// Tests a property and calls `panic!` on failure.
138    ///
139    /// The `panic!` message will include a (hopefully) minimal witness of
140    /// failure.
141    ///
142    /// It is appropriate to use this method with Rust's unit testing
143    /// infrastructure.
144    ///
145    /// # Example
146    ///
147    /// ```rust
148    /// use qcheck::QuickCheck;
149    ///
150    /// fn prop_reverse_reverse() {
151    ///     fn revrev(xs: Vec<usize>) -> bool {
152    ///         let rev: Vec<_> = xs.clone().into_iter().rev().collect();
153    ///         let revrev: Vec<_> = rev.into_iter().rev().collect();
154    ///         xs == revrev
155    ///     }
156    ///     QuickCheck::new().quickcheck(revrev as fn(Vec<usize>) -> bool);
157    /// }
158    /// ```
159    pub fn quickcheck<A>(&mut self, f: A)
160    where
161        A: Testable,
162    {
163        let n_tests_passed = match self.quicktest(f) {
164            Ok(n_tests_passed) => n_tests_passed,
165            Err(result) => panic!("{}", result.failed_msg()),
166        };
167
168        if n_tests_passed >= self.min_tests_passed {
169            // Do nothing.
170        } else {
171            panic!(
172                "(Unable to generate enough tests, {} not discarded.)",
173                n_tests_passed
174            )
175        }
176    }
177}
178
179impl Default for QuickCheck {
180    fn default() -> Self {
181        Self::new()
182    }
183}
184
185/// Convenience function for running QuickCheck.
186///
187/// This is an alias for `QuickCheck::new().quickcheck(f)`.
188pub fn quickcheck<A: Testable>(f: A) {
189    QuickCheck::new().quickcheck(f)
190}
191
192/// Describes the status of a single instance of a test.
193///
194/// All testable things must be capable of producing a `TestResult`.
195#[derive(Clone, Debug)]
196pub struct TestResult {
197    status: Status,
198    arguments: Option<Vec<String>>,
199    err: Option<String>,
200}
201
202/// Whether a test has passed, failed or been discarded.
203#[derive(Clone, Debug)]
204enum Status {
205    Pass,
206    Fail,
207    Discard,
208}
209
210impl TestResult {
211    /// Produces a test result that indicates the current test has passed.
212    pub fn passed() -> TestResult {
213        TestResult::from_bool(true)
214    }
215
216    /// Produces a test result that indicates the current test has failed.
217    pub fn failed() -> TestResult {
218        TestResult::from_bool(false)
219    }
220
221    /// Produces a test result that indicates failure from a runtime error.
222    pub fn error<S: Into<String>>(msg: S) -> TestResult {
223        let mut r = TestResult::from_bool(false);
224        r.err = Some(msg.into());
225        r
226    }
227
228    /// Produces a test result that instructs `quickcheck` to ignore it.
229    /// This is useful for restricting the domain of your properties.
230    /// When a test is discarded, `quickcheck` will replace it with a
231    /// fresh one (up to a certain limit).
232    pub fn discard() -> TestResult {
233        TestResult {
234            status: Discard,
235            arguments: None,
236            err: None,
237        }
238    }
239
240    /// Converts a `bool` to a `TestResult`. A `true` value indicates that
241    /// the test has passed and a `false` value indicates that the test
242    /// has failed.
243    pub fn from_bool(b: bool) -> TestResult {
244        TestResult {
245            status: if b { Pass } else { Fail },
246            arguments: None,
247            err: None,
248        }
249    }
250
251    /// Tests if a "procedure" fails when executed. The test passes only if
252    /// `f` generates a task failure during its execution.
253    pub fn must_fail<T, F>(f: F) -> TestResult
254    where
255        F: FnOnce() -> T,
256        F: 'static,
257        T: 'static,
258    {
259        let f = panic::AssertUnwindSafe(f);
260        TestResult::from_bool(panic::catch_unwind(f).is_err())
261    }
262
263    /// Returns `true` if and only if this test result describes a failing
264    /// test.
265    pub fn is_failure(&self) -> bool {
266        match self.status {
267            Fail => true,
268            Pass | Discard => false,
269        }
270    }
271
272    /// Returns `true` if and only if this test result describes a failing
273    /// test as a result of a run time error.
274    pub fn is_error(&self) -> bool {
275        self.is_failure() && self.err.is_some()
276    }
277
278    fn failed_msg(&self) -> String {
279        let arguments_msg = match self.arguments {
280            None => String::from("No Arguments Provided"),
281            Some(ref args) => format!("Arguments: ({})", args.join(", ")),
282        };
283        match self.err {
284            None => format!("[quickcheck] TEST FAILED. {}", arguments_msg),
285            Some(ref err) => format!(
286                "[quickcheck] TEST FAILED (runtime error). {}\nError: {}",
287                arguments_msg, err
288            ),
289        }
290    }
291}
292
293/// `Testable` describes types (e.g., a function) whose values can be
294/// tested.
295///
296/// Anything that can be tested must be capable of producing a `TestResult`
297/// given a random number generator. This is trivial for types like `bool`,
298/// which are just converted to either a passing or failing test result.
299///
300/// For functions, an implementation must generate random arguments
301/// and potentially shrink those arguments if they produce a failure.
302///
303/// It's unlikely that you'll have to implement this trait yourself.
304pub trait Testable: 'static {
305    fn result(&self, _: &mut Gen) -> TestResult;
306}
307
308impl Testable for bool {
309    fn result(&self, _: &mut Gen) -> TestResult {
310        TestResult::from_bool(*self)
311    }
312}
313
314impl Testable for () {
315    fn result(&self, _: &mut Gen) -> TestResult {
316        TestResult::passed()
317    }
318}
319
320impl Testable for TestResult {
321    fn result(&self, _: &mut Gen) -> TestResult {
322        self.clone()
323    }
324}
325
326impl<A, E> Testable for Result<A, E>
327where
328    A: Testable,
329    E: Debug + 'static,
330{
331    fn result(&self, g: &mut Gen) -> TestResult {
332        match *self {
333            Ok(ref r) => r.result(g),
334            Err(ref err) => TestResult::error(format!("{:?}", err)),
335        }
336    }
337}
338
339/// Return a vector of the debug formatting of each item in `args`
340fn debug_reprs(args: &[&dyn Debug]) -> Vec<String> {
341    args.iter().map(|x| format!("{:?}", x)).collect()
342}
343
344macro_rules! testable_fn {
345    ($($name: ident),*) => {
346
347impl<T: Testable,
348     $($name: Arbitrary + Debug),*> Testable for fn($($name),*) -> T {
349    #[allow(non_snake_case)]
350    fn result(&self, g: &mut Gen) -> TestResult {
351        fn shrink_failure<T: Testable, $($name: Arbitrary + Debug),*>(
352            g: &mut Gen,
353            self_: fn($($name),*) -> T,
354            a: ($($name,)*),
355        ) -> Option<TestResult> {
356            for t in a.shrink() {
357                let ($($name,)*) = t.clone();
358                let mut r_new = safe(move || {self_($($name),*)}).result(g);
359                if r_new.is_failure() {
360                    {
361                        let ($(ref $name,)*) : ($($name,)*) = t;
362                        r_new.arguments = Some(debug_reprs(&[$($name),*]));
363                    }
364
365                    // The shrunk value *does* witness a failure, so keep
366                    // trying to shrink it.
367                    let shrunk = shrink_failure(g, self_, t);
368
369                    // If we couldn't witness a failure on any shrunk value,
370                    // then return the failure we already have.
371                    return Some(shrunk.unwrap_or(r_new))
372                }
373            }
374            None
375        }
376
377        let self_ = *self;
378        let a: ($($name,)*) = Arbitrary::arbitrary(g);
379        let ( $($name,)* ) = a.clone();
380        let r = safe(move || {self_($($name),*)}).result(g);
381        match r.status {
382            Pass|Discard => r,
383            Fail => {
384                shrink_failure(g, self_, a).unwrap_or(r)
385            }
386        }
387    }
388}}}
389
390testable_fn!();
391testable_fn!(A);
392testable_fn!(A, B);
393testable_fn!(A, B, C);
394testable_fn!(A, B, C, D);
395testable_fn!(A, B, C, D, E);
396testable_fn!(A, B, C, D, E, F);
397testable_fn!(A, B, C, D, E, F, G);
398testable_fn!(A, B, C, D, E, F, G, H);
399
400fn safe<T, F>(fun: F) -> Result<T, String>
401where
402    F: FnOnce() -> T,
403    F: 'static,
404    T: 'static,
405{
406    panic::catch_unwind(panic::AssertUnwindSafe(fun)).map_err(|any_err| {
407        // Extract common types of panic payload:
408        // panic and assert produce &str or String
409        if let Some(&s) = any_err.downcast_ref::<&str>() {
410            s.to_owned()
411        } else if let Some(s) = any_err.downcast_ref::<String>() {
412            s.to_owned()
413        } else {
414            "UNABLE TO SHOW RESULT OF PANIC.".to_owned()
415        }
416    })
417}
418
419/// Convenient aliases.
420trait AShow: Arbitrary + Debug {}
421impl<A: Arbitrary + Debug> AShow for A {}
422
423#[cfg(test)]
424mod test {
425    use crate::{Gen, QuickCheck};
426
427    #[test]
428    fn shrinking_regression_issue_126() {
429        fn thetest(vals: Vec<bool>) -> bool {
430            vals.iter().filter(|&v| *v).count() < 2
431        }
432        let failing_case = QuickCheck::new()
433            .quicktest(thetest as fn(vals: Vec<bool>) -> bool)
434            .unwrap_err();
435        let expected_argument = format!("{:?}", [true, true]);
436        assert_eq!(failing_case.arguments, Some(vec![expected_argument]));
437    }
438
439    #[test]
440    fn size_for_small_types_issue_143() {
441        fn t(_: i8) -> bool {
442            true
443        }
444        QuickCheck::new()
445            .gen(Gen::new(129))
446            .quickcheck(t as fn(i8) -> bool);
447    }
448
449    #[test]
450    fn regression_signed_shrinker_panic() {
451        fn foo_can_shrink(v: i8) -> bool {
452            let _ = crate::Arbitrary::shrink(&v).take(100).count();
453            true
454        }
455        crate::quickcheck(foo_can_shrink as fn(i8) -> bool);
456    }
457}