noprop 0.0.4

Imperative property-based testing with no dependencies and no macro DSL
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
//! Error and result types for [`Runner::run`](crate::Runner::run) and
//! [`Runner::run_feedback_guided`](crate::Runner::run_feedback_guided).

use std::panic::Location;

use crate::GeneratedValue;
use crate::rng::Feature;
use crate::runner::Stats;

/// Result alias for `#[test]` functions and property closures.
///
/// `T` defaults to `()`, so a plain `noprop::TestResult` is
/// `Result<(), Box<dyn std::error::Error>>`. Every failure a test can
/// hit — a [`RunError`] from the runner, or a config error from the
/// env helpers — converts into the boxed error via `?`.
pub type TestResult<T = ()> = std::result::Result<T, Box<dyn std::error::Error>>;

/// Result alias for [`Runner::run`](crate::Runner::run) and
/// [`Runner::run_feedback_guided`](crate::Runner::run_feedback_guided).
///
/// Unlike [`TestResult`], the error stays type-safe: callers can
/// inspect the seed, case index, generated-value trace, stats, and
/// reproduce hint via [`RunError`]'s accessors and dispatch on the
/// failure kind via [`RunError::kind`].
pub type RunResult = std::result::Result<(), RunError>;

/// Failure information from a [`Runner::run`](crate::Runner::run) or
/// [`Runner::run_feedback_guided`](crate::Runner::run_feedback_guided)
/// invocation.
///
/// A property failure (panic or returned `Err`) is deterministically
/// reproducible: rerunning
/// `noprop::Runner::new(err.seed()).run(N, |ctx| ...)` (or
/// `.run_feedback_guided(N, |ctx| ...)` for a feedback-guided failure)
/// with the `N` printed by the reproduce hint will hit the same
/// failure again. The hint reuses the original case budget so the
/// rerun also hits the same rejection cap.
///
/// A `TooManyRejections` failure — raised when
/// [`TestCaseContext::reject_case`](crate::TestCaseContext::reject_case) fires so often that
/// the internal global limit is reached — reports the number of
/// accepted cases that completed before the runner gave up as
/// `case_index()`, so the same seed and case budget reproduce the
/// same exit.
///
/// A feedback-guided failure report additionally carries the semantic
/// features the failing case reported and a one-based candidate index.
/// The candidate index counts every attempt — accepted, rejected, and
/// the failing case itself — so it relates to the zero-based
/// `case_index()` (accepted cases only) as
/// `candidate_index = case_index + 1 + rejections before the failure`.
/// For `TooManyRejections`, the candidate index is the ordinal of the
/// last rejected attempt (`accepted + rejected`) and the semantic
/// features are those of that last rejected case.
///
/// `generated()` returns the sequence of values every primitive
/// generator produced during the failing case, together with each call
/// site's source location. For `TooManyRejections`, `generated()`
/// returns the (discarded) trace of the last rejected iteration, since
/// no accepted iteration produced the failure.
///
/// The `Debug` and `Display` output includes the failure message
/// captured from the user's closure along with the generated-value
/// list, so returning this from a `#[test]` function prints a
/// self-contained failure report through the standard test harness.
/// Both formats also print a reproduce hint reusing the original
/// case budget. `run`'s hint prints:
///
/// ```text
/// reproduce with: noprop::Runner::new(0x...).run(N, |ctx| ...)
/// ```
///
/// while a feedback-guided failure prints:
///
/// ```text
/// reproduce with: noprop::Runner::new(0x...).run_feedback_guided(N, |ctx| ...)
/// ```
///
/// In each case the closure body is a placeholder: the caller
/// substitutes the original property closure before rerunning, and the
/// re-run size never needs to be recomputed by hand.
pub struct RunError {
    seed: u64,
    case_index: usize,
    /// The case budget the failing run was given. The reproduce
    /// hint reuses it so reruns hit the same rejection cap (a
    /// `case_index + 1` hint would shrink the cap and turn the failure
    /// into `TooManyRejections`).
    cases: usize,
    kind: ErrorKind,
    generated: Vec<GeneratedValue>,
    // Boxed to keep `RunError` small: with the corpus stats fields
    // inline, `RunError` is exactly 128 bytes, which already triggers
    // clippy's `result_large_err` (its threshold comparison is
    // `>= 128`). Boxed, `RunError` is 96 bytes — and its size no longer
    // grows when `Stats` gains fields.
    stats: Box<Stats>,
    /// The runner entry point that produced this failure. Switches the
    /// reproduce hint.
    policy: SearchPolicy,
    /// Semantic details of the failing case (feedback-guided runs only;
    /// `None` otherwise).
    semantic: Option<Box<SemanticFailureReport>>,
}

/// Semantic details carried by a feedback-guided failure report.
///
/// Boxed so `RunError` stays small: the fields are only populated on
/// the feedback-guided failure path, and the uniform entry point never
/// touches them.
struct SemanticFailureReport {
    /// Semantic features the failing case reported.
    features: Vec<Feature>,
    /// The one-based index of the failing candidate across accepted
    /// and rejected cases (the failing case itself included).
    candidate_index: usize,
}

/// The runner entry point that produced an [`RunError`].
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum SearchPolicy {
    /// [`Runner::run`](crate::Runner::run).
    Uniform,
    /// [`Runner::run_feedback_guided`](crate::Runner::run_feedback_guided).
    FeedbackGuided,
}

/// The kind of a [`RunError`], for type-safe dispatch on the failure
/// mode.
///
/// Field-less on purpose: the payload (e.g. the rejected-iteration
/// count and last reject location of [`RunErrorKind::TooManyRejections`])
/// stays on [`RunError`] itself, so the failure report keeps its full
/// detail while the kind stays cheap to compare.
///
/// New kinds are added as breaking changes (no `#[non_exhaustive]`), so
/// future failure axes extend this enum deliberately.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RunErrorKind {
    /// The property closure returned `Err` or panicked in a case.
    PropertyFailure,
    /// The internal global rejection limit was reached before
    /// `Runner::run`'s `cases` budget of accepted cases completed.
    TooManyRejections,
}

/// The internal, payload-carrying failure mode of a [`RunError`].
enum ErrorKind {
    /// The property closure panicked in this case (typically via
    /// `assert!` / `assert_eq!` or an explicit `panic!`).
    Panic { message: String },
    /// The internal global rejection limit was reached before
    /// `Runner::run`'s `cases` budget of accepted cases completed.
    /// Rejected
    /// cases do not count toward the budget, but the
    /// runner keeps track of total attempts (accepted + rejected) via
    /// a crate-private constant so a generator that always rejects
    /// still terminates.
    TooManyRejections {
        rejected_cases: usize,
        last_reject_location: &'static Location<'static>,
    },
}

impl RunErrorKind {
    fn of(kind: &ErrorKind) -> Self {
        match kind {
            ErrorKind::Panic { .. } => RunErrorKind::PropertyFailure,
            ErrorKind::TooManyRejections { .. } => RunErrorKind::TooManyRejections,
        }
    }
}

impl RunError {
    /// Build a property-failure error from the recorded run stats.
    ///
    /// The case index is taken from `stats.accepted_cases`: the caller
    /// must have recorded the progress counters (via `record_stats` /
    /// `record_corpus_stats`) before constructing the error.
    pub(crate) fn from_panic(
        seed: u64,
        cases: usize,
        message: String,
        generated: Vec<GeneratedValue>,
        stats: Stats,
        policy: SearchPolicy,
    ) -> Self {
        Self::new(
            seed,
            stats.accepted_cases,
            cases,
            ErrorKind::Panic { message },
            generated,
            stats,
            policy,
        )
    }

    /// Attach the semantic features and candidate index of a failing
    /// feedback-guided case. Used by
    /// [`Runner::run_feedback_guided`](crate::Runner::run_feedback_guided)
    /// and the too-many-rejections exit path.
    pub(crate) fn with_semantic(mut self, features: Vec<Feature>, candidate_index: usize) -> Self {
        self.semantic = Some(Box::new(SemanticFailureReport {
            features,
            candidate_index,
        }));
        self
    }

    /// Build a too-many-rejections error from the recorded run stats.
    ///
    /// The accepted and rejected case counts are taken from
    /// `stats.accepted_cases` / `stats.rejected_cases`: the caller must
    /// have recorded the progress counters before constructing the
    /// error.
    pub(crate) fn from_too_many_rejections(
        seed: u64,
        cases: usize,
        last_reject_location: &'static Location<'static>,
        generated: Vec<GeneratedValue>,
        stats: Stats,
        policy: SearchPolicy,
    ) -> Self {
        Self::new(
            seed,
            stats.accepted_cases,
            cases,
            ErrorKind::TooManyRejections {
                rejected_cases: stats.rejected_cases,
                last_reject_location,
            },
            generated,
            stats,
            policy,
        )
    }

    fn new(
        seed: u64,
        case_index: usize,
        cases: usize,
        kind: ErrorKind,
        generated: Vec<GeneratedValue>,
        stats: Stats,
        policy: SearchPolicy,
    ) -> Self {
        Self {
            seed,
            case_index,
            cases,
            kind,
            generated,
            stats: Box::new(stats),
            policy,
            semantic: None,
        }
    }

    /// The seed that was passed to the [`Runner`](crate::Runner) that
    /// produced this failure.
    pub fn seed(&self) -> u64 {
        self.seed
    }

    /// The zero-based index of the accepted iteration this failure is
    /// tied to. For a property panic / returned `Err`, this is the
    /// index of the failing iteration. For `TooManyRejections`, this
    /// is the count of accepted cases that ran before the runner
    /// gave up (i.e. the index of the iteration that could not be
    /// accepted).
    pub fn case_index(&self) -> usize {
        self.case_index
    }

    /// The generated values recorded during the failing case, in call
    /// order. This is a debugging trace, not a stack backtrace.
    pub fn generated(&self) -> &[GeneratedValue] {
        &self.generated
    }

    /// Observability counters accumulated up to (and including) the
    /// failing case. `accepted_cases` matches
    /// [`case_index`](Self::case_index). The corpus fields
    /// (`discovered_features` / `max_corpus_size`) do not include the
    /// failing case's features (the failing case is not admitted).
    pub fn stats(&self) -> Stats {
        *self.stats
    }

    /// The failure kind of this error, for type-safe dispatch.
    ///
    /// Prefer this over string-matching the `Display` / `Debug` output
    /// when branching on the failure mode.
    pub fn kind(&self) -> RunErrorKind {
        RunErrorKind::of(&self.kind)
    }
}

impl RunError {
    /// The reproduce command shared by
    /// [`Debug`](std::fmt::Debug) and [`Display`](std::fmt::Display).
    /// Reuses the original case budget so reruns hit the same
    /// rejection cap. In feedback-guided mode the closure body
    /// is a placeholder: the caller substitutes the original property
    /// closure.
    fn reproduce_command(&self) -> String {
        match self.policy {
            SearchPolicy::Uniform => format!(
                "noprop::Runner::new({:#018x}).run({}, |ctx| ...)",
                self.seed, self.cases
            ),
            SearchPolicy::FeedbackGuided => format!(
                "noprop::Runner::new({:#018x}).run_feedback_guided({}, |ctx| ...)",
                self.seed, self.cases
            ),
        }
    }
}

impl std::fmt::Debug for RunError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        writeln!(f, "RunError {{")?;
        writeln!(f, "    seed: {:#018x},", self.seed)?;
        writeln!(f, "    case_index: {},", self.case_index)?;
        if let Some(report) = &self.semantic {
            writeln!(f, "    candidate_index: {},", report.candidate_index)?;
        }
        writeln!(f, "    policy: {:?},", self.policy)?;
        match &self.kind {
            ErrorKind::Panic { message } => {
                writeln!(f, "    panic: {message:?},")?;
            }
            ErrorKind::TooManyRejections {
                rejected_cases,
                last_reject_location,
            } => {
                writeln!(
                    f,
                    "    too_many_rejections: {{ rejected: {rejected_cases}, last_reject_at: {}:{} }},",
                    last_reject_location.file(),
                    last_reject_location.line(),
                )?;
            }
        }
        writeln!(f, "    reproduce: {},", self.reproduce_command())?;
        writeln!(
            f,
            "    stats: {{ accepted: {}, rejected: {}, total_samples: {}, discovered_features: {}, max_corpus_size: {} }},",
            self.stats.accepted_cases,
            self.stats.rejected_cases,
            self.stats.total_samples,
            self.stats.discovered_features,
            self.stats.max_corpus_size,
        )?;
        if self.generated.is_empty() {
            writeln!(f, "    generated: [],")?;
        } else {
            writeln!(f, "    generated: [")?;
            for entry in &self.generated {
                writeln!(f, "        {entry:?}")?;
            }
            writeln!(f, "    ],")?;
        }
        if let Some(report) = &self.semantic
            && !report.features.is_empty()
        {
            writeln!(f, "    semantic_features: [")?;
            for feature in &report.features {
                writeln!(f, "        {},", feature.display_repr())?;
            }
            writeln!(f, "    ],")?;
        }
        write!(f, "}}")
    }
}

impl std::fmt::Display for RunError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match &self.kind {
            ErrorKind::Panic { message } => {
                writeln!(
                    f,
                    "noprop failure at case {} (seed={:#018x}): {}",
                    self.case_index, self.seed, message
                )?;
            }
            ErrorKind::TooManyRejections {
                rejected_cases,
                last_reject_location,
            } => {
                writeln!(
                    f,
                    "noprop too many rejections at case {} (seed={:#018x}): \
                     {rejected_cases} rejected case(s), last reject at {}:{}",
                    self.case_index,
                    self.seed,
                    last_reject_location.file(),
                    last_reject_location.line(),
                )?;
            }
        }
        writeln!(f, "reproduce with: {}", self.reproduce_command())?;
        writeln!(
            f,
            "stats: accepted={}, rejected={}, total_samples={}, discovered_features={}, max_corpus_size={}",
            self.stats.accepted_cases,
            self.stats.rejected_cases,
            self.stats.total_samples,
            self.stats.discovered_features,
            self.stats.max_corpus_size,
        )?;
        if !self.generated.is_empty() {
            writeln!(f, "Generated values:")?;
            for entry in &self.generated {
                writeln!(f, "  {entry:?}")?;
            }
        }
        if let Some(report) = &self.semantic
            && !report.features.is_empty()
        {
            writeln!(f, "Semantic features:")?;
            for feature in &report.features {
                writeln!(f, "  {}", feature.display_repr())?;
            }
        }
        Ok(())
    }
}

impl std::error::Error for RunError {}