use std::{panic::resume_unwind, thread};
use proptest::{
option,
prelude::*,
sample::{Index, select, subsequence},
test_runner::{Config, FileFailurePersistence, TestCaseError, TestRunner}
};
use rand::{SeedableRng, rngs::StdRng};
use crate::{
EvaluationError, Evaluator, Function as IrFunction, HistogramBuilder,
Optimizer as _, Parser, Passes, StandardOptimizer,
ast::Function,
compiler::{CompilationError, compile, compile_unoptimized},
diagnostics::diagnose,
s_expr::{SExpressible, SExpressibleOptions, read_s_expr},
serial,
support::{SMALL_STACK_SIZE, on_small_stack},
tests::corpus::TOKENS
};
const CASES: u32 = 2_000;
const RUNNER_STACK_SIZE: usize = 64 * 1024 * 1024;
#[test]
fn test_programs()
{
on_small_stack(|| check(program, |source| check_program(&source)));
}
#[test]
fn test_mutated_programs()
{
on_small_stack(|| check(mutated_program, |source| check_source(&source)));
}
#[test]
fn test_evaluated_programs()
{
on_small_stack(|| {
check(
|| {
(
program(),
[binding(), binding()],
[binding(), binding(), binding(), binding(), binding()],
budget(),
any::<u64>()
)
},
|(source, args, externals, budget, seed)| {
check_evaluation(&source, &args, &externals, budget, seed)
}
)
});
}
const HISTOGRAM_BUDGET: u64 = 20_000;
#[test]
fn test_optimized_programs()
{
on_small_stack(|| {
check(
|| {
(
program(),
[binding(), binding()],
[binding(), binding(), binding(), binding(), binding()]
)
},
|(source, args, externals)| {
check_optimization(&source, &args, &externals)
}
)
});
}
fn check<S>(
strategy: impl FnOnce() -> S + Send,
property: impl Fn(S::Value) -> Result<(), TestCaseError> + Sync
) where
S: Strategy,
S::Value: Send
{
let config = Config {
cases: CASES,
source_file: Some(file!()),
failure_persistence: Some(Box::new(
FileFailurePersistence::SourceParallel("proptest-regressions")
)),
..Config::default()
};
on_stack(RUNNER_STACK_SIZE, || {
let result = TestRunner::new(config).run(&strategy(), |value| {
on_stack(SMALL_STACK_SIZE, || property(value))
});
if let Err(e) = result
{
panic!("{}", e);
}
})
}
fn on_stack<R: Send>(size: usize, f: impl FnOnce() -> R + Send) -> R
{
thread::scope(|scope| {
thread::Builder::new()
.stack_size(size)
.spawn_scoped(scope, f)
.unwrap()
.join()
.unwrap_or_else(|payload| resume_unwind(payload))
})
}
fn check_program(source: &str) -> Result<(), TestCaseError>
{
let function = Parser::parse(source)
.map_err(|e| TestCaseError::fail(format!("program fails:\n{}", e)))?;
let rendered = function.to_string();
prop_assert_eq!(reparse(&rendered)?.to_string(), rendered.as_str());
let options = SExpressibleOptions::default()
.with_spans(true)
.with_groups(true);
let s_expr = function.to_s_expr(options);
let read = read_s_expr(&s_expr).map_err(|e| {
TestCaseError::fail(format!("S-expression {:?} fails: {}", s_expr, e))
})?;
prop_assert!(
read == function,
"S-expression {:?} reads back differently",
s_expr
);
check_source(source)
}
fn reparse(rendered: &str) -> Result<Function<'_>, TestCaseError>
{
Parser::parse(rendered).map_err(|e| {
TestCaseError::fail(format!("rendering {:?} fails:\n{}", rendered, e))
})
}
fn check_source(source: &str) -> Result<(), TestCaseError>
{
let parsed = Parser::parse(source);
match (&parsed, compile(source))
{
(Err(expected), Err(CompilationError::ParseError(actual))) =>
{
prop_assert_eq!(&actual, expected)
},
(Err(_), _) => prop_assert!(false, "compile parsed a failing source"),
(Ok(_), Err(CompilationError::ParseError(e))) =>
{
prop_assert!(false, "compile failed to parse:\n{}", e)
},
(Ok(_), _) =>
{}
}
let diagnosis = diagnose(source);
match &parsed
{
Ok(_) =>
{
prop_assert_eq!(diagnosis.corrected_source.as_deref(), Some(source))
},
Err(_) => prop_assert!(
!diagnosis.diagnostics.is_empty(),
"no diagnostics for a failing source"
)
}
if let Some(corrected) = &diagnosis.corrected_source
{
prop_assert!(
Parser::parse(corrected).is_ok(),
"corrected source {:?} fails",
corrected
);
}
Ok(())
}
fn check_evaluation(
source: &str,
args: &[i32],
externals: &[i32],
budget: u64,
seed: u64
) -> Result<(), TestCaseError>
{
let Ok(function) = compile(source)
else
{
return Ok(())
};
let mut evaluator = Evaluator::new(function);
let args = &args[..evaluator.function.arity()];
let externals = NAMES
.iter()
.zip(externals)
.filter(|(name, _)| {
evaluator.function.externals.contains(&name.to_string())
})
.map(|(name, value)| (*name, *value))
.collect::<Vec<_>>();
for (name, value) in &externals
{
evaluator.bind(name, *value).unwrap();
}
let bounds = evaluator
.bounds_over(
args.iter().map(|arg| Some((*arg).into())),
externals
.iter()
.map(|(name, value)| (*name, (*value).into()))
)
.unwrap();
let metered = evaluator.evaluate_metered(
args.iter().copied(),
&mut StdRng::seed_from_u64(seed),
budget
);
match &metered
{
Ok(evaluation) =>
{
prop_assert!(evaluation.dice <= budget, "over budget");
prop_assert!(evaluation.dice <= bounds.dice, "over worst case");
let unmetered = evaluator
.evaluate(
args.iter().copied(),
&mut StdRng::seed_from_u64(seed)
)
.unwrap();
prop_assert_eq!(evaluation, &unmetered);
},
Err(EvaluationError::DiceBudgetExhausted {
requested,
remaining,
consumed
}) =>
{
prop_assert!(requested > remaining, "refused an affordable roll");
prop_assert_eq!(consumed.checked_add(*remaining), Some(budget));
prop_assert!(bounds.dice > budget, "refused within the worst case");
},
Err(e) => prop_assert!(false, "evaluation failed: {}", e)
}
let builder = serial::HistogramBuilder::new(evaluator);
match builder.build_metered(args.iter().copied(), budget)
{
Ok(histogram) =>
{
prop_assert!(histogram.total() > 0, "empty histogram");
for outcome in histogram.keys()
{
prop_assert!(
bounds.value.contains(*outcome),
"outcome {} out of bounds {}",
outcome,
bounds.value
);
}
if let Ok(evaluation) = &metered
{
prop_assert!(
histogram.get(evaluation.result) > 0,
"histogram lacks the evaluated result {}",
evaluation.result
);
}
},
Err(EvaluationError::HistogramBudgetExhausted {
requested,
remaining,
consumed
}) =>
{
prop_assert!(requested > remaining, "refused affordable branches");
prop_assert_eq!(consumed.checked_add(remaining), Some(budget));
},
Err(e) => prop_assert!(false, "histogram failed: {}", e)
}
Ok(())
}
fn check_optimization(
source: &str,
args: &[i32],
externals: &[i32]
) -> Result<(), TestCaseError>
{
let Ok(unoptimized) = compile_unoptimized(source)
else
{
return Ok(())
};
let optimized = StandardOptimizer::new(Passes::all())
.optimize(unoptimized.clone())
.unwrap();
let args = &args[..unoptimized.arity()];
let evaluator = |function: &IrFunction| {
let mut evaluator = Evaluator::new(function.clone());
let externals = NAMES
.iter()
.zip(externals)
.filter(|(name, _)| function.externals.contains(&name.to_string()))
.map(|(name, value)| (*name, *value))
.collect::<Vec<_>>();
for (name, value) in &externals
{
evaluator.bind(name, *value).unwrap();
}
let bounds = evaluator
.bounds_over(
args.iter().map(|arg| Some((*arg).into())),
externals
.iter()
.map(|(name, value)| (*name, (*value).into()))
)
.unwrap();
(evaluator, bounds.value)
};
let (unoptimized_evaluator, unoptimized_bounds) = evaluator(&unoptimized);
let (optimized_evaluator, optimized_bounds) = evaluator(&optimized);
prop_assert!(
unoptimized_bounds.min <= optimized_bounds.min
&& optimized_bounds.max <= unoptimized_bounds.max,
"optimized bounds {} escape unoptimized bounds {}\nunoptimized:\n{}\n\
optimized:\n{}",
optimized_bounds,
unoptimized_bounds,
unoptimized,
optimized
);
let histogram = |evaluator| {
serial::HistogramBuilder::new(evaluator)
.build_metered(args.iter().copied(), HISTOGRAM_BUDGET)
};
let (Ok(before), Ok(after)) = (
histogram(unoptimized_evaluator),
histogram(optimized_evaluator)
)
else
{
return Ok(())
};
let (total_before, total_after) =
(before.total() as u128, after.total() as u128);
prop_assert_eq!(
before.outcomes(),
after.outcomes(),
"outcomes differ\nunoptimized:\n{}\noptimized:\n{}",
unoptimized,
optimized
);
for outcome in before.outcomes()
{
prop_assert_eq!(
before.get(outcome) as u128 * total_after,
after.get(outcome) as u128 * total_before,
"probability of {} differs\nunoptimized:\n{}\noptimized:\n{}",
outcome,
unoptimized,
optimized
);
}
Ok(())
}
const NAMES: &[&str] =
&["x", "hit points", "weapon: 2/3", "1d6 drop lowest", "Ω-(1)"];
const WHITESPACE: &[&str] = &[" ", " ", "\t", "\n", "\u{A0}"];
const OPERATORS: &[&str] = &["+", "-", "*", "/", "%", "^", "×", "÷"];
const MAX_DEPTH: u32 = 8;
const TARGET_SIZE: u32 = 128;
fn program() -> impl Strategy<Value = String>
{
let parameters =
(subsequence(&NAMES[..2], 1..=2), [spelling(), spelling()]).prop_map(
|(names, spellings)| {
names
.iter()
.zip(&spellings)
.map(|(name, spelling)| spelling.braced(name))
.collect::<Vec<_>>()
.join(", ")
}
);
(option::of(parameters), expression()).prop_map(|(parameters, body)| {
match parameters
{
Some(parameters) => format!("{}: {}", parameters, body),
None => body
}
})
}
fn expression() -> impl Strategy<Value = String>
{
let leaf = prop_oneof![constant(), variable()];
leaf.prop_recursive(MAX_DEPTH, TARGET_SIZE, 3, |inner| {
prop_oneof![
(inner.clone(), select(OPERATORS), inner.clone()).prop_map(
|(left, op, right)| format!("{} {} {}", left, op, right)
),
inner.clone().prop_map(|operand| format!("-{}", operand)),
inner.clone().prop_map(|e| format!("({})", e)),
(variable(), inner.clone())
.prop_map(|(name, e)| format!("{}@({})", name, e)),
(inner.clone(), inner.clone())
.prop_map(|(start, end)| format!("[{}:{}]", start, end)),
dice(inner)
]
})
}
fn dice(inner: BoxedStrategy<String>) -> impl Strategy<Value = String>
{
let faces = prop_oneof![
atom(inner.clone()),
prop::collection::vec(-3i32..=20, 1..6).prop_map(|faces| {
let faces = faces.iter().map(i32::to_string).collect::<Vec<_>>();
format!("[{}]", faces.join(", "))
})
];
let clause = (
select(&["lowest", "highest"][..]),
option::of(atom(inner.clone()))
)
.prop_map(|(direction, drop)| match drop
{
Some(drop) => format!(" drop {} {}", direction, drop),
None => format!(" drop {}", direction)
});
(
atom(inner),
select(&["d", "D"][..]),
faces,
prop::collection::vec(clause, 0..3)
)
.prop_map(|(count, operator, faces, clauses)| {
format!("{}{}{}{}", count, operator, faces, clauses.concat())
})
}
fn atom(inner: BoxedStrategy<String>) -> impl Strategy<Value = String>
{
prop_oneof![
constant(),
variable(),
inner.clone().prop_map(|e| format!("({})", e)),
(variable(), inner).prop_map(|(name, e)| format!("{}@({})", name, e))
]
}
fn constant() -> impl Strategy<Value = String>
{
prop_oneof![
4 => (-3i32..=20).prop_map(|n| n.to_string()),
1 => any::<i32>().prop_map(|n| n.to_string())
]
}
fn variable() -> impl Strategy<Value = String>
{
(select(NAMES), spelling())
.prop_map(|(name, spelling)| spelling.braced(name))
}
#[derive(Clone, Debug)]
struct Spelling
{
leading: &'static str,
gaps: Vec<&'static str>,
trailing: &'static str
}
impl Spelling
{
fn braced(&self, name: &str) -> String
{
let mut words = name.split(' ');
let mut spelling = format!("{{{}", self.leading);
spelling.extend(words.next());
for (gap, word) in self.gaps.iter().zip(words)
{
spelling.push_str(gap);
spelling.push_str(word);
}
spelling.push_str(self.trailing);
spelling.push('}');
spelling
}
}
fn spelling() -> impl Strategy<Value = Spelling>
{
let run = || select(WHITESPACE);
let gap = prop_oneof![3 => Just(" "), 1 => run()];
let gaps = NAMES
.iter()
.map(|name| name.matches(' ').count())
.max()
.unwrap_or_default();
(
option::weighted(0.25, run()),
prop::collection::vec(gap, gaps),
option::weighted(0.25, run())
)
.prop_map(|(leading, gaps, trailing)| Spelling {
leading: leading.unwrap_or_default(),
gaps,
trailing: trailing.unwrap_or_default()
})
}
fn binding() -> impl Strategy<Value = i32>
{
prop_oneof![
select(&[i32::MIN, -1, 0, 1, i32::MAX][..]),
-3i32..=20,
any::<i32>()
]
}
fn budget() -> impl Strategy<Value = u64>
{
prop_oneof![0u64..=16, 0u64..=10_000]
}
#[derive(Clone, Debug)]
enum Mutation
{
Insert(Index, &'static str),
Delete(Index),
Replace(Index, &'static str),
Truncate(Index)
}
impl Mutation
{
fn apply(&self, source: &mut String)
{
let boundaries = source
.char_indices()
.map(|(i, _)| i)
.chain([source.len()])
.collect::<Vec<_>>();
let at = |index: &Index| boundaries[index.index(boundaries.len())];
let end = |start: usize| {
source[start..]
.chars()
.next()
.map_or(start, |c| start + c.len_utf8())
};
match self
{
Mutation::Insert(index, token) =>
{
source.insert_str(at(index), token)
},
Mutation::Delete(index) =>
{
let start = at(index);
source.replace_range(start..end(start), "");
},
Mutation::Replace(index, token) =>
{
let start = at(index);
source.replace_range(start..end(start), token);
},
Mutation::Truncate(index) => source.truncate(at(index))
}
}
}
fn mutation() -> impl Strategy<Value = Mutation>
{
prop_oneof![
3 => (any::<Index>(), select(TOKENS))
.prop_map(|(index, token)| Mutation::Insert(index, token)),
3 => any::<Index>().prop_map(Mutation::Delete),
3 => (any::<Index>(), select(TOKENS))
.prop_map(|(index, token)| Mutation::Replace(index, token)),
1 => any::<Index>().prop_map(Mutation::Truncate)
]
}
fn mutated_program() -> impl Strategy<Value = String>
{
(program(), prop::collection::vec(mutation(), 1..=3)).prop_map(
|(mut source, mutations)| {
for mutation in &mutations
{
mutation.apply(&mut source);
}
source
}
)
}