pub(crate) mod bitwise;
mod cases;
pub(super) mod casts;
pub(super) mod compares;
pub(super) mod demanded;
pub(super) mod forms;
mod gf2;
pub(super) mod invert;
pub(super) mod linear;
pub(super) mod linear_mba;
#[cfg(feature = "mba")]
pub(super) mod mba;
mod order;
mod poly;
pub(super) mod residue;
pub(super) mod shuffle;
#[cfg(test)]
mod tests;
pub(super) mod xor;
use super::{Accept, By, Fin, Reject, Runner, Step, Stop};
use crate::engine::Exhausted;
use crate::engine::budget::Counter;
use crate::expr::{Context, OpCode};
use crate::facts::{Facts, KnownBits, Reliance};
pub(crate) struct Marks {
stamp: Vec<u32>,
epoch: u32,
}
impl Default for Marks {
fn default() -> Self {
Marks {
stamp: Vec::new(),
epoch: 1,
}
}
}
impl Marks {
pub(crate) fn begin(&mut self, len: usize) {
if self.stamp.len() < len {
self.stamp.resize(len, 0);
}
self.clear();
}
pub(crate) fn clear(&mut self) {
self.epoch = self.epoch.wrapping_add(1);
if self.epoch == 0 {
self.stamp.fill(0);
self.epoch = 1;
}
}
#[inline]
pub(crate) fn insert(&mut self, i: u32) -> bool {
let i = i as usize;
if i >= self.stamp.len() {
self.stamp.resize(i + 1, 0);
}
let fresh = self.stamp[i] != self.epoch;
self.stamp[i] = self.epoch;
fresh
}
#[inline]
pub(crate) fn remove(&mut self, i: u32) -> bool {
match self.stamp.get_mut(i as usize) {
Some(s) if *s == self.epoch => {
*s = 0;
true
}
_ => false,
}
}
#[inline]
pub(crate) fn contains(&self, i: u32) -> bool {
self.stamp.get(i as usize) == Some(&self.epoch)
}
}
pub(crate) struct Counts {
stamp: Vec<u32>,
val: Vec<u32>,
epoch: u32,
}
impl Default for Counts {
fn default() -> Self {
Counts {
stamp: Vec::new(),
val: Vec::new(),
epoch: 1,
}
}
}
impl Counts {
pub(crate) fn begin(&mut self, len: usize) {
if self.stamp.len() < len {
self.stamp.resize(len, 0);
self.val.resize(len, 0);
}
self.epoch = self.epoch.wrapping_add(1);
if self.epoch == 0 {
self.stamp.fill(0);
self.epoch = 1;
}
}
#[inline]
pub(crate) fn get(&self, i: u32) -> Option<u32> {
let i = i as usize;
(self.stamp.get(i) == Some(&self.epoch)).then(|| self.val[i])
}
#[inline]
pub(crate) fn add(&mut self, i: u32) {
let i = i as usize;
if i >= self.stamp.len() {
self.stamp.resize(i + 1, 0);
self.val.resize(i + 1, 0);
}
if self.stamp[i] != self.epoch {
self.stamp[i] = self.epoch;
self.val[i] = 0;
}
self.val[i] += 1;
}
#[inline]
pub(crate) fn set(&mut self, i: u32, v: u32) {
let i = i as usize;
if i >= self.stamp.len() {
self.stamp.resize(i + 1, 0);
self.val.resize(i + 1, 0);
}
self.stamp[i] = self.epoch;
self.val[i] = v;
}
}
#[derive(Default)]
pub(crate) struct Scratch {
region: Marks,
seen: Marks,
kept: Marks,
dying: Counts,
local: Counts,
atoms: Marks,
zeros: Vec<u32>,
queue: std::collections::VecDeque<u32>,
stack: Vec<u32>,
dead: Vec<u32>,
decided: Marks,
}
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub(crate) enum PassKind {
FactFold,
Linear,
Bitwise,
Xor,
Compares,
Casts,
Demanded,
LinearMba,
Shuffle,
Invert,
#[cfg_attr(not(feature = "mba"), allow(dead_code))]
Mba,
}
impl PassKind {
pub(crate) fn name(self) -> &'static str {
match self {
PassKind::FactFold => "fact_fold",
PassKind::Linear => "linear",
PassKind::Bitwise => "bitwise",
PassKind::Xor => "xor",
PassKind::Compares => "compares",
PassKind::Casts => "casts",
PassKind::Demanded => "demanded",
PassKind::LinearMba => "linear_mba",
PassKind::Shuffle => "shuffle",
PassKind::Invert => "invert",
PassKind::Mba => "mba",
}
}
}
pub(super) fn step(
r: &mut Runner<'_, '_>,
cx: &mut Context,
kind: PassKind,
n: u32,
) -> Result<Step, Stop> {
r.stats.passes.entry(kind.name()).or_default().calls += 1;
r.meter.charge(Counter::PassWork, 1)?;
match kind {
PassKind::FactFold => fold(r, cx, n),
PassKind::Linear => linear::step(r, cx, n),
PassKind::Bitwise => bitwise::step(r, cx, n),
PassKind::Xor => xor::step(r, cx, n),
PassKind::Compares => compares::step(r, cx, n),
PassKind::Casts => casts::step(r, cx, n),
PassKind::Demanded => demanded::step(r, cx, n),
PassKind::LinearMba => linear_mba::step(r, cx, n),
PassKind::Shuffle => shuffle::step(r, cx, n),
PassKind::Invert => invert::step(r, cx, n),
PassKind::Mba => Ok(Step::Normal(Fin::FINAL)),
}
}
#[cfg(feature = "mba")]
pub(super) fn mba_step(
r: &mut Runner<'_, '_>,
cx: &mut Context,
cfg: &crate::mba::MbaConfig,
n: u32,
) -> Result<Step, Stop> {
if r.quarantined_passes.contains(&PassKind::Mba) {
return Ok(Step::Normal(Fin::PROVISIONAL));
}
if let Some(Some(fin)) = r.first.get(&n) {
return Ok(Step::Normal(*fin));
}
let additive = |op: OpCode| matches!(op, OpCode::Add | OpCode::Sub | OpCode::Neg | OpCode::Shl);
if let Some(&p) = r.chain_parent.get(&n) {
let (op, up) = (cx.node(n).op, cx.node(p).op);
if ((matches!(op, OpCode::Add | OpCode::Sub) && additive(up))
|| (op == OpCode::Mul && up == OpCode::Mul))
&& mba::asks(r, cx, cfg, p)?
{
return Ok(Step::Normal(Fin::PROVISIONAL));
}
}
r.stats.passes.entry("mba").or_default().calls += 1;
r.meter.charge(Counter::PassWork, 1)?;
mba::step(r, cx, cfg, n)
}
#[cfg(feature = "mba")]
pub(super) fn mba_first(
r: &mut Runner<'_, '_>,
cx: &mut Context,
cfg: &crate::mba::MbaConfig,
n: u32,
) -> Result<Option<(u32, Fin)>, Stop> {
let top = match r.chain_parent.get(&n) {
None => true,
Some(&p) => !mba::asks(r, cx, cfg, p)?,
};
if !top || !mba::asks(r, cx, cfg, n)? {
r.first.insert(n, None);
return Ok(None);
}
match mba_step(r, cx, cfg, n)? {
Step::To(x, fin) => {
r.first.insert(n, None);
Ok(Some((x, fin)))
}
Step::Normal(fin) => {
r.first.insert(n, Some(fin));
Ok(None)
}
}
}
pub(super) fn facts(
r: &mut Runner<'_, '_>,
cx: &mut Context,
n: u32,
) -> Result<(Option<Facts>, Fin), Stop> {
if let Some(v) = cx.const_val(n) {
return Ok((Some(Facts::constant(&v)), Fin::FINAL));
}
query(r, cx, n, |f| f, Context::try_facts_cap)
}
pub(super) fn known(
r: &mut Runner<'_, '_>,
cx: &mut Context,
n: u32,
) -> Result<(Option<KnownBits>, Fin), Stop> {
if let Some(v) = cx.const_val(n) {
return Ok((Some(KnownBits::constant(&v)), Fin::FINAL));
}
query(r, cx, n, |f| f.known(), Context::try_known_cap)
}
pub(super) fn known_words(
r: &mut Runner<'_, '_>,
cx: &mut Context,
n: u32,
) -> Result<(Option<[u64; 2]>, Fin), Stop> {
let words = |k: &KnownBits| [k.known_zero().limbs()[0], k.known_one().limbs()[0]];
if let Some(v) = cx.const_val(n) {
return Ok((Some(words(&KnownBits::constant(&v))), Fin::FINAL));
}
query(
r,
cx,
n,
|f| words(&f.known()),
Context::try_known_words_cap,
)
}
pub(super) fn disjoint(
r: &mut Runner<'_, '_>,
cx: &mut Context,
a: u32,
b: u32,
) -> Result<(bool, Fin, Fin), Stop> {
let bits = cx.width_of(a).bits();
if bits <= 64 {
let (ka, fa) = known_words(r, cx, a)?;
let (kb, fb) = known_words(r, cx, b)?;
let mask = u64::MAX >> (64 - bits);
let d = matches!((ka, kb), (Some([za, _]), Some([zb, _])) if !za & !zb & mask == 0);
return Ok((d, fa, fb));
}
let (ka, fa) = known(r, cx, a)?;
let (kb, fb) = known(r, cx, b)?;
let d = match (ka, kb) {
(Some(x), Some(y)) => crate::facts::known::bv_and(&x.maybe_one(), &y.maybe_one()).is_zero(),
_ => false,
};
Ok((d, fa, fb))
}
fn query<T>(
r: &mut Runner<'_, '_>,
cx: &mut Context,
n: u32,
of: impl FnOnce(Facts) -> T,
base: impl FnOnce(&mut Context, crate::Expr, u32) -> Result<Option<T>, crate::Error>,
) -> Result<(Option<T>, Fin), Stop> {
let left = r.meter.left().fact_work;
let cap = u32::try_from(left).unwrap_or(u32::MAX);
if cap == 0 {
return Err(Stop::Exhausted(Exhausted::FactWork));
}
let (work0, capped0) = (cx.facts.work, cx.facts.capped);
let e = cx.handle(n);
let (f, rel) = match r.assumptions {
Some(a) => match cx.facts_under_cap(e, a, cap).map_err(Stop::Error)? {
Ok((f, rel)) => (Some(of(f)), rel),
Err(_) => (None, Reliance::NONE),
},
None => (base(cx, e, cap).map_err(Stop::Error)?, Reliance::NONE),
};
r.meter.spent.fact_work += cx.facts.work - work0;
if cx.facts.capped != capped0 {
if cap < cx.config().fact_work {
return Err(Stop::Exhausted(Exhausted::FactWork));
}
r.stats.degraded += 1;
return Ok((None, Fin::capped(Exhausted::FactWork)));
}
r.meter.check()?;
Ok((f, Fin::relying(rel)))
}
fn fold(r: &mut Runner<'_, '_>, cx: &mut Context, n: u32) -> Result<Step, Stop> {
if cx.const_val(n).is_some() {
return Ok(Step::Normal(Fin::FINAL));
}
let (f, fin) = facts(r, cx, n)?;
let Some(v) = f.and_then(|f| f.as_constant()) else {
r.stats.passes.entry("fact_fold").or_default().noop += 1;
return Ok(Step::Normal(fin));
};
if let Some(h) = r.hooks
&& !h.fold_known(cx, cx.handle(n))
{
r.stats.hook_vetoes += 1;
return Ok(Step::Normal(fin));
}
let before = cx.len() as u32;
let c = r.build(cx, |cx| cx.mk_const(&v))?;
finish(r, cx, PassKind::FactFold, n, c, before, &[], fin)
}
fn refresh_uses(r: &mut Runner<'_, '_>, cx: &Context) -> Result<(), Stop> {
if !r.uses_on {
r.live.clear();
r.scratch.seen.begin(cx.len());
let others = others_hold(r, cx);
let mut stack: Vec<u32> = if others {
if let Err(e) = charge_each(&mut r.meter, r.live.others.nodes as u64) {
r.live.clear();
return Err(e.into());
}
r.live.base = r.live.others.nodes > 0;
r.live_roots.get(r.active).copied().into_iter().collect()
} else {
r.live_roots.clone()
};
while let Some(i) = stack.pop() {
if (others && r.live.others.reach.contains(i))
|| !r.scratch.seen.insert(i)
|| r.dead.contains(i)
{
continue;
}
if let Err(e) = r.meter.charge(Counter::PassWork, 1) {
r.live.clear();
return Err(e.into());
}
for c in cx.node(i).children() {
r.live.add(c);
stack.push(c);
}
r.live.count(i);
}
r.uses_on = true;
r.uses_upto = r.phase_start;
}
let len = cx.len() as u32;
let from = r.uses_upto.min(len);
r.meter.charge(Counter::PassWork, u64::from(len - from))?;
let mut stack: Vec<u32> = (from..len)
.rev()
.filter(|&i| !r.dead.contains(i) && !r.live.counted(i))
.collect();
while let Some(i) = stack.pop() {
if !r.live.count(i) {
continue;
}
r.dead.remove(i);
r.meter.charge(Counter::PassWork, 1)?;
for c in cx.node(i).children() {
r.live.add(c);
if !r.live.counted(c) {
stack.push(c);
}
}
}
r.uses_upto = len;
Ok(())
}
fn charge_each(meter: &mut crate::engine::budget::Meter, n: u64) -> Result<(), Exhausted> {
if meter.charge(Counter::PassWork, n).is_err() {
for _ in 0..n {
meter.charge(Counter::PassWork, 1)?;
}
}
Ok(())
}
#[derive(Default)]
pub(crate) struct Others {
active: Option<usize>,
reach: Marks,
nodes: u32,
uses: Counts,
}
#[derive(Default)]
pub(crate) struct Live {
others: Others,
base: bool,
counted: Marks,
uncounted: Marks,
stamp: Vec<u32>,
delta: Vec<i32>,
epoch: u32,
}
impl Live {
pub(crate) fn clear(&mut self) {
self.base = false;
self.counted.clear();
self.uncounted.clear();
self.epoch = self.epoch.wrapping_add(1);
if self.epoch == 0 {
self.stamp.fill(0);
self.epoch = 1;
}
}
pub(crate) fn forget_others(&mut self) {
self.others.active = None;
}
#[inline]
fn in_base(&self, i: u32) -> bool {
self.base && self.others.reach.contains(i)
}
#[inline]
fn delta(&self, i: u32) -> Option<i32> {
let i = i as usize;
(self.stamp.get(i) == Some(&self.epoch)).then(|| self.delta[i])
}
#[inline]
fn change(&mut self, i: u32, by: i32) {
let i = i as usize;
if i >= self.stamp.len() {
self.stamp.resize(i + 1, 0);
self.delta.resize(i + 1, 0);
}
if self.stamp[i] != self.epoch {
self.stamp[i] = self.epoch;
self.delta[i] = 0;
}
self.delta[i] += by;
}
#[inline]
pub(crate) fn uses(&self, i: u32) -> Option<u32> {
let d = self.delta(i);
if !self.base {
return d.map(|d| d.max(0) as u32);
}
match (self.others.uses.get(i), d) {
(None, None) => None,
(b, d) => Some((i64::from(b.unwrap_or(0)) + i64::from(d.unwrap_or(0))).max(0) as u32),
}
}
#[inline]
pub(crate) fn add(&mut self, i: u32) {
self.change(i, 1);
}
#[inline]
pub(crate) fn dec(&mut self, i: u32) -> Option<u32> {
let n = self.uses(i)?;
if n > 0 {
self.change(i, -1);
}
Some(n.saturating_sub(1))
}
#[inline]
pub(crate) fn counted(&self, i: u32) -> bool {
(self.in_base(i) && !self.uncounted.contains(i)) || self.counted.contains(i)
}
#[inline]
pub(crate) fn count(&mut self, i: u32) -> bool {
if self.counted(i) {
return false;
}
if self.in_base(i) {
self.uncounted.remove(i);
} else {
self.counted.insert(i);
}
true
}
#[inline]
pub(crate) fn uncount(&mut self, i: u32) -> bool {
if self.counted.remove(i) {
return true;
}
self.in_base(i) && self.uncounted.insert(i)
}
}
fn others_hold(r: &mut Runner<'_, '_>, cx: &Context) -> bool {
let o = &mut r.live.others;
if o.active != Some(r.active) {
o.active = Some(r.active);
o.reach.clear();
o.uses.begin(0);
o.nodes = 0;
let mut stack: Vec<u32> = r
.live_roots
.iter()
.enumerate()
.filter(|&(k, _)| k != r.active)
.map(|(_, &i)| i)
.collect();
while let Some(i) = stack.pop() {
if !o.reach.insert(i) {
continue;
}
o.nodes += 1;
for c in cx.node(i).children() {
o.uses.add(c);
stack.push(c);
}
}
}
!r.dead_log.iter().any(|&d| o.reach.contains(d))
}
fn region(r: &mut Runner<'_, '_>, cx: &Context, n: u32, atoms: &[u32]) -> Result<(), Stop> {
let sc = &mut r.scratch;
sc.atoms.begin(cx.len());
for &a in atoms {
sc.atoms.insert(a);
}
sc.region.begin(cx.len());
sc.local.begin(cx.len());
sc.zeros.clear();
sc.queue.clear();
sc.queue.push_back(n);
let mut size = 0u32;
let mut last = None;
while let Some(i) = sc.queue.pop_front() {
if !sc.region.insert(i) {
continue;
}
r.meter.charge(Counter::PassWork, 1)?;
if r.uses_on && r.live.uses(i) == Some(0) {
sc.zeros.push(i);
}
size += 1;
if size >= REGION_CAP {
last = Some(i);
break;
}
for c in cx.node(i).children() {
if !sc.atoms.contains(c) {
sc.local.add(c);
sc.queue.push_back(c);
}
}
}
if let Some(i) = last {
for c in cx.node(i).children() {
if sc.region.contains(c) {
sc.local.add(c);
}
}
}
Ok(())
}
fn dying_in(
r: &mut Runner<'_, '_>,
cx: &Context,
n: u32,
kept: bool,
limit: u32,
shared: bool,
) -> Result<u32, Stop> {
if shared {
refresh_uses(r, cx)?;
}
let sc = &mut r.scratch;
sc.dying.begin(cx.len());
sc.decided.begin(cx.len());
sc.dead.clear();
let uses = |sc: &Scratch, live: &Live, i: u32| {
if shared {
live.uses(i)
} else {
sc.local.get(i)
}
};
if !(kept && sc.kept.contains(n)) {
sc.decided.insert(n);
sc.dead.push(n);
}
if shared {
for k in 0..sc.zeros.len() {
let z = sc.zeros[k];
if !(kept && sc.kept.contains(z))
&& uses(sc, &r.live, z) == Some(0)
&& sc.decided.insert(z)
{
sc.dead.push(z);
}
}
}
let mut count = 0u32;
while let Some(i) = sc.dead.pop() {
count += 1;
if count >= limit {
return Ok(count);
}
for c in cx.node(i).children() {
if !sc.region.contains(c) {
continue;
}
sc.dying.add(c);
if sc.decided.contains(c) || (kept && sc.kept.contains(c)) {
continue;
}
if sc.dying.get(c).unwrap_or(0) >= uses(sc, &r.live, c).unwrap_or(u32::MAX) {
sc.decided.insert(c);
sc.dead.push(c);
}
}
}
Ok(count)
}
fn needed(r: &mut Runner<'_, '_>, cx: &Context, e: u32) -> Result<Option<u32>, Stop> {
let sc = &mut r.scratch;
sc.seen.begin(cx.len());
sc.kept.begin(cx.len());
sc.stack.clear();
sc.stack.push(e);
let mut new = 0u32;
while let Some(i) = sc.stack.pop() {
if !sc.seen.insert(i) || sc.atoms.contains(i) {
continue;
}
r.meter.charge(Counter::PassWork, 1)?;
if sc.region.contains(i) {
sc.kept.insert(i);
continue;
}
new += 1;
if new > REGION_CAP {
r.meter.check()?;
return Ok(None);
}
sc.stack.extend(cx.node(i).children());
}
r.meter.check()?;
Ok(Some(new))
}
pub(super) const REGION_CAP: u32 = 1024;
pub(super) fn worth_building(
r: &mut Runner<'_, '_>,
cx: &Context,
n: u32,
atoms: &[u32],
estimate: u32,
) -> Result<Option<Fin>, Stop> {
region(r, cx, n, atoms)?;
r.meter.check()?;
if dying_in(r, cx, n, false, estimate.saturating_add(1), true)? > estimate {
return Ok(None);
}
let alone = dying_in(r, cx, n, false, estimate.saturating_add(1), false)?;
Ok(Some(if alone > estimate {
Fin::PROVISIONAL
} else {
Fin::FINAL
}))
}
pub(super) fn shrinks(
r: &mut Runner<'_, '_>,
cx: &Context,
n: u32,
e: u32,
atoms: &[u32],
) -> Result<Result<(), Fin>, Stop> {
if cx.node(e).op == OpCode::Const && cx.node(n).op != OpCode::Const {
return Ok(Ok(()));
}
region(r, cx, n, atoms)?;
r.meter.check()?;
let Some(new) = needed(r, cx, e)? else {
return Ok(Err(Fin::FINAL));
};
let freed = dying_in(r, cx, n, true, new.saturating_add(1), true)?;
if new >= freed {
let alone = dying_in(r, cx, n, true, new.saturating_add(1), false)?;
return Ok(Err(if new < alone {
Fin::PROVISIONAL
} else {
Fin::FINAL
}));
}
Ok(Ok(()))
}
#[allow(clippy::too_many_arguments)]
pub(super) fn finish(
r: &mut Runner<'_, '_>,
cx: &mut Context,
kind: PassKind,
n: u32,
e: u32,
before: u32,
atoms: &[u32],
fin: Fin,
) -> Result<Step, Stop> {
let name = kind.name();
if e == n {
r.stats.passes.entry(name).or_default().noop += 1;
return Ok(Step::Normal(fin));
}
if let Err(more) = shrinks(r, cx, n, e, atoms)? {
r.stats.passes.entry(name).or_default().rejected_cost += 1;
discard(r, cx, before);
return Ok(Step::Normal(fin.and(more)));
}
match r.accept(cx, By::Pass(name), true, n, e, fin.rel)? {
Accept::Yes => {
r.stats.passes.entry(name).or_default().changed += 1;
Ok(Step::To(e, fin))
}
Accept::Vetoed => {
r.stats.passes.entry(name).or_default().rejected += 1;
discard(r, cx, before);
Ok(Step::Normal(fin))
}
Accept::Rejected(reason) => {
r.stats.passes.entry(name).or_default().rejected += 1;
discard(r, cx, before);
if matches!(reason, Reject::Width | Reject::Verify | Reject::Tripwire) {
r.quarantined_passes.push(kind);
r.stats.quarantined += 1;
}
Ok(Step::Normal(fin))
}
}
}
pub(super) fn discard(r: &mut Runner<'_, '_>, cx: &Context, before: u32) {
for i in before..cx.len() as u32 {
r.retire(cx, i);
}
}