use rucc_ir::{
Block, Builder, Def, Extra, Flags, Func, Inst, InstData, IntPred, MemInfo, Opcode, Type, Value,
};
use crate::cfg::Cfg;
use crate::discharge::{Question, operand_of, yes};
use crate::dom::Dominators;
use crate::loops::{LoopId, Loops};
use crate::rules::safety;
use crate::scev::{Assumption, Count, Invariant, Scev};
use crate::{Analyses, Analysis, Fuel, Pass, Preserved, Stats};
const HOISTED: &str = "bounds check taken out of a loop, one check in front of it covers every \
iteration";
const NO_FUEL: &str = "bounds check kept, the pass ran out of fuel";
const NO_PREHEADER: &str = "loop left alone, it has no block in front of it to put a check in";
const ANOTHER_WAY_OUT: &str =
"loop left alone, it can be left somewhere other than its bottom test";
const A_LOOP_INSIDE: &str = "loop left alone, it has another loop inside it";
const A_CALL_INSIDE: &str = "loop left alone, a call in it might not come back";
const NOT_COUNTED: &str = "loop left alone, how many times it runs is not settled before it starts";
const NOT_SIGNED: &str = "loop left alone, how many times it runs is not read as a signed number";
const COUNT_TOO_WIDE: &str =
"bounds check kept, how many bytes the loop covers might not fit in sixty four bits";
const NOT_A_SWEEP: &str = "bounds check kept, its address does not walk the loop by a constant";
const ALREADY_COMPUTED: &str =
"bounds check kept, how many bytes it covers is a number only the program has";
const BACKWARDS: &str = "bounds check kept, its address walks the loop from high to low";
const NOT_EVERY_TIME: &str = "bounds check kept, an iteration can finish without reaching it";
const MISALIGNED: &str = "bounds check kept, its step is not a whole number of its alignment";
const TOO_WIDE: &str = "bounds check kept, the range the loop sweeps is too wide for the rule";
#[derive(Debug)]
pub struct Hoist;
impl Pass for Hoist {
fn name(&self) -> &'static str {
"hoist"
}
fn describe(&self) -> &'static str {
"a bounds check in a counted loop becomes one check in front of the loop"
}
fn preserves(&self) -> Preserved {
Preserved::ALL.without(Analysis::Liveness).without(Analysis::Pressure)
}
fn run(&self, func: &mut Func, an: &mut Analyses, fuel: &mut Fuel) -> Stats {
let mut stats = Stats::new();
if func.entry().is_none() {
return stats;
}
let cfg = an.cfg(func).clone();
let doms = an.dominators(func).clone();
let loops = an.loops(func).clone();
if loops.count() == 0 {
return stats;
}
let mut plans = Vec::new();
{
let mut scev = Scev::new(func, &cfg, &loops);
for id in loops.all() {
sweep(func, &cfg, &doms, &loops, &mut scev, id, &mut plans, &mut stats);
}
}
for plan in plans {
if !fuel.take() {
stats.missed(NO_FUEL);
continue;
}
apply(func, &plan);
stats.optimized(HOISTED);
}
stats
}
}
#[derive(Debug)]
struct Plan {
preheader: Block,
base: Value,
offset: i128,
span: Extent,
info: MemInfo,
check: Inst,
}
#[derive(Clone, Copy, Debug)]
enum Extent {
Bytes(u64),
Computed { count: Invariant, step: i128, reach: i128 },
}
#[derive(Clone, Copy, Debug)]
enum Around {
Number(i128),
Computed(Invariant),
}
#[expect(clippy::too_many_arguments, reason = "three analyses, a plan list and a report to fill")]
fn sweep(
func: &Func,
cfg: &Cfg,
doms: &Dominators,
loops: &Loops,
scev: &mut Scev<'_>,
id: LoopId,
plans: &mut Vec<Plan>,
stats: &mut Stats,
) {
let checks: Vec<Inst> = loops
.blocks(id)
.iter()
.filter(|&&block| loops.innermost(block) == Some(id))
.flat_map(|&block| func.insts(block).collect::<Vec<Inst>>())
.filter(|&inst| func[inst].opcode == Opcode::CheckBounds)
.collect();
if checks.is_empty() {
return;
}
let (preheader, guard) = match shaped(func, cfg, doms, loops, id) {
Ok(shape) => shape,
Err(why) => {
stats.missed(why);
return;
}
};
let around = match counted(scev, id) {
Ok(around) => around,
Err(why) => {
stats.missed(why);
return;
}
};
for check in checks {
match planned(func, doms, scev, id, preheader, guard, around, check) {
Ok(plan) => plans.push(plan),
Err(why) => stats.missed(why),
}
}
}
fn counted(scev: &mut Scev<'_>, id: LoopId) -> Result<Around, &'static str> {
let bound = scev.bound(id).ok_or(NOT_COUNTED)?;
if let Some(Count::Exact(exact)) = bound.under_undefined_overflow() {
return i128::try_from(exact).map(Around::Number).map_err(|_| NOT_COUNTED);
}
let (Count::Symbolic(count), assumptions) = bound.parts() else {
return Err(NOT_COUNTED);
};
if !assumptions.contains(&Assumption::StrictOverflow) {
return Err(NOT_SIGNED);
}
let known = assumptions
.iter()
.all(|rests_on| matches!(rests_on, Assumption::StrictOverflow | Assumption::Approaching));
if !known {
return Err(NOT_COUNTED);
}
Ok(Around::Computed(count))
}
fn shaped(
func: &Func,
cfg: &Cfg,
doms: &Dominators,
loops: &Loops,
id: LoopId,
) -> Result<(Block, Block), &'static str> {
let Some(preheader) = loops.preheader(cfg, id) else {
return Err(NO_PREHEADER);
};
let [latch] = loops.latches(id) else {
return Err(ANOTHER_WAY_OUT);
};
let [exit] = loops.exits(id) else {
return Err(ANOTHER_WAY_OUT);
};
if !doms.dominates(exit.from, *latch) {
return Err(ANOTHER_WAY_OUT);
}
for &block in loops.blocks(id) {
if loops.innermost(block) != Some(id) {
return Err(A_LOOP_INSIDE);
}
if cfg.successors(block).is_empty() {
return Err(ANOTHER_WAY_OUT);
}
for inst in func.insts(block) {
if matches!(
func[inst].opcode,
Opcode::Call
| Opcode::CallIndirect
| Opcode::TailCall
| Opcode::InlineAsm
| Opcode::MetaEnd
| Opcode::MetaTransfer
) {
return Err(A_CALL_INSIDE);
}
}
}
Ok((preheader, exit.from))
}
#[expect(clippy::too_many_arguments, reason = "each one is a separate thing the answer rests on")]
fn planned(
func: &Func,
doms: &Dominators,
scev: &mut Scev<'_>,
id: LoopId,
preheader: Block,
guard: Block,
around: Around,
check: Inst,
) -> Result<Plan, &'static str> {
let block = func.block_of(check).ok_or(NOT_EVERY_TIME)?;
if !doms.dominates(block, guard) {
return Err(NOT_EVERY_TIME);
}
let args = &func[func[check].args];
if args.len() > 2 {
return Err(ALREADY_COMPUTED);
}
let (Some(&capability), Some(&pointer)) = (args.first(), args.get(1)) else {
return Err(NOT_A_SWEEP);
};
if operand_of(func, capability, Opcode::CapOf, 0) != Some(pointer) {
return Err(NOT_A_SWEEP);
}
let Extra::Mem(held) = func[check].extra else { return Err(NOT_A_SWEEP) };
let info = func[held];
let Some(chrec) = scev.evolution(id, pointer).chrec() else {
return Err(NOT_A_SWEEP);
};
let Some(step) = chrec.step.as_number() else {
return Err(NOT_A_SWEEP);
};
if step <= 0 {
return Err(BACKWARDS);
}
let (Some(base), 1) = (chrec.base.value, chrec.base.scale) else {
return Err(NOT_A_SWEEP);
};
let offset = chrec.base.offset;
if step % i128::from(info.align) != 0 {
return Err(MISALIGNED);
}
let reach = i128::from(info.size);
let span = match around {
Around::Number(around) => {
let far = around.checked_mul(step).ok_or(TOO_WIDE)?;
let span = far.checked_add(reach).ok_or(TOO_WIDE)?;
if !swept(span, far, reach) {
return Err(TOO_WIDE);
}
Extent::Bytes(u64::try_from(span).map_err(|_| TOO_WIDE)?)
}
Around::Computed(count) => {
fits(func, count, step, reach)?;
if !swept_sym(reach) {
return Err(TOO_WIDE);
}
Extent::Computed { count, step, reach }
}
};
Ok(Plan { preheader, base, offset, span, info, check })
}
fn fits(func: &Func, count: Invariant, step: i128, reach: i128) -> Result<(), &'static str> {
let value = count.value.ok_or(COUNT_TOO_WIDE)?;
let ty = func[value].ty;
if !ty.is_int() || ty.bits() >= 64 {
return Err(COUNT_TOO_WIDE);
}
let most = 1i128 << (ty.bits() - 1);
let reached = count
.scale
.checked_abs()
.and_then(|scale| scale.checked_mul(most))
.and_then(|far| far.checked_add(count.offset.checked_abs()?))
.ok_or(COUNT_TOO_WIDE)?;
let span =
reached.checked_mul(step).and_then(|far| far.checked_add(reach)).ok_or(COUNT_TOO_WIDE)?;
if span > i128::from(i64::MAX) {
return Err(COUNT_TOO_WIDE);
}
Ok(())
}
fn swept(span: i128, far: i128, reach: i128) -> bool {
let mut question = Question::default();
let at = question.opaque();
let at = question.app("value.i64", &[at]);
let span = question.number(span);
let span = question.app("iconst.i64", &[span]);
let far = question.number(far);
let far = question.app("iconst.i64", &[far]);
let reach = question.number(reach);
let reach = question.app("iconst.i64", &[reach]);
let delta = question.opaque();
let delta = question.app("value.i64", &[delta]);
let term = question.app("swept.i64", &[at, span, far, reach, delta]);
match safety::TABLE.find(&question, term) {
Some(found) => yes(&safety::TABLE, found.rule),
None => false,
}
}
fn swept_sym(reach: i128) -> bool {
let mut question = Question::default();
let at = question.opaque();
let at = question.app("value.i64", &[at]);
let span = question.opaque();
let span = question.app("value.i64", &[span]);
let far = question.opaque();
let far = question.app("value.i64", &[far]);
let reach = question.number(reach);
let reach = question.app("iconst.i64", &[reach]);
let delta = question.opaque();
let delta = question.app("value.i64", &[delta]);
let term = question.app("swept.sym.i64", &[at, span, far, reach, delta]);
match safety::TABLE.find(&question, term) {
Some(found) => yes(&safety::TABLE, found.rule),
None => false,
}
}
fn apply(func: &mut Func, plan: &Plan) {
let term = func.terminator(plan.preheader).expect("a preheader ends in a jump to the header");
let mut made = Vec::new();
let mut build = Builder::new(func, plan.preheader);
let first = if plan.offset == 0 {
plan.base
} else {
let by = build.iconst(Type::int(64), plan.offset);
made.push(by);
let args = build.func().push_values(&[plan.base, by]);
let sum = build.value(InstData { args, ..InstData::new(Opcode::PtrAdd) }, Type::PTR);
made.push(sum);
sum
};
let args = build.func().push_values(&[first]);
let capability = build.value(InstData { args, ..InstData::new(Opcode::CapOf) }, Type::CAP);
made.push(capability);
let (size, extent) = match plan.span {
Extent::Bytes(bytes) => (bytes, None),
Extent::Computed { count, step, reach } => {
(plan.info.size, Some(computed(&mut build, &mut made, count, step, reach)))
}
};
let info = MemInfo { size, ..plan.info };
let extra = Extra::Mem(build.func().add_mem(info));
let operands: Vec<Value> = match extent {
Some(bytes) => vec![capability, first, bytes],
None => vec![capability, first],
};
let args = build.func().push_values(&operands);
let check = build.inst(InstData { args, extra, ..InstData::new(Opcode::CheckBounds) }, &[]);
for value in made {
let inst = inst_of(func, value);
func.remove_inst(inst);
func.insert_before(inst, term);
}
func.remove_inst(check);
func.insert_before(check, term);
func.remove_inst(plan.check);
}
fn computed(
build: &mut Builder<'_>,
made: &mut Vec<Value>,
count: Invariant,
step: i128,
reach: i128,
) -> Value {
let word = Type::int(64);
let value = count.value.expect("a count that is an expression is built on a value");
let mut wide = build.unary(Opcode::SExt, value, word);
made.push(wide);
if count.scale != 1 {
let scale = build.iconst(word, count.scale);
made.push(scale);
wide = build.binary(Opcode::Mul, wide, scale, Flags::NSW);
made.push(wide);
}
if count.offset != 0 {
let offset = build.iconst(word, count.offset);
made.push(offset);
wide = build.binary(Opcode::Add, wide, offset, Flags::NSW);
made.push(wide);
}
let zero = build.iconst(word, 0);
made.push(zero);
let entered = build.icmp(IntPred::Sgt, wide, zero);
made.push(entered);
let mut span = build.select(entered, wide, zero);
made.push(span);
if step != 1 {
let by = build.iconst(word, step);
made.push(by);
span = build.binary(Opcode::Mul, span, by, Flags::NSW);
made.push(span);
}
if reach != 0 {
let last = build.iconst(word, reach);
made.push(last);
span = build.binary(Opcode::Add, span, last, Flags::NSW);
made.push(span);
}
span
}
fn inst_of(func: &Func, value: Value) -> Inst {
let Def::Result { inst, .. } = func[value].def else {
unreachable!("the builder was just asked for an instruction that produces this")
};
inst
}
#[cfg(test)]
mod tests {
use rucc_base::Interner;
use rucc_ir::{Flags, IntPred, MemInfo, MemOrder, Module, Restrict, Signature, verify_func};
use rucc_target::{TargetInfo, Triple};
use super::{HOISTED, Hoist};
use crate::canon::Canon;
use crate::stats::Kind;
use crate::{Analyses, Fuel, Pass, Stats};
use rucc_ir::{Block, Builder, Extra, Func, Inst, InstData, Opcode, Type, Value};
const WIDTH: i128 = 4;
fn walking(trips: i128, step: i128, size: u64, align: u32) -> (Interner, Func, Vec<Block>) {
promising(trips, step, size, align, Flags::NSW)
}
fn promising(
trips: i128,
step: i128,
size: u64,
align: u32,
flags: Flags,
) -> (Interner, Func, Vec<Block>) {
let mut names = Interner::new();
let signature = Signature::new().with_params(&[Type::PTR]);
let mut func = Func::new(names.intern("f"), signature);
let entry = func.create_block();
let head = func.create_block();
let done = func.create_block();
let array = func.append_param(entry, Type::PTR);
let counter = func.append_param(head, Type::int(64));
let zero = Builder::new(&mut func, entry).iconst(Type::int(64), 0);
Builder::new(&mut func, entry).jump(head, &[zero]);
let mut build = Builder::new(&mut func, head);
let by = build.iconst(Type::int(64), step);
let scaled = build.binary(Opcode::Mul, counter, by, Flags::NSW);
let args = build.func().push_values(&[array, scaled]);
let pointer = build.value(InstData { args, ..InstData::new(Opcode::PtrAdd) }, Type::PTR);
check(&mut build, pointer, size, align);
let one = build.iconst(Type::int(64), 1);
let next = build.binary(Opcode::Add, counter, one, flags);
let limit = build.iconst(Type::int(64), trips);
let again = build.icmp(IntPred::Slt, next, limit);
build.br_if(again, head, &[next], done, &[]);
Builder::new(&mut func, done).ret(&[]);
(names, func, vec![entry, head, done])
}
fn check(build: &mut Builder<'_>, pointer: Value, size: u64, align: u32) {
let args = build.func().push_values(&[pointer]);
let capability = build.value(InstData { args, ..InstData::new(Opcode::CapOf) }, Type::CAP);
let info = MemInfo {
size,
align,
order: MemOrder::NotAtomic,
tbaa: None,
restrict: Restrict::NONE,
};
let args = build.func().push_values(&[capability, pointer]);
let extra = Extra::Mem(build.func().add_mem(info));
build.inst(InstData { args, extra, ..InstData::new(Opcode::CheckBounds) }, &[]);
}
fn hoisted(func: &mut Func) -> Stats {
let mut an = Analyses::new();
Canon.run(func, &mut an, &mut Fuel::unlimited());
Hoist.run(func, &mut an, &mut Fuel::unlimited())
}
fn checks(func: &Func) -> Vec<(Block, Inst)> {
func.blocks()
.flat_map(|block| func.insts(block).map(move |inst| (block, inst)).collect::<Vec<_>>())
.filter(|&(_, inst)| func[inst].opcode == Opcode::CheckBounds)
.collect()
}
fn extent(func: &Func, check: Inst) -> u64 {
let Extra::Mem(info) = func[check].extra else { panic!("a check carries a payload") };
func[info].size
}
fn sound(func: &Func, names: &mut Interner) {
let target = TargetInfo::new("x86_64-unknown-linux-gnu".parse::<Triple>().unwrap());
let module = Module::new(names.intern("t.c"), &target);
if let Err(errors) = verify_func(&module, func, names) {
panic!("{errors:#?}");
}
}
#[test]
fn a_check_that_walks_a_counted_loop_becomes_one_check_in_front_of_it() {
let (mut names, mut func, _) = walking(16, WIDTH, 4, 4);
let stats = hoisted(&mut func);
assert_eq!(stats.count(Kind::Optimized, HOISTED), 1);
let left = checks(&func);
assert_eq!(left.len(), 1, "one check, and it is the one that was put in front");
assert_eq!(extent(&func, left[0].1), 64, "fifteen steps of four, plus the last read");
sound(&func, &mut names);
}
#[test]
fn a_loop_whose_counter_promises_nothing_keeps_its_check() {
let (_, mut func, _) = promising(16, WIDTH, 4, 4, Flags::NONE);
let stats = hoisted(&mut func);
assert!(!stats.changed());
assert_eq!(stats.count(Kind::Missed, super::NOT_COUNTED), 1);
assert_eq!(checks(&func).len(), 1, "and it is still in the body");
}
#[test]
fn the_check_that_is_left_is_outside_the_loop() {
let (_, mut func, _) = walking(16, WIDTH, 4, 4);
hoisted(&mut func);
let (block, _) = checks(&func)[0];
let (cfg, doms, loops) = forest(&func);
let _ = doms;
let id = loops.all().next().expect("there is a loop");
assert!(!loops.contains(id, block), "the check is not in the loop any more");
assert_eq!(loops.preheader(&cfg, id), Some(block), "it is in the preheader");
}
fn forest(func: &Func) -> (crate::Cfg, crate::Dominators, crate::Loops) {
let cfg = crate::Cfg::new(func);
let doms = crate::Dominators::new(&cfg);
let loops = crate::Loops::new(&cfg, &doms);
(cfg, doms, loops)
}
#[test]
fn a_walk_whose_step_is_wider_than_its_access_covers_the_gaps_too() {
let (mut names, mut func, _) = walking(8, 16, 4, 4);
assert_eq!(hoisted(&mut func).count(Kind::Optimized, HOISTED), 1);
assert_eq!(extent(&func, checks(&func)[0].1), 116, "seven steps of sixteen, plus four");
sound(&func, &mut names);
}
fn unknown(ty: Type, pred: IntPred, flags: Flags) -> (Interner, Func, Vec<Block>) {
let mut names = Interner::new();
let signature = Signature::new().with_params(&[Type::PTR, ty]);
let mut func = Func::new(names.intern("f"), signature);
let entry = func.create_block();
let head = func.create_block();
let done = func.create_block();
let array = func.append_param(entry, Type::PTR);
let limit = func.append_param(entry, ty);
let counter = func.append_param(head, ty);
let zero = Builder::new(&mut func, entry).iconst(ty, 0);
Builder::new(&mut func, entry).jump(head, &[zero]);
let mut build = Builder::new(&mut func, head);
let wide = if ty == Type::int(64) {
counter
} else {
build.unary(Opcode::SExt, counter, Type::int(64))
};
let by = build.iconst(Type::int(64), WIDTH);
let scaled = build.binary(Opcode::Mul, wide, by, Flags::NSW);
let args = build.func().push_values(&[array, scaled]);
let pointer = build.value(InstData { args, ..InstData::new(Opcode::PtrAdd) }, Type::PTR);
check(&mut build, pointer, 4, 4);
let one = build.iconst(ty, 1);
let next = build.binary(Opcode::Add, counter, one, flags);
let again = build.icmp(pred, next, limit);
build.br_if(again, head, &[next], done, &[]);
Builder::new(&mut func, done).ret(&[]);
(names, func, vec![entry, head, done])
}
fn operands(func: &Func, check: Inst) -> Vec<Value> {
func[func[check].args].to_vec()
}
#[test]
fn a_loop_that_runs_a_number_of_times_nobody_knows_gets_a_check_that_works_it_out() {
let (mut names, mut func, _) = unknown(Type::int(32), IntPred::Slt, Flags::NSW);
let stats = hoisted(&mut func);
assert_eq!(stats.count(Kind::Optimized, HOISTED), 1);
let left = checks(&func);
assert_eq!(left.len(), 1, "one check, and it is the one that was put in front");
assert_eq!(operands(&func, left[0].1).len(), 3, "its extent is an operand");
assert_eq!(extent(&func, left[0].1), 4, "and its payload is one element of the walk");
sound(&func, &mut names);
}
#[test]
fn the_extent_a_loop_of_unknown_length_gets_is_the_one_the_arithmetic_says() {
let (_, mut func, blocks) = unknown(Type::int(32), IntPred::Slt, Flags::NSW);
hoisted(&mut func);
let (block, check) = checks(&func)[0];
assert_ne!(block, blocks[1], "the check is out of the body");
let bytes = operands(&func, check)[2];
let steps: Vec<Opcode> = func
.insts(block)
.map(|inst| func[inst].opcode)
.filter(|&opcode| {
matches!(opcode, Opcode::SExt | Opcode::Add | Opcode::ICmp | Opcode::Select)
})
.collect();
assert_eq!(
steps,
[Opcode::SExt, Opcode::Add, Opcode::ICmp, Opcode::Select, Opcode::Add],
"sign extend, take one off, clamp at zero, and add the last read back on"
);
assert_eq!(func[bytes].ty, Type::int(64), "the extent is a word wide");
}
#[test]
fn a_loop_counted_as_wide_as_the_arithmetic_keeps_its_check() {
let (_, mut func, _) = unknown(Type::int(64), IntPred::Slt, Flags::NSW);
let stats = hoisted(&mut func);
assert!(!stats.changed());
assert_eq!(stats.count(Kind::Missed, super::COUNT_TOO_WIDE), 1);
assert_eq!(checks(&func).len(), 1, "and it is still in the body");
}
#[test]
fn a_loop_whose_exit_test_is_unsigned_keeps_its_check() {
let (_, mut func, _) = unknown(Type::int(32), IntPred::Ult, Flags::NSW);
let stats = hoisted(&mut func);
assert!(!stats.changed());
assert_eq!(stats.count(Kind::Missed, super::NOT_SIGNED), 1);
}
#[test]
fn a_loop_whose_counter_of_unknown_length_promises_nothing_keeps_its_check() {
let (_, mut func, _) = unknown(Type::int(32), IntPred::Slt, Flags::NONE);
let stats = hoisted(&mut func);
assert!(!stats.changed());
assert_eq!(stats.count(Kind::Missed, super::NOT_COUNTED), 1);
}
#[test]
fn a_loop_with_a_call_in_it_keeps_its_check() {
let (mut names, mut func, blocks) = walking(16, WIDTH, 4, 4);
let head = blocks[1];
let term = func.terminator(head).expect("the header branches");
let callee = names.intern("might_not_return");
let signature = func.add_signature(Signature::new());
let call = Builder::new(&mut func, head).call(callee, signature, &[]);
func.remove_inst(call);
func.insert_before(call, term);
let stats = hoisted(&mut func);
assert!(!stats.changed());
assert_eq!(stats.count(Kind::Missed, super::A_CALL_INSIDE), 1);
}
#[test]
fn a_loop_that_can_be_left_early_keeps_its_check() {
let (mut names, mut func, blocks) = walking(16, WIDTH, 4, 4);
let (head, done) = (blocks[1], blocks[2]);
let split = func.create_block();
let term = func.terminator(head).expect("the header branches");
let mut build = Builder::new(&mut func, head);
let counter = build.func()[head].params[0];
let seven = build.iconst(Type::int(64), 7);
let bail = build.icmp(IntPred::Eq, counter, seven);
let leave = build.br_if(bail, done, &[], split, &[]);
for inst in [inst_of(&func, seven), inst_of(&func, bail), leave] {
func.remove_inst(inst);
func.insert_before(inst, term);
}
let rest: Vec<Inst> = func.insts(head).skip_while(|&inst| inst != term).collect();
for inst in rest {
func.remove_inst(inst);
Builder::new(&mut func, split).func();
func.append_inst(split, inst);
}
let stats = hoisted(&mut func);
assert!(!stats.changed());
assert_eq!(stats.count(Kind::Missed, super::ANOTHER_WAY_OUT), 1);
sound(&func, &mut names);
}
#[test]
fn a_check_an_iteration_can_finish_without_reaching_stays() {
let (mut names, mut func, _) = guarded();
let stats = hoisted(&mut func);
assert!(!stats.changed());
assert_eq!(stats.count(Kind::Missed, super::NOT_EVERY_TIME), 1);
sound(&func, &mut names);
}
fn guarded() -> (Interner, Func, Vec<Block>) {
let mut names = Interner::new();
let signature = Signature::new().with_params(&[Type::PTR, Type::int(64)]);
let mut func = Func::new(names.intern("f"), signature);
let entry = func.create_block();
let head = func.create_block();
let read = func.create_block();
let tail = func.create_block();
let done = func.create_block();
let array = func.append_param(entry, Type::PTR);
let choice = func.append_param(entry, Type::int(64));
let counter = func.append_param(head, Type::int(64));
let zero = Builder::new(&mut func, entry).iconst(Type::int(64), 0);
Builder::new(&mut func, entry).jump(head, &[zero]);
let mut build = Builder::new(&mut func, head);
let take = build.icmp(IntPred::Ne, choice, zero);
build.br_if(take, read, &[], tail, &[]);
let mut build = Builder::new(&mut func, read);
let by = build.iconst(Type::int(64), 4);
let scaled = build.binary(Opcode::Mul, counter, by, Flags::NSW);
let args = build.func().push_values(&[array, scaled]);
let pointer = build.value(InstData { args, ..InstData::new(Opcode::PtrAdd) }, Type::PTR);
check(&mut build, pointer, 4, 4);
build.jump(tail, &[]);
let mut build = Builder::new(&mut func, tail);
let one = build.iconst(Type::int(64), 1);
let next = build.binary(Opcode::Add, counter, one, Flags::NSW);
let limit = build.iconst(Type::int(64), 16);
let again = build.icmp(IntPred::Slt, next, limit);
build.br_if(again, head, &[next], done, &[]);
Builder::new(&mut func, done).ret(&[]);
(names, func, vec![entry, head, read, tail, done])
}
#[test]
fn a_check_whose_step_does_not_keep_its_alignment_stays() {
let (_, mut func, _) = walking(8, 3, 4, 4);
let stats = hoisted(&mut func);
assert!(!stats.changed());
assert_eq!(stats.count(Kind::Missed, super::MISALIGNED), 1);
}
#[test]
fn a_check_whose_address_walks_backwards_stays() {
let (_, mut func, _) = walking(8, -4, 4, 4);
let stats = hoisted(&mut func);
assert!(!stats.changed());
assert_eq!(stats.count(Kind::Missed, super::BACKWARDS), 1);
}
#[test]
fn a_check_through_a_pointer_the_loop_does_not_move_is_not_this_pass_to_take_out() {
let mut names = Interner::new();
let signature = Signature::new().with_params(&[Type::PTR]);
let mut func = Func::new(names.intern("f"), signature);
let entry = func.create_block();
let head = func.create_block();
let done = func.create_block();
let array = func.append_param(entry, Type::PTR);
let counter = func.append_param(head, Type::int(64));
let zero = Builder::new(&mut func, entry).iconst(Type::int(64), 0);
Builder::new(&mut func, entry).jump(head, &[zero]);
let mut build = Builder::new(&mut func, head);
check(&mut build, array, 4, 4);
let one = build.iconst(Type::int(64), 1);
let next = build.binary(Opcode::Add, counter, one, Flags::NSW);
let limit = build.iconst(Type::int(64), 16);
let again = build.icmp(IntPred::Slt, next, limit);
build.br_if(again, head, &[next], done, &[]);
Builder::new(&mut func, done).ret(&[]);
let stats = hoisted(&mut func);
assert!(!stats.changed());
assert_eq!(stats.count(Kind::Missed, super::NOT_A_SWEEP), 1);
}
#[test]
fn a_check_that_already_covers_a_computed_range_is_left_where_it_is() {
let (mut names, mut func, blocks) = walking(16, WIDTH, 4, 4);
let head = blocks[1];
let check = func
.insts(head)
.find(|&inst| func[inst].opcode == Opcode::CheckBounds)
.expect("the body has a check");
let [capability, pointer] = func[func[check].args] else { panic!("two operands") };
let bytes = Builder::new(&mut func, head).iconst(Type::int(64), 64);
let moved = inst_of(&func, bytes);
func.remove_inst(moved);
func.insert_before(moved, check);
func[check].args = func.push_values(&[capability, pointer, bytes]);
let stats = hoisted(&mut func);
assert!(!stats.changed());
assert_eq!(stats.count(Kind::Missed, super::ALREADY_COMPUTED), 1);
sound(&func, &mut names);
}
#[test]
fn fuel_stops_the_hoist_where_it_stands() {
let (mut names, mut func, _) = walking(16, WIDTH, 4, 4);
let mut an = Analyses::new();
Canon.run(&mut func, &mut an, &mut Fuel::unlimited());
let stats = Hoist.run(&mut func, &mut an, &mut Fuel::of(0));
assert_eq!(stats.count(Kind::Optimized, HOISTED), 0);
assert_eq!(stats.count(Kind::Missed, super::NO_FUEL), 1);
assert_eq!(checks(&func).len(), 1, "and the check is where it was");
sound(&func, &mut names);
}
#[test]
fn a_loop_that_sweeps_further_than_the_rule_goes_keeps_its_check() {
let (_, mut func, _) = walking(2, 1 << 33, 4, 1);
let stats = hoisted(&mut func);
assert!(!stats.changed());
assert_eq!(stats.count(Kind::Missed, super::TOO_WIDE), 1);
}
fn inst_of(func: &Func, value: Value) -> Inst {
super::inst_of(func, value)
}
}