use super::*;
impl<'a, 'b> StructuredBodyLowerer<'a, 'b> {
pub(super) fn build_loop_state_plan(
&self,
candidate: &LoopCandidate,
preheader: Option<BlockRef>,
exit: BlockRef,
excluded_regs: &[Reg],
target_overrides: &BTreeMap<TempId, HirLValue>,
) -> Option<LoopStatePlan> {
let excluded = excluded_regs.iter().copied().collect::<BTreeSet<_>>();
let mut plan = LoopStatePlan::default();
for value in Self::header_values(candidate) {
if excluded.contains(&value.reg) {
continue;
}
let init = match self.loop_entry_expr(preheader?, value, target_overrides) {
Some(init) => init,
None => {
if value.outside_arm.defs().count() == 0 {
if Self::exit_value_for_reg(candidate, exit, value.reg).is_some() {
HirExpr::Nil
} else {
continue;
}
} else {
return None;
}
}
};
let temp = *self.lowering.bindings.phi_temps.get(value.phi_id.index())?;
let target = self.loop_state_target(candidate, exit, value.reg, temp, target_overrides);
plan.backedge_target_overrides.insert(temp, target.clone());
if self.lowering.dataflow.phi_use_count(value.phi_id) > 0
|| Self::exit_value_for_reg(candidate, exit, value.reg).is_some()
{
for def in value.inside_arm.defs() {
let def_temp = *self.lowering.bindings.fixed_temps.get(def.index())?;
plan.backedge_target_overrides
.insert(def_temp, target.clone());
}
}
plan.states.push(LoopStateSlot {
phi_id: value.phi_id,
reg: value.reg,
temp,
target,
init,
});
}
let inside_exit_blocks = self
.loop_state_inside_exit_blocks(candidate, exit)
.unwrap_or_else(|| candidate.blocks.clone());
for value in Self::exit_values(candidate, exit) {
if excluded.contains(&value.reg)
|| plan.states.iter().any(|state| state.reg == value.reg)
|| !loop_value_has_inside_and_outside_incoming(value)
|| self.exit_value_is_owned_by_inherited_state(value, target_overrides)
{
continue;
}
let Some(init) = self.loop_exit_entry_expr_with_inside_blocks(
value,
&inside_exit_blocks,
target_overrides,
) else {
continue;
};
let temp = *self.lowering.bindings.phi_temps.get(value.phi_id.index())?;
let target = self.loop_state_target(candidate, exit, value.reg, temp, target_overrides);
plan.backedge_target_overrides.insert(temp, target.clone());
if self.lowering.dataflow.phi_use_count(value.phi_id) > 0 {
for def in value.inside_arm.defs() {
let def_temp = *self.lowering.bindings.fixed_temps.get(def.index())?;
plan.backedge_target_overrides
.insert(def_temp, target.clone());
}
}
plan.states.push(LoopStateSlot {
phi_id: value.phi_id,
reg: value.reg,
temp,
target,
init,
});
}
Some(plan)
}
fn loop_entry_expr(
&self,
preheader: BlockRef,
value: &LoopValueMerge,
target_overrides: &BTreeMap<TempId, HirLValue>,
) -> Option<HirExpr> {
let incoming = value.outside_arm.incoming_for_pred(preheader)?;
self.loop_incoming_expr(
preheader,
value.reg,
incoming.defs.iter().copied(),
target_overrides,
)
}
fn loop_exit_entry_expr_with_inside_blocks(
&self,
value: &LoopValueMerge,
inside_blocks: &BTreeSet<BlockRef>,
target_overrides: &BTreeMap<TempId, HirLValue>,
) -> Option<HirExpr> {
let mut init_expr = None;
for incoming in value
.inside_arm
.incomings
.iter()
.chain(value.outside_arm.incomings.iter())
.filter(|incoming| !inside_blocks.contains(&incoming.pred))
{
let expr = self.loop_incoming_expr(
incoming.pred,
value.reg,
incoming.defs.iter().copied(),
target_overrides,
)?;
if init_expr
.as_ref()
.is_some_and(|known_expr: &HirExpr| *known_expr != expr)
{
return None;
}
init_expr = Some(expr);
}
init_expr
}
fn loop_incoming_expr(
&self,
pred: BlockRef,
reg: Reg,
defs: impl IntoIterator<Item = crate::cfg::DefId>,
target_overrides: &BTreeMap<TempId, HirLValue>,
) -> Option<HirExpr> {
let defs = defs.into_iter().collect::<Vec<_>>();
if let Some(expr) = self.overrides.carried_entry_expr(pred, reg) {
return Some(expr.clone());
}
if let Some(expr) = shared_expr_for_defs(
&self.lowering.bindings.fixed_temps,
defs.iter().copied(),
target_overrides,
) {
return Some(expr);
}
if let Some(expr) = single_fixed_def_expr(self.lowering, defs.iter().copied()) {
return Some(expr);
}
if defs.len() > 1 {
let mut expr = expr_for_reg_at_block_exit(self.lowering, pred, reg);
rewrite_expr_temps(&mut expr, &temp_expr_overrides(target_overrides));
return Some(expr);
}
if let Some(expr) = self.reaching_phi_override_expr(pred, reg, target_overrides) {
return Some(expr);
}
if defs.is_empty() && self.pred_has_no_reaching_value(pred, reg) {
return Some(HirExpr::Nil);
}
None
}
fn pred_has_no_reaching_value(&self, pred: BlockRef, reg: Reg) -> bool {
let range = self.lowering.cfg.blocks[pred.index()].instrs;
if range.is_empty() {
return false;
}
let values = self.lowering.dataflow.reaching_values_at(range.start);
let entry_empty = values.get(reg).is_none_or(|set| set.is_empty());
if !entry_empty {
return false;
}
!(range.start.index()..range.end()).any(|instr_index| {
let effect = &self.lowering.dataflow.instr_effects[instr_index];
effect.fixed_must_defs.contains(®) || effect.fixed_may_defs.contains(®)
})
}
fn reaching_phi_override_expr(
&self,
pred: BlockRef,
reg: Reg,
target_overrides: &BTreeMap<TempId, HirLValue>,
) -> Option<HirExpr> {
use crate::cfg::SsaValue;
let first_instr = self.lowering.cfg.blocks[pred.index()].instrs.start;
let reaching = self.lowering.dataflow.reaching_values_at(first_instr);
let values = reaching.get(reg)?;
let mut phi_ids = values.iter().filter_map(|v| match v {
SsaValue::Phi(phi_id) => Some(phi_id),
SsaValue::Def(_) => None,
});
let phi_id = phi_ids.next()?;
if phi_ids.next().is_some() {
return None;
}
let temp = *self.lowering.bindings.phi_temps.get(phi_id.index())?;
let lvalue = target_overrides.get(&temp)?;
lvalue_as_expr(lvalue)
}
pub(super) fn install_loop_exit_bindings(
&mut self,
candidate: &LoopCandidate,
exit: BlockRef,
plan: &LoopStatePlan,
target_overrides: &BTreeMap<TempId, HirLValue>,
) {
for state in &plan.states {
let Some(state_expr) = lvalue_as_expr(&state.target) else {
continue;
};
self.install_entry_override(exit, state.reg, state_expr);
}
let inside_exit_blocks = self
.loop_state_inside_exit_blocks(candidate, exit)
.unwrap_or_else(|| candidate.blocks.clone());
self.apply_exit_phi_bindings(candidate, exit, &inside_exit_blocks, plan, target_overrides);
if let Some(downstream) = self.normalized_post_loop_successor(exit) {
let mut inside_with_pad = inside_exit_blocks.clone();
inside_with_pad.insert(exit);
self.apply_exit_phi_bindings(
candidate,
downstream,
&inside_with_pad,
plan,
target_overrides,
);
}
}
fn apply_exit_phi_bindings(
&mut self,
candidate: &LoopCandidate,
at_block: BlockRef,
inside_exit_blocks: &BTreeSet<BlockRef>,
plan: &LoopStatePlan,
target_overrides: &BTreeMap<TempId, HirLValue>,
) {
for value in Self::exit_values(candidate, at_block) {
if let Some(state) = plan.states.iter().find(|state| state.reg == value.reg) {
let Some(state_expr) = lvalue_as_expr(&state.target) else {
continue;
};
if loop_value_incoming_all_within_blocks(value, inside_exit_blocks) {
self.replace_phi_with_target_expr(
at_block,
value.phi_id,
state.temp,
state_expr,
);
continue;
}
let Some(exit_init) = self.loop_exit_entry_expr_with_inside_blocks(
value,
inside_exit_blocks,
target_overrides,
) else {
continue;
};
if exit_init != state.init {
continue;
}
self.replace_phi_with_target_expr(at_block, value.phi_id, state.temp, state_expr);
continue;
}
if let Some(target) = self.inherited_exit_target_for_value(value, target_overrides)
&& let Some(expr) = lvalue_as_expr(&target)
{
self.overrides.insert_phi_expr(at_block, value.phi_id, expr);
}
}
}
fn inherited_exit_target_for_value(
&self,
value: &LoopValueMerge,
target_overrides: &BTreeMap<TempId, HirLValue>,
) -> Option<HirLValue> {
let fixed_temps = &self.lowering.bindings.fixed_temps;
let combined_defs = value
.inside_arm
.defs()
.chain(value.outside_arm.defs())
.collect::<Vec<_>>();
if combined_defs.is_empty() {
return None;
}
shared_lvalue_for_defs(fixed_temps, combined_defs, target_overrides)
}
fn loop_state_target(
&self,
candidate: &LoopCandidate,
exit: BlockRef,
reg: Reg,
temp: TempId,
target_overrides: &BTreeMap<TempId, HirLValue>,
) -> HirLValue {
if let Some(target) = target_overrides
.get(&temp)
.filter(|target| lvalue_as_expr(target).is_some())
{
return target.clone();
}
if let Some(target) =
self.uniform_loop_header_target_override(candidate, reg, target_overrides)
{
return target;
}
if let Some(target) =
self.uniform_loop_exit_target_override(candidate, exit, reg, target_overrides)
{
return target;
}
if let Some(preheader) = unique_loop_preheader(candidate)
&& let Some(target) =
self.reaching_phi_lvalue_override(preheader, reg, target_overrides)
{
return target;
}
HirLValue::Temp(temp)
}
fn reaching_phi_lvalue_override(
&self,
block: BlockRef,
reg: Reg,
target_overrides: &BTreeMap<TempId, HirLValue>,
) -> Option<HirLValue> {
use crate::cfg::SsaValue;
let first_instr = self.lowering.cfg.blocks[block.index()].instrs.start;
let reaching = self.lowering.dataflow.reaching_values_at(first_instr);
let values = reaching.get(reg)?;
let mut phi_ids = values.iter().filter_map(|v| match v {
SsaValue::Phi(phi_id) => Some(phi_id),
SsaValue::Def(_) => None,
});
let phi_id = phi_ids.next()?;
if phi_ids.next().is_some() {
return None;
}
let temp = *self.lowering.bindings.phi_temps.get(phi_id.index())?;
let lvalue = target_overrides.get(&temp)?;
lvalue_as_expr(lvalue)?;
Some(lvalue.clone())
}
fn exit_value_is_owned_by_inherited_state(
&self,
value: &LoopValueMerge,
target_overrides: &BTreeMap<TempId, HirLValue>,
) -> bool {
let phi_temp = self.lowering.bindings.phi_temps[value.phi_id.index()];
if target_overrides.contains_key(&phi_temp) {
return true;
}
for def in value.inside_arm.defs() {
let def_temp = self.lowering.bindings.fixed_temps[def.index()];
if target_overrides.contains_key(&def_temp) {
return true;
}
}
false
}
fn uniform_loop_header_target_override(
&self,
candidate: &LoopCandidate,
reg: Reg,
target_overrides: &BTreeMap<TempId, HirLValue>,
) -> Option<HirLValue> {
let value = Self::header_value_for_reg(candidate, reg)?;
self.shared_loop_inside_target(&value.inside_arm, target_overrides)
}
fn uniform_loop_exit_target_override(
&self,
candidate: &LoopCandidate,
exit: BlockRef,
reg: Reg,
target_overrides: &BTreeMap<TempId, HirLValue>,
) -> Option<HirLValue> {
if let Some(value) = Self::exit_value_for_reg(candidate, exit, reg) {
if !loop_value_has_inside_and_outside_incoming(value) {
return None;
}
if let Some(target) =
self.shared_loop_inside_target(&value.inside_arm, target_overrides)
{
return Some(target);
}
}
None
}
fn shared_loop_inside_target(
&self,
arm: &LoopValueArm,
target_overrides: &BTreeMap<TempId, HirLValue>,
) -> Option<HirLValue> {
shared_lvalue_for_defs(
&self.lowering.bindings.fixed_temps,
arm.defs(),
target_overrides,
)
}
pub(super) fn header_values(
candidate: &LoopCandidate,
) -> impl Iterator<Item = &LoopValueMerge> {
candidate.header_value_merges.iter()
}
pub(super) fn header_value_for_reg(
candidate: &LoopCandidate,
reg: Reg,
) -> Option<&LoopValueMerge> {
Self::header_values(candidate).find(|value| value.reg == reg)
}
pub(super) fn exit_values(
candidate: &LoopCandidate,
exit: BlockRef,
) -> impl Iterator<Item = &LoopValueMerge> {
candidate
.exit_value_merges
.iter()
.find(|candidate| candidate.exit == exit)
.into_iter()
.flat_map(|candidate| candidate.values.iter())
}
pub(super) fn exit_value_for_reg(
candidate: &LoopCandidate,
exit: BlockRef,
reg: Reg,
) -> Option<&LoopValueMerge> {
Self::exit_values(candidate, exit).find(|value| value.reg == reg)
}
pub(super) fn build_active_loop_context(
&self,
candidate: &LoopCandidate,
post_loop: BlockRef,
target_overrides: &BTreeMap<TempId, HirLValue>,
states: &[LoopStateSlot],
) -> Option<ActiveLoopContext> {
let downstream_post_loop = self.normalized_post_loop_successor(post_loop);
let mut break_exits = BTreeMap::new();
for exit in candidate
.exits
.iter()
.copied()
.filter(|exit| *exit != post_loop)
{
if block_is_terminal_exit(self.lowering, exit) {
continue;
}
if downstream_post_loop == Some(exit) {
continue;
}
break_exits.insert(
exit,
self.lower_break_exit_pad(
exit,
post_loop,
downstream_post_loop,
target_overrides,
states,
)?,
);
}
let continue_target = candidate.continue_target;
let continue_sources = continue_target
.map(|target| {
self.lowering
.structure
.goto_requirements
.iter()
.filter(|requirement| {
requirement.reason == crate::structure::GotoReason::UnstructuredContinueLike
&& requirement.to == target
&& candidate.blocks.contains(&requirement.from)
})
.map(|requirement| requirement.from)
.collect::<BTreeSet<_>>()
})
.unwrap_or_default();
Some(ActiveLoopContext {
header: candidate.header,
loop_blocks: BTreeSet::new(),
post_loop,
downstream_post_loop,
continue_target,
continue_sources,
break_exits,
state_slots: Vec::new(),
})
}
fn normalized_post_loop_successor(&self, post_loop: BlockRef) -> Option<BlockRef> {
let (_instr_ref, instr) = self.block_terminator(post_loop)?;
let LowInstr::Jump(jump) = instr else {
return None;
};
let target = self.lowering.cfg.instr_to_block[jump.target.index()];
self.lower_block_prefix(post_loop, false, &BTreeMap::new())?;
Some(target)
}
fn loop_state_inside_exit_blocks(
&self,
candidate: &LoopCandidate,
post_loop: BlockRef,
) -> Option<BTreeSet<BlockRef>> {
let downstream_post_loop = self.normalized_post_loop_successor(post_loop);
let mut inside_blocks = candidate.blocks.clone();
for exit in candidate
.exits
.iter()
.copied()
.filter(|exit| *exit != post_loop)
{
if block_is_terminal_exit(self.lowering, exit) {
continue;
}
if downstream_post_loop == Some(exit) {
continue;
}
self.lower_break_exit_pad(
exit,
post_loop,
downstream_post_loop,
&BTreeMap::new(),
&[],
)?;
inside_blocks.insert(exit);
}
Some(inside_blocks)
}
pub(super) fn repeat_backedge_pad(
&self,
header: BlockRef,
loop_backedge_target: BlockRef,
target_overrides: &BTreeMap<TempId, HirLValue>,
) -> Option<Option<BlockRef>> {
if loop_backedge_target == header {
return Some(None);
}
if self
.lowering
.cfg
.unique_reachable_successor(loop_backedge_target)
!= Some(header)
{
return None;
}
if !self
.lower_block_prefix(loop_backedge_target, false, target_overrides)?
.is_empty()
{
return None;
}
Some(Some(loop_backedge_target))
}
fn lower_break_exit_pad(
&self,
block: BlockRef,
post_loop: BlockRef,
downstream_post_loop: Option<BlockRef>,
target_overrides: &BTreeMap<TempId, HirLValue>,
states: &[LoopStateSlot],
) -> Option<BreakExitBlock> {
let mut combined = target_overrides.clone();
if !states.is_empty() {
let range = self.lowering.cfg.blocks[block.index()].instrs;
for instr_index in range.start.index()..range.end() {
for def_id in &self.lowering.dataflow.instr_defs[instr_index] {
let def = &self.lowering.dataflow.defs[def_id.index()];
if let Some(state) = states.iter().find(|s| s.reg == def.reg) {
let temp = self.lowering.bindings.fixed_temps[def_id.index()];
combined.insert(temp, state.target.clone());
}
}
}
}
if matches!(
self.block_terminator(block),
Some((_instr_ref, LowInstr::Branch(_)))
) {
return self.lower_branch_break_exit_pad(
block,
post_loop,
downstream_post_loop,
&combined,
);
}
let mut stmts = self.lower_block_prefix(block, false, &combined)?;
let target = match self.block_terminator(block) {
Some((_instr_ref, LowInstr::Jump(jump))) => {
self.lowering.cfg.instr_to_block[jump.target.index()]
}
Some((_instr_ref, instr)) if !is_control_terminator(instr) => {
self.lowering.cfg.unique_reachable_successor(block)?
}
None => self.lowering.cfg.unique_reachable_successor(block)?,
Some(_) => return None,
};
if target != post_loop && Some(target) != downstream_post_loop {
return None;
}
stmts.push(HirStmt::Break);
Some(BreakExitBlock {
block: HirBlock { stmts },
blocks: BTreeSet::from([block]),
})
}
fn lower_branch_break_exit_pad(
&self,
block: BlockRef,
post_loop: BlockRef,
downstream_post_loop: Option<BlockRef>,
target_overrides: &BTreeMap<TempId, HirLValue>,
) -> Option<BreakExitBlock> {
let candidate = *self.branch_by_header.get(&block)?;
let merge = candidate.merge?;
if merge != post_loop && Some(merge) != downstream_post_loop {
return None;
}
let mut blocks = BTreeSet::from([block]);
let mut stmts = self.lower_block_prefix(block, true, target_overrides)?;
let mut cond = self.lower_candidate_cond(block, candidate)?;
if let Some(entry_expr_overrides) = self.block_entry_expr_overrides(block) {
rewrite_expr_temps(&mut cond, entry_expr_overrides);
}
let then_pad = self.lower_break_exit_pad_arm(
candidate.then_entry,
merge,
post_loop,
downstream_post_loop,
target_overrides,
)?;
blocks.extend(then_pad.blocks.iter().copied());
let else_pad = match candidate.else_entry {
Some(else_entry) => {
let pad = self.lower_break_exit_pad_arm(
else_entry,
merge,
post_loop,
downstream_post_loop,
target_overrides,
)?;
blocks.extend(pad.blocks.iter().copied());
Some(pad.block)
}
None => None,
};
stmts.push(branch_stmt(cond, then_pad.block, else_pad));
stmts.push(HirStmt::Break);
Some(BreakExitBlock {
block: HirBlock { stmts },
blocks,
})
}
fn lower_break_exit_pad_arm(
&self,
block: BlockRef,
merge: BlockRef,
post_loop: BlockRef,
downstream_post_loop: Option<BlockRef>,
target_overrides: &BTreeMap<TempId, HirLValue>,
) -> Option<BreakExitBlock> {
if block == merge || block == post_loop || Some(block) == downstream_post_loop {
return Some(BreakExitBlock {
block: HirBlock::default(),
blocks: BTreeSet::new(),
});
}
let stmts = self.lower_block_prefix(block, false, target_overrides)?;
let target = match self.block_terminator(block) {
Some((_instr_ref, LowInstr::Jump(jump))) => {
self.lowering.cfg.instr_to_block[jump.target.index()]
}
Some((_instr_ref, instr)) if !is_control_terminator(instr) => {
self.lowering.cfg.unique_reachable_successor(block)?
}
None => self.lowering.cfg.unique_reachable_successor(block)?,
Some(_) => return None,
};
if target != merge && target != post_loop && Some(target) != downstream_post_loop {
return None;
}
Some(BreakExitBlock {
block: HirBlock { stmts },
blocks: BTreeSet::from([block]),
})
}
pub(super) fn generic_for_header_instrs(
&self,
header: BlockRef,
) -> Option<(
InstrRef,
crate::transformer::GenericForCallInstr,
crate::transformer::GenericForLoopInstr,
)> {
let range = self.lowering.cfg.blocks[header.index()].instrs;
if range.len < 2 {
return None;
}
let call_instr_ref = InstrRef(range.end() - 2);
let loop_instr_ref = InstrRef(range.end() - 1);
let LowInstr::GenericForCall(call) =
self.lowering.proto.instrs.get(call_instr_ref.index())?
else {
return None;
};
let LowInstr::GenericForLoop(loop_instr) =
self.lowering.proto.instrs.get(loop_instr_ref.index())?
else {
return None;
};
Some((call_instr_ref, *call, *loop_instr))
}
pub(super) fn lower_generic_for_iterator(
&self,
header: BlockRef,
call_instr_ref: InstrRef,
call: crate::transformer::GenericForCallInstr,
) -> Vec<HirExpr> {
(0..call.state.len)
.map(|offset| {
expr_for_reg_use(
self.lowering,
header,
call_instr_ref,
Reg(call.state.start.index() + offset),
)
})
.collect()
}
pub(crate) fn block_prefix_temp_expr_overrides(
&self,
block: BlockRef,
) -> (BTreeMap<TempId, HirExpr>, BTreeSet<TempId>) {
let range = self.lowering.cfg.blocks[block.index()].instrs;
if range.is_empty() {
return (BTreeMap::new(), BTreeSet::new());
}
let end = if let Some((_instr_ref, instr)) = self.block_terminator(block) {
if is_control_terminator(instr) {
range.end() - 1
} else {
range.end()
}
} else {
range.end()
};
let mut expr_overrides = BTreeMap::new();
let mut all_prefix_temps = BTreeSet::new();
for instr_index in range.start.index()..end {
let instr_ref = InstrRef(instr_index);
if self.overrides.instr_is_suppressed(instr_ref) {
continue;
}
for def in &self.lowering.dataflow.instr_defs[instr_index] {
let temp = self.lowering.bindings.fixed_temps[def.index()];
all_prefix_temps.insert(temp);
let Some(mut expr) = expr_for_fixed_def(self.lowering, *def) else {
continue;
};
rewrite_expr_temps(&mut expr, &expr_overrides);
expr_overrides.insert(temp, expr);
}
}
(expr_overrides, all_prefix_temps)
}
}