use super::*;
pub(in crate::passes) struct PrivAggAccess {
pub(in crate::passes) remove: Vec<(usize, usize)>, pub(in crate::passes) anchor: (usize, usize), pub(in crate::passes) is_store: bool,
pub(in crate::passes) store_val: Word, pub(in crate::passes) load_result: Word, pub(in crate::passes) load_result_ty: Word, pub(in crate::passes) var: Word, pub(in crate::passes) elem_ty: Word, pub(in crate::passes) ptr_elem_ty: Word, pub(in crate::passes) start_elem: u32, pub(in crate::passes) lanes: u32, }
pub(in crate::passes) fn lower_private_byte_aggregate_reinterpret(
ctx: &mut Ctx,
entry_idx: usize,
) -> Result<(), String> {
let mut ptr_info: HashMap<Word, (StorageClass, Word)> = HashMap::new();
let mut defs: HashMap<Word, Instruction> = HashMap::new();
for inst in ctx
.new_globals
.iter()
.chain(ctx.module.types_global_values.iter())
{
if let Some(r) = inst.result_id {
defs.insert(r, inst.clone());
}
if inst.class.opcode == Op::TypePointer {
if let (Some(id), Some(Operand::StorageClass(s)), Some(Operand::IdRef(p))) =
(inst.result_id, inst.operands.first(), inst.operands.get(1))
{
ptr_info.insert(id, (*s, *p));
}
}
}
let mut def_pos: HashMap<Word, (usize, usize)> = HashMap::new();
let mut users: HashMap<Word, Vec<(usize, usize)>> = HashMap::new();
for (bi, block) in ctx.module.functions[entry_idx].blocks.iter().enumerate() {
for (ii, inst) in block.instructions.iter().enumerate() {
if let Some(r) = inst.result_id {
def_pos.insert(r, (bi, ii));
}
for op in &inst.operands {
if let Operand::IdRef(id) = op {
users.entry(*id).or_default().push((bi, ii));
}
}
}
}
let mut accesses: Vec<PrivAggAccess> = Vec::new();
let mut need_len: HashMap<Word, (u32, Word)> = HashMap::new();
for (bi, block) in ctx.module.functions[entry_idx].blocks.iter().enumerate() {
for (ii, inst) in block.instructions.iter().enumerate() {
if inst.class.opcode != Op::PtrAccessChain {
continue;
}
let Some(rp) = inst.result_id else {
continue;
};
let Some(rpt) = inst.result_type else {
continue;
};
let Some(&(storage, pac_pointee)) = ptr_info.get(&rpt) else {
continue;
};
if storage != StorageClass::Private {
continue;
}
let Some(pw) = direct_scalar_width(ctx, pac_pointee) else {
continue;
};
if inst.operands.len() != 2 {
continue;
}
let Some(Operand::IdRef(base)) = inst.operands.first() else {
continue;
};
let Some(Operand::IdRef(c_id)) = inst.operands.get(1) else {
continue;
};
let Some(c) = const_u32(ctx, *c_id) else {
continue;
};
let Some(&(bbi, bii)) = def_pos.get(base) else {
continue;
};
let bdef = &ctx.module.functions[entry_idx].blocks[bbi].instructions[bii];
if !matches!(bdef.class.opcode, Op::InBoundsAccessChain | Op::AccessChain) {
continue;
}
let Some(bpt) = bdef.result_type else {
continue;
};
let Some(&(bstorage, elem_ty)) = ptr_info.get(&bpt) else {
continue;
};
if bstorage != StorageClass::Private {
continue;
}
let Some(elem_bits) = direct_scalar_width(ctx, elem_ty) else {
continue;
};
if pw == 0 || pw % 8 != 0 || elem_bits == 0 || pw >= elem_bits {
continue;
}
if bdef.operands.len() != 2 {
continue;
}
let Some(Operand::IdRef(var)) = bdef.operands.first() else {
continue;
};
let Some(Operand::IdRef(e0_id)) = bdef.operands.get(1) else {
continue;
};
let Some(e0) = const_u32(ctx, *e0_id) else {
continue;
};
let Some(vdef) = defs.get(var) else {
continue;
};
if vdef.class.opcode != Op::Variable {
continue;
}
let Some(vpt) = vdef.result_type else {
continue;
};
let Some(&(vstorage, varr)) = ptr_info.get(&vpt) else {
continue;
};
if vstorage != StorageClass::Private {
continue;
}
let Some(arr_def) = defs.get(&varr) else {
continue;
};
if arr_def.class.opcode != Op::TypeArray {
continue;
}
let (Some(Operand::IdRef(arr_elem)), Some(Operand::IdRef(_len_c))) =
(arr_def.operands.first(), arr_def.operands.get(1))
else {
continue;
};
if *arr_elem != elem_ty {
continue;
}
let elem_bytes = elem_bits / 8;
let byteoff = c * (pw / 8);
if byteoff % elem_bytes != 0 {
continue;
}
let start_elem = e0 + byteoff / elem_bytes;
let Some(rp_users) = users.get(&rp) else {
continue;
};
if rp_users.len() != 1 {
continue;
}
let (ubi, uii) = rp_users[0];
let udef = &ctx.module.functions[entry_idx].blocks[ubi].instructions[uii];
if udef.class.opcode == Op::Store {
let Some(Operand::IdRef(sptr)) = udef.operands.first() else {
continue;
};
if *sptr != rp {
continue;
}
let Some(Operand::IdRef(val)) = udef.operands.get(1) else {
continue;
};
let Some(vty) = value_result_type(ctx, *val) else {
continue;
};
let Some(lanes) = vector_lanes_of(ctx, vty, elem_ty) else {
continue;
};
let entry = need_len.entry(*var).or_insert((0, elem_ty));
entry.0 = entry.0.max(start_elem + lanes);
accesses.push(PrivAggAccess {
remove: vec![(bi, ii)],
anchor: (ubi, uii),
is_store: true,
store_val: *val,
load_result: 0,
load_result_ty: 0,
var: *var,
elem_ty,
ptr_elem_ty: bpt,
start_elem,
lanes,
});
} else if udef.class.opcode == Op::Bitcast {
let Some(vp) = udef.result_id else {
continue;
};
let Some(vpt2) = udef.result_type else {
continue;
};
let Some(&(vp_storage, vec_pointee)) = ptr_info.get(&vpt2) else {
continue;
};
if vp_storage != StorageClass::Private {
continue;
}
let Some(lanes) = vector_lanes_of(ctx, vec_pointee, elem_ty) else {
continue;
};
let Some(vp_users) = users.get(&vp) else {
continue;
};
if vp_users.len() != 1 {
continue;
}
let (lbi, lii) = vp_users[0];
let ldef = &ctx.module.functions[entry_idx].blocks[lbi].instructions[lii];
if ldef.class.opcode != Op::Load {
continue;
}
let Some(Operand::IdRef(lptr)) = ldef.operands.first() else {
continue;
};
if *lptr != vp {
continue;
}
let (Some(lres), Some(lty)) = (ldef.result_id, ldef.result_type) else {
continue;
};
if lty != vec_pointee {
continue;
}
let entry = need_len.entry(*var).or_insert((0, elem_ty));
entry.0 = entry.0.max(start_elem + lanes);
accesses.push(PrivAggAccess {
remove: vec![(bi, ii), (ubi, uii)],
anchor: (lbi, lii),
is_store: false,
store_val: 0,
load_result: lres,
load_result_ty: lty,
var: *var,
elem_ty,
ptr_elem_ty: bpt,
start_elem,
lanes,
});
}
}
}
if accesses.is_empty() {
return Ok(());
}
for (&var, &(needed, elem_ty)) in &need_len {
let Some(vdef) = defs.get(&var) else {
continue;
};
let Some(vpt) = vdef.result_type else {
continue;
};
let Some(&(_, varr)) = ptr_info.get(&vpt) else {
continue;
};
let Some(arr_def) = defs.get(&varr) else {
continue;
};
let Some(Operand::IdRef(len_c)) = arr_def.operands.get(1) else {
continue;
};
let Some(n) = const_u32(ctx, *len_c) else {
continue;
};
if needed <= n {
continue;
}
let new_glob_start = ctx.new_globals.len();
let len_needed_c = ctx.const_uint(needed);
let arr_ty = ctx.ty_array(elem_ty, needed);
let new_var_ptr = ctx.ty_ptr(StorageClass::Private, arr_ty);
let mut synthesized: Vec<Instruction> = ctx.new_globals.drain(new_glob_start..).collect();
if let Some(p) = ctx
.module
.types_global_values
.iter()
.position(|i| i.class.opcode == Op::Variable && i.result_id == Some(var))
{
if let Some(q) = ctx
.module
.types_global_values
.iter()
.position(|i| i.result_id == Some(len_needed_c))
{
if q >= p {
let len_inst = ctx.module.types_global_values.remove(q);
synthesized.insert(0, len_inst);
}
}
let p = ctx
.module
.types_global_values
.iter()
.position(|i| i.class.opcode == Op::Variable && i.result_id == Some(var))
.ok_or("private byte aggregate: variable not found in globals")?;
ctx.module.types_global_values[p].result_type = Some(new_var_ptr);
let tail = ctx.module.types_global_values.split_off(p);
ctx.module.types_global_values.extend(synthesized);
ctx.module.types_global_values.extend(tail);
} else if let Some(p) = ctx
.new_globals
.iter()
.position(|i| i.class.opcode == Op::Variable && i.result_id == Some(var))
{
if let Some(q) = ctx
.new_globals
.iter()
.position(|i| i.result_id == Some(len_needed_c))
{
if q >= p {
let len_inst = ctx.new_globals.remove(q);
synthesized.insert(0, len_inst);
}
}
let p = ctx
.new_globals
.iter()
.position(|i| i.class.opcode == Op::Variable && i.result_id == Some(var))
.ok_or("private byte aggregate: variable not found in globals")?;
ctx.new_globals[p].result_type = Some(new_var_ptr);
let tail = ctx.new_globals.split_off(p);
ctx.new_globals.extend(synthesized);
ctx.new_globals.extend(tail);
} else {
ctx.new_globals.extend(synthesized);
}
}
let mut remove: HashSet<(usize, usize)> = HashSet::new();
let mut store_anchor: HashMap<(usize, usize), usize> = HashMap::new();
let mut load_anchor: HashMap<(usize, usize), usize> = HashMap::new();
for (ai, acc) in accesses.iter().enumerate() {
for &r in &acc.remove {
remove.insert(r);
}
if acc.is_store {
store_anchor.insert(acc.anchor, ai);
} else {
load_anchor.insert(acc.anchor, ai);
}
}
let nblocks = ctx.module.functions[entry_idx].blocks.len();
for bi in 0..nblocks {
let touched = (0..ctx.module.functions[entry_idx].blocks[bi]
.instructions
.len())
.any(|ii| {
remove.contains(&(bi, ii))
|| store_anchor.contains_key(&(bi, ii))
|| load_anchor.contains_key(&(bi, ii))
});
if !touched {
continue;
}
let insts = ctx.module.functions[entry_idx].blocks[bi]
.instructions
.clone();
let mut out: Vec<Instruction> = Vec::with_capacity(insts.len());
for (ii, inst) in insts.into_iter().enumerate() {
if remove.contains(&(bi, ii)) {
continue;
}
if let Some(&ai) = store_anchor.get(&(bi, ii)) {
let acc = &accesses[ai];
for k in 0..acc.lanes {
let idx_c = ctx.const_uint(acc.start_elem + k);
let pk = ctx.module.fresh_id();
out.push(Instruction::new(
Op::InBoundsAccessChain,
Some(acc.ptr_elem_ty),
Some(pk),
vec![Operand::IdRef(acc.var), Operand::IdRef(idx_c)],
));
let ek = if acc.lanes == 1 {
acc.store_val
} else {
let id = ctx.module.fresh_id();
out.push(Instruction::new(
Op::CompositeExtract,
Some(acc.elem_ty),
Some(id),
vec![Operand::IdRef(acc.store_val), Operand::LiteralBit32(k)],
));
id
};
out.push(Instruction::new(
Op::Store,
None,
None,
vec![Operand::IdRef(pk), Operand::IdRef(ek)],
));
}
continue;
}
if let Some(&ai) = load_anchor.get(&(bi, ii)) {
let acc = &accesses[ai];
let mut elems: Vec<Operand> = Vec::with_capacity(acc.lanes as usize);
for k in 0..acc.lanes {
let idx_c = ctx.const_uint(acc.start_elem + k);
let pk = ctx.module.fresh_id();
out.push(Instruction::new(
Op::InBoundsAccessChain,
Some(acc.ptr_elem_ty),
Some(pk),
vec![Operand::IdRef(acc.var), Operand::IdRef(idx_c)],
));
let lk = ctx.module.fresh_id();
out.push(Instruction::new(
Op::Load,
Some(acc.elem_ty),
Some(lk),
vec![Operand::IdRef(pk)],
));
elems.push(Operand::IdRef(lk));
}
if acc.lanes == 1 {
out.push(Instruction::new(
Op::CopyObject,
Some(acc.load_result_ty),
Some(acc.load_result),
vec![elems[0].clone()],
));
} else {
out.push(Instruction::new(
Op::CompositeConstruct,
Some(acc.load_result_ty),
Some(acc.load_result),
elems,
));
}
continue;
}
out.push(inst);
}
ctx.module.functions[entry_idx].blocks[bi].instructions = out;
}
Ok(())
}
pub(in crate::passes) fn retype_demoted_copymemory_placeholder(ctx: &mut Ctx, entry_idx: usize) {
let mut ptr_info: HashMap<Word, (StorageClass, Word)> = HashMap::new();
let mut defs: HashMap<Word, Instruction> = HashMap::new();
for inst in ctx
.new_globals
.iter()
.chain(ctx.module.types_global_values.iter())
{
if let Some(r) = inst.result_id {
defs.insert(r, inst.clone());
}
if inst.class.opcode == Op::TypePointer {
if let (Some(id), Some(Operand::StorageClass(s)), Some(Operand::IdRef(p))) =
(inst.result_id, inst.operands.first(), inst.operands.get(1))
{
ptr_info.insert(id, (*s, *p));
}
}
}
let mut use_count: HashMap<Word, usize> = HashMap::new();
for block in &ctx.module.functions[entry_idx].blocks {
for inst in &block.instructions {
for op in &inst.operands {
if let Operand::IdRef(id) = op {
*use_count.entry(*id).or_default() += 1;
}
}
}
}
let mut plans: Vec<(Word, Word)> = Vec::new();
for block in &ctx.module.functions[entry_idx].blocks {
for inst in &block.instructions {
if inst.class.opcode != Op::CopyMemory {
continue;
}
let (Some(Operand::IdRef(target)), Some(Operand::IdRef(source))) =
(inst.operands.first(), inst.operands.get(1))
else {
continue;
};
let (Some(tpt), Some(spt)) = (
value_result_type(ctx, *target),
value_result_type(ctx, *source),
) else {
continue;
};
let (Some(&(t_storage, t_pointee)), Some(&(_, s_pointee))) =
(ptr_info.get(&tpt), ptr_info.get(&spt))
else {
continue;
};
if t_pointee == s_pointee {
continue; }
if t_storage != StorageClass::Private {
continue;
}
let Some(tdef) = defs.get(target) else {
continue;
};
if tdef.class.opcode != Op::Variable {
continue;
}
if use_count.get(target).copied().unwrap_or(0) != 1 {
continue;
}
plans.push((*target, s_pointee));
}
}
if plans.is_empty() {
return;
}
for (var, new_pointee) in plans {
let new_glob_start = ctx.new_globals.len();
let new_ptr = ctx.ty_ptr(StorageClass::Private, new_pointee);
let synthesized: Vec<Instruction> = ctx.new_globals.drain(new_glob_start..).collect();
if let Some(p) = ctx
.module
.types_global_values
.iter()
.position(|i| i.class.opcode == Op::Variable && i.result_id == Some(var))
{
ctx.module.types_global_values[p].result_type = Some(new_ptr);
ctx.module.types_global_values[p].operands =
vec![Operand::StorageClass(StorageClass::Private)];
let tail = ctx.module.types_global_values.split_off(p);
ctx.module.types_global_values.extend(synthesized);
ctx.module.types_global_values.extend(tail);
} else if let Some(p) = ctx
.new_globals
.iter()
.position(|i| i.class.opcode == Op::Variable && i.result_id == Some(var))
{
ctx.new_globals[p].result_type = Some(new_ptr);
ctx.new_globals[p].operands = vec![Operand::StorageClass(StorageClass::Private)];
let tail = ctx.new_globals.split_off(p);
ctx.new_globals.extend(synthesized);
ctx.new_globals.extend(tail);
} else {
ctx.new_globals.extend(synthesized);
}
}
}
pub(in crate::passes) fn rewrite_strided_descent_access_chains(ctx: &mut Ctx, entry_idx: usize) {
let mut ptr_info: HashMap<Word, (StorageClass, Word)> = HashMap::new();
for inst in ctx
.new_globals
.iter()
.chain(ctx.module.types_global_values.iter())
{
if inst.class.opcode == Op::TypePointer {
if let (Some(id), Some(Operand::StorageClass(s)), Some(Operand::IdRef(p))) =
(inst.result_id, inst.operands.first(), inst.operands.get(1))
{
ptr_info.insert(id, (*s, *p));
}
}
}
let mut rewrites: Vec<(usize, usize)> = Vec::new();
for (bi, block) in ctx.module.functions[entry_idx].blocks.iter().enumerate() {
for (ii, inst) in block.instructions.iter().enumerate() {
if !matches!(inst.class.opcode, Op::InBoundsAccessChain | Op::AccessChain) {
continue;
}
let Some(result_type) = inst.result_type else {
continue;
};
let Some((storage, result_pointee)) = ptr_info.get(&result_type).copied() else {
continue;
};
if !ptr_access_chain_allowed_storage(storage) {
continue;
}
if inst.operands.len() < 3 {
continue;
}
let Some(Operand::IdRef(base)) = inst.operands.first() else {
continue;
};
let Some(base_ptr_ty) = value_result_type(ctx, *base) else {
continue;
};
let Some(&(_, base_pointee)) = ptr_info.get(&base_ptr_ty) else {
continue;
};
let indices = &inst.operands[1..];
if walk_into_type(ctx, base_pointee, indices).is_some() {
continue;
}
if walk_into_type(ctx, base_pointee, &indices[1..]) != Some(result_pointee) {
continue;
}
rewrites.push((bi, ii));
}
}
for (bi, ii) in rewrites {
let inst = &mut ctx.module.functions[entry_idx].blocks[bi].instructions[ii];
*inst = Instruction::new(
Op::PtrAccessChain,
inst.result_type,
inst.result_id,
inst.operands.clone(),
);
}
}
pub(in crate::passes) fn direct_scalar_width(ctx: &Ctx, ty: Word) -> Option<u32> {
type_def_of(ctx, ty).and_then(|d| match d.class.opcode {
Op::TypeInt | Op::TypeFloat => match d.operands.first() {
Some(Operand::LiteralBit32(b)) => Some(*b),
_ => None,
},
_ => None,
})
}