Enum cranelift_codegen::ir::instructions::InstructionData
source · pub enum InstructionData {
Show 40 variants
AtomicCas {
opcode: Opcode,
args: [Value; 3],
flags: MemFlags,
},
AtomicRmw {
opcode: Opcode,
args: [Value; 2],
flags: MemFlags,
op: AtomicRmwOp,
},
Binary {
opcode: Opcode,
args: [Value; 2],
},
BinaryImm64 {
opcode: Opcode,
arg: Value,
imm: Imm64,
},
BinaryImm8 {
opcode: Opcode,
arg: Value,
imm: Uimm8,
},
Branch {
opcode: Opcode,
args: ValueList,
destination: Block,
},
BranchTable {
opcode: Opcode,
arg: Value,
destination: Block,
table: JumpTable,
},
Call {
opcode: Opcode,
args: ValueList,
func_ref: FuncRef,
},
CallIndirect {
opcode: Opcode,
args: ValueList,
sig_ref: SigRef,
},
CondTrap {
opcode: Opcode,
arg: Value,
code: TrapCode,
},
DynamicStackLoad {
opcode: Opcode,
dynamic_stack_slot: DynamicStackSlot,
},
DynamicStackStore {
opcode: Opcode,
arg: Value,
dynamic_stack_slot: DynamicStackSlot,
},
FloatCompare {
opcode: Opcode,
args: [Value; 2],
cond: FloatCC,
},
FuncAddr {
opcode: Opcode,
func_ref: FuncRef,
},
HeapAddr {
opcode: Opcode,
arg: Value,
heap: Heap,
offset: Uimm32,
size: Uimm8,
},
HeapLoad {
opcode: Opcode,
arg: Value,
heap_imm: HeapImm,
},
HeapStore {
opcode: Opcode,
args: [Value; 2],
heap_imm: HeapImm,
},
IntAddTrap {
opcode: Opcode,
args: [Value; 2],
code: TrapCode,
},
IntCompare {
opcode: Opcode,
args: [Value; 2],
cond: IntCC,
},
IntCompareImm {
opcode: Opcode,
arg: Value,
cond: IntCC,
imm: Imm64,
},
Jump {
opcode: Opcode,
args: ValueList,
destination: Block,
},
Load {
opcode: Opcode,
arg: Value,
flags: MemFlags,
offset: Offset32,
},
LoadNoOffset {
opcode: Opcode,
arg: Value,
flags: MemFlags,
},
MultiAry {
opcode: Opcode,
args: ValueList,
},
NullAry {
opcode: Opcode,
},
Shuffle {
opcode: Opcode,
args: [Value; 2],
imm: Immediate,
},
StackLoad {
opcode: Opcode,
stack_slot: StackSlot,
offset: Offset32,
},
StackStore {
opcode: Opcode,
arg: Value,
stack_slot: StackSlot,
offset: Offset32,
},
Store {
opcode: Opcode,
args: [Value; 2],
flags: MemFlags,
offset: Offset32,
},
StoreNoOffset {
opcode: Opcode,
args: [Value; 2],
flags: MemFlags,
},
TableAddr {
opcode: Opcode,
arg: Value,
table: Table,
offset: Offset32,
},
Ternary {
opcode: Opcode,
args: [Value; 3],
},
TernaryImm8 {
opcode: Opcode,
args: [Value; 2],
imm: Uimm8,
},
Trap {
opcode: Opcode,
code: TrapCode,
},
Unary {
opcode: Opcode,
arg: Value,
},
UnaryConst {
opcode: Opcode,
constant_handle: Constant,
},
UnaryGlobalValue {
opcode: Opcode,
global_value: GlobalValue,
},
UnaryIeee32 {
opcode: Opcode,
imm: Ieee32,
},
UnaryIeee64 {
opcode: Opcode,
imm: Ieee64,
},
UnaryImm {
opcode: Opcode,
imm: Imm64,
},
}Variants§
AtomicCas
AtomicRmw
Binary
BinaryImm64
BinaryImm8
Branch
BranchTable
Call
CallIndirect
CondTrap
DynamicStackLoad
DynamicStackStore
FloatCompare
FuncAddr
HeapAddr
HeapLoad
HeapStore
IntAddTrap
IntCompare
IntCompareImm
Jump
Load
LoadNoOffset
MultiAry
NullAry
Shuffle
StackLoad
StackStore
Store
StoreNoOffset
TableAddr
Ternary
TernaryImm8
Trap
Unary
UnaryConst
UnaryGlobalValue
UnaryIeee32
UnaryIeee64
UnaryImm
Implementations§
source§impl InstructionData
impl InstructionData
sourcepub fn opcode(&self) -> Opcode
pub fn opcode(&self) -> Opcode
Get the opcode of this instruction.
Examples found in repository?
src/simple_gvn.rs (line 28)
25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150
fn is_load_and_not_readonly(inst_data: &InstructionData) -> bool {
match *inst_data {
InstructionData::Load { flags, .. } => !flags.readonly(),
_ => inst_data.opcode().can_load(),
}
}
/// Wrapper around `InstructionData` which implements `Eq` and `Hash`
#[derive(Clone)]
struct HashKey<'a, 'f: 'a> {
inst: InstructionData,
ty: Type,
pos: &'a RefCell<FuncCursor<'f>>,
}
impl<'a, 'f: 'a> Hash for HashKey<'a, 'f> {
fn hash<H: Hasher>(&self, state: &mut H) {
let pool = &self.pos.borrow().func.dfg.value_lists;
self.inst.hash(state, pool);
self.ty.hash(state);
}
}
impl<'a, 'f: 'a> PartialEq for HashKey<'a, 'f> {
fn eq(&self, other: &Self) -> bool {
let pool = &self.pos.borrow().func.dfg.value_lists;
self.inst.eq(&other.inst, pool) && self.ty == other.ty
}
}
impl<'a, 'f: 'a> Eq for HashKey<'a, 'f> {}
/// Perform simple GVN on `func`.
///
pub fn do_simple_gvn(func: &mut Function, domtree: &mut DominatorTree) {
let _tt = timing::gvn();
debug_assert!(domtree.is_valid());
// Visit blocks in a reverse post-order.
//
// The RefCell here is a bit ugly since the HashKeys in the ScopedHashMap
// need a reference to the function.
let pos = RefCell::new(FuncCursor::new(func));
let mut visible_values: ScopedHashMap<HashKey, Inst> = ScopedHashMap::new();
let mut scope_stack: Vec<Inst> = Vec::new();
for &block in domtree.cfg_postorder().iter().rev() {
{
// Pop any scopes that we just exited.
let layout = &pos.borrow().func.layout;
loop {
if let Some(current) = scope_stack.last() {
if domtree.dominates(*current, block, layout) {
break;
}
} else {
break;
}
scope_stack.pop();
visible_values.decrement_depth();
}
// Push a scope for the current block.
scope_stack.push(layout.first_inst(block).unwrap());
visible_values.increment_depth();
}
pos.borrow_mut().goto_top(block);
while let Some(inst) = {
let mut pos = pos.borrow_mut();
pos.next_inst()
} {
// Resolve aliases, particularly aliases we created earlier.
pos.borrow_mut().func.dfg.resolve_aliases_in_arguments(inst);
let func = Ref::map(pos.borrow(), |pos| &pos.func);
let opcode = func.dfg[inst].opcode();
if opcode.is_branch() && !opcode.is_terminator() {
scope_stack.push(func.layout.next_inst(inst).unwrap());
visible_values.increment_depth();
}
if trivially_unsafe_for_gvn(opcode) {
continue;
}
// These are split up to separate concerns.
if is_load_and_not_readonly(&func.dfg[inst]) {
continue;
}
let ctrl_typevar = func.dfg.ctrl_typevar(inst);
let key = HashKey {
inst: func.dfg[inst],
ty: ctrl_typevar,
pos: &pos,
};
use crate::scoped_hash_map::Entry::*;
match visible_values.entry(key) {
Occupied(entry) => {
#[allow(clippy::debug_assert_with_mut_call)]
{
// Clippy incorrectly believes `&func.layout` should not be used here:
// https://github.com/rust-lang/rust-clippy/issues/4737
debug_assert!(domtree.dominates(*entry.get(), inst, &func.layout));
}
// If the redundant instruction is representing the current
// scope, pick a new representative.
let old = scope_stack.last_mut().unwrap();
if *old == inst {
*old = func.layout.next_inst(inst).unwrap();
}
// Replace the redundant instruction and remove it.
drop(func);
let mut pos = pos.borrow_mut();
pos.func.dfg.replace_with_aliases(inst, *entry.get());
pos.remove_inst_and_step_back();
}
Vacant(entry) => {
entry.insert(inst);
}
}
}
}
}More examples
src/inst_predicates.rs (line 44)
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
pub fn has_side_effect(func: &Function, inst: Inst) -> bool {
let data = &func.dfg[inst];
let opcode = data.opcode();
trivially_has_side_effects(opcode) || is_load_with_defined_trapping(opcode, data)
}
/// Does the given instruction have any side-effect as per [has_side_effect], or else is a load,
/// but not the get_pinned_reg opcode?
pub fn has_lowering_side_effect(func: &Function, inst: Inst) -> bool {
let op = func.dfg[inst].opcode();
op != Opcode::GetPinnedReg && (has_side_effect(func, inst) || op.can_load())
}
/// Is the given instruction a constant value (`iconst`, `fconst`) that can be
/// represented in 64 bits?
pub fn is_constant_64bit(func: &Function, inst: Inst) -> Option<u64> {
let data = &func.dfg[inst];
if data.opcode() == Opcode::Null {
return Some(0);
}
match data {
&InstructionData::UnaryImm { imm, .. } => Some(imm.bits() as u64),
&InstructionData::UnaryIeee32 { imm, .. } => Some(imm.bits() as u64),
&InstructionData::UnaryIeee64 { imm, .. } => Some(imm.bits()),
_ => None,
}
}
/// Get the address, offset, and access type from the given instruction, if any.
pub fn inst_addr_offset_type(func: &Function, inst: Inst) -> Option<(Value, Offset32, Type)> {
let data = &func.dfg[inst];
match data {
InstructionData::Load { arg, offset, .. } => {
let ty = func.dfg.value_type(func.dfg.inst_results(inst)[0]);
Some((*arg, *offset, ty))
}
InstructionData::LoadNoOffset { arg, .. } => {
let ty = func.dfg.value_type(func.dfg.inst_results(inst)[0]);
Some((*arg, 0.into(), ty))
}
InstructionData::Store { args, offset, .. } => {
let ty = func.dfg.value_type(args[0]);
Some((args[1], *offset, ty))
}
InstructionData::StoreNoOffset { args, .. } => {
let ty = func.dfg.value_type(args[0]);
Some((args[1], 0.into(), ty))
}
_ => None,
}
}
/// Get the store data, if any, from an instruction.
pub fn inst_store_data(func: &Function, inst: Inst) -> Option<Value> {
let data = &func.dfg[inst];
match data {
InstructionData::Store { args, .. } | InstructionData::StoreNoOffset { args, .. } => {
Some(args[0])
}
_ => None,
}
}
/// Determine whether this opcode behaves as a memory fence, i.e.,
/// prohibits any moving of memory accesses across it.
pub fn has_memory_fence_semantics(op: Opcode) -> bool {
match op {
Opcode::AtomicRmw
| Opcode::AtomicCas
| Opcode::AtomicLoad
| Opcode::AtomicStore
| Opcode::Fence
| Opcode::Debugtrap => true,
Opcode::Call | Opcode::CallIndirect => true,
op if op.can_trap() => true,
_ => false,
}
}
/// Visit all successors of a block with a given visitor closure. The closure
/// arguments are the branch instruction that is used to reach the successor,
/// the successor block itself, and a flag indicating whether the block is
/// branched to via a table entry.
pub(crate) fn visit_block_succs<F: FnMut(Inst, Block, bool)>(
f: &Function,
block: Block,
mut visit: F,
) {
for inst in f.layout.block_likely_branches(block) {
if f.dfg[inst].opcode().is_branch() {
visit_branch_targets(f, inst, &mut visit);
}
}
}src/licm.rs (line 151)
148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
fn is_unsafe_load(inst_data: &InstructionData) -> bool {
match *inst_data {
InstructionData::Load { flags, .. } => !flags.readonly() || !flags.notrap(),
_ => inst_data.opcode().can_load(),
}
}
/// Test whether the given instruction is loop-invariant.
fn is_loop_invariant(inst: Inst, dfg: &DataFlowGraph, loop_values: &FxHashSet<Value>) -> bool {
if trivially_unsafe_for_licm(dfg[inst].opcode()) {
return false;
}
if is_unsafe_load(&dfg[inst]) {
return false;
}
let inst_args = dfg.inst_args(inst);
for arg in inst_args {
let arg = dfg.resolve_aliases(*arg);
if loop_values.contains(&arg) {
return false;
}
}
true
}src/ir/dfg.rs (line 627)
625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209
pub fn inst_fixed_args(&self, inst: Inst) -> &[Value] {
let num_fixed_args = self[inst]
.opcode()
.constraints()
.num_fixed_value_arguments();
&self.inst_args(inst)[..num_fixed_args]
}
/// Get the fixed value arguments on `inst` as a mutable slice.
pub fn inst_fixed_args_mut(&mut self, inst: Inst) -> &mut [Value] {
let num_fixed_args = self[inst]
.opcode()
.constraints()
.num_fixed_value_arguments();
&mut self.inst_args_mut(inst)[..num_fixed_args]
}
/// Get the variable value arguments on `inst` as a slice.
pub fn inst_variable_args(&self, inst: Inst) -> &[Value] {
let num_fixed_args = self[inst]
.opcode()
.constraints()
.num_fixed_value_arguments();
&self.inst_args(inst)[num_fixed_args..]
}
/// Get the variable value arguments on `inst` as a mutable slice.
pub fn inst_variable_args_mut(&mut self, inst: Inst) -> &mut [Value] {
let num_fixed_args = self[inst]
.opcode()
.constraints()
.num_fixed_value_arguments();
&mut self.inst_args_mut(inst)[num_fixed_args..]
}
/// Create result values for an instruction that produces multiple results.
///
/// Instructions that produce no result values only need to be created with `make_inst`,
/// otherwise call `make_inst_results` to allocate value table entries for the results.
///
/// The result value types are determined from the instruction's value type constraints and the
/// provided `ctrl_typevar` type for polymorphic instructions. For non-polymorphic
/// instructions, `ctrl_typevar` is ignored, and `INVALID` can be used.
///
/// The type of the first result value is also set, even if it was already set in the
/// `InstructionData` passed to `make_inst`. If this function is called with a single-result
/// instruction, that is the only effect.
pub fn make_inst_results(&mut self, inst: Inst, ctrl_typevar: Type) -> usize {
self.make_inst_results_reusing(inst, ctrl_typevar, iter::empty())
}
/// Create result values for `inst`, reusing the provided detached values.
///
/// Create a new set of result values for `inst` using `ctrl_typevar` to determine the result
/// types. Any values provided by `reuse` will be reused. When `reuse` is exhausted or when it
/// produces `None`, a new value is created.
pub fn make_inst_results_reusing<I>(
&mut self,
inst: Inst,
ctrl_typevar: Type,
reuse: I,
) -> usize
where
I: Iterator<Item = Option<Value>>,
{
let mut reuse = reuse.fuse();
self.results[inst].clear(&mut self.value_lists);
// Get the call signature if this is a function call.
if let Some(sig) = self.call_signature(inst) {
// Create result values corresponding to the call return types.
debug_assert_eq!(
self.insts[inst].opcode().constraints().num_fixed_results(),
0
);
let num_results = self.signatures[sig].returns.len();
for res_idx in 0..num_results {
let ty = self.signatures[sig].returns[res_idx].value_type;
if let Some(Some(v)) = reuse.next() {
debug_assert_eq!(self.value_type(v), ty, "Reused {} is wrong type", ty);
self.attach_result(inst, v);
} else {
self.append_result(inst, ty);
}
}
num_results
} else {
// Create result values corresponding to the opcode's constraints.
let constraints = self.insts[inst].opcode().constraints();
let num_results = constraints.num_fixed_results();
for res_idx in 0..num_results {
let ty = constraints.result_type(res_idx, ctrl_typevar);
if let Some(Some(v)) = reuse.next() {
debug_assert_eq!(self.value_type(v), ty, "Reused {} is wrong type", ty);
self.attach_result(inst, v);
} else {
self.append_result(inst, ty);
}
}
num_results
}
}
/// Create a `ReplaceBuilder` that will replace `inst` with a new instruction in place.
pub fn replace(&mut self, inst: Inst) -> ReplaceBuilder {
ReplaceBuilder::new(self, inst)
}
/// Detach the list of result values from `inst` and return it.
///
/// This leaves `inst` without any result values. New result values can be created by calling
/// `make_inst_results` or by using a `replace(inst)` builder.
pub fn detach_results(&mut self, inst: Inst) -> ValueList {
self.results[inst].take()
}
/// Clear the list of result values from `inst`.
///
/// This leaves `inst` without any result values. New result values can be created by calling
/// `make_inst_results` or by using a `replace(inst)` builder.
pub fn clear_results(&mut self, inst: Inst) {
self.results[inst].clear(&mut self.value_lists)
}
/// Attach an existing value to the result value list for `inst`.
///
/// The `res` value is appended to the end of the result list.
///
/// This is a very low-level operation. Usually, instruction results with the correct types are
/// created automatically. The `res` value must not be attached to anything else.
pub fn attach_result(&mut self, inst: Inst, res: Value) {
debug_assert!(!self.value_is_attached(res));
let num = self.results[inst].push(res, &mut self.value_lists);
debug_assert!(num <= u16::MAX as usize, "Too many result values");
let ty = self.value_type(res);
self.values[res] = ValueData::Inst {
ty,
num: num as u16,
inst,
}
.into();
}
/// Replace an instruction result with a new value of type `new_type`.
///
/// The `old_value` must be an attached instruction result.
///
/// The old value is left detached, so it should probably be changed into something else.
///
/// Returns the new value.
pub fn replace_result(&mut self, old_value: Value, new_type: Type) -> Value {
let (num, inst) = match ValueData::from(self.values[old_value]) {
ValueData::Inst { num, inst, .. } => (num, inst),
_ => panic!("{} is not an instruction result value", old_value),
};
let new_value = self.make_value(ValueData::Inst {
ty: new_type,
num,
inst,
});
let num = num as usize;
let attached = mem::replace(
self.results[inst]
.get_mut(num, &mut self.value_lists)
.expect("Replacing detached result"),
new_value,
);
debug_assert_eq!(
attached,
old_value,
"{} wasn't detached from {}",
old_value,
self.display_inst(inst)
);
new_value
}
/// Append a new instruction result value to `inst`.
pub fn append_result(&mut self, inst: Inst, ty: Type) -> Value {
let res = self.values.next_key();
let num = self.results[inst].push(res, &mut self.value_lists);
debug_assert!(num <= u16::MAX as usize, "Too many result values");
self.make_value(ValueData::Inst {
ty,
inst,
num: num as u16,
})
}
/// Append a new value argument to an instruction.
///
/// Panics if the instruction doesn't support arguments.
pub fn append_inst_arg(&mut self, inst: Inst, new_arg: Value) {
let mut branch_values = self.insts[inst]
.take_value_list()
.expect("the instruction doesn't have value arguments");
branch_values.push(new_arg, &mut self.value_lists);
self.insts[inst].put_value_list(branch_values)
}
/// Get the first result of an instruction.
///
/// This function panics if the instruction doesn't have any result.
pub fn first_result(&self, inst: Inst) -> Value {
self.results[inst]
.first(&self.value_lists)
.expect("Instruction has no results")
}
/// Test if `inst` has any result values currently.
pub fn has_results(&self, inst: Inst) -> bool {
!self.results[inst].is_empty()
}
/// Return all the results of an instruction.
pub fn inst_results(&self, inst: Inst) -> &[Value] {
self.results[inst].as_slice(&self.value_lists)
}
/// Return all the results of an instruction as ValueList.
pub fn inst_results_list(&self, inst: Inst) -> ValueList {
self.results[inst]
}
/// Get the call signature of a direct or indirect call instruction.
/// Returns `None` if `inst` is not a call instruction.
pub fn call_signature(&self, inst: Inst) -> Option<SigRef> {
match self.insts[inst].analyze_call(&self.value_lists) {
CallInfo::NotACall => None,
CallInfo::Direct(f, _) => Some(self.ext_funcs[f].signature),
CallInfo::Indirect(s, _) => Some(s),
}
}
/// Check if `inst` is a branch.
pub fn analyze_branch(&self, inst: Inst) -> BranchInfo {
self.insts[inst].analyze_branch(&self.value_lists)
}
/// Compute the type of an instruction result from opcode constraints and call signatures.
///
/// This computes the same sequence of result types that `make_inst_results()` above would
/// assign to the created result values, but it does not depend on `make_inst_results()` being
/// called first.
///
/// Returns `None` if asked about a result index that is too large.
pub fn compute_result_type(
&self,
inst: Inst,
result_idx: usize,
ctrl_typevar: Type,
) -> Option<Type> {
let constraints = self.insts[inst].opcode().constraints();
let num_fixed_results = constraints.num_fixed_results();
if result_idx < num_fixed_results {
return Some(constraints.result_type(result_idx, ctrl_typevar));
}
// Not a fixed result, try to extract a return type from the call signature.
self.call_signature(inst).and_then(|sigref| {
self.signatures[sigref]
.returns
.get(result_idx - num_fixed_results)
.map(|&arg| arg.value_type)
})
}
/// Get the controlling type variable, or `INVALID` if `inst` isn't polymorphic.
pub fn ctrl_typevar(&self, inst: Inst) -> Type {
let constraints = self[inst].opcode().constraints();
if !constraints.is_polymorphic() {
types::INVALID
} else if constraints.requires_typevar_operand() {
// Not all instruction formats have a designated operand, but in that case
// `requires_typevar_operand()` should never be true.
self.value_type(
self[inst]
.typevar_operand(&self.value_lists)
.unwrap_or_else(|| {
panic!(
"Instruction format for {:?} doesn't have a designated operand",
self[inst]
)
}),
)
} else {
self.value_type(self.first_result(inst))
}
}
}
/// Allow immutable access to instructions via indexing.
impl Index<Inst> for DataFlowGraph {
type Output = InstructionData;
fn index(&self, inst: Inst) -> &InstructionData {
&self.insts[inst]
}
}
/// Allow mutable access to instructions via indexing.
impl IndexMut<Inst> for DataFlowGraph {
fn index_mut(&mut self, inst: Inst) -> &mut InstructionData {
&mut self.insts[inst]
}
}
/// basic blocks.
impl DataFlowGraph {
/// Create a new basic block.
pub fn make_block(&mut self) -> Block {
self.blocks.push(BlockData::new())
}
/// Get the number of parameters on `block`.
pub fn num_block_params(&self, block: Block) -> usize {
self.blocks[block].params.len(&self.value_lists)
}
/// Get the parameters on `block`.
pub fn block_params(&self, block: Block) -> &[Value] {
self.blocks[block].params.as_slice(&self.value_lists)
}
/// Get the types of the parameters on `block`.
pub fn block_param_types(&self, block: Block) -> impl Iterator<Item = Type> + '_ {
self.block_params(block).iter().map(|&v| self.value_type(v))
}
/// Append a parameter with type `ty` to `block`.
pub fn append_block_param(&mut self, block: Block, ty: Type) -> Value {
let param = self.values.next_key();
let num = self.blocks[block].params.push(param, &mut self.value_lists);
debug_assert!(num <= u16::MAX as usize, "Too many parameters on block");
self.make_value(ValueData::Param {
ty,
num: num as u16,
block,
})
}
/// Removes `val` from `block`'s parameters by swapping it with the last parameter on `block`.
/// Returns the position of `val` before removal.
///
/// *Important*: to ensure O(1) deletion, this method swaps the removed parameter with the
/// last `block` parameter. This can disrupt all the branch instructions jumping to this
/// `block` for which you have to change the branch argument order if necessary.
///
/// Panics if `val` is not a block parameter.
pub fn swap_remove_block_param(&mut self, val: Value) -> usize {
let (block, num) =
if let ValueData::Param { num, block, .. } = ValueData::from(self.values[val]) {
(block, num)
} else {
panic!("{} must be a block parameter", val);
};
self.blocks[block]
.params
.swap_remove(num as usize, &mut self.value_lists);
if let Some(last_arg_val) = self.blocks[block]
.params
.get(num as usize, &self.value_lists)
{
// We update the position of the old last arg.
let mut last_arg_data = ValueData::from(self.values[last_arg_val]);
if let ValueData::Param {
num: ref mut old_num,
..
} = &mut last_arg_data
{
*old_num = num;
self.values[last_arg_val] = last_arg_data.into();
} else {
panic!("{} should be a Block parameter", last_arg_val);
}
}
num as usize
}
/// Removes `val` from `block`'s parameters by a standard linear time list removal which
/// preserves ordering. Also updates the values' data.
pub fn remove_block_param(&mut self, val: Value) {
let (block, num) =
if let ValueData::Param { num, block, .. } = ValueData::from(self.values[val]) {
(block, num)
} else {
panic!("{} must be a block parameter", val);
};
self.blocks[block]
.params
.remove(num as usize, &mut self.value_lists);
for index in num..(self.num_block_params(block) as u16) {
let packed = &mut self.values[self.blocks[block]
.params
.get(index as usize, &self.value_lists)
.unwrap()];
let mut data = ValueData::from(*packed);
match &mut data {
ValueData::Param { ref mut num, .. } => {
*num -= 1;
*packed = data.into();
}
_ => panic!(
"{} must be a block parameter",
self.blocks[block]
.params
.get(index as usize, &self.value_lists)
.unwrap()
),
}
}
}
/// Append an existing value to `block`'s parameters.
///
/// The appended value can't already be attached to something else.
///
/// In almost all cases, you should be using `append_block_param()` instead of this method.
pub fn attach_block_param(&mut self, block: Block, param: Value) {
debug_assert!(!self.value_is_attached(param));
let num = self.blocks[block].params.push(param, &mut self.value_lists);
debug_assert!(num <= u16::MAX as usize, "Too many parameters on block");
let ty = self.value_type(param);
self.values[param] = ValueData::Param {
ty,
num: num as u16,
block,
}
.into();
}
/// Replace a block parameter with a new value of type `ty`.
///
/// The `old_value` must be an attached block parameter. It is removed from its place in the list
/// of parameters and replaced by a new value of type `new_type`. The new value gets the same
/// position in the list, and other parameters are not disturbed.
///
/// The old value is left detached, so it should probably be changed into something else.
///
/// Returns the new value.
pub fn replace_block_param(&mut self, old_value: Value, new_type: Type) -> Value {
// Create new value identical to the old one except for the type.
let (block, num) =
if let ValueData::Param { num, block, .. } = ValueData::from(self.values[old_value]) {
(block, num)
} else {
panic!("{} must be a block parameter", old_value);
};
let new_arg = self.make_value(ValueData::Param {
ty: new_type,
num,
block,
});
self.blocks[block]
.params
.as_mut_slice(&mut self.value_lists)[num as usize] = new_arg;
new_arg
}
/// Detach all the parameters from `block` and return them as a `ValueList`.
///
/// This is a quite low-level operation. Sensible things to do with the detached block parameters
/// is to put them back on the same block with `attach_block_param()` or change them into aliases
/// with `change_to_alias()`.
pub fn detach_block_params(&mut self, block: Block) -> ValueList {
self.blocks[block].params.take()
}
}
/// Contents of a basic block.
///
/// Parameters on a basic block are values that dominate everything in the block. All
/// branches to this block must provide matching arguments, and the arguments to the entry block must
/// match the function arguments.
#[derive(Clone, PartialEq, Hash)]
#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
struct BlockData {
/// List of parameters to this block.
params: ValueList,
}
impl BlockData {
fn new() -> Self {
Self {
params: ValueList::new(),
}
}
}
/// Object that can display an instruction.
pub struct DisplayInst<'a>(&'a DataFlowGraph, Inst);
impl<'a> fmt::Display for DisplayInst<'a> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let dfg = self.0;
let inst = self.1;
if let Some((first, rest)) = dfg.inst_results(inst).split_first() {
write!(f, "{}", first)?;
for v in rest {
write!(f, ", {}", v)?;
}
write!(f, " = ")?;
}
let typevar = dfg.ctrl_typevar(inst);
if typevar.is_invalid() {
write!(f, "{}", dfg[inst].opcode())?;
} else {
write!(f, "{}.{}", dfg[inst].opcode(), typevar)?;
}
write_operands(f, dfg, inst)
}
}
/// Parser routines. These routines should not be used outside the parser.
impl DataFlowGraph {
/// Set the type of a value. This is only for use in the parser, which needs
/// to create invalid values for index padding which may be reassigned later.
#[cold]
fn set_value_type_for_parser(&mut self, v: Value, t: Type) {
assert_eq!(
self.value_type(v),
types::INVALID,
"this function is only for assigning types to previously invalid values"
);
self.values[v].set_type(t);
}
/// Check that the given concrete `Type` has been defined in the function.
pub fn check_dynamic_type(&mut self, ty: Type) -> Option<Type> {
debug_assert!(ty.is_dynamic_vector());
if self
.dynamic_types
.values()
.any(|dyn_ty_data| dyn_ty_data.concrete().unwrap() == ty)
{
Some(ty)
} else {
None
}
}
/// Create result values for `inst`, reusing the provided detached values.
/// This is similar to `make_inst_results_reusing` except it's only for use
/// in the parser, which needs to reuse previously invalid values.
#[cold]
pub fn make_inst_results_for_parser(
&mut self,
inst: Inst,
ctrl_typevar: Type,
reuse: &[Value],
) -> usize {
// Get the call signature if this is a function call.
if let Some(sig) = self.call_signature(inst) {
assert_eq!(
self.insts[inst].opcode().constraints().num_fixed_results(),
0
);
for res_idx in 0..self.signatures[sig].returns.len() {
let ty = self.signatures[sig].returns[res_idx].value_type;
if let Some(v) = reuse.get(res_idx) {
self.set_value_type_for_parser(*v, ty);
}
}
} else {
let constraints = self.insts[inst].opcode().constraints();
for res_idx in 0..constraints.num_fixed_results() {
let ty = constraints.result_type(res_idx, ctrl_typevar);
if ty.is_dynamic_vector() {
self.check_dynamic_type(ty)
.unwrap_or_else(|| panic!("Use of undeclared dynamic type: {}", ty));
}
if let Some(v) = reuse.get(res_idx) {
self.set_value_type_for_parser(*v, ty);
}
}
}
self.make_inst_results_reusing(inst, ctrl_typevar, reuse.iter().map(|x| Some(*x)))
}src/isa/x64/lower.rs (line 37)
33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613
fn matches_input(ctx: &mut Lower<Inst>, input: InsnInput, op: Opcode) -> Option<IRInst> {
let inputs = ctx.get_input_as_source_or_const(input.insn, input.input);
inputs.inst.as_inst().and_then(|(src_inst, _)| {
let data = ctx.data(src_inst);
if data.opcode() == op {
return Some(src_inst);
}
None
})
}
/// Emits instruction(s) to generate the given 64-bit constant value into a newly-allocated
/// temporary register, returning that register.
fn generate_constant(ctx: &mut Lower<Inst>, ty: Type, c: u64) -> ValueRegs<Reg> {
let from_bits = ty_bits(ty);
let masked = if from_bits < 64 {
c & ((1u64 << from_bits) - 1)
} else {
c
};
let cst_copy = ctx.alloc_tmp(ty);
for inst in Inst::gen_constant(cst_copy, masked as u128, ty, |ty| {
ctx.alloc_tmp(ty).only_reg().unwrap()
})
.into_iter()
{
ctx.emit(inst);
}
non_writable_value_regs(cst_copy)
}
/// Put the given input into possibly multiple registers, and mark it as used (side-effect).
fn put_input_in_regs(ctx: &mut Lower<Inst>, spec: InsnInput) -> ValueRegs<Reg> {
let ty = ctx.input_ty(spec.insn, spec.input);
let input = ctx.get_input_as_source_or_const(spec.insn, spec.input);
if let Some(c) = input.constant {
// Generate constants fresh at each use to minimize long-range register pressure.
generate_constant(ctx, ty, c)
} else {
ctx.put_input_in_regs(spec.insn, spec.input)
}
}
/// Put the given input into a register, and mark it as used (side-effect).
fn put_input_in_reg(ctx: &mut Lower<Inst>, spec: InsnInput) -> Reg {
put_input_in_regs(ctx, spec)
.only_reg()
.expect("Multi-register value not expected")
}
/// Determines whether a load operation (indicated by `src_insn`) can be merged
/// into the current lowering point. If so, returns the address-base source (as
/// an `InsnInput`) and an offset from that address from which to perform the
/// load.
fn is_mergeable_load(ctx: &mut Lower<Inst>, src_insn: IRInst) -> Option<(InsnInput, i32)> {
let insn_data = ctx.data(src_insn);
let inputs = ctx.num_inputs(src_insn);
if inputs != 1 {
return None;
}
let load_ty = ctx.output_ty(src_insn, 0);
if ty_bits(load_ty) < 32 {
// Narrower values are handled by ALU insts that are at least 32 bits
// wide, which is normally OK as we ignore upper buts; but, if we
// generate, e.g., a direct-from-memory 32-bit add for a byte value and
// the byte is the last byte in a page, the extra data that we load is
// incorrectly accessed. So we only allow loads to merge for
// 32-bit-and-above widths.
return None;
}
// SIMD instructions can only be load-coalesced when the loaded value comes
// from an aligned address.
if load_ty.is_vector() && !insn_data.memflags().map_or(false, |f| f.aligned()) {
return None;
}
// Just testing the opcode is enough, because the width will always match if
// the type does (and the type should match if the CLIF is properly
// constructed).
if insn_data.opcode() == Opcode::Load {
let offset = insn_data
.load_store_offset()
.expect("load should have offset");
Some((
InsnInput {
insn: src_insn,
input: 0,
},
offset,
))
} else {
None
}
}
fn input_to_imm(ctx: &mut Lower<Inst>, spec: InsnInput) -> Option<u64> {
ctx.get_input_as_source_or_const(spec.insn, spec.input)
.constant
}
fn emit_vm_call(
ctx: &mut Lower<Inst>,
flags: &Flags,
triple: &Triple,
libcall: LibCall,
inputs: &[Reg],
outputs: &[Writable<Reg>],
) -> CodegenResult<()> {
let extname = ExternalName::LibCall(libcall);
let dist = if flags.use_colocated_libcalls() {
RelocDistance::Near
} else {
RelocDistance::Far
};
// TODO avoid recreating signatures for every single Libcall function.
let call_conv = CallConv::for_libcall(flags, CallConv::triple_default(triple));
let sig = libcall.signature(call_conv);
let caller_conv = ctx.abi().call_conv(ctx.sigs());
if !ctx.sigs().have_abi_sig_for_signature(&sig) {
ctx.sigs_mut()
.make_abi_sig_from_ir_signature::<X64ABIMachineSpec>(sig.clone(), flags)?;
}
let mut abi =
X64Caller::from_libcall(ctx.sigs(), &sig, &extname, dist, caller_conv, flags.clone())?;
abi.emit_stack_pre_adjust(ctx);
assert_eq!(inputs.len(), abi.num_args(ctx.sigs()));
for (i, input) in inputs.iter().enumerate() {
for inst in abi.gen_arg(ctx, i, ValueRegs::one(*input)) {
ctx.emit(inst);
}
}
let mut retval_insts: SmallInstVec<_> = smallvec![];
for (i, output) in outputs.iter().enumerate() {
retval_insts.extend(abi.gen_retval(ctx, i, ValueRegs::one(*output)).into_iter());
}
abi.emit_call(ctx);
for inst in retval_insts {
ctx.emit(inst);
}
abi.emit_stack_post_adjust(ctx);
Ok(())
}
/// Returns whether the given input is a shift by a constant value less or equal than 3.
/// The goal is to embed it within an address mode.
fn matches_small_constant_shift(ctx: &mut Lower<Inst>, spec: InsnInput) -> Option<(InsnInput, u8)> {
matches_input(ctx, spec, Opcode::Ishl).and_then(|shift| {
match input_to_imm(
ctx,
InsnInput {
insn: shift,
input: 1,
},
) {
Some(shift_amt) if shift_amt <= 3 => Some((
InsnInput {
insn: shift,
input: 0,
},
shift_amt as u8,
)),
_ => None,
}
})
}
/// Lowers an instruction to one of the x86 addressing modes.
///
/// Note: the 32-bit offset in Cranelift has to be sign-extended, which maps x86's behavior.
fn lower_to_amode(ctx: &mut Lower<Inst>, spec: InsnInput, offset: i32) -> Amode {
let flags = ctx
.memflags(spec.insn)
.expect("Instruction with amode should have memflags");
// We now either have an add that we must materialize, or some other input; as well as the
// final offset.
if let Some(add) = matches_input(ctx, spec, Opcode::Iadd) {
debug_assert_eq!(ctx.output_ty(add, 0), types::I64);
let add_inputs = &[
InsnInput {
insn: add,
input: 0,
},
InsnInput {
insn: add,
input: 1,
},
];
// TODO heap_addr legalization generates a uext64 *after* the shift, so these optimizations
// aren't happening in the wasm case. We could do better, given some range analysis.
let (base, index, shift) = if let Some((shift_input, shift_amt)) =
matches_small_constant_shift(ctx, add_inputs[0])
{
(
put_input_in_reg(ctx, add_inputs[1]),
put_input_in_reg(ctx, shift_input),
shift_amt,
)
} else if let Some((shift_input, shift_amt)) =
matches_small_constant_shift(ctx, add_inputs[1])
{
(
put_input_in_reg(ctx, add_inputs[0]),
put_input_in_reg(ctx, shift_input),
shift_amt,
)
} else {
for i in 0..=1 {
// Try to pierce through uextend.
if let Some(uextend) = matches_input(
ctx,
InsnInput {
insn: add,
input: i,
},
Opcode::Uextend,
) {
if let Some(cst) = ctx.get_input_as_source_or_const(uextend, 0).constant {
// Zero the upper bits.
let input_size = ctx.input_ty(uextend, 0).bits() as u64;
let shift: u64 = 64 - input_size;
let uext_cst: u64 = (cst << shift) >> shift;
let final_offset = (offset as i64).wrapping_add(uext_cst as i64);
if low32_will_sign_extend_to_64(final_offset as u64) {
let base = put_input_in_reg(ctx, add_inputs[1 - i]);
return Amode::imm_reg(final_offset as u32, base).with_flags(flags);
}
}
}
// If it's a constant, add it directly!
if let Some(cst) = ctx.get_input_as_source_or_const(add, i).constant {
let final_offset = (offset as i64).wrapping_add(cst as i64);
if low32_will_sign_extend_to_64(final_offset as u64) {
let base = put_input_in_reg(ctx, add_inputs[1 - i]);
return Amode::imm_reg(final_offset as u32, base).with_flags(flags);
}
}
}
(
put_input_in_reg(ctx, add_inputs[0]),
put_input_in_reg(ctx, add_inputs[1]),
0,
)
};
return Amode::imm_reg_reg_shift(
offset as u32,
Gpr::new(base).unwrap(),
Gpr::new(index).unwrap(),
shift,
)
.with_flags(flags);
}
let input = put_input_in_reg(ctx, spec);
Amode::imm_reg(offset as u32, input).with_flags(flags)
}
//=============================================================================
// Top-level instruction lowering entry point, for one instruction.
/// Actually codegen an instruction's results into registers.
fn lower_insn_to_regs(
ctx: &mut Lower<Inst>,
insn: IRInst,
flags: &Flags,
isa_flags: &x64_settings::Flags,
triple: &Triple,
) -> CodegenResult<()> {
let outputs: SmallVec<[InsnOutput; 2]> = (0..ctx.num_outputs(insn))
.map(|i| InsnOutput { insn, output: i })
.collect();
if let Ok(()) = isle::lower(ctx, triple, flags, isa_flags, &outputs, insn) {
return Ok(());
}
let op = ctx.data(insn).opcode();
match op {
Opcode::Iconst
| Opcode::F32const
| Opcode::F64const
| Opcode::Null
| Opcode::Iadd
| Opcode::IaddIfcout
| Opcode::SaddSat
| Opcode::UaddSat
| Opcode::Isub
| Opcode::SsubSat
| Opcode::UsubSat
| Opcode::AvgRound
| Opcode::Band
| Opcode::Bor
| Opcode::Bxor
| Opcode::Imul
| Opcode::BandNot
| Opcode::Iabs
| Opcode::Smax
| Opcode::Umax
| Opcode::Smin
| Opcode::Umin
| Opcode::Bnot
| Opcode::Bitselect
| Opcode::Vselect
| Opcode::Ushr
| Opcode::Sshr
| Opcode::Ishl
| Opcode::Rotl
| Opcode::Rotr
| Opcode::Ineg
| Opcode::Trap
| Opcode::ResumableTrap
| Opcode::Clz
| Opcode::Ctz
| Opcode::Popcnt
| Opcode::Bitrev
| Opcode::Bswap
| Opcode::IsNull
| Opcode::IsInvalid
| Opcode::Uextend
| Opcode::Sextend
| Opcode::Ireduce
| Opcode::Debugtrap
| Opcode::WideningPairwiseDotProductS
| Opcode::Fadd
| Opcode::Fsub
| Opcode::Fmul
| Opcode::Fdiv
| Opcode::Fmin
| Opcode::Fmax
| Opcode::FminPseudo
| Opcode::FmaxPseudo
| Opcode::Sqrt
| Opcode::Fpromote
| Opcode::FvpromoteLow
| Opcode::Fdemote
| Opcode::Fvdemote
| Opcode::Fma
| Opcode::Icmp
| Opcode::Fcmp
| Opcode::Load
| Opcode::Uload8
| Opcode::Sload8
| Opcode::Uload16
| Opcode::Sload16
| Opcode::Uload32
| Opcode::Sload32
| Opcode::Sload8x8
| Opcode::Uload8x8
| Opcode::Sload16x4
| Opcode::Uload16x4
| Opcode::Sload32x2
| Opcode::Uload32x2
| Opcode::Store
| Opcode::Istore8
| Opcode::Istore16
| Opcode::Istore32
| Opcode::AtomicRmw
| Opcode::AtomicCas
| Opcode::AtomicLoad
| Opcode::AtomicStore
| Opcode::Fence
| Opcode::FuncAddr
| Opcode::SymbolValue
| Opcode::Return
| Opcode::Call
| Opcode::CallIndirect
| Opcode::GetFramePointer
| Opcode::GetStackPointer
| Opcode::GetReturnAddress
| Opcode::Select
| Opcode::SelectSpectreGuard
| Opcode::FcvtFromSint
| Opcode::FcvtLowFromSint
| Opcode::FcvtFromUint
| Opcode::FcvtToUint
| Opcode::FcvtToSint
| Opcode::FcvtToUintSat
| Opcode::FcvtToSintSat
| Opcode::IaddPairwise
| Opcode::UwidenHigh
| Opcode::UwidenLow
| Opcode::SwidenHigh
| Opcode::SwidenLow
| Opcode::Snarrow
| Opcode::Unarrow
| Opcode::Bitcast
| Opcode::Fabs
| Opcode::Fneg
| Opcode::Fcopysign
| Opcode::Ceil
| Opcode::Floor
| Opcode::Nearest
| Opcode::Trunc
| Opcode::StackAddr
| Opcode::Udiv
| Opcode::Urem
| Opcode::Sdiv
| Opcode::Srem
| Opcode::Umulhi
| Opcode::Smulhi
| Opcode::GetPinnedReg
| Opcode::SetPinnedReg
| Opcode::Vconst
| Opcode::Insertlane
| Opcode::Shuffle
| Opcode::Swizzle
| Opcode::Extractlane
| Opcode::ScalarToVector
| Opcode::Splat
| Opcode::VanyTrue
| Opcode::VallTrue
| Opcode::VhighBits
| Opcode::Iconcat
| Opcode::Isplit
| Opcode::TlsValue
| Opcode::SqmulRoundSat
| Opcode::Uunarrow
| Opcode::Nop
| Opcode::Bmask => {
let ty = if outputs.len() > 0 {
Some(ctx.output_ty(insn, 0))
} else {
None
};
unreachable!(
"implemented in ISLE: inst = `{}`, type = `{:?}`",
ctx.dfg().display_inst(insn),
ty
)
}
Opcode::DynamicStackAddr => unimplemented!("DynamicStackAddr"),
// Unimplemented opcodes below. These are not currently used by Wasm
// lowering or other known embeddings, but should be either supported or
// removed eventually
Opcode::ExtractVector => {
unimplemented!("ExtractVector not supported");
}
Opcode::Cls => unimplemented!("Cls not supported"),
Opcode::BorNot | Opcode::BxorNot => {
unimplemented!("or-not / xor-not opcodes not implemented");
}
Opcode::Vsplit | Opcode::Vconcat => {
unimplemented!("Vector split/concat ops not implemented.");
}
// Opcodes that should be removed by legalization. These should
// eventually be removed if/when we replace in-situ legalization with
// something better.
Opcode::Ifcmp | Opcode::Ffcmp => {
panic!("Should never reach ifcmp/ffcmp as isel root!");
}
Opcode::IaddImm
| Opcode::ImulImm
| Opcode::UdivImm
| Opcode::SdivImm
| Opcode::UremImm
| Opcode::SremImm
| Opcode::IrsubImm
| Opcode::IaddCin
| Opcode::IaddIfcin
| Opcode::IaddCout
| Opcode::IaddCarry
| Opcode::IaddIfcarry
| Opcode::IsubBin
| Opcode::IsubIfbin
| Opcode::IsubBout
| Opcode::IsubIfbout
| Opcode::IsubBorrow
| Opcode::IsubIfborrow
| Opcode::UaddOverflowTrap
| Opcode::BandImm
| Opcode::BorImm
| Opcode::BxorImm
| Opcode::RotlImm
| Opcode::RotrImm
| Opcode::IshlImm
| Opcode::UshrImm
| Opcode::SshrImm
| Opcode::IcmpImm
| Opcode::IfcmpImm => {
panic!("ALU+imm and ALU+carry ops should not appear here!");
}
Opcode::StackLoad
| Opcode::StackStore
| Opcode::DynamicStackStore
| Opcode::DynamicStackLoad => {
panic!("Direct stack memory access not supported; should have been legalized");
}
Opcode::GlobalValue => {
panic!("global_value should have been removed by legalization!");
}
Opcode::HeapLoad | Opcode::HeapStore | Opcode::HeapAddr => {
panic!("heap access instructions should have been removed by legalization!");
}
Opcode::TableAddr => {
panic!("table_addr should have been removed by legalization!");
}
Opcode::Trapz | Opcode::Trapnz | Opcode::ResumableTrapnz => {
panic!("trapz / trapnz / resumable_trapnz should have been removed by legalization!");
}
Opcode::Jump | Opcode::Brz | Opcode::Brnz | Opcode::BrTable => {
panic!("Branch opcode reached non-branch lowering logic!");
}
}
}
//=============================================================================
// Lowering-backend trait implementation.
impl LowerBackend for X64Backend {
type MInst = Inst;
fn lower(&self, ctx: &mut Lower<Inst>, ir_inst: IRInst) -> CodegenResult<()> {
lower_insn_to_regs(ctx, ir_inst, &self.flags, &self.x64_flags, &self.triple)
}
fn lower_branch_group(
&self,
ctx: &mut Lower<Inst>,
branches: &[IRInst],
targets: &[MachLabel],
) -> CodegenResult<()> {
// A block should end with at most two branches. The first may be a
// conditional branch; a conditional branch can be followed only by an
// unconditional branch or fallthrough. Otherwise, if only one branch,
// it may be an unconditional branch, a fallthrough, a return, or a
// trap. These conditions are verified by `is_ebb_basic()` during the
// verifier pass.
assert!(branches.len() <= 2);
if branches.len() == 2 {
let op1 = ctx.data(branches[1]).opcode();
assert!(op1 == Opcode::Jump);
}
if let Ok(()) = isle::lower_branch(
ctx,
&self.triple,
&self.flags,
&self.x64_flags,
branches[0],
targets,
) {
return Ok(());
}
unreachable!(
"implemented in ISLE: branch = `{}`",
ctx.dfg().display_inst(branches[0]),
);
}sourcepub fn typevar_operand(&self, pool: &ValueListPool) -> Option<Value>
pub fn typevar_operand(&self, pool: &ValueListPool) -> Option<Value>
Get the controlling type variable operand.
Examples found in repository?
src/ir/dfg.rs (line 905)
895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916
pub fn ctrl_typevar(&self, inst: Inst) -> Type {
let constraints = self[inst].opcode().constraints();
if !constraints.is_polymorphic() {
types::INVALID
} else if constraints.requires_typevar_operand() {
// Not all instruction formats have a designated operand, but in that case
// `requires_typevar_operand()` should never be true.
self.value_type(
self[inst]
.typevar_operand(&self.value_lists)
.unwrap_or_else(|| {
panic!(
"Instruction format for {:?} doesn't have a designated operand",
self[inst]
)
}),
)
} else {
self.value_type(self.first_result(inst))
}
}More examples
src/write.rs (line 297)
286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313
fn type_suffix(func: &Function, inst: Inst) -> Option<Type> {
let inst_data = &func.dfg[inst];
let constraints = inst_data.opcode().constraints();
if !constraints.is_polymorphic() {
return None;
}
// If the controlling type variable can be inferred from the type of the designated value input
// operand, we don't need the type suffix.
if constraints.use_typevar_operand() {
let ctrl_var = inst_data.typevar_operand(&func.dfg.value_lists).unwrap();
let def_block = match func.dfg.value_def(ctrl_var) {
ValueDef::Result(instr, _) => func.layout.inst_block(instr),
ValueDef::Param(block, _) => Some(block),
};
if def_block.is_some() && def_block == func.layout.inst_block(inst) {
return None;
}
}
let rtype = func.dfg.ctrl_typevar(inst);
assert!(
!rtype.is_invalid(),
"Polymorphic instruction must produce a result"
);
Some(rtype)
}sourcepub fn arguments<'a>(&'a self, pool: &'a ValueListPool) -> &[Value]
pub fn arguments<'a>(&'a self, pool: &'a ValueListPool) -> &[Value]
Get the value arguments to this instruction.
sourcepub fn arguments_mut<'a>(
&'a mut self,
pool: &'a mut ValueListPool
) -> &mut [Value]
pub fn arguments_mut<'a>(
&'a mut self,
pool: &'a mut ValueListPool
) -> &mut [Value]
Get mutable references to the value arguments to this instruction.
Examples found in repository?
src/ir/dfg.rs (line 331)
330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622
pub fn resolve_aliases_in_arguments(&mut self, inst: Inst) {
for arg in self.insts[inst].arguments_mut(&mut self.value_lists) {
let resolved = resolve_aliases(&self.values, *arg);
if resolved != *arg {
*arg = resolved;
}
}
}
/// Turn a value into an alias of another.
///
/// Change the `dest` value to behave as an alias of `src`. This means that all uses of `dest`
/// will behave as if they used that value `src`.
///
/// The `dest` value can't be attached to an instruction or block.
pub fn change_to_alias(&mut self, dest: Value, src: Value) {
debug_assert!(!self.value_is_attached(dest));
// Try to create short alias chains by finding the original source value.
// This also avoids the creation of loops.
let original = self.resolve_aliases(src);
debug_assert_ne!(
dest, original,
"Aliasing {} to {} would create a loop",
dest, src
);
let ty = self.value_type(original);
debug_assert_eq!(
self.value_type(dest),
ty,
"Aliasing {} to {} would change its type {} to {}",
dest,
src,
self.value_type(dest),
ty
);
debug_assert_ne!(ty, types::INVALID);
self.values[dest] = ValueData::Alias { ty, original }.into();
}
/// Replace the results of one instruction with aliases to the results of another.
///
/// Change all the results of `dest_inst` to behave as aliases of
/// corresponding results of `src_inst`, as if calling change_to_alias for
/// each.
///
/// After calling this instruction, `dest_inst` will have had its results
/// cleared, so it likely needs to be removed from the graph.
///
pub fn replace_with_aliases(&mut self, dest_inst: Inst, src_inst: Inst) {
debug_assert_ne!(
dest_inst, src_inst,
"Replacing {} with itself would create a loop",
dest_inst
);
debug_assert_eq!(
self.results[dest_inst].len(&self.value_lists),
self.results[src_inst].len(&self.value_lists),
"Replacing {} with {} would produce a different number of results.",
dest_inst,
src_inst
);
for (&dest, &src) in self.results[dest_inst]
.as_slice(&self.value_lists)
.iter()
.zip(self.results[src_inst].as_slice(&self.value_lists))
{
let original = src;
let ty = self.value_type(original);
debug_assert_eq!(
self.value_type(dest),
ty,
"Aliasing {} to {} would change its type {} to {}",
dest,
src,
self.value_type(dest),
ty
);
debug_assert_ne!(ty, types::INVALID);
self.values[dest] = ValueData::Alias { ty, original }.into();
}
self.clear_results(dest_inst);
}
}
/// Where did a value come from?
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ValueDef {
/// Value is the n'th result of an instruction.
Result(Inst, usize),
/// Value is the n'th parameter to a block.
Param(Block, usize),
}
impl ValueDef {
/// Unwrap the instruction where the value was defined, or panic.
pub fn unwrap_inst(&self) -> Inst {
self.inst().expect("Value is not an instruction result")
}
/// Get the instruction where the value was defined, if any.
pub fn inst(&self) -> Option<Inst> {
match *self {
Self::Result(inst, _) => Some(inst),
_ => None,
}
}
/// Unwrap the block there the parameter is defined, or panic.
pub fn unwrap_block(&self) -> Block {
match *self {
Self::Param(block, _) => block,
_ => panic!("Value is not a block parameter"),
}
}
/// Get the program point where the value was defined.
pub fn pp(self) -> ir::ExpandedProgramPoint {
self.into()
}
/// Get the number component of this definition.
///
/// When multiple values are defined at the same program point, this indicates the index of
/// this value.
pub fn num(self) -> usize {
match self {
Self::Result(_, n) | Self::Param(_, n) => n,
}
}
}
/// Internal table storage for extended values.
#[derive(Clone, Debug, PartialEq, Hash)]
#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
enum ValueData {
/// Value is defined by an instruction.
Inst { ty: Type, num: u16, inst: Inst },
/// Value is a block parameter.
Param { ty: Type, num: u16, block: Block },
/// Value is an alias of another value.
/// An alias value can't be linked as an instruction result or block parameter. It is used as a
/// placeholder when the original instruction or block has been rewritten or modified.
Alias { ty: Type, original: Value },
}
/// Bit-packed version of ValueData, for efficiency.
///
/// Layout:
///
/// ```plain
/// | tag:2 | type:14 | num:16 | index:32 |
/// ```
#[derive(Clone, Copy, Debug, PartialEq, Hash)]
#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
struct ValueDataPacked(u64);
impl ValueDataPacked {
const INDEX_SHIFT: u64 = 0;
const INDEX_BITS: u64 = 32;
const NUM_SHIFT: u64 = Self::INDEX_SHIFT + Self::INDEX_BITS;
const NUM_BITS: u64 = 16;
const TYPE_SHIFT: u64 = Self::NUM_SHIFT + Self::NUM_BITS;
const TYPE_BITS: u64 = 14;
const TAG_SHIFT: u64 = Self::TYPE_SHIFT + Self::TYPE_BITS;
const TAG_BITS: u64 = 2;
const TAG_INST: u64 = 1;
const TAG_PARAM: u64 = 2;
const TAG_ALIAS: u64 = 3;
fn make(tag: u64, ty: Type, num: u16, index: u32) -> ValueDataPacked {
debug_assert!(tag < (1 << Self::TAG_BITS));
debug_assert!(ty.repr() < (1 << Self::TYPE_BITS));
ValueDataPacked(
(tag << Self::TAG_SHIFT)
| ((ty.repr() as u64) << Self::TYPE_SHIFT)
| ((num as u64) << Self::NUM_SHIFT)
| ((index as u64) << Self::INDEX_SHIFT),
)
}
#[inline(always)]
fn field(self, shift: u64, bits: u64) -> u64 {
(self.0 >> shift) & ((1 << bits) - 1)
}
#[inline(always)]
fn ty(self) -> Type {
let ty = self.field(ValueDataPacked::TYPE_SHIFT, ValueDataPacked::TYPE_BITS) as u16;
Type::from_repr(ty)
}
#[inline(always)]
fn set_type(&mut self, ty: Type) {
self.0 &= !(((1 << Self::TYPE_BITS) - 1) << Self::TYPE_SHIFT);
self.0 |= (ty.repr() as u64) << Self::TYPE_SHIFT;
}
}
impl From<ValueData> for ValueDataPacked {
fn from(data: ValueData) -> Self {
match data {
ValueData::Inst { ty, num, inst } => {
Self::make(Self::TAG_INST, ty, num, inst.as_bits())
}
ValueData::Param { ty, num, block } => {
Self::make(Self::TAG_PARAM, ty, num, block.as_bits())
}
ValueData::Alias { ty, original } => {
Self::make(Self::TAG_ALIAS, ty, 0, original.as_bits())
}
}
}
}
impl From<ValueDataPacked> for ValueData {
fn from(data: ValueDataPacked) -> Self {
let tag = data.field(ValueDataPacked::TAG_SHIFT, ValueDataPacked::TAG_BITS);
let ty = data.field(ValueDataPacked::TYPE_SHIFT, ValueDataPacked::TYPE_BITS) as u16;
let num = data.field(ValueDataPacked::NUM_SHIFT, ValueDataPacked::NUM_BITS) as u16;
let index = data.field(ValueDataPacked::INDEX_SHIFT, ValueDataPacked::INDEX_BITS) as u32;
let ty = Type::from_repr(ty);
match tag {
ValueDataPacked::TAG_INST => ValueData::Inst {
ty,
num,
inst: Inst::from_bits(index),
},
ValueDataPacked::TAG_PARAM => ValueData::Param {
ty,
num,
block: Block::from_bits(index),
},
ValueDataPacked::TAG_ALIAS => ValueData::Alias {
ty,
original: Value::from_bits(index),
},
_ => panic!("Invalid tag {} in ValueDataPacked 0x{:x}", tag, data.0),
}
}
}
/// Instructions.
///
impl DataFlowGraph {
/// Create a new instruction.
///
/// The type of the first result is indicated by `data.ty`. If the instruction produces
/// multiple results, also call `make_inst_results` to allocate value table entries.
pub fn make_inst(&mut self, data: InstructionData) -> Inst {
let n = self.num_insts() + 1;
self.results.resize(n);
self.insts.push(data)
}
/// Declares a dynamic vector type
pub fn make_dynamic_ty(&mut self, data: DynamicTypeData) -> DynamicType {
self.dynamic_types.push(data)
}
/// Returns an object that displays `inst`.
pub fn display_inst<'a>(&'a self, inst: Inst) -> DisplayInst<'a> {
DisplayInst(self, inst)
}
/// Returns an object that displays the given `value`'s defining instruction.
///
/// Panics if the value is not defined by an instruction (i.e. it is a basic
/// block argument).
pub fn display_value_inst(&self, value: Value) -> DisplayInst<'_> {
match self.value_def(value) {
ir::ValueDef::Result(inst, _) => self.display_inst(inst),
ir::ValueDef::Param(_, _) => panic!("value is not defined by an instruction"),
}
}
/// Get all value arguments on `inst` as a slice.
pub fn inst_args(&self, inst: Inst) -> &[Value] {
self.insts[inst].arguments(&self.value_lists)
}
/// Get all value arguments on `inst` as a mutable slice.
pub fn inst_args_mut(&mut self, inst: Inst) -> &mut [Value] {
self.insts[inst].arguments_mut(&mut self.value_lists)
}sourcepub fn take_value_list(&mut self) -> Option<ValueList>
pub fn take_value_list(&mut self) -> Option<ValueList>
Take out the value list with all the value arguments and return it.
This leaves the value list in the instruction empty. Use
put_value_list to put the value list back.
Examples found in repository?
More examples
src/remove_constant_phis.rs (line 381)
218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425
pub fn do_remove_constant_phis(func: &mut Function, domtree: &mut DominatorTree) {
let _tt = timing::remove_constant_phis();
debug_assert!(domtree.is_valid());
// Phase 1 of 3: for each block, make a summary containing all relevant
// info. The solver will iterate over the summaries, rather than having
// to inspect each instruction in each block.
let bump =
Bump::with_capacity(domtree.cfg_postorder().len() * 4 * std::mem::size_of::<Value>());
let mut summaries =
SecondaryMap::<Block, BlockSummary>::with_capacity(domtree.cfg_postorder().len());
for b in domtree.cfg_postorder().iter().rev().copied() {
let formals = func.dfg.block_params(b);
let mut summary = BlockSummary::new(&bump, formals);
for inst in func.layout.block_insts(b) {
let idetails = &func.dfg[inst];
// Note that multi-dest transfers (i.e., branch tables) don't
// carry parameters in our IR, so we only have to care about
// `SingleDest` here.
if let BranchInfo::SingleDest(dest, _) = idetails.analyze_branch(&func.dfg.value_lists)
{
if let Some(edge) = OutEdge::new(&bump, &func.dfg, inst, dest) {
summary.dests.push(edge);
}
}
}
// Ensure the invariant that all blocks (except for the entry) appear
// in the summary, *unless* they have neither formals nor any
// param-carrying branches/jumps.
if formals.len() > 0 || summary.dests.len() > 0 {
summaries[b] = summary;
}
}
// Phase 2 of 3: iterate over the summaries in reverse postorder,
// computing new `AbstractValue`s for each tracked `Value`. The set of
// tracked `Value`s is exactly Group A as described above.
let entry_block = func
.layout
.entry_block()
.expect("remove_constant_phis: entry block unknown");
// Set up initial solver state
let mut state = SolverState::new();
for b in domtree.cfg_postorder().iter().rev().copied() {
// For each block, get the formals
if b == entry_block {
continue;
}
let formals = func.dfg.block_params(b);
for formal in formals {
let mb_old_absval = state.absvals.insert(*formal, AbstractValue::None);
assert!(mb_old_absval.is_none());
}
}
// Solve: repeatedly traverse the blocks in reverse postorder, until there
// are no changes.
let mut iter_no = 0;
loop {
iter_no += 1;
let mut changed = false;
for src in domtree.cfg_postorder().iter().rev().copied() {
let src_summary = &summaries[src];
for edge in &src_summary.dests {
assert!(edge.block != entry_block);
// By contrast, the dst block must have a summary. Phase 1
// will have only included an entry in `src_summary.dests` if
// that branch/jump carried at least one parameter. So the
// dst block does take parameters, so it must have a summary.
let dst_summary = &summaries[edge.block];
let dst_formals = &dst_summary.formals;
assert_eq!(edge.args.len(), dst_formals.len());
for (formal, actual) in dst_formals.iter().zip(edge.args) {
// Find the abstract value for `actual`. If it is a block
// formal parameter then the most recent abstract value is
// to be found in the solver state. If not, then it's a
// real value defining point (not a phi), in which case
// return it itself.
let actual_absval = match state.maybe_get(*actual) {
Some(pt) => *pt,
None => AbstractValue::One(*actual),
};
// And `join` the new value with the old.
let formal_absval_old = state.get(*formal);
let formal_absval_new = formal_absval_old.join(actual_absval);
if formal_absval_new != formal_absval_old {
changed = true;
state.set(*formal, formal_absval_new);
}
}
}
}
if !changed {
break;
}
}
let mut n_consts = 0;
for absval in state.absvals.values() {
if absval.is_one() {
n_consts += 1;
}
}
// Phase 3 of 3: edit the function to remove constant formals, using the
// summaries and the final solver state as a guide.
// Make up a set of blocks that need editing.
let mut need_editing = FxHashSet::<Block>::default();
for (block, summary) in summaries.iter() {
if block == entry_block {
continue;
}
for formal in summary.formals {
let formal_absval = state.get(*formal);
if formal_absval.is_one() {
need_editing.insert(block);
break;
}
}
}
// Firstly, deal with the formals. For each formal which is redundant,
// remove it, and also add a reroute from it to the constant value which
// it we know it to be.
for b in &need_editing {
let mut del_these = SmallVec::<[(Value, Value); 32]>::new();
let formals: &[Value] = func.dfg.block_params(*b);
for formal in formals {
// The state must give an absval for `formal`.
if let AbstractValue::One(replacement_val) = state.get(*formal) {
del_these.push((*formal, replacement_val));
}
}
// We can delete the formals in any order. However,
// `remove_block_param` works by sliding backwards all arguments to
// the right of the value it is asked to delete. Hence when removing more
// than one formal, it is significantly more efficient to ask it to
// remove the rightmost formal first, and hence this `rev()`.
for (redundant_formal, replacement_val) in del_these.into_iter().rev() {
func.dfg.remove_block_param(redundant_formal);
func.dfg.change_to_alias(redundant_formal, replacement_val);
}
}
// Secondly, visit all branch insns. If the destination has had its
// formals changed, change the actuals accordingly. Don't scan all insns,
// rather just visit those as listed in the summaries we prepared earlier.
for summary in summaries.values() {
for edge in &summary.dests {
if !need_editing.contains(&edge.block) {
continue;
}
let old_actuals = func.dfg[edge.inst].take_value_list().unwrap();
let num_old_actuals = old_actuals.len(&func.dfg.value_lists);
let num_fixed_actuals = func.dfg[edge.inst]
.opcode()
.constraints()
.num_fixed_value_arguments();
let dst_summary = &summaries[edge.block];
// Check that the numbers of arguments make sense.
assert!(num_fixed_actuals <= num_old_actuals);
assert_eq!(
num_fixed_actuals + dst_summary.formals.len(),
num_old_actuals
);
// Create a new value list.
let mut new_actuals = EntityList::<Value>::new();
// Copy the fixed args to the new list
for i in 0..num_fixed_actuals {
let val = old_actuals.get(i, &func.dfg.value_lists).unwrap();
new_actuals.push(val, &mut func.dfg.value_lists);
}
// Copy the variable args (the actual block params) to the new
// list, filtering out redundant ones.
for (i, formal_i) in dst_summary.formals.iter().enumerate() {
let actual_i = old_actuals
.get(num_fixed_actuals + i, &func.dfg.value_lists)
.unwrap();
let is_redundant = state.get(*formal_i).is_one();
if !is_redundant {
new_actuals.push(actual_i, &mut func.dfg.value_lists);
}
}
func.dfg[edge.inst].put_value_list(new_actuals);
}
}
log::debug!(
"do_remove_constant_phis: done, {} iters. {} formals, of which {} const.",
iter_no,
state.absvals.len(),
n_consts
);
}sourcepub fn put_value_list(&mut self, vlist: ValueList)
pub fn put_value_list(&mut self, vlist: ValueList)
Put back a value list.
After removing a value list with take_value_list(), use this
method to put it back. It is required that this instruction has
a format that accepts a value list, and that the existing value
list is empty. This avoids leaking list pool memory.
Examples found in repository?
More examples
src/remove_constant_phis.rs (line 415)
218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425
pub fn do_remove_constant_phis(func: &mut Function, domtree: &mut DominatorTree) {
let _tt = timing::remove_constant_phis();
debug_assert!(domtree.is_valid());
// Phase 1 of 3: for each block, make a summary containing all relevant
// info. The solver will iterate over the summaries, rather than having
// to inspect each instruction in each block.
let bump =
Bump::with_capacity(domtree.cfg_postorder().len() * 4 * std::mem::size_of::<Value>());
let mut summaries =
SecondaryMap::<Block, BlockSummary>::with_capacity(domtree.cfg_postorder().len());
for b in domtree.cfg_postorder().iter().rev().copied() {
let formals = func.dfg.block_params(b);
let mut summary = BlockSummary::new(&bump, formals);
for inst in func.layout.block_insts(b) {
let idetails = &func.dfg[inst];
// Note that multi-dest transfers (i.e., branch tables) don't
// carry parameters in our IR, so we only have to care about
// `SingleDest` here.
if let BranchInfo::SingleDest(dest, _) = idetails.analyze_branch(&func.dfg.value_lists)
{
if let Some(edge) = OutEdge::new(&bump, &func.dfg, inst, dest) {
summary.dests.push(edge);
}
}
}
// Ensure the invariant that all blocks (except for the entry) appear
// in the summary, *unless* they have neither formals nor any
// param-carrying branches/jumps.
if formals.len() > 0 || summary.dests.len() > 0 {
summaries[b] = summary;
}
}
// Phase 2 of 3: iterate over the summaries in reverse postorder,
// computing new `AbstractValue`s for each tracked `Value`. The set of
// tracked `Value`s is exactly Group A as described above.
let entry_block = func
.layout
.entry_block()
.expect("remove_constant_phis: entry block unknown");
// Set up initial solver state
let mut state = SolverState::new();
for b in domtree.cfg_postorder().iter().rev().copied() {
// For each block, get the formals
if b == entry_block {
continue;
}
let formals = func.dfg.block_params(b);
for formal in formals {
let mb_old_absval = state.absvals.insert(*formal, AbstractValue::None);
assert!(mb_old_absval.is_none());
}
}
// Solve: repeatedly traverse the blocks in reverse postorder, until there
// are no changes.
let mut iter_no = 0;
loop {
iter_no += 1;
let mut changed = false;
for src in domtree.cfg_postorder().iter().rev().copied() {
let src_summary = &summaries[src];
for edge in &src_summary.dests {
assert!(edge.block != entry_block);
// By contrast, the dst block must have a summary. Phase 1
// will have only included an entry in `src_summary.dests` if
// that branch/jump carried at least one parameter. So the
// dst block does take parameters, so it must have a summary.
let dst_summary = &summaries[edge.block];
let dst_formals = &dst_summary.formals;
assert_eq!(edge.args.len(), dst_formals.len());
for (formal, actual) in dst_formals.iter().zip(edge.args) {
// Find the abstract value for `actual`. If it is a block
// formal parameter then the most recent abstract value is
// to be found in the solver state. If not, then it's a
// real value defining point (not a phi), in which case
// return it itself.
let actual_absval = match state.maybe_get(*actual) {
Some(pt) => *pt,
None => AbstractValue::One(*actual),
};
// And `join` the new value with the old.
let formal_absval_old = state.get(*formal);
let formal_absval_new = formal_absval_old.join(actual_absval);
if formal_absval_new != formal_absval_old {
changed = true;
state.set(*formal, formal_absval_new);
}
}
}
}
if !changed {
break;
}
}
let mut n_consts = 0;
for absval in state.absvals.values() {
if absval.is_one() {
n_consts += 1;
}
}
// Phase 3 of 3: edit the function to remove constant formals, using the
// summaries and the final solver state as a guide.
// Make up a set of blocks that need editing.
let mut need_editing = FxHashSet::<Block>::default();
for (block, summary) in summaries.iter() {
if block == entry_block {
continue;
}
for formal in summary.formals {
let formal_absval = state.get(*formal);
if formal_absval.is_one() {
need_editing.insert(block);
break;
}
}
}
// Firstly, deal with the formals. For each formal which is redundant,
// remove it, and also add a reroute from it to the constant value which
// it we know it to be.
for b in &need_editing {
let mut del_these = SmallVec::<[(Value, Value); 32]>::new();
let formals: &[Value] = func.dfg.block_params(*b);
for formal in formals {
// The state must give an absval for `formal`.
if let AbstractValue::One(replacement_val) = state.get(*formal) {
del_these.push((*formal, replacement_val));
}
}
// We can delete the formals in any order. However,
// `remove_block_param` works by sliding backwards all arguments to
// the right of the value it is asked to delete. Hence when removing more
// than one formal, it is significantly more efficient to ask it to
// remove the rightmost formal first, and hence this `rev()`.
for (redundant_formal, replacement_val) in del_these.into_iter().rev() {
func.dfg.remove_block_param(redundant_formal);
func.dfg.change_to_alias(redundant_formal, replacement_val);
}
}
// Secondly, visit all branch insns. If the destination has had its
// formals changed, change the actuals accordingly. Don't scan all insns,
// rather just visit those as listed in the summaries we prepared earlier.
for summary in summaries.values() {
for edge in &summary.dests {
if !need_editing.contains(&edge.block) {
continue;
}
let old_actuals = func.dfg[edge.inst].take_value_list().unwrap();
let num_old_actuals = old_actuals.len(&func.dfg.value_lists);
let num_fixed_actuals = func.dfg[edge.inst]
.opcode()
.constraints()
.num_fixed_value_arguments();
let dst_summary = &summaries[edge.block];
// Check that the numbers of arguments make sense.
assert!(num_fixed_actuals <= num_old_actuals);
assert_eq!(
num_fixed_actuals + dst_summary.formals.len(),
num_old_actuals
);
// Create a new value list.
let mut new_actuals = EntityList::<Value>::new();
// Copy the fixed args to the new list
for i in 0..num_fixed_actuals {
let val = old_actuals.get(i, &func.dfg.value_lists).unwrap();
new_actuals.push(val, &mut func.dfg.value_lists);
}
// Copy the variable args (the actual block params) to the new
// list, filtering out redundant ones.
for (i, formal_i) in dst_summary.formals.iter().enumerate() {
let actual_i = old_actuals
.get(num_fixed_actuals + i, &func.dfg.value_lists)
.unwrap();
let is_redundant = state.get(*formal_i).is_one();
if !is_redundant {
new_actuals.push(actual_i, &mut func.dfg.value_lists);
}
}
func.dfg[edge.inst].put_value_list(new_actuals);
}
}
log::debug!(
"do_remove_constant_phis: done, {} iters. {} formals, of which {} const.",
iter_no,
state.absvals.len(),
n_consts
);
}sourcepub fn eq(&self, other: &Self, pool: &ValueListPool) -> bool
pub fn eq(&self, other: &Self, pool: &ValueListPool) -> bool
Compare two InstructionData for equality.
This operation requires a reference to a ValueListPool to
determine if the contents of any ValueLists are equal.
sourcepub fn hash<H: Hasher>(&self, state: &mut H, pool: &ValueListPool)
pub fn hash<H: Hasher>(&self, state: &mut H, pool: &ValueListPool)
Hash an InstructionData.
This operation requires a reference to a ValueListPool to
hash the contents of any ValueLists.
source§impl InstructionData
impl InstructionData
Analyzing an instruction.
Avoid large matches on instruction formats by using the methods defined here to examine instructions.
sourcepub fn analyze_branch<'a>(&'a self, pool: &'a ValueListPool) -> BranchInfo<'a>
pub fn analyze_branch<'a>(&'a self, pool: &'a ValueListPool) -> BranchInfo<'a>
Return information about the destination of a branch or jump instruction.
Any instruction that can transfer control to another block reveals its possible destinations here.
Examples found in repository?
More examples
src/inst_predicates.rs (line 138)
137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154
fn visit_branch_targets<F: FnMut(Inst, Block, bool)>(f: &Function, inst: Inst, visit: &mut F) {
match f.dfg[inst].analyze_branch(&f.dfg.value_lists) {
BranchInfo::NotABranch => {}
BranchInfo::SingleDest(dest, _) => {
visit(inst, dest, false);
}
BranchInfo::Table(table, maybe_dest) => {
if let Some(dest) = maybe_dest {
// The default block is reached via a direct conditional branch,
// so it is not part of the table.
visit(inst, dest, false);
}
for &dest in f.jump_tables[table].as_slice() {
visit(inst, dest, true);
}
}
}
}src/remove_constant_phis.rs (line 239)
218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425
pub fn do_remove_constant_phis(func: &mut Function, domtree: &mut DominatorTree) {
let _tt = timing::remove_constant_phis();
debug_assert!(domtree.is_valid());
// Phase 1 of 3: for each block, make a summary containing all relevant
// info. The solver will iterate over the summaries, rather than having
// to inspect each instruction in each block.
let bump =
Bump::with_capacity(domtree.cfg_postorder().len() * 4 * std::mem::size_of::<Value>());
let mut summaries =
SecondaryMap::<Block, BlockSummary>::with_capacity(domtree.cfg_postorder().len());
for b in domtree.cfg_postorder().iter().rev().copied() {
let formals = func.dfg.block_params(b);
let mut summary = BlockSummary::new(&bump, formals);
for inst in func.layout.block_insts(b) {
let idetails = &func.dfg[inst];
// Note that multi-dest transfers (i.e., branch tables) don't
// carry parameters in our IR, so we only have to care about
// `SingleDest` here.
if let BranchInfo::SingleDest(dest, _) = idetails.analyze_branch(&func.dfg.value_lists)
{
if let Some(edge) = OutEdge::new(&bump, &func.dfg, inst, dest) {
summary.dests.push(edge);
}
}
}
// Ensure the invariant that all blocks (except for the entry) appear
// in the summary, *unless* they have neither formals nor any
// param-carrying branches/jumps.
if formals.len() > 0 || summary.dests.len() > 0 {
summaries[b] = summary;
}
}
// Phase 2 of 3: iterate over the summaries in reverse postorder,
// computing new `AbstractValue`s for each tracked `Value`. The set of
// tracked `Value`s is exactly Group A as described above.
let entry_block = func
.layout
.entry_block()
.expect("remove_constant_phis: entry block unknown");
// Set up initial solver state
let mut state = SolverState::new();
for b in domtree.cfg_postorder().iter().rev().copied() {
// For each block, get the formals
if b == entry_block {
continue;
}
let formals = func.dfg.block_params(b);
for formal in formals {
let mb_old_absval = state.absvals.insert(*formal, AbstractValue::None);
assert!(mb_old_absval.is_none());
}
}
// Solve: repeatedly traverse the blocks in reverse postorder, until there
// are no changes.
let mut iter_no = 0;
loop {
iter_no += 1;
let mut changed = false;
for src in domtree.cfg_postorder().iter().rev().copied() {
let src_summary = &summaries[src];
for edge in &src_summary.dests {
assert!(edge.block != entry_block);
// By contrast, the dst block must have a summary. Phase 1
// will have only included an entry in `src_summary.dests` if
// that branch/jump carried at least one parameter. So the
// dst block does take parameters, so it must have a summary.
let dst_summary = &summaries[edge.block];
let dst_formals = &dst_summary.formals;
assert_eq!(edge.args.len(), dst_formals.len());
for (formal, actual) in dst_formals.iter().zip(edge.args) {
// Find the abstract value for `actual`. If it is a block
// formal parameter then the most recent abstract value is
// to be found in the solver state. If not, then it's a
// real value defining point (not a phi), in which case
// return it itself.
let actual_absval = match state.maybe_get(*actual) {
Some(pt) => *pt,
None => AbstractValue::One(*actual),
};
// And `join` the new value with the old.
let formal_absval_old = state.get(*formal);
let formal_absval_new = formal_absval_old.join(actual_absval);
if formal_absval_new != formal_absval_old {
changed = true;
state.set(*formal, formal_absval_new);
}
}
}
}
if !changed {
break;
}
}
let mut n_consts = 0;
for absval in state.absvals.values() {
if absval.is_one() {
n_consts += 1;
}
}
// Phase 3 of 3: edit the function to remove constant formals, using the
// summaries and the final solver state as a guide.
// Make up a set of blocks that need editing.
let mut need_editing = FxHashSet::<Block>::default();
for (block, summary) in summaries.iter() {
if block == entry_block {
continue;
}
for formal in summary.formals {
let formal_absval = state.get(*formal);
if formal_absval.is_one() {
need_editing.insert(block);
break;
}
}
}
// Firstly, deal with the formals. For each formal which is redundant,
// remove it, and also add a reroute from it to the constant value which
// it we know it to be.
for b in &need_editing {
let mut del_these = SmallVec::<[(Value, Value); 32]>::new();
let formals: &[Value] = func.dfg.block_params(*b);
for formal in formals {
// The state must give an absval for `formal`.
if let AbstractValue::One(replacement_val) = state.get(*formal) {
del_these.push((*formal, replacement_val));
}
}
// We can delete the formals in any order. However,
// `remove_block_param` works by sliding backwards all arguments to
// the right of the value it is asked to delete. Hence when removing more
// than one formal, it is significantly more efficient to ask it to
// remove the rightmost formal first, and hence this `rev()`.
for (redundant_formal, replacement_val) in del_these.into_iter().rev() {
func.dfg.remove_block_param(redundant_formal);
func.dfg.change_to_alias(redundant_formal, replacement_val);
}
}
// Secondly, visit all branch insns. If the destination has had its
// formals changed, change the actuals accordingly. Don't scan all insns,
// rather just visit those as listed in the summaries we prepared earlier.
for summary in summaries.values() {
for edge in &summary.dests {
if !need_editing.contains(&edge.block) {
continue;
}
let old_actuals = func.dfg[edge.inst].take_value_list().unwrap();
let num_old_actuals = old_actuals.len(&func.dfg.value_lists);
let num_fixed_actuals = func.dfg[edge.inst]
.opcode()
.constraints()
.num_fixed_value_arguments();
let dst_summary = &summaries[edge.block];
// Check that the numbers of arguments make sense.
assert!(num_fixed_actuals <= num_old_actuals);
assert_eq!(
num_fixed_actuals + dst_summary.formals.len(),
num_old_actuals
);
// Create a new value list.
let mut new_actuals = EntityList::<Value>::new();
// Copy the fixed args to the new list
for i in 0..num_fixed_actuals {
let val = old_actuals.get(i, &func.dfg.value_lists).unwrap();
new_actuals.push(val, &mut func.dfg.value_lists);
}
// Copy the variable args (the actual block params) to the new
// list, filtering out redundant ones.
for (i, formal_i) in dst_summary.formals.iter().enumerate() {
let actual_i = old_actuals
.get(num_fixed_actuals + i, &func.dfg.value_lists)
.unwrap();
let is_redundant = state.get(*formal_i).is_one();
if !is_redundant {
new_actuals.push(actual_i, &mut func.dfg.value_lists);
}
}
func.dfg[edge.inst].put_value_list(new_actuals);
}
}
log::debug!(
"do_remove_constant_phis: done, {} iters. {} formals, of which {} const.",
iter_no,
state.absvals.len(),
n_consts
);
}sourcepub fn branch_destination(&self) -> Option<Block>
pub fn branch_destination(&self) -> Option<Block>
Get the single destination of this branch instruction, if it is a single destination branch or jump.
Multi-destination branches like br_table return None.
Examples found in repository?
src/simple_preopt.rs (line 505)
479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574
fn branch_order(pos: &mut FuncCursor, cfg: &mut ControlFlowGraph, block: Block, inst: Inst) {
let (term_inst, term_inst_args, term_dest, cond_inst, cond_inst_args, cond_dest, kind) =
match pos.func.dfg[inst] {
InstructionData::Jump {
opcode: Opcode::Jump,
destination,
ref args,
} => {
let next_block = if let Some(next_block) = pos.func.layout.next_block(block) {
next_block
} else {
return;
};
if destination == next_block {
return;
}
let prev_inst = if let Some(prev_inst) = pos.func.layout.prev_inst(inst) {
prev_inst
} else {
return;
};
let prev_inst_data = &pos.func.dfg[prev_inst];
if let Some(prev_dest) = prev_inst_data.branch_destination() {
if prev_dest != next_block {
return;
}
} else {
return;
}
match prev_inst_data {
InstructionData::Branch {
opcode,
args: ref prev_args,
destination: cond_dest,
} => {
let cond_arg = {
let args = pos.func.dfg.inst_args(prev_inst);
args[0]
};
let kind = match opcode {
Opcode::Brz => BranchOrderKind::BrzToBrnz(cond_arg),
Opcode::Brnz => BranchOrderKind::BrnzToBrz(cond_arg),
_ => panic!("unexpected opcode"),
};
(
inst,
args.clone(),
destination,
prev_inst,
prev_args.clone(),
*cond_dest,
kind,
)
}
_ => return,
}
}
_ => return,
};
let cond_args = cond_inst_args.as_slice(&pos.func.dfg.value_lists).to_vec();
let term_args = term_inst_args.as_slice(&pos.func.dfg.value_lists).to_vec();
match kind {
BranchOrderKind::BrnzToBrz(cond_arg) => {
pos.func
.dfg
.replace(term_inst)
.jump(cond_dest, &cond_args[1..]);
pos.func
.dfg
.replace(cond_inst)
.brz(cond_arg, term_dest, &term_args);
}
BranchOrderKind::BrzToBrnz(cond_arg) => {
pos.func
.dfg
.replace(term_inst)
.jump(cond_dest, &cond_args[1..]);
pos.func
.dfg
.replace(cond_inst)
.brnz(cond_arg, term_dest, &term_args);
}
}
cfg.recompute_block(pos.func, block);
}sourcepub fn branch_destination_mut(&mut self) -> Option<&mut Block>
pub fn branch_destination_mut(&mut self) -> Option<&mut Block>
Get a mutable reference to the single destination of this branch instruction, if it is a single destination branch or jump.
Multi-destination branches like br_table return None.
sourcepub fn trap_code(&self) -> Option<TrapCode>
pub fn trap_code(&self) -> Option<TrapCode>
If this is a trapping instruction, get its trap code. Otherwise, return
None.
sourcepub fn cond_code(&self) -> Option<IntCC>
pub fn cond_code(&self) -> Option<IntCC>
If this is a control-flow instruction depending on an integer condition, gets its
condition. Otherwise, return None.
sourcepub fn fp_cond_code(&self) -> Option<FloatCC>
pub fn fp_cond_code(&self) -> Option<FloatCC>
If this is a control-flow instruction depending on a floating-point condition, gets its
condition. Otherwise, return None.
sourcepub fn trap_code_mut(&mut self) -> Option<&mut TrapCode>
pub fn trap_code_mut(&mut self) -> Option<&mut TrapCode>
If this is a trapping instruction, get an exclusive reference to its
trap code. Otherwise, return None.
sourcepub fn atomic_rmw_op(&self) -> Option<AtomicRmwOp>
pub fn atomic_rmw_op(&self) -> Option<AtomicRmwOp>
If this is an atomic read/modify/write instruction, return its subopcode.
sourcepub fn load_store_offset(&self) -> Option<i32>
pub fn load_store_offset(&self) -> Option<i32>
If this is a load/store instruction, returns its immediate offset.
Examples found in repository?
src/isa/x64/lower.rs (line 118)
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130
fn is_mergeable_load(ctx: &mut Lower<Inst>, src_insn: IRInst) -> Option<(InsnInput, i32)> {
let insn_data = ctx.data(src_insn);
let inputs = ctx.num_inputs(src_insn);
if inputs != 1 {
return None;
}
let load_ty = ctx.output_ty(src_insn, 0);
if ty_bits(load_ty) < 32 {
// Narrower values are handled by ALU insts that are at least 32 bits
// wide, which is normally OK as we ignore upper buts; but, if we
// generate, e.g., a direct-from-memory 32-bit add for a byte value and
// the byte is the last byte in a page, the extra data that we load is
// incorrectly accessed. So we only allow loads to merge for
// 32-bit-and-above widths.
return None;
}
// SIMD instructions can only be load-coalesced when the loaded value comes
// from an aligned address.
if load_ty.is_vector() && !insn_data.memflags().map_or(false, |f| f.aligned()) {
return None;
}
// Just testing the opcode is enough, because the width will always match if
// the type does (and the type should match if the CLIF is properly
// constructed).
if insn_data.opcode() == Opcode::Load {
let offset = insn_data
.load_store_offset()
.expect("load should have offset");
Some((
InsnInput {
insn: src_insn,
input: 0,
},
offset,
))
} else {
None
}
}sourcepub fn memflags(&self) -> Option<MemFlags>
pub fn memflags(&self) -> Option<MemFlags>
If this is a load/store instruction, return its memory flags.
Examples found in repository?
src/egraph/stores.rs (line 135)
127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
fn update(&mut self, func: &Function, inst: Inst) {
let opcode = func.dfg[inst].opcode();
if has_memory_fence_semantics(opcode) {
self.heap = MemoryState::AfterInst(inst);
self.table = MemoryState::AfterInst(inst);
self.vmctx = MemoryState::AfterInst(inst);
self.other = MemoryState::AfterInst(inst);
} else if opcode.can_store() {
if let Some(memflags) = func.dfg[inst].memflags() {
*self.for_flags(memflags) = MemoryState::Store(inst);
} else {
self.heap = MemoryState::AfterInst(inst);
self.table = MemoryState::AfterInst(inst);
self.vmctx = MemoryState::AfterInst(inst);
self.other = MemoryState::AfterInst(inst);
}
}
}More examples
src/alias_analysis.rs (line 95)
87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130
fn update(&mut self, func: &Function, inst: Inst) {
let opcode = func.dfg[inst].opcode();
if has_memory_fence_semantics(opcode) {
self.heap = inst.into();
self.table = inst.into();
self.vmctx = inst.into();
self.other = inst.into();
} else if opcode.can_store() {
if let Some(memflags) = func.dfg[inst].memflags() {
if memflags.heap() {
self.heap = inst.into();
} else if memflags.table() {
self.table = inst.into();
} else if memflags.vmctx() {
self.vmctx = inst.into();
} else {
self.other = inst.into();
}
} else {
self.heap = inst.into();
self.table = inst.into();
self.vmctx = inst.into();
self.other = inst.into();
}
}
}
fn get_last_store(&self, func: &Function, inst: Inst) -> PackedOption<Inst> {
if let Some(memflags) = func.dfg[inst].memflags() {
if memflags.heap() {
self.heap
} else if memflags.table() {
self.table
} else if memflags.vmctx() {
self.vmctx
} else {
self.other
}
} else if func.dfg[inst].opcode().can_load() || func.dfg[inst].opcode().can_store() {
inst.into()
} else {
PackedOption::default()
}
}src/isa/x64/lower.rs (line 109)
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130
fn is_mergeable_load(ctx: &mut Lower<Inst>, src_insn: IRInst) -> Option<(InsnInput, i32)> {
let insn_data = ctx.data(src_insn);
let inputs = ctx.num_inputs(src_insn);
if inputs != 1 {
return None;
}
let load_ty = ctx.output_ty(src_insn, 0);
if ty_bits(load_ty) < 32 {
// Narrower values are handled by ALU insts that are at least 32 bits
// wide, which is normally OK as we ignore upper buts; but, if we
// generate, e.g., a direct-from-memory 32-bit add for a byte value and
// the byte is the last byte in a page, the extra data that we load is
// incorrectly accessed. So we only allow loads to merge for
// 32-bit-and-above widths.
return None;
}
// SIMD instructions can only be load-coalesced when the loaded value comes
// from an aligned address.
if load_ty.is_vector() && !insn_data.memflags().map_or(false, |f| f.aligned()) {
return None;
}
// Just testing the opcode is enough, because the width will always match if
// the type does (and the type should match if the CLIF is properly
// constructed).
if insn_data.opcode() == Opcode::Load {
let offset = insn_data
.load_store_offset()
.expect("load should have offset");
Some((
InsnInput {
insn: src_insn,
input: 0,
},
offset,
))
} else {
None
}
}sourcepub fn stack_slot(&self) -> Option<StackSlot>
pub fn stack_slot(&self) -> Option<StackSlot>
If this instruction references a stack slot, return it
sourcepub fn analyze_call<'a>(&'a self, pool: &'a ValueListPool) -> CallInfo<'a>
pub fn analyze_call<'a>(&'a self, pool: &'a ValueListPool) -> CallInfo<'a>
Return information about a call instruction.
Any instruction that can call another function reveals its call signature here.
Examples found in repository?
More examples
src/verifier/mod.rs (line 1402)
1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421
fn typecheck_variable_args(
&self,
inst: Inst,
errors: &mut VerifierErrors,
) -> VerifierStepResult<()> {
match self.func.dfg.analyze_branch(inst) {
BranchInfo::SingleDest(block, _) => {
let iter = self
.func
.dfg
.block_params(block)
.iter()
.map(|&v| self.func.dfg.value_type(v));
self.typecheck_variable_args_iterator(inst, iter, errors)?;
}
BranchInfo::Table(table, block) => {
if let Some(block) = block {
let arg_count = self.func.dfg.num_block_params(block);
if arg_count != 0 {
return errors.nonfatal((
inst,
self.context(inst),
format!(
"takes no arguments, but had target {} with {} arguments",
block, arg_count,
),
));
}
}
for block in self.func.jump_tables[table].iter() {
let arg_count = self.func.dfg.num_block_params(*block);
if arg_count != 0 {
return errors.nonfatal((
inst,
self.context(inst),
format!(
"takes no arguments, but had target {} with {} arguments",
block, arg_count,
),
));
}
}
}
BranchInfo::NotABranch => {}
}
match self.func.dfg[inst].analyze_call(&self.func.dfg.value_lists) {
CallInfo::Direct(func_ref, _) => {
let sig_ref = self.func.dfg.ext_funcs[func_ref].signature;
let arg_types = self.func.dfg.signatures[sig_ref]
.params
.iter()
.map(|a| a.value_type);
self.typecheck_variable_args_iterator(inst, arg_types, errors)?;
}
CallInfo::Indirect(sig_ref, _) => {
let arg_types = self.func.dfg.signatures[sig_ref]
.params
.iter()
.map(|a| a.value_type);
self.typecheck_variable_args_iterator(inst, arg_types, errors)?;
}
CallInfo::NotACall => {}
}
Ok(())
}Trait Implementations§
source§impl Clone for InstructionData
impl Clone for InstructionData
source§fn clone(&self) -> InstructionData
fn clone(&self) -> InstructionData
Returns a copy of the value. Read more
1.0.0 · source§fn clone_from(&mut self, source: &Self)
fn clone_from(&mut self, source: &Self)
Performs copy-assignment from
source. Read moresource§impl Debug for InstructionData
impl Debug for InstructionData
source§impl<'a> From<&'a InstructionData> for InstructionFormat
impl<'a> From<&'a InstructionData> for InstructionFormat
source§fn from(inst: &'a InstructionData) -> Self
fn from(inst: &'a InstructionData) -> Self
Converts to this type from the input type.
source§impl From<&InstructionData> for InstructionImms
impl From<&InstructionData> for InstructionImms
source§fn from(data: &InstructionData) -> InstructionImms
fn from(data: &InstructionData) -> InstructionImms
Convert an InstructionData into an InstructionImms.
source§impl Hash for InstructionData
impl Hash for InstructionData
source§impl PartialEq<InstructionData> for InstructionData
impl PartialEq<InstructionData> for InstructionData
source§fn eq(&self, other: &InstructionData) -> bool
fn eq(&self, other: &InstructionData) -> bool
This method tests for
self and other values to be equal, and is used
by ==.