1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
//! Breaks outgoing critical edges for call terminators in the MIR.
//!
//! Critical edges are edges that are neither the only edge leaving a
//! block, nor the only edge entering one.
//!
//! When you want something to happen "along" an edge, you can either
//! do at the end of the predecessor block, or at the start of the
//! successor block. Critical edges have to be broken in order to prevent
//! "edge actions" from affecting other edges. We need this for calls that are
//! codegened to LLVM invoke instructions, because invoke is a block terminator
//! in LLVM so we can't insert any code to handle the call's result into the
//! block that performs the call.
//!
//! This function will break those edges by inserting new blocks along them.
//!
//! NOTE: Simplify CFG will happily undo most of the work this pass does.
// `#![no_std]`: these arrive with the standard prelude and name no path, so a `std::`
// search cannot see them - and a `#[derive]` can use them without the name appearing
// in this file at all, which is why they are not trimmed by inspection.
use alloc::borrow::ToOwned;
use alloc::boxed::Box;
use alloc::format;
use alloc::string::{String, ToString};
use alloc::vec;
use alloc::vec::Vec;
use crate::rustc_data_structures::thin_vec::ThinVec;
use crate::rustc_index::{Idx, IndexVec};
use crate::rustc_middle::mir::*;
use crate::rustc_middle::ty::TyCtxt;
use tracing::debug;
use crate::rustc_mir_transform::PassPolicy;
#[derive(PartialEq)]
pub(super) enum AddCallGuards {
AllCallEdges,
CriticalCallEdges,
}
pub(super) use self::AddCallGuards::*;
impl<'tcx> crate::rustc_mir_transform::MirPass<'tcx> for AddCallGuards {
fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
let mut pred_count = IndexVec::from_elem(0u8, &body.basic_blocks);
for (_, data) in body.basic_blocks.iter_enumerated() {
for succ in data.terminator().successors() {
pred_count[succ] = pred_count[succ].saturating_add(1);
}
}
enum Action {
Call,
Asm { target_index: usize },
}
let mut work = Vec::with_capacity(body.basic_blocks.len());
for (bb, block) in body.basic_blocks.iter_enumerated() {
let term = block.terminator();
match term.kind {
TerminatorKind::Call { target: Some(destination), unwind, .. }
if pred_count[destination] > 1
&& (generates_invoke(unwind) || self == &AllCallEdges) =>
{
// It's a critical edge, break it
work.push((bb, Action::Call));
}
TerminatorKind::InlineAsm {
asm_macro: InlineAsmMacro::Asm,
ref targets,
ref operands,
unwind,
..
} if self == &CriticalCallEdges => {
let has_outputs = operands.iter().any(|op| {
matches!(op, InlineAsmOperand::InOut { .. } | InlineAsmOperand::Out { .. })
});
let has_labels =
operands.iter().any(|op| matches!(op, InlineAsmOperand::Label { .. }));
if has_outputs && (has_labels || generates_invoke(unwind)) {
for (target_index, target) in targets.iter().enumerate() {
if pred_count[*target] > 1 {
work.push((bb, Action::Asm { target_index }));
}
}
}
}
_ => {}
}
}
if work.is_empty() {
return;
}
// We need a place to store the new blocks generated
let mut new_blocks = Vec::with_capacity(work.len());
let cur_len = body.basic_blocks.len();
let mut new_block = |source_info: SourceInfo, is_cleanup: bool, target: BasicBlock| {
let block = BasicBlockData::new(
Some(Terminator {
source_info,
kind: TerminatorKind::Goto { target },
attributes: ThinVec::new(),
}),
is_cleanup,
);
let idx = cur_len + new_blocks.len();
new_blocks.push(block);
BasicBlock::new(idx)
};
let basic_blocks = body.basic_blocks.as_mut();
for (source, action) in work {
let block = &mut basic_blocks[source];
let is_cleanup = block.is_cleanup;
let term = block.terminator_mut();
let source_info = term.source_info;
let destination = match action {
Action::Call => {
let TerminatorKind::Call { target: Some(ref mut destination), .. } = term.kind
else {
unreachable!()
};
destination
}
Action::Asm { target_index } => {
let TerminatorKind::InlineAsm { ref mut targets, .. } = term.kind else {
unreachable!()
};
&mut targets[target_index]
}
};
*destination = new_block(source_info, is_cleanup, *destination);
}
debug!("Broke {} N edges", new_blocks.len());
basic_blocks.extend(new_blocks);
}
fn policy(&self, _sess: &crate::rustc_session::Session) -> PassPolicy {
// Breaks critical edges so codegen can place edge-specific actions without affecting
// other control-flow edges.
PassPolicy::Required
}
}
/// Returns true if this unwind action is code generated as an invoke as opposed to a call.
fn generates_invoke(unwind: UnwindAction) -> bool {
match unwind {
UnwindAction::Continue | UnwindAction::Unreachable => false,
UnwindAction::Cleanup(_) | UnwindAction::Terminate(_) => true,
}
}