1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
// `#![no_std]`: these arrive with the standard prelude and name no path, so a `std::`
// search cannot see them - and a `#[derive]` can use them without the name appearing
// in this file at all, which is why they are not trimmed by inspection.
use alloc::borrow::ToOwned;
use alloc::boxed::Box;
use alloc::format;
use alloc::string::{String, ToString};
use alloc::vec;
use alloc::vec::Vec;
use crate::rustc_hir::attrs::InlineAttr;
use crate::rustc_hir::def::DefKind;
use crate::rustc_hir::def_id::LocalDefId;
use crate::rustc_hir::{self as hir, find_attr};
use crate::bug;
use crate::rustc_middle::mir::visit::Visitor;
use crate::rustc_middle::mir::*;
use crate::rustc_middle::query::Providers;
use crate::rustc_middle::ty::TyCtxt;
use crate::rustc_session::config::{InliningThreshold, OptLevel};
use crate::rustc_mir_transform::{inline, pass_manager as pm};
pub(super) fn provide(providers: &mut Providers) {
providers.cross_crate_inlinable = cross_crate_inlinable;
}
fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id);
// If this has an extern indicator, then this function is globally shared and thus will not
// generate cgu-internal copies which would make it cross-crate inlinable.
if codegen_fn_attrs.contains_extern_indicator() {
return false;
}
// This just reproduces the logic from Instance::requires_inline.
match tcx.def_kind(def_id) {
DefKind::Ctor(..) | DefKind::Closure | DefKind::SyntheticCoroutineBody => return true,
DefKind::Fn | DefKind::AssocFn => {}
_ => return false,
}
// From this point on, it is valid to return true or false.
if tcx.sess.opts.unstable_opts.cross_crate_inline_threshold == InliningThreshold::Always {
return true;
}
if find_attr!(tcx, def_id, RustcIntrinsic) {
// Intrinsic fallback bodies are always cross-crate inlineable.
// To ensure that the MIR inliner doesn't cluelessly try to inline fallback
// bodies even when the backend would implement something better, we stop
// the MIR inliner from ever inlining an intrinsic.
return true;
}
if let hir::Constness::Const { always: true } = tcx.constness(def_id) {
// Comptime functions only exist during const eval and can never be passed
// to codegen. The const eval MIR pipeline also doesn't inline anything at all.
return false;
}
// Obey source annotations first; this is important because it means we can use
// #[inline(never)] to force code generation.
match codegen_fn_attrs.inline {
InlineAttr::Never => return false,
InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. } => return true,
_ => {}
}
// If the crate is likely to be mostly unused, use cross-crate inlining to defer codegen until
// the function is referenced, in order to skip codegen for unused functions. This is
// intentionally after the check for `inline(never)`, so that `inline(never)` wins.
if tcx.sess.opts.unstable_opts.hint_mostly_unused {
return true;
}
let sig = tcx.fn_sig(def_id).instantiate_identity().skip_norm_wip();
for ty in sig.inputs().skip_binder().iter().chain(core::iter::once(&sig.output().skip_binder()))
{
// FIXME(f16_f128): in order to avoid crashes building `core`, always inline to skip
// codegen if the function is not used.
if ty == &tcx.types.f16 || ty == &tcx.types.f128 {
return true;
}
}
// Don't do any inference when incremental compilation is enabled; the additional inlining that
// inference permits also creates more work for small edits.
if tcx.sess.opts.incremental.is_some() {
return false;
}
// Don't do any inference if codegen optimizations are disabled and also MIR inlining is not
// enabled. This ensures that we do inference even if someone only passes -Zinline-mir,
// which is less confusing than having to also enable -Copt-level=1.
let inliner_will_run = pm::should_run_pass(tcx, &inline::Inline, pm::Optimizations::Allowed)
|| inline::ForceInline::should_run_pass_for_callee(tcx, def_id.to_def_id());
if matches!(tcx.sess.opts.optimize, OptLevel::No) && !inliner_will_run {
return false;
}
if !tcx.is_mir_available(def_id) {
return false;
}
let threshold = match tcx.sess.opts.unstable_opts.cross_crate_inline_threshold {
InliningThreshold::Always => return true,
InliningThreshold::Sometimes(threshold) => threshold,
InliningThreshold::Never => return false,
};
let mir = tcx.optimized_mir(def_id);
let mut checker =
CostChecker { tcx, callee_body: mir, calls: 0, statements: 0, landing_pads: 0, resumes: 0 };
checker.visit_body(mir);
checker.calls == 0
&& checker.resumes == 0
&& checker.landing_pads == 0
&& checker.statements <= threshold
}
// The threshold that CostChecker computes is balancing the desire to make more things
// inlinable cross crates against the growth in incremental CGU size that happens when too many
// things in the sysroot are made inlinable.
// Permitting calls causes the size of some incremental CGUs to grow, because more functions are
// made inlinable out of the sysroot or dependencies.
// Assert terminators are similar to calls, but do not have the same impact on compile time, so
// those are just treated as statements.
// A threshold exists at all because we don't want to blindly mark a huge function as inlinable.
struct CostChecker<'b, 'tcx> {
tcx: TyCtxt<'tcx>,
callee_body: &'b Body<'tcx>,
calls: usize,
statements: usize,
landing_pads: usize,
resumes: usize,
}
impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) {
// Don't count StorageLive/StorageDead in the inlining cost.
match statement.kind {
StatementKind::StorageLive(_) | StatementKind::StorageDead(_) | StatementKind::Nop => {}
_ => self.statements += 1,
}
}
fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, _: Location) {
self.statements += 1;
let tcx = self.tcx;
match &terminator.kind {
TerminatorKind::Drop { place, unwind, .. } => {
let ty = place.ty(self.callee_body, tcx).ty;
if !ty.is_trivially_pure_clone_copy() {
self.calls += 1;
if let UnwindAction::Cleanup(_) = unwind {
self.landing_pads += 1;
}
}
}
TerminatorKind::Call { func, unwind, .. } => {
// We track calls because they make our function not a leaf (and in theory, the
// number of calls indicates how likely this function is to perturb other CGUs).
// But there are a handful of intrinsics such as raw_eq that should not block
// cross-crate-inlining. Adding a broad exception for all intrinsics benchmarks well
// and seems more sustainable than an ever-growing list of intrinsics to ignore.
if let Some((fn_def_id, _)) = func.const_fn_def()
&& find_attr!(tcx, fn_def_id, RustcIntrinsic)
{
return;
}
self.calls += 1;
if let UnwindAction::Cleanup(_) = unwind {
self.landing_pads += 1;
}
}
TerminatorKind::TailCall { .. } => {
self.calls += 1;
}
TerminatorKind::Assert { unwind, .. } => {
if let UnwindAction::Cleanup(_) = unwind {
self.landing_pads += 1;
}
}
TerminatorKind::UnwindResume => self.resumes += 1,
TerminatorKind::InlineAsm { unwind, .. } => {
if let UnwindAction::Cleanup(_) = unwind {
self.landing_pads += 1;
}
}
TerminatorKind::Return
| TerminatorKind::Goto { .. }
| TerminatorKind::SwitchInt { .. }
| TerminatorKind::Unreachable
| TerminatorKind::UnwindTerminate(_) => {}
kind @ (TerminatorKind::FalseUnwind { .. }
| TerminatorKind::FalseEdge { .. }
| TerminatorKind::Yield { .. }
| TerminatorKind::CoroutineDrop) => {
bug!("{kind:?} should not be in runtime MIR");
}
}
}
}