1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
#![cfg(not(feature = "mangled-names"))]
#![cfg(not(windows))]
#![cfg(any(target_arch = "x86_64", target_arch = "x86"))]
extern "C" {
pub fn __rust_probestack();
}
#[cfg(not(any(target_vendor = "apple", target_os = "uefi")))]
macro_rules! define_rust_probestack {
($body: expr) => {
concat!(
"
.pushsection .text.__rust_probestack
.globl __rust_probestack
.type __rust_probestack, @function
.hidden __rust_probestack
__rust_probestack:
",
$body,
"
.size __rust_probestack, . - __rust_probestack
.popsection
"
)
};
}
#[cfg(target_os = "uefi")]
macro_rules! define_rust_probestack {
($body: expr) => {
concat!(
"
.globl __rust_probestack
__rust_probestack:
",
$body
)
};
}
#[cfg(target_vendor = "apple")]
macro_rules! define_rust_probestack {
($body: expr) => {
concat!(
"
.globl ___rust_probestack
___rust_probestack:
",
$body
)
};
}
#[cfg(target_arch = "x86_64")]
global_asm!(define_rust_probestack!(
"
.cfi_startproc
pushq %rbp
.cfi_adjust_cfa_offset 8
.cfi_offset %rbp, -16
movq %rsp, %rbp
.cfi_def_cfa_register %rbp
mov %rax,%r11 // duplicate %rax as we're clobbering %r11
// Main loop, taken in one page increments. We're decrementing rsp by
// a page each time until there's less than a page remaining. We're
// guaranteed that this function isn't called unless there's more than a
// page needed.
//
// Note that we're also testing against `8(%rsp)` to account for the 8
// bytes pushed on the stack orginally with our return address. Using
// `8(%rsp)` simulates us testing the stack pointer in the caller's
// context.
// It's usually called when %rax >= 0x1000, but that's not always true.
// Dynamic stack allocation, which is needed to implement unsized
// rvalues, triggers stackprobe even if %rax < 0x1000.
// Thus we have to check %r11 first to avoid segfault.
cmp $0x1000,%r11
jna 3f
2:
sub $0x1000,%rsp
test %rsp,8(%rsp)
sub $0x1000,%r11
cmp $0x1000,%r11
ja 2b
3:
// Finish up the last remaining stack space requested, getting the last
// bits out of r11
sub %r11,%rsp
test %rsp,8(%rsp)
// Restore the stack pointer to what it previously was when entering
// this function. The caller will readjust the stack pointer after we
// return.
add %rax,%rsp
leave
.cfi_def_cfa_register %rsp
.cfi_adjust_cfa_offset -8
ret
.cfi_endproc
"
));
#[cfg(target_arch = "x86")]
global_asm!(define_rust_probestack!(
"
.cfi_startproc
push %ebp
.cfi_adjust_cfa_offset 4
.cfi_offset %ebp, -8
mov %esp, %ebp
.cfi_def_cfa_register %ebp
push %ecx
mov %eax,%ecx
cmp $0x1000,%ecx
jna 3f
2:
sub $0x1000,%esp
test %esp,8(%esp)
sub $0x1000,%ecx
cmp $0x1000,%ecx
ja 2b
3:
sub %ecx,%esp
test %esp,8(%esp)
add %eax,%esp
pop %ecx
leave
.cfi_def_cfa_register %esp
.cfi_adjust_cfa_offset -4
ret
.cfi_endproc
"
));