winch_codegen/codegen/call.rs
1//! Function call emission. For more details around the ABI and
2//! calling convention, see [ABI].
3//!
4//! This module exposes a single function [`FnCall::emit`], which is responsible
5//! of orchestrating the emission of calls. In general such orchestration
6//! takes place in 6 steps:
7//!
8//! 1. [`Callee`] resolution.
9//! 2. Mapping of the [`Callee`] to the [`CalleeKind`].
10//! 3. Spilling the value stack.
11//! 4. Calculate the return area, for 1+ results.
12//! 5. Emission.
13//! 6. Stack space cleanup.
14//!
15//! The stack space consumed by the function call is the amount
16//! of space used by any memory entries in the value stack present
17//! at the callsite (after spilling the value stack), that will be
18//! used as arguments for the function call. Any memory values in the
19//! value stack that are needed as part of the function
20//! arguments will be consumed by the function call (either by
21//! assigning those values to a register or by storing those
22//! values in a memory location if the callee argument is on
23//! the stack).
24//! This could also be done when assigning arguments every time a
25//! memory entry needs to be assigned to a particular location,
26//! but doing so will emit more instructions (e.g. a pop per
27//! argument that needs to be assigned); it's more efficient to
28//! calculate the space used by those memory values and reclaim it
29//! at once when cleaning up the stack after the call has been
30//! emitted.
31//!
32//! The machine stack throughout the function call is as follows:
33//! ┌──────────────────────────────────────────────────┐
34//! │ │
35//! │ Stack space created by any previous spills │
36//! │ from the value stack; and which memory values │
37//! │ are used as function arguments. │
38//! │ │
39//! ├──────────────────────────────────────────────────┤ ---> The Wasm value stack at this point in time would look like:
40//! │ │
41//! │ Stack space created by spilling locals and |
42//! │ registers at the callsite. │
43//! │ │
44//! ├─────────────────────────────────────────────────┬┤
45//! │ │
46//! │ Return Area (Multi-value results) │
47//! │ │
48//! │ │
49//! ├─────────────────────────────────────────────────┬┤ ---> The Wasm value stack at this point in time would look like:
50//! │ │ [ Mem(offset) | Mem(offset) | Mem(offset) | Mem(offset) ]
51//! │ │ Assuming that the callee takes 4 arguments, we calculate
52//! │ │ 4 memory values; all of which will be used as arguments to
53//! │ Stack space allocated for │ the call via `assign_args`, thus the sum of the size of the
54//! │ the callee function arguments in the stack; │ memory they represent is considered to be consumed by the call.
55//! │ represented by `arg_stack_space` │
56//! │ │
57//! │ │
58//! │ │
59//! └──────────────────────────────────────────────────┘ ------> Stack pointer when emitting the call
60
61use crate::{
62 FuncEnv, Result,
63 abi::{ABIOperand, ABISig, RetArea, vmctx},
64 codegen::{BuiltinFunction, BuiltinType, Callee, CodeGenContext, CodeGenError, Emission},
65 ensure,
66 masm::{
67 CalleeKind, ContextArgs, IntScratch, MacroAssembler, MemMoveDirection, OperandSize,
68 SPOffset, VMContextLoc,
69 },
70 reg::{Reg, writable},
71 stack::Val,
72};
73use wasmtime_environ::{DefinedFuncIndex, FuncIndex, PtrSize, VMOffsets};
74
75/// All the information needed to emit a function call.
76#[derive(Copy, Clone)]
77pub(crate) struct FnCall {}
78
79impl FnCall {
80 /// Orchestrates the emission of a function call:
81 /// 1. Resolves the [`Callee`] through the given callback.
82 /// 2. Lowers the resolved [`Callee`] to a ([`CalleeKind`], [ContextArgs])
83 /// 3. Spills the value stack.
84 /// 4. Creates the stack space needed for the return area.
85 /// 5. Emits the call.
86 /// 6. Cleans up the stack space.
87 pub fn emit<M: MacroAssembler>(
88 env: &mut FuncEnv<M::Ptr>,
89 masm: &mut M,
90 context: &mut CodeGenContext<Emission>,
91 callee: Callee,
92 ) -> Result<()> {
93 let (kind, callee_context) = Self::lower(env, context.vmoffsets, &callee, context, masm)?;
94
95 let sig = env.callee_sig::<M::ABI>(&callee)?;
96 context.spill(masm)?;
97 let ret_area = Self::make_ret_area(&sig, masm)?;
98 let arg_stack_space = sig.params_stack_size();
99 let reserved_stack = masm.call(arg_stack_space, |masm| {
100 Self::assign(sig, &callee_context, ret_area.as_ref(), context, masm)?;
101 Ok((kind, sig.call_conv))
102 })?;
103
104 Self::cleanup(
105 sig,
106 &callee_context,
107 &kind,
108 reserved_stack,
109 ret_area,
110 masm,
111 context,
112 )
113 }
114
115 /// Calculates the return area for the callee, if any.
116 fn make_ret_area<M: MacroAssembler>(
117 callee_sig: &ABISig,
118 masm: &mut M,
119 ) -> Result<Option<RetArea>> {
120 if callee_sig.has_stack_results() {
121 let base = masm.sp_offset()?.as_u32();
122 let end = base + callee_sig.results_stack_size();
123 if end > base {
124 masm.reserve_stack(end - base)?;
125 }
126 Ok(Some(RetArea::sp(SPOffset::from_u32(end))))
127 } else {
128 Ok(None)
129 }
130 }
131
132 /// Lowers the high-level [`Callee`] to a [`CalleeKind`] and
133 /// [ContextArgs] pair which contains all the metadata needed for
134 /// emission.
135 fn lower<M: MacroAssembler>(
136 env: &mut FuncEnv<M::Ptr>,
137 vmoffsets: &VMOffsets<u8>,
138 callee: &Callee,
139 context: &mut CodeGenContext<Emission>,
140 masm: &mut M,
141 ) -> Result<(CalleeKind, ContextArgs)> {
142 let ptr = vmoffsets.ptr.size();
143 match callee {
144 Callee::Builtin(b) => Ok(Self::lower_builtin(env, b, None)),
145 Callee::BuiltinWithDifferentVmctx(b, offset) => {
146 Ok(Self::lower_builtin(env, b, Some(*offset)))
147 }
148 Callee::FuncRef(_) => {
149 Self::lower_funcref(env.callee_sig::<M::ABI>(callee)?, ptr, context, masm)
150 }
151 Callee::Local(i) => {
152 let f = env.translation.module.defined_func_index(*i).unwrap();
153 Ok(Self::lower_local(env, f))
154 }
155 Callee::Import(i) => {
156 let sig = env.callee_sig::<M::ABI>(callee)?;
157 Self::lower_import(*i, sig, context, masm, vmoffsets)
158 }
159 }
160 }
161
162 /// Lowers a builtin function by loading its address to the next available
163 /// register.
164 fn lower_builtin<P: PtrSize>(
165 env: &mut FuncEnv<P>,
166 builtin: &BuiltinFunction,
167 vmctx_offset: Option<u32>,
168 ) -> (CalleeKind, ContextArgs) {
169 match builtin.ty() {
170 BuiltinType::Builtin(idx) => (
171 CalleeKind::direct(env.name_builtin(idx)),
172 match vmctx_offset {
173 Some(offset) => ContextArgs::offset_from_pinned_vmctx(offset),
174 None => ContextArgs::pinned_vmctx(),
175 },
176 ),
177 }
178 }
179
180 /// Lower a local function to a [`CalleeKind`] and [ContextArgs] pair.
181 fn lower_local<P: PtrSize>(
182 env: &mut FuncEnv<P>,
183 index: DefinedFuncIndex,
184 ) -> (CalleeKind, ContextArgs) {
185 (
186 CalleeKind::direct(env.name_wasm(index)),
187 ContextArgs::pinned_callee_and_caller_vmctx(),
188 )
189 }
190
191 /// Lowers a function import by loading its address to the next available
192 /// register.
193 fn lower_import<M: MacroAssembler, P: PtrSize>(
194 index: FuncIndex,
195 sig: &ABISig,
196 context: &mut CodeGenContext<Emission>,
197 masm: &mut M,
198 vmoffsets: &VMOffsets<P>,
199 ) -> Result<(CalleeKind, ContextArgs)> {
200 let (callee, callee_vmctx) =
201 context.without::<Result<(Reg, Reg)>, M, _>(&sig.regs, masm, |context, masm| {
202 Ok((context.any_gpr(masm)?, context.any_gpr(masm)?))
203 })??;
204 let callee_vmctx_offset = vmoffsets.vmctx_vmfunction_import_vmctx(index);
205 let callee_vmctx_addr = masm.address_at_vmctx(callee_vmctx_offset)?;
206 masm.load_ptr(callee_vmctx_addr, writable!(callee_vmctx))?;
207
208 let callee_body_offset = vmoffsets.vmctx_vmfunction_import_wasm_call(index);
209 let callee_addr = masm.address_at_vmctx(callee_body_offset)?;
210 masm.load_ptr(callee_addr, writable!(callee))?;
211
212 Ok((
213 CalleeKind::indirect(callee),
214 ContextArgs::with_callee_and_pinned_caller(callee_vmctx),
215 ))
216 }
217
218 /// Lowers a function reference by loading its address into the next
219 /// available register.
220 fn lower_funcref<M: MacroAssembler>(
221 sig: &ABISig,
222 ptr: impl PtrSize,
223 context: &mut CodeGenContext<Emission>,
224 masm: &mut M,
225 ) -> Result<(CalleeKind, ContextArgs)> {
226 // Pop the funcref pointer to a register and allocate a register to hold the
227 // address of the funcref. Since the callee is not addressed from a global non
228 // allocatable register (like the vmctx in the case of an import), we load the
229 // funcref to a register ensuring that it doesn't get assigned to a register
230 // used in the callee's signature.
231 let (funcref_ptr, funcref, callee_vmctx) = context
232 .without::<Result<(Reg, Reg, Reg)>, M, _>(&sig.regs, masm, |cx, masm| {
233 Ok((
234 cx.pop_to_reg(masm, None)?.into(),
235 cx.any_gpr(masm)?,
236 cx.any_gpr(masm)?,
237 ))
238 })??;
239
240 // Load the callee VMContext, that will be passed as first argument to
241 // the function call.
242 masm.load_ptr(
243 masm.address_at_reg(funcref_ptr, ptr.vm_func_ref_vmctx().into())?,
244 writable!(callee_vmctx),
245 )?;
246
247 // Load the function pointer to be called.
248 masm.load_ptr(
249 masm.address_at_reg(funcref_ptr, ptr.vm_func_ref_wasm_call().into())?,
250 writable!(funcref),
251 )?;
252 context.free_reg(funcref_ptr);
253
254 Ok((
255 CalleeKind::indirect(funcref),
256 ContextArgs::with_callee_and_pinned_caller(callee_vmctx),
257 ))
258 }
259
260 /// Materializes any [ContextArgs] as a function argument.
261 fn assign_context_args<M: MacroAssembler>(
262 sig: &ABISig,
263 context: &ContextArgs,
264 masm: &mut M,
265 ) -> Result<()> {
266 ensure!(
267 sig.params().len() >= context.len(),
268 CodeGenError::vmcontext_arg_expected(),
269 );
270 for (context_arg, operand) in context
271 .as_slice()
272 .iter()
273 .zip(sig.params_without_retptr().iter().take(context.len()))
274 {
275 match (context_arg, operand) {
276 (VMContextLoc::Pinned, ABIOperand::Reg { ty, reg, .. }) => {
277 masm.mov(writable!(*reg), vmctx!(M).into(), (*ty).try_into()?)?;
278 }
279 (VMContextLoc::Pinned, ABIOperand::Stack { ty, offset, .. }) => {
280 let addr = masm.address_at_sp(SPOffset::from_u32(*offset))?;
281 masm.store(vmctx!(M).into(), addr, (*ty).try_into()?)?;
282 }
283 (VMContextLoc::OffsetFromPinned(offset), ABIOperand::Reg { ty, reg, .. }) => {
284 let addr = masm.address_at_vmctx(*offset)?;
285 masm.load(addr, writable!(*reg), (*ty).try_into()?)?;
286 }
287 (VMContextLoc::OffsetFromPinned(_), ABIOperand::Stack { .. }) => {
288 crate::bail!("unimplemented load from vmctx into stack");
289 }
290
291 (VMContextLoc::Reg(src), ABIOperand::Reg { ty, reg, .. }) => {
292 masm.mov(writable!(*reg), (*src).into(), (*ty).try_into()?)?;
293 }
294
295 (VMContextLoc::Reg(src), ABIOperand::Stack { ty, offset, .. }) => {
296 let addr = masm.address_at_sp(SPOffset::from_u32(*offset))?;
297 masm.store((*src).into(), addr, (*ty).try_into()?)?;
298 }
299 }
300 }
301 Ok(())
302 }
303
304 /// Assign arguments for the function call.
305 fn assign<M: MacroAssembler>(
306 sig: &ABISig,
307 callee_context: &ContextArgs,
308 ret_area: Option<&RetArea>,
309 context: &mut CodeGenContext<Emission>,
310 masm: &mut M,
311 ) -> Result<()> {
312 let arg_count = sig.params.len_without_retptr();
313 debug_assert!(arg_count >= callee_context.len());
314 let stack = &context.stack;
315 let stack_values = stack.peekn(arg_count - callee_context.len());
316
317 if callee_context.len() > 0 {
318 Self::assign_context_args(&sig, &callee_context, masm)?;
319 }
320
321 for (arg, val) in sig
322 .params_without_retptr()
323 .iter()
324 .skip(callee_context.len())
325 .zip(stack_values)
326 {
327 match arg {
328 &ABIOperand::Reg { reg, .. } => {
329 context.move_val_to_reg(&val, reg, masm)?;
330 }
331 &ABIOperand::Stack { ty, offset, .. } => {
332 let addr = masm.address_at_sp(SPOffset::from_u32(offset))?;
333 let size: OperandSize = ty.try_into()?;
334 masm.with_scratch_for(ty, |masm, scratch| {
335 context.move_val_to_reg(val, scratch.inner(), masm)?;
336 masm.store(scratch.inner().into(), addr, size)
337 })?;
338 }
339 }
340 }
341
342 if sig.has_stack_results() {
343 let operand = sig.params.unwrap_results_area_operand();
344 let base = ret_area.unwrap().unwrap_sp();
345 let addr = masm.address_from_sp(base)?;
346
347 match operand {
348 &ABIOperand::Reg { ty, reg, .. } => {
349 masm.compute_addr(addr, writable!(reg), ty.try_into()?)?;
350 }
351 &ABIOperand::Stack { ty, offset, .. } => {
352 let slot = masm.address_at_sp(SPOffset::from_u32(offset))?;
353 // Don't rely on `ABI::scratch_for` as we always use
354 // an int register as the return pointer.
355 masm.with_scratch::<IntScratch, _>(|masm, scratch| {
356 masm.compute_addr(addr, scratch.writable(), ty.try_into()?)?;
357 masm.store(scratch.inner().into(), slot, ty.try_into()?)
358 })?;
359 }
360 }
361 }
362 Ok(())
363 }
364
365 /// Cleanup stack space, handle multiple results, and free registers after
366 /// emitting the call.
367 fn cleanup<M: MacroAssembler>(
368 sig: &ABISig,
369 callee_context: &ContextArgs,
370 callee_kind: &CalleeKind,
371 reserved_space: u32,
372 ret_area: Option<RetArea>,
373 masm: &mut M,
374 context: &mut CodeGenContext<Emission>,
375 ) -> Result<()> {
376 // Free any registers holding any function references.
377 match callee_kind {
378 CalleeKind::Indirect(r) => context.free_reg(*r),
379 _ => {}
380 }
381
382 // Free any registers used as part of the [ContextArgs].
383 for loc in callee_context.as_slice() {
384 match loc {
385 VMContextLoc::Reg(r) => context.free_reg(*r),
386 _ => {}
387 }
388 }
389 // Deallocate the reserved space for stack arguments and for alignment,
390 // which was allocated last.
391 masm.free_stack(reserved_space)?;
392
393 ensure!(
394 sig.params.len_without_retptr() >= callee_context.len(),
395 CodeGenError::vmcontext_arg_expected()
396 );
397
398 // Drop params from value stack and calculate amount of machine stack
399 // space they consumed.
400 let mut stack_consumed = 0;
401 context.drop_last(
402 sig.params.len_without_retptr() - callee_context.len(),
403 |_regalloc, v| {
404 ensure!(
405 v.is_mem() || v.is_const(),
406 CodeGenError::unexpected_value_in_value_stack()
407 );
408 if let Val::Memory(mem) = v {
409 stack_consumed += mem.slot.size;
410 }
411 Ok(())
412 },
413 )?;
414
415 if let Some(ret_area) = ret_area {
416 if stack_consumed > 0 {
417 // Perform a memory move, by shuffling the result area to
418 // higher addresses. This is needed because the result area
419 // is located after any memory addresses located on the stack,
420 // and after spilled values consumed by the call.
421 let sp = ret_area.unwrap_sp();
422 let result_bytes = sig.results_stack_size();
423 ensure!(
424 sp.as_u32() >= stack_consumed + result_bytes,
425 CodeGenError::invalid_sp_offset(),
426 );
427 let dst = SPOffset::from_u32(sp.as_u32() - stack_consumed);
428 masm.memmove(sp, dst, result_bytes, MemMoveDirection::LowToHigh)?;
429 }
430 };
431
432 // Free the bytes consumed by the call.
433 masm.free_stack(stack_consumed)?;
434
435 let mut calculated_ret_area = None;
436
437 if let Some(area) = ret_area {
438 if stack_consumed > 0 {
439 // If there's a return area and stack space was consumed by the
440 // call, adjust the return area to be to the current stack
441 // pointer offset.
442 calculated_ret_area = Some(RetArea::sp(masm.sp_offset()?));
443 } else {
444 // Else if no stack space was consumed by the call, simply use
445 // the previously calculated area.
446 ensure!(
447 area.unwrap_sp() == masm.sp_offset()?,
448 CodeGenError::invalid_sp_offset()
449 );
450 calculated_ret_area = Some(area);
451 }
452 }
453
454 // In the case of [Callee], there's no need to set the [RetArea] of the
455 // signature, as it's only used here to push abi results.
456 context.push_abi_results(&sig.results, masm, |_, _, _| calculated_ret_area)?;
457 // Reload the [VMContext] pointer into the corresponding pinned
458 // register. Winch currently doesn't have any callee-saved registers in
459 // the default ABI. So the callee might clobber the designated pinned
460 // register.
461 context.load_vmctx(masm)
462 }
463}