1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
//! Support for implementing the [`RuntimeLinearMemory`] trait in terms of a
//! platform mmap primitive.
use crate::prelude::*;
use crate::runtime::vm::memory::RuntimeLinearMemory;
use crate::runtime::vm::{HostAlignedByteCount, Mmap, mmap::AlignedLength};
use alloc::sync::Arc;
use wasmtime_environ::Tunables;
use super::MemoryBase;
/// A linear memory instance.
#[derive(Debug)]
pub struct MmapMemory {
// The underlying allocation.
mmap: Arc<Mmap<AlignedLength>>,
// The current length of this Wasm memory, in bytes.
//
// This region starts at `pre_guard_size` offset from the base of `mmap`. It
// is always accessible, which means that if the Wasm page size is smaller
// than the host page size, there may be some trailing region in the `mmap`
// that is accessible but should not be accessed. (We rely on explicit
// bounds checks in the compiled code to protect this region.)
len: usize,
// The optional maximum accessible size, in bytes, for this linear memory.
//
// Note that this maximum does not factor in guard pages, so this isn't the
// maximum size of the linear address space reservation for this memory.
//
// This is *not* always a multiple of the host page size, and
// `self.accessible()` may go past `self.maximum` when Wasm is using a small
// custom page size due to `self.accessible()`'s rounding up to the host
// page size.
maximum: Option<usize>,
// The amount of extra bytes to reserve whenever memory grows. This is
// specified so that the cost of repeated growth is amortized.
extra_to_reserve_on_growth: HostAlignedByteCount,
// Size in bytes of extra guard pages before the start and after the end to
// optimize loads and stores with constant offsets.
pre_guard_size: HostAlignedByteCount,
offset_guard_size: HostAlignedByteCount,
}
impl MmapMemory {
/// Create a new linear memory instance with specified minimum and maximum
/// number of wasm pages.
pub fn new(
ty: &wasmtime_environ::Memory,
tunables: &Tunables,
minimum: usize,
maximum: Option<usize>,
) -> Result<Self> {
// It's a programmer error for these two configuration values to exceed
// the host available address space, so panic if such a configuration is
// found (mostly an issue for hypothetical 32-bit hosts).
//
// Also be sure to round up to the host page size for this value.
let offset_guard_bytes =
HostAlignedByteCount::new_rounded_up_u64(tunables.memory_guard_size)
.context("tunable.memory_guard_size overflows")?;
let pre_guard_bytes = if tunables.guard_before_linear_memory {
offset_guard_bytes
} else {
HostAlignedByteCount::ZERO
};
// Calculate how much is going to be allocated for this linear memory in
// addition to how much extra space we're reserving to grow into.
//
// If the minimum size of this linear memory fits within the initial
// allocation (tunables.memory_reservation) then that's how many bytes
// are going to be allocated. If the maximum size of linear memory
// additionally fits within the entire allocation then there's no need
// to reserve any extra for growth.
//
// If the minimum size doesn't fit within this linear memory.
let mut alloc_bytes = tunables.memory_reservation;
let mut extra_to_reserve_on_growth = tunables.memory_reservation_for_growth;
let minimum_u64 = u64::try_from(minimum).unwrap();
if minimum_u64 <= alloc_bytes {
if let Ok(max) = ty.maximum_byte_size() {
if max <= alloc_bytes {
extra_to_reserve_on_growth = 0;
}
}
} else {
alloc_bytes = minimum_u64.saturating_add(extra_to_reserve_on_growth);
}
// Convert `alloc_bytes` and `extra_to_reserve_on_growth` to
// page-aligned `usize` values.
let alloc_bytes = HostAlignedByteCount::new_rounded_up_u64(alloc_bytes)
.context("tunables.memory_reservation overflows")?;
let extra_to_reserve_on_growth =
HostAlignedByteCount::new_rounded_up_u64(extra_to_reserve_on_growth)
.context("tunables.memory_reservation_for_growth overflows")?;
let request_bytes = pre_guard_bytes
.checked_add(alloc_bytes)
.and_then(|i| i.checked_add(offset_guard_bytes))
.with_context(|| format!("cannot allocate {minimum} with guard regions"))?;
let mmap = Mmap::accessible_reserved(HostAlignedByteCount::ZERO, request_bytes)?;
if minimum > 0 {
let accessible = HostAlignedByteCount::new_rounded_up(minimum)?;
// SAFETY: mmap is not in use right now so it's safe to make it accessible.
unsafe {
mmap.make_accessible(pre_guard_bytes, accessible)?;
}
}
Ok(Self {
mmap: try_new::<Arc<_>>(mmap)?,
len: minimum,
maximum,
pre_guard_size: pre_guard_bytes,
offset_guard_size: offset_guard_bytes,
extra_to_reserve_on_growth,
})
}
/// Get the length of the accessible portion of the underlying `mmap`. This
/// is the same region as `self.len` but rounded up to a multiple of the
/// host page size.
fn accessible(&self) -> HostAlignedByteCount {
let accessible = HostAlignedByteCount::new_rounded_up(self.len)
.expect("accessible region always fits in usize");
debug_assert!(accessible <= self.current_capacity());
accessible
}
/// Get the amount to which this memory can grow.
fn current_capacity(&self) -> HostAlignedByteCount {
let mmap_len = self.mmap.len_aligned();
mmap_len
.checked_sub(self.offset_guard_size)
.and_then(|i| i.checked_sub(self.pre_guard_size))
.expect("guard regions fit in mmap.len")
}
}
impl RuntimeLinearMemory for MmapMemory {
fn byte_size(&self) -> usize {
self.len
}
fn byte_capacity(&self) -> usize {
self.current_capacity().byte_count()
}
fn grow_to(&mut self, new_size: usize) -> Result<()> {
let new_accessible = HostAlignedByteCount::new_rounded_up(new_size)?;
let current_capacity = self.current_capacity();
if new_accessible > current_capacity {
// If the new size of this heap exceeds the current size of the
// allocation we have, then this must be a dynamic heap. Use
// `new_size` to calculate a new size of an allocation, allocate it,
// and then copy over the memory from before.
let request_bytes = self
.pre_guard_size
.checked_add(new_accessible)
.and_then(|s| s.checked_add(self.extra_to_reserve_on_growth))
.and_then(|s| s.checked_add(self.offset_guard_size))
.context("overflow calculating size of memory allocation")?;
let mut new_mmap =
Mmap::accessible_reserved(HostAlignedByteCount::ZERO, request_bytes)?;
// SAFETY: new_mmap is not in use right now so it's safe to make it
// accessible.
unsafe {
new_mmap.make_accessible(self.pre_guard_size, new_accessible)?;
}
// This method has an exclusive reference to `self.mmap` and just
// created `new_mmap` so it should be safe to acquire references
// into both of them and copy between them.
unsafe {
let range =
self.pre_guard_size.byte_count()..(self.pre_guard_size.byte_count() + self.len);
let src = self.mmap.slice(range.clone());
let dst = new_mmap.slice_mut(range);
dst.copy_from_slice(src);
}
self.mmap = Arc::new(new_mmap);
} else {
// If the new size of this heap fits within the existing allocation
// then all we need to do is to make the new pages accessible. This
// can happen either for "static" heaps which always hit this case,
// or "dynamic" heaps which have some space reserved after the
// initial allocation to grow into before the heap is moved in
// memory.
assert!(new_size <= current_capacity.byte_count());
assert!(self.maximum.map_or(true, |max| new_size <= max));
// If the Wasm memory's page size is smaller than the host's page
// size, then we might not need to actually change permissions,
// since we are forced to round our accessible range up to the
// host's page size.
if let Ok(difference) = new_accessible.checked_sub(self.accessible()) {
// SAFETY: the difference was previously inaccessible so we
// never handed out any references to within it.
unsafe {
self.mmap.make_accessible(
self.pre_guard_size
.checked_add(self.accessible())
.context("overflow calculating new accessible region")?,
difference,
)?;
}
}
}
self.len = new_size;
Ok(())
}
fn set_byte_size(&mut self, len: usize) {
self.len = len;
}
fn base(&self) -> MemoryBase {
MemoryBase::Mmap(
self.mmap
.offset(self.pre_guard_size)
.expect("pre_guard_size is in bounds"),
)
}
fn vmmemory(&self) -> crate::vm::VMMemoryDefinition {
let pre_guard_size = self.pre_guard_size.byte_count();
assert!(pre_guard_size <= self.mmap.len());
let pre_guard_size = isize::try_from(pre_guard_size).unwrap();
let mmap = self.mmap.as_non_null();
let base = unsafe {
// Safety: `pre_guard_size` is within the mmap allocation.
mmap.offset(pre_guard_size)
};
crate::vm::VMMemoryDefinition {
base: base.into(),
current_length: self.len.into(),
}
}
}