1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
use super::header_metadata::HeaderMetadataSpec;
use crate::util::metadata::metadata_val_traits::*;
use crate::util::metadata::side_metadata::SideMetadataSpec;
use crate::util::ObjectReference;
use crate::vm::ObjectModel;
use crate::vm::VMBinding;
use atomic::Ordering;
/// This struct stores the specification of a metadata bit-set.
/// It is used as an input to the (inline) functions provided by the side metadata module.
///
/// Each plan or policy which uses a metadata bit-set, needs to create an instance of this struct.
///
/// For performance reasons, objects of this struct should be constants.
#[derive(Clone, Copy, Debug)]
pub enum MetadataSpec {
/// In-header metadata uses bits from an object header.
InHeader(HeaderMetadataSpec),
/// On-side metadata uses a side table.
OnSide(SideMetadataSpec),
}
impl MetadataSpec {
/// Is this metadata stored in the side table?
pub const fn is_on_side(&self) -> bool {
matches!(self, &MetadataSpec::OnSide(_))
}
/// Is this metadata stored in the object header?
pub const fn is_in_header(&self) -> bool {
matches!(self, &MetadataSpec::InHeader(_))
}
/// Extract SideMetadataSpec from a MetadataSpec. Panics if this is not side metadata.
pub const fn extract_side_spec(&self) -> &SideMetadataSpec {
match self {
MetadataSpec::OnSide(spec) => spec,
MetadataSpec::InHeader(_) => panic!("Expect a side spec"),
}
}
/// A function to non-atomically load the specified metadata's content.
/// Returns the metadata value.
///
/// # Arguments:
///
/// * `object`: is a reference to the target object.
/// * `mask`: is an optional mask value for the metadata. This value is used in cases like the forwarding pointer metadata, where some of the bits are reused by other metadata such as the forwarding bits.
///
/// # Safety
/// This is a non-atomic load, thus not thread-safe.
pub unsafe fn load<VM: VMBinding, T: MetadataValue>(
&self,
object: ObjectReference,
mask: Option<T>,
) -> T {
match self {
MetadataSpec::OnSide(metadata_spec) => metadata_spec.load(object.to_raw_address()),
MetadataSpec::InHeader(metadata_spec) => {
VM::VMObjectModel::load_metadata::<T>(metadata_spec, object, mask)
}
}
}
/// A function to atomically load the specified metadata's content.
/// Returns the metadata value.
///
/// # Arguments:
///
/// * `object`: is a reference to the target object.
/// * `mask`: is an optional mask value for the metadata. This value is used in cases like the forwarding pointer metadata, where some of the bits are reused by other metadata such as the forwarding bits.
/// * `atomic_ordering`: is the ordering for the load operation.
///
pub fn load_atomic<VM: VMBinding, T: MetadataValue>(
&self,
object: ObjectReference,
mask: Option<T>,
ordering: Ordering,
) -> T {
match self {
MetadataSpec::OnSide(metadata_spec) => {
metadata_spec.load_atomic(object.to_raw_address(), ordering)
}
MetadataSpec::InHeader(metadata_spec) => {
VM::VMObjectModel::load_metadata_atomic::<T>(metadata_spec, object, mask, ordering)
}
}
}
/// A function to non-atomically store a value to the specified metadata.
///
/// # Arguments:
///
/// * `object`: is a reference to the target object.
/// * `val`: is the new metadata value to be stored.
/// * `mask`: is an optional mask value for the metadata. This value is used in cases like the forwarding pointer metadata, where some of the bits are reused by other metadata such as the forwarding bits.
///
/// # Safety
/// This is a non-atomic store, thus not thread-safe.
pub unsafe fn store<VM: VMBinding, T: MetadataValue>(
&self,
object: ObjectReference,
val: T,
mask: Option<T>,
) {
match self {
MetadataSpec::OnSide(metadata_spec) => {
metadata_spec.store(object.to_raw_address(), val);
}
MetadataSpec::InHeader(metadata_spec) => {
VM::VMObjectModel::store_metadata::<T>(metadata_spec, object, val, mask)
}
}
}
/// A function to atomically store a value to the specified metadata.
///
/// # Arguments:
///
/// * `object`: is a reference to the target object.
/// * `val`: is the new metadata value to be stored.
/// * `mask`: is an optional mask value for the metadata. This value is used in cases like the forwarding pointer metadata, where some of the bits are reused by other metadata such as the forwarding bits.
/// * `ordering`: is the ordering for the store operation.
pub fn store_atomic<VM: VMBinding, T: MetadataValue>(
&self,
object: ObjectReference,
val: T,
mask: Option<T>,
ordering: Ordering,
) {
match self {
MetadataSpec::OnSide(metadata_spec) => {
metadata_spec.store_atomic(object.to_raw_address(), val, ordering);
}
MetadataSpec::InHeader(metadata_spec) => VM::VMObjectModel::store_metadata_atomic::<T>(
metadata_spec,
object,
val,
mask,
ordering,
),
}
}
/// A function to atomically compare-and-exchange the specified metadata's content.
/// Returns `true` if the operation is successful, and `false` otherwise.
///
/// # Arguments:
///
/// * `object`: is a reference to the target object.
/// * `old_val`: is the expected current value of the metadata.
/// * `new_val`: is the new metadata value to be stored if the compare-and-exchange operation is successful.
/// * `mask`: is an optional mask value for the metadata. This value is used in cases like the forwarding pointer metadata, where some of the bits are reused by other metadata such as the forwarding bits.
/// * `success_order`: is the atomic ordering used if the operation is successful.
/// * `failure_order`: is the atomic ordering used if the operation fails.
///
pub fn compare_exchange_metadata<VM: VMBinding, T: MetadataValue>(
&self,
object: ObjectReference,
old_val: T,
new_val: T,
mask: Option<T>,
success_order: Ordering,
failure_order: Ordering,
) -> std::result::Result<T, T> {
match self {
MetadataSpec::OnSide(metadata_spec) => metadata_spec.compare_exchange_atomic(
object.to_raw_address(),
old_val,
new_val,
success_order,
failure_order,
),
MetadataSpec::InHeader(metadata_spec) => {
VM::VMObjectModel::compare_exchange_metadata::<T>(
metadata_spec,
object,
old_val,
new_val,
mask,
success_order,
failure_order,
)
}
}
}
/// A function to atomically perform an add operation on the specified metadata's content.
/// Returns the old metadata value.
///
/// # Arguments:
///
/// * `object`: is a reference to the target object.
/// * `val`: is the value to be added to the current value of the metadata.
/// * `order`: is the atomic ordering of the fetch-and-add operation.
///
pub fn fetch_add_metadata<VM: VMBinding, T: MetadataValue>(
&self,
object: ObjectReference,
val: T,
order: Ordering,
) -> T {
match self {
MetadataSpec::OnSide(metadata_spec) => {
metadata_spec.fetch_add_atomic(object.to_raw_address(), val, order)
}
MetadataSpec::InHeader(metadata_spec) => {
VM::VMObjectModel::fetch_add_metadata::<T>(metadata_spec, object, val, order)
}
}
}
/// A function to atomically perform a subtract operation on the specified metadata's content.
/// Returns the old metadata value.
///
/// # Arguments:
///
/// * `object`: is a reference to the target object.
/// * `val`: is the value to be subtracted from the current value of the metadata.
/// * `order`: is the atomic ordering of the fetch-and-add operation.
///
pub fn fetch_sub_metadata<VM: VMBinding, T: MetadataValue>(
&self,
object: ObjectReference,
val: T,
order: Ordering,
) -> T {
match self {
MetadataSpec::OnSide(metadata_spec) => {
metadata_spec.fetch_sub_atomic(object.to_raw_address(), val, order)
}
MetadataSpec::InHeader(metadata_spec) => {
VM::VMObjectModel::fetch_sub_metadata::<T>(metadata_spec, object, val, order)
}
}
}
/// A function to atomically perform a bit-and operation on the specified metadata's content.
/// Returns the old metadata value.
///
/// # Arguments:
///
/// * `object`: is a reference to the target object.
/// * `val`: is the value to bit-and with the current value of the metadata.
/// * `order`: is the atomic ordering of the fetch-and-add operation.
///
pub fn fetch_and_metadata<VM: VMBinding, T: MetadataValue>(
&self,
object: ObjectReference,
val: T,
order: Ordering,
) -> T {
match self {
MetadataSpec::OnSide(metadata_spec) => {
metadata_spec.fetch_and_atomic(object.to_raw_address(), val, order)
}
MetadataSpec::InHeader(metadata_spec) => {
VM::VMObjectModel::fetch_and_metadata::<T>(metadata_spec, object, val, order)
}
}
}
/// A function to atomically perform a bit-or operation on the specified metadata's content.
/// Returns the old metadata value.
///
/// # Arguments:
///
/// * `object`: is a reference to the target object.
/// * `val`: is the value to bit-or with the current value of the metadata.
/// * `order`: is the atomic ordering of the fetch-and-add operation.
///
pub fn fetch_or_metadata<VM: VMBinding, T: MetadataValue>(
&self,
object: ObjectReference,
val: T,
order: Ordering,
) -> T {
match self {
MetadataSpec::OnSide(metadata_spec) => {
metadata_spec.fetch_or_atomic(object.to_raw_address(), val, order)
}
MetadataSpec::InHeader(metadata_spec) => {
VM::VMObjectModel::fetch_or_metadata::<T>(metadata_spec, object, val, order)
}
}
}
/// A function to atomically perform an update operation on the specified metadata's content. The semantics are the same as Rust's `fetch_update` on atomic types.
/// Returns a Result of Ok(previous_value) if the function returned Some(_), else Err(previous_value).
///
/// # Arguments:
///
/// * `object`: is a reference to the target object.
/// * `val`: is the value to bit-or with the current value of the metadata.
/// * `order`: is the atomic ordering of the fetch-and-add operation.
/// * `f`: the update function. The function may be called multiple times.
///
pub fn fetch_update_metadata<
VM: VMBinding,
T: MetadataValue,
F: FnMut(T) -> Option<T> + Copy,
>(
&self,
object: ObjectReference,
set_order: Ordering,
fetch_order: Ordering,
f: F,
) -> std::result::Result<T, T> {
match self {
MetadataSpec::OnSide(metadata_spec) => metadata_spec.fetch_update_atomic(
object.to_raw_address(),
set_order,
fetch_order,
f,
),
MetadataSpec::InHeader(metadata_spec) => VM::VMObjectModel::fetch_update_metadata(
metadata_spec,
object,
set_order,
fetch_order,
f,
),
}
}
}
/// Given a slice of metadata specifications, returns a vector of the specs which are on side.
///
/// # Arguments:
/// * `specs` is the input slice of on-side and/or in-header metadata specifications.
///
pub(crate) fn extract_side_metadata(specs: &[MetadataSpec]) -> Vec<SideMetadataSpec> {
let mut side_specs = vec![];
for spec in specs {
if let MetadataSpec::OnSide(ss) = *spec {
side_specs.push(ss);
}
}
side_specs
}