1use core::mem::{align_of, offset_of, size_of};
5use core::num::{NonZeroU16, NonZeroUsize};
6
7#[cfg_attr(target_arch = "x86_64", path = "arch/amd64/layout.rs")]
8#[cfg_attr(target_arch = "aarch64", path = "arch/aarch64/layout.rs")]
9mod arch;
10
11pub use arch::{
12 SCRATCH_TOP_GPA, SCRATCH_TOP_GVA, SNAPSHOT_PT_GVA_MAX, SNAPSHOT_PT_GVA_MIN,
13 VIRTQ_BUFFER_GVA_END, VIRTQ_BUFFER_GVA_START, io_page,
14};
15#[cfg(target_arch = "aarch64")]
16pub use arch::{WHP_GICD_BASE_GPA, WHP_GICR_BASE_GPA, WHP_GITS_TRANSLATOR_BASE_GPA};
17
18use crate::virtq;
19
20const EXN_STACK_ALIGNMENT: usize = 16;
21pub const SCRATCH_TOP_RESERVED_PAGES: usize = 12;
23
24#[repr(C)]
27struct ScratchTopMetadata {
28 heap_executable: u64,
30 h2g_buffer_size: u64,
32 h2g_pool_pages: u64,
34 h2g_queue_size: u64,
36 g2h_buffer_size: u64,
38 g2h_pool_pages: u64,
40 g2h_queue_size: u64,
42 transport_arena_gpa: u64,
44 guest_log_level: u64,
46 libc_rng_seed: u64,
48 snapshot_generation: u64,
50 snapshot_pt_gpa_base: u64,
52 allocator: u64,
54 scratch_size: u64,
56}
57
58const fn scratch_top_offset(field_offset: usize) -> u64 {
59 (size_of::<ScratchTopMetadata>() - field_offset) as u64
60}
61
62pub const SCRATCH_TOP_G2H_QUEUE_SIZE_OFFSET: u64 =
63 scratch_top_offset(offset_of!(ScratchTopMetadata, g2h_queue_size));
64pub const SCRATCH_TOP_G2H_POOL_PAGES_OFFSET: u64 =
65 scratch_top_offset(offset_of!(ScratchTopMetadata, g2h_pool_pages));
66pub const SCRATCH_TOP_G2H_BUFFER_SIZE_OFFSET: u64 =
67 scratch_top_offset(offset_of!(ScratchTopMetadata, g2h_buffer_size));
68pub const SCRATCH_TOP_H2G_QUEUE_SIZE_OFFSET: u64 =
69 scratch_top_offset(offset_of!(ScratchTopMetadata, h2g_queue_size));
70pub const SCRATCH_TOP_H2G_POOL_PAGES_OFFSET: u64 =
71 scratch_top_offset(offset_of!(ScratchTopMetadata, h2g_pool_pages));
72pub const SCRATCH_TOP_H2G_BUFFER_SIZE_OFFSET: u64 =
73 scratch_top_offset(offset_of!(ScratchTopMetadata, h2g_buffer_size));
74pub const SCRATCH_TOP_TRANSPORT_ARENA_GPA_OFFSET: u64 =
75 scratch_top_offset(offset_of!(ScratchTopMetadata, transport_arena_gpa));
76pub const SCRATCH_TOP_SIZE_OFFSET: u64 =
77 scratch_top_offset(offset_of!(ScratchTopMetadata, scratch_size));
78pub const SCRATCH_TOP_ALLOCATOR_OFFSET: u64 =
79 scratch_top_offset(offset_of!(ScratchTopMetadata, allocator));
80pub const SCRATCH_TOP_SNAPSHOT_PT_GPA_BASE_OFFSET: u64 =
81 scratch_top_offset(offset_of!(ScratchTopMetadata, snapshot_pt_gpa_base));
82pub const SCRATCH_TOP_SNAPSHOT_GENERATION_OFFSET: u64 =
83 scratch_top_offset(offset_of!(ScratchTopMetadata, snapshot_generation));
84pub const SCRATCH_TOP_LIBC_RNG_SEED_OFFSET: u64 =
85 scratch_top_offset(offset_of!(ScratchTopMetadata, libc_rng_seed));
86pub const SCRATCH_TOP_GUEST_LOG_LEVEL_OFFSET: u64 =
87 scratch_top_offset(offset_of!(ScratchTopMetadata, guest_log_level));
88pub const SCRATCH_TOP_HEAP_EXECUTABLE_OFFSET: u64 =
89 scratch_top_offset(offset_of!(ScratchTopMetadata, heap_executable));
90pub const SCRATCH_TOP_EXN_STACK_OFFSET: u64 = size_of::<ScratchTopMetadata>() as u64;
91
92const _: () = {
93 assert!(
94 size_of::<ScratchTopMetadata>().is_multiple_of(EXN_STACK_ALIGNMENT),
95 "pad ScratchTopMetadata at its low end to keep the exception stack 16-byte aligned"
96 );
97 assert!(SCRATCH_TOP_SIZE_OFFSET == 0x08);
98 assert!(SCRATCH_TOP_ALLOCATOR_OFFSET == 0x10);
99 assert!(SCRATCH_TOP_SNAPSHOT_PT_GPA_BASE_OFFSET == 0x18);
100 assert!(SCRATCH_TOP_SNAPSHOT_GENERATION_OFFSET == 0x20);
101 assert!(SCRATCH_TOP_LIBC_RNG_SEED_OFFSET == 0x28);
102 assert!(SCRATCH_TOP_GUEST_LOG_LEVEL_OFFSET == 0x30);
103 assert!(SCRATCH_TOP_TRANSPORT_ARENA_GPA_OFFSET == 0x38);
104 assert!(SCRATCH_TOP_G2H_QUEUE_SIZE_OFFSET == 0x40);
105 assert!(SCRATCH_TOP_G2H_POOL_PAGES_OFFSET == 0x48);
106 assert!(SCRATCH_TOP_G2H_BUFFER_SIZE_OFFSET == 0x50);
107 assert!(SCRATCH_TOP_H2G_QUEUE_SIZE_OFFSET == 0x58);
108 assert!(SCRATCH_TOP_H2G_POOL_PAGES_OFFSET == 0x60);
109 assert!(SCRATCH_TOP_H2G_BUFFER_SIZE_OFFSET == 0x68);
110 assert!(SCRATCH_TOP_HEAP_EXECUTABLE_OFFSET == 0x70);
111 assert!(SCRATCH_TOP_EXN_STACK_OFFSET == 0x70);
112
113 assert!(VIRTQ_BUFFER_GVA_START < VIRTQ_BUFFER_GVA_END);
114 assert!((VIRTQ_BUFFER_GVA_START as usize).is_multiple_of(crate::vmem::PAGE_SIZE));
115 assert!((VIRTQ_BUFFER_GVA_END as usize).is_multiple_of(crate::vmem::PAGE_SIZE));
116 assert!(VIRTQ_BUFFER_GVA_START > SNAPSHOT_PT_GVA_MAX as u64);
117
118 assert!(VIRTQ_BUFFER_GVA_END < scratch_base_gva(16 * 1024 * 1024 * 1024));
119 assert!((SCRATCH_TOP_GPA as u64) < VIRTQ_BUFFER_GVA_END - VIRTQ_BUFFER_GVA_START);
120};
121
122pub const SCRATCH_TRANSPORT_GAP_PAGES: usize = 1;
127
128pub const fn scratch_reserved_top_gpa() -> u64 {
132 (SCRATCH_TOP_GPA + 1 - SCRATCH_TOP_RESERVED_PAGES * crate::vmem::PAGE_SIZE) as u64
133}
134
135pub const fn scratch_reserved_top_gva() -> u64 {
137 SCRATCH_TOP_GVA as u64 + 1 - (SCRATCH_TOP_RESERVED_PAGES * crate::vmem::PAGE_SIZE) as u64
138}
139
140pub const fn scratch_transport_end_gpa() -> u64 {
143 scratch_reserved_top_gpa() - (SCRATCH_TRANSPORT_GAP_PAGES * crate::vmem::PAGE_SIZE) as u64
144}
145
146pub const fn scratch_transport_end_gva() -> u64 {
148 scratch_reserved_top_gva() - (SCRATCH_TRANSPORT_GAP_PAGES * crate::vmem::PAGE_SIZE) as u64
149}
150
151pub fn scratch_base_gpa(size: usize) -> u64 {
152 (SCRATCH_TOP_GPA - size + 1) as u64
153}
154pub const fn scratch_base_gva(size: usize) -> u64 {
155 (SCRATCH_TOP_GVA - size + 1) as u64
156}
157
158pub fn min_scratch_size(transport_len: usize) -> usize {
164 arch::min_scratch_size()
165 .and_then(|fixed| fixed.checked_add(transport_len))
166 .unwrap_or(usize::MAX)
167}
168
169#[derive(Clone, Copy, Debug, PartialEq, Eq)]
171pub struct QueueDims {
172 size: NonZeroU16,
173 pool_pages: NonZeroUsize,
174}
175
176impl QueueDims {
177 pub fn new(size: usize, pool_pages: usize) -> Option<Self> {
179 let size = u16::try_from(size).ok()?;
180 let size = NonZeroU16::new(size)?;
181
182 if !size.get().is_power_of_two() {
183 return None;
184 }
185
186 let pool_pages = NonZeroUsize::new(pool_pages)?;
187 pool_pages.get().checked_mul(crate::vmem::PAGE_SIZE)?;
188 Some(Self { size, pool_pages })
189 }
190
191 pub const fn size(&self) -> NonZeroU16 {
193 self.size
194 }
195
196 pub const fn pool_pages(&self) -> NonZeroUsize {
198 self.pool_pages
199 }
200
201 pub fn ring_len(&self) -> usize {
203 virtq::Layout::query_size(usize::from(self.size.get()))
204 }
205
206 pub fn pool_len(&self) -> usize {
208 self.pool_pages.get() * crate::vmem::PAGE_SIZE
209 }
210}
211
212#[derive(Clone, Copy, Debug, PartialEq, Eq)]
223pub struct TransportArena {
224 g2h_ring_addr: u64,
226 h2g_ring_addr: u64,
228 mbx_addr: u64,
230 g2h_pool_addr: u64,
232 h2g_pool_addr: u64,
234 ring_span_len: usize,
236 len: usize,
238}
239
240impl TransportArena {
241 pub fn new(base_addr: u64, g2h: QueueDims, h2g: QueueDims) -> Option<Self> {
243 if !base_addr.is_multiple_of(crate::vmem::PAGE_SIZE as u64) {
244 return None;
245 }
246
247 let h2g_ring_offset = g2h
248 .ring_len()
249 .checked_next_multiple_of(virtq::Descriptor::ALIGN)?;
250
251 let mbx_offset = h2g_ring_offset
252 .checked_add(h2g.ring_len())?
253 .checked_next_multiple_of(align_of::<u64>())?;
254
255 let g2h_pool_offset = mbx_offset
256 .checked_add(size_of::<u64>())?
257 .checked_next_multiple_of(crate::vmem::PAGE_SIZE)?;
258
259 let g2h_pool_len = g2h.pool_len();
260 let h2g_pool_offset = g2h_pool_offset.checked_add(g2h_pool_len)?;
261
262 let h2g_pool_len = h2g.pool_len();
263 let len = h2g_pool_offset.checked_add(h2g_pool_len)?;
264
265 let addr = |offset: usize| base_addr.checked_add(u64::try_from(offset).ok()?);
266 let _end_addr = addr(len)?;
267
268 Some(Self {
269 g2h_ring_addr: base_addr,
270 h2g_ring_addr: addr(h2g_ring_offset)?,
271 mbx_addr: addr(mbx_offset)?,
272 g2h_pool_addr: addr(g2h_pool_offset)?,
273 h2g_pool_addr: addr(h2g_pool_offset)?,
274 ring_span_len: g2h_pool_offset,
275 len,
276 })
277 }
278
279 pub const fn base_addr(&self) -> u64 {
281 self.g2h_ring_addr
282 }
283
284 pub const fn g2h_ring_addr(&self) -> u64 {
286 self.g2h_ring_addr
287 }
288
289 pub const fn h2g_ring_addr(&self) -> u64 {
291 self.h2g_ring_addr
292 }
293
294 pub const fn mbx_addr(&self) -> u64 {
296 self.mbx_addr
297 }
298
299 pub const fn mbx_offset(&self) -> usize {
301 (self.mbx_addr - self.g2h_ring_addr) as usize
303 }
304
305 pub const fn g2h_pool_addr(&self) -> u64 {
307 self.g2h_pool_addr
308 }
309
310 pub const fn h2g_pool_addr(&self) -> u64 {
312 self.h2g_pool_addr
313 }
314
315 pub const fn ring_span_len(&self) -> usize {
317 self.ring_span_len
318 }
319
320 pub const fn size(&self) -> usize {
322 self.len
323 }
324
325 pub const fn end_addr(&self) -> u64 {
327 self.g2h_ring_addr + self.len as u64
328 }
329
330 pub fn to_offsets(&self) -> (usize, usize, usize, usize, usize) {
332 #[allow(clippy::unwrap_used)] let to_offset = |addr| usize::try_from(addr - self.g2h_ring_addr).unwrap();
334
335 (
336 to_offset(self.h2g_ring_addr),
337 to_offset(self.mbx_addr),
338 to_offset(self.g2h_pool_addr),
339 to_offset(self.h2g_pool_addr),
340 self.len,
341 )
342 }
343}
344
345#[cfg(test)]
346mod tests {
347 use super::*;
348
349 #[test]
350 fn queue_dims_validate_byte_lengths() {
351 let max_pages = usize::MAX / crate::vmem::PAGE_SIZE;
352 let dims = QueueDims::new(32768, max_pages).unwrap();
353
354 assert_eq!(dims.ring_len(), 0x80008);
355 assert_eq!(dims.pool_len(), max_pages * crate::vmem::PAGE_SIZE);
356
357 for (size, pages) in [
358 (0, 1),
359 (3, 1),
360 (usize::MAX, 1),
361 (64, 0),
362 (64, max_pages + 1),
363 ] {
364 assert_eq!(QueueDims::new(size, pages), None);
365 }
366 }
367
368 #[test]
369 fn transport_arena_derives_aligned_regions() {
370 let base = 0x1_0000;
371 let g2h = QueueDims::new(64, 8).unwrap();
372 let h2g = QueueDims::new(32, 4).unwrap();
373 let arena = TransportArena::new(base, g2h, h2g).unwrap();
374
375 assert_eq!(arena.g2h_ring_addr(), base);
376 assert!(
377 arena
378 .h2g_ring_addr()
379 .is_multiple_of(virtq::Descriptor::ALIGN as u64)
380 );
381 assert!(arena.mbx_addr().is_multiple_of(align_of::<u64>() as u64));
382 assert!(
383 arena
384 .g2h_pool_addr()
385 .is_multiple_of(crate::vmem::PAGE_SIZE as u64)
386 );
387 assert_eq!(
388 arena.h2g_pool_addr(),
389 base + 9 * crate::vmem::PAGE_SIZE as u64
390 );
391 assert_eq!(arena.end_addr(), base + 13 * crate::vmem::PAGE_SIZE as u64);
392 assert_eq!(
393 arena.to_offsets(),
394 (
395 0x410,
396 0x618,
397 crate::vmem::PAGE_SIZE,
398 9 * crate::vmem::PAGE_SIZE,
399 13 * crate::vmem::PAGE_SIZE,
400 )
401 );
402 assert_eq!(arena.ring_span_len(), crate::vmem::PAGE_SIZE);
403 assert_eq!(arena.size(), 13 * crate::vmem::PAGE_SIZE);
404 assert_eq!(TransportArena::new(base + 1, g2h, h2g), None);
405
406 let oversized = QueueDims::new(64, usize::MAX / crate::vmem::PAGE_SIZE).unwrap();
407 assert_eq!(TransportArena::new(base, oversized, h2g), None);
408 assert_eq!(
409 TransportArena::new(u64::MAX - crate::vmem::PAGE_SIZE as u64 + 1, g2h, h2g,),
410 None
411 );
412 }
413
414 #[test]
415 fn transport_arena_mailbox_offset_at_high_base() {
416 let base = u64::MAX - 16 * crate::vmem::PAGE_SIZE as u64 + 1;
418 let g2h = QueueDims::new(64, 8).unwrap();
419 let h2g = QueueDims::new(32, 4).unwrap();
420 let arena = TransportArena::new(base, g2h, h2g).unwrap();
421
422 assert_eq!(arena.mbx_offset(), 0x618);
423 assert_eq!(base + arena.mbx_offset() as u64, arena.mbx_addr());
424 assert!(arena.mbx_offset() + size_of::<u64>() <= arena.ring_span_len());
425 }
426
427 #[test]
428 fn transport_arena_mailbox_offset_with_largest_queues() {
429 let dims = QueueDims::new(32768, 1).unwrap();
431 let arena = TransportArena::new(0, dims, dims).unwrap();
432
433 assert_eq!(arena.mbx_offset(), 0x100018);
434 assert_eq!(arena.mbx_offset() as u64, arena.mbx_addr());
435 assert!(arena.mbx_offset() + size_of::<u64>() <= arena.ring_span_len());
436 }
437
438 #[test]
439 fn minimum_scratch_includes_transport() {
440 let fixed = arch::min_scratch_size().unwrap();
441
442 for transport_len in [0, (1 + 8 + 4) * crate::vmem::PAGE_SIZE, 5 << 20] {
443 assert_eq!(fixed + transport_len, min_scratch_size(transport_len));
444 }
445 }
446
447 #[test]
448 fn minimum_scratch_saturates_on_overflow() {
449 assert_eq!(usize::MAX, min_scratch_size(usize::MAX));
450 }
451}