1use core::mem::{align_of, offset_of, size_of};
5use core::num::{NonZeroU16, NonZeroUsize};
6
7#[cfg_attr(target_arch = "x86_64", path = "arch/amd64/layout.rs")]
8#[cfg_attr(target_arch = "aarch64", path = "arch/aarch64/layout.rs")]
9mod arch;
10
11pub use arch::{
12 SCRATCH_TOP_GPA, SCRATCH_TOP_GVA, SNAPSHOT_PT_GVA_MAX, SNAPSHOT_PT_GVA_MIN,
13 VIRTQ_BUFFER_GVA_END, VIRTQ_BUFFER_GVA_START, io_page,
14};
15#[cfg(target_arch = "aarch64")]
16pub use arch::{WHP_GICD_BASE_GPA, WHP_GICR_BASE_GPA, WHP_GITS_TRANSLATOR_BASE_GPA};
17
18use crate::virtq;
19
20const EXN_STACK_ALIGNMENT: usize = 16;
21pub const SCRATCH_TOP_RESERVED_PAGES: usize = 12;
23
24#[repr(C)]
27struct ScratchTopMetadata {
28 _alignment_padding: [u8; 8],
30 h2g_buffer_size: u64,
32 h2g_pool_pages: u64,
34 h2g_queue_size: u64,
36 g2h_buffer_size: u64,
38 g2h_pool_pages: u64,
40 g2h_queue_size: u64,
42 transport_arena_gpa: u64,
44 guest_log_level: u64,
46 libc_rng_seed: u64,
48 snapshot_generation: u64,
50 snapshot_pt_gpa_base: u64,
52 allocator: u64,
54 scratch_size: u64,
56}
57
58const fn scratch_top_offset(field_offset: usize) -> u64 {
59 (size_of::<ScratchTopMetadata>() - field_offset) as u64
60}
61
62pub const SCRATCH_TOP_G2H_QUEUE_SIZE_OFFSET: u64 =
63 scratch_top_offset(offset_of!(ScratchTopMetadata, g2h_queue_size));
64pub const SCRATCH_TOP_G2H_POOL_PAGES_OFFSET: u64 =
65 scratch_top_offset(offset_of!(ScratchTopMetadata, g2h_pool_pages));
66pub const SCRATCH_TOP_G2H_BUFFER_SIZE_OFFSET: u64 =
67 scratch_top_offset(offset_of!(ScratchTopMetadata, g2h_buffer_size));
68pub const SCRATCH_TOP_H2G_QUEUE_SIZE_OFFSET: u64 =
69 scratch_top_offset(offset_of!(ScratchTopMetadata, h2g_queue_size));
70pub const SCRATCH_TOP_H2G_POOL_PAGES_OFFSET: u64 =
71 scratch_top_offset(offset_of!(ScratchTopMetadata, h2g_pool_pages));
72pub const SCRATCH_TOP_H2G_BUFFER_SIZE_OFFSET: u64 =
73 scratch_top_offset(offset_of!(ScratchTopMetadata, h2g_buffer_size));
74pub const SCRATCH_TOP_TRANSPORT_ARENA_GPA_OFFSET: u64 =
75 scratch_top_offset(offset_of!(ScratchTopMetadata, transport_arena_gpa));
76pub const SCRATCH_TOP_SIZE_OFFSET: u64 =
77 scratch_top_offset(offset_of!(ScratchTopMetadata, scratch_size));
78pub const SCRATCH_TOP_ALLOCATOR_OFFSET: u64 =
79 scratch_top_offset(offset_of!(ScratchTopMetadata, allocator));
80pub const SCRATCH_TOP_SNAPSHOT_PT_GPA_BASE_OFFSET: u64 =
81 scratch_top_offset(offset_of!(ScratchTopMetadata, snapshot_pt_gpa_base));
82pub const SCRATCH_TOP_SNAPSHOT_GENERATION_OFFSET: u64 =
83 scratch_top_offset(offset_of!(ScratchTopMetadata, snapshot_generation));
84pub const SCRATCH_TOP_LIBC_RNG_SEED_OFFSET: u64 =
85 scratch_top_offset(offset_of!(ScratchTopMetadata, libc_rng_seed));
86pub const SCRATCH_TOP_GUEST_LOG_LEVEL_OFFSET: u64 =
87 scratch_top_offset(offset_of!(ScratchTopMetadata, guest_log_level));
88pub const SCRATCH_TOP_EXN_STACK_OFFSET: u64 = size_of::<ScratchTopMetadata>() as u64;
89
90const _: () = {
91 assert!(size_of::<ScratchTopMetadata>().is_multiple_of(EXN_STACK_ALIGNMENT));
92 assert!(SCRATCH_TOP_SIZE_OFFSET == 0x08);
93 assert!(SCRATCH_TOP_ALLOCATOR_OFFSET == 0x10);
94 assert!(SCRATCH_TOP_SNAPSHOT_PT_GPA_BASE_OFFSET == 0x18);
95 assert!(SCRATCH_TOP_SNAPSHOT_GENERATION_OFFSET == 0x20);
96 assert!(SCRATCH_TOP_LIBC_RNG_SEED_OFFSET == 0x28);
97 assert!(SCRATCH_TOP_GUEST_LOG_LEVEL_OFFSET == 0x30);
98 assert!(SCRATCH_TOP_TRANSPORT_ARENA_GPA_OFFSET == 0x38);
99 assert!(SCRATCH_TOP_G2H_QUEUE_SIZE_OFFSET == 0x40);
100 assert!(SCRATCH_TOP_G2H_POOL_PAGES_OFFSET == 0x48);
101 assert!(SCRATCH_TOP_G2H_BUFFER_SIZE_OFFSET == 0x50);
102 assert!(SCRATCH_TOP_H2G_QUEUE_SIZE_OFFSET == 0x58);
103 assert!(SCRATCH_TOP_H2G_POOL_PAGES_OFFSET == 0x60);
104 assert!(SCRATCH_TOP_H2G_BUFFER_SIZE_OFFSET == 0x68);
105 assert!(SCRATCH_TOP_EXN_STACK_OFFSET == 0x70);
106
107 assert!(VIRTQ_BUFFER_GVA_START < VIRTQ_BUFFER_GVA_END);
108 assert!((VIRTQ_BUFFER_GVA_START as usize).is_multiple_of(crate::vmem::PAGE_SIZE));
109 assert!((VIRTQ_BUFFER_GVA_END as usize).is_multiple_of(crate::vmem::PAGE_SIZE));
110 assert!(VIRTQ_BUFFER_GVA_START > SNAPSHOT_PT_GVA_MAX as u64);
111
112 assert!(VIRTQ_BUFFER_GVA_END < scratch_base_gva(16 * 1024 * 1024 * 1024));
113 assert!((SCRATCH_TOP_GPA as u64) < VIRTQ_BUFFER_GVA_END - VIRTQ_BUFFER_GVA_START);
114};
115
116pub const fn scratch_allocator_limit_gpa() -> u64 {
118 (SCRATCH_TOP_GPA + 1 - SCRATCH_TOP_RESERVED_PAGES * crate::vmem::PAGE_SIZE) as u64
119}
120
121pub fn scratch_base_gpa(size: usize) -> u64 {
122 (SCRATCH_TOP_GPA - size + 1) as u64
123}
124pub const fn scratch_base_gva(size: usize) -> u64 {
125 (SCRATCH_TOP_GVA - size + 1) as u64
126}
127
128pub fn min_scratch_size(transport_len: usize) -> usize {
133 arch::min_scratch_size()
134 .and_then(|fixed| fixed.checked_add(transport_len))
135 .unwrap_or(usize::MAX)
136}
137
138#[derive(Clone, Copy, Debug, PartialEq, Eq)]
140pub struct QueueDims {
141 size: NonZeroU16,
142 pool_pages: NonZeroUsize,
143}
144
145impl QueueDims {
146 pub fn new(size: usize, pool_pages: usize) -> Option<Self> {
148 let size = u16::try_from(size).ok()?;
149 let size = NonZeroU16::new(size)?;
150
151 if !size.get().is_power_of_two() {
152 return None;
153 }
154
155 let pool_pages = NonZeroUsize::new(pool_pages)?;
156 pool_pages.get().checked_mul(crate::vmem::PAGE_SIZE)?;
157 Some(Self { size, pool_pages })
158 }
159
160 pub const fn size(&self) -> NonZeroU16 {
162 self.size
163 }
164
165 pub const fn pool_pages(&self) -> NonZeroUsize {
167 self.pool_pages
168 }
169
170 pub fn ring_len(&self) -> usize {
172 virtq::Layout::query_size(usize::from(self.size.get()))
173 }
174
175 pub fn pool_len(&self) -> usize {
177 self.pool_pages.get() * crate::vmem::PAGE_SIZE
178 }
179}
180
181#[derive(Clone, Copy, Debug, PartialEq, Eq)]
192pub struct TransportArena {
193 g2h_ring_addr: u64,
195 h2g_ring_addr: u64,
197 mbx_addr: u64,
199 g2h_pool_addr: u64,
201 h2g_pool_addr: u64,
203 ring_span_len: usize,
205 len: usize,
207}
208
209impl TransportArena {
210 pub fn new(base_addr: u64, g2h: QueueDims, h2g: QueueDims) -> Option<Self> {
212 if !base_addr.is_multiple_of(crate::vmem::PAGE_SIZE as u64) {
213 return None;
214 }
215
216 let h2g_ring_offset = g2h
217 .ring_len()
218 .checked_next_multiple_of(virtq::Descriptor::ALIGN)?;
219
220 let mbx_offset = h2g_ring_offset
221 .checked_add(h2g.ring_len())?
222 .checked_next_multiple_of(align_of::<u64>())?;
223
224 let g2h_pool_offset = mbx_offset
225 .checked_add(size_of::<u64>())?
226 .checked_next_multiple_of(crate::vmem::PAGE_SIZE)?;
227
228 let g2h_pool_len = g2h.pool_len();
229 let h2g_pool_offset = g2h_pool_offset.checked_add(g2h_pool_len)?;
230
231 let h2g_pool_len = h2g.pool_len();
232 let len = h2g_pool_offset.checked_add(h2g_pool_len)?;
233
234 let addr = |offset: usize| base_addr.checked_add(u64::try_from(offset).ok()?);
235 let _end_addr = addr(len)?;
236
237 Some(Self {
238 g2h_ring_addr: base_addr,
239 h2g_ring_addr: addr(h2g_ring_offset)?,
240 mbx_addr: addr(mbx_offset)?,
241 g2h_pool_addr: addr(g2h_pool_offset)?,
242 h2g_pool_addr: addr(h2g_pool_offset)?,
243 ring_span_len: g2h_pool_offset,
244 len,
245 })
246 }
247
248 pub const fn base_addr(&self) -> u64 {
250 self.g2h_ring_addr
251 }
252
253 pub const fn g2h_ring_addr(&self) -> u64 {
255 self.g2h_ring_addr
256 }
257
258 pub const fn h2g_ring_addr(&self) -> u64 {
260 self.h2g_ring_addr
261 }
262
263 pub const fn mbx_addr(&self) -> u64 {
265 self.mbx_addr
266 }
267
268 pub const fn mbx_offset(&self) -> usize {
270 (self.mbx_addr - self.g2h_ring_addr) as usize
272 }
273
274 pub const fn g2h_pool_addr(&self) -> u64 {
276 self.g2h_pool_addr
277 }
278
279 pub const fn h2g_pool_addr(&self) -> u64 {
281 self.h2g_pool_addr
282 }
283
284 pub const fn ring_span_len(&self) -> usize {
286 self.ring_span_len
287 }
288
289 pub const fn size(&self) -> usize {
291 self.len
292 }
293
294 pub const fn end_addr(&self) -> u64 {
296 self.g2h_ring_addr + self.len as u64
297 }
298
299 pub fn to_offsets(&self) -> (usize, usize, usize, usize, usize) {
301 #[allow(clippy::unwrap_used)] let to_offset = |addr| usize::try_from(addr - self.g2h_ring_addr).unwrap();
303
304 (
305 to_offset(self.h2g_ring_addr),
306 to_offset(self.mbx_addr),
307 to_offset(self.g2h_pool_addr),
308 to_offset(self.h2g_pool_addr),
309 self.len,
310 )
311 }
312}
313
314#[cfg(test)]
315mod tests {
316 use super::*;
317
318 #[test]
319 fn queue_dims_validate_byte_lengths() {
320 let max_pages = usize::MAX / crate::vmem::PAGE_SIZE;
321 let dims = QueueDims::new(32768, max_pages).unwrap();
322
323 assert_eq!(dims.ring_len(), 0x80008);
324 assert_eq!(dims.pool_len(), max_pages * crate::vmem::PAGE_SIZE);
325
326 for (size, pages) in [
327 (0, 1),
328 (3, 1),
329 (usize::MAX, 1),
330 (64, 0),
331 (64, max_pages + 1),
332 ] {
333 assert_eq!(QueueDims::new(size, pages), None);
334 }
335 }
336
337 #[test]
338 fn transport_arena_derives_aligned_regions() {
339 let base = 0x1_0000;
340 let g2h = QueueDims::new(64, 8).unwrap();
341 let h2g = QueueDims::new(32, 4).unwrap();
342 let arena = TransportArena::new(base, g2h, h2g).unwrap();
343
344 assert_eq!(arena.g2h_ring_addr(), base);
345 assert!(
346 arena
347 .h2g_ring_addr()
348 .is_multiple_of(virtq::Descriptor::ALIGN as u64)
349 );
350 assert!(arena.mbx_addr().is_multiple_of(align_of::<u64>() as u64));
351 assert!(
352 arena
353 .g2h_pool_addr()
354 .is_multiple_of(crate::vmem::PAGE_SIZE as u64)
355 );
356 assert_eq!(
357 arena.h2g_pool_addr(),
358 base + 9 * crate::vmem::PAGE_SIZE as u64
359 );
360 assert_eq!(arena.end_addr(), base + 13 * crate::vmem::PAGE_SIZE as u64);
361 assert_eq!(
362 arena.to_offsets(),
363 (
364 0x410,
365 0x618,
366 crate::vmem::PAGE_SIZE,
367 9 * crate::vmem::PAGE_SIZE,
368 13 * crate::vmem::PAGE_SIZE,
369 )
370 );
371 assert_eq!(arena.ring_span_len(), crate::vmem::PAGE_SIZE);
372 assert_eq!(arena.size(), 13 * crate::vmem::PAGE_SIZE);
373 assert_eq!(TransportArena::new(base + 1, g2h, h2g), None);
374
375 let oversized = QueueDims::new(64, usize::MAX / crate::vmem::PAGE_SIZE).unwrap();
376 assert_eq!(TransportArena::new(base, oversized, h2g), None);
377 assert_eq!(
378 TransportArena::new(u64::MAX - crate::vmem::PAGE_SIZE as u64 + 1, g2h, h2g,),
379 None
380 );
381 }
382
383 #[test]
384 fn transport_arena_mailbox_offset_at_high_base() {
385 let base = u64::MAX - 16 * crate::vmem::PAGE_SIZE as u64 + 1;
387 let g2h = QueueDims::new(64, 8).unwrap();
388 let h2g = QueueDims::new(32, 4).unwrap();
389 let arena = TransportArena::new(base, g2h, h2g).unwrap();
390
391 assert_eq!(arena.mbx_offset(), 0x618);
392 assert_eq!(base + arena.mbx_offset() as u64, arena.mbx_addr());
393 assert!(arena.mbx_offset() + size_of::<u64>() <= arena.ring_span_len());
394 }
395
396 #[test]
397 fn transport_arena_mailbox_offset_with_largest_queues() {
398 let dims = QueueDims::new(32768, 1).unwrap();
400 let arena = TransportArena::new(0, dims, dims).unwrap();
401
402 assert_eq!(arena.mbx_offset(), 0x100018);
403 assert_eq!(arena.mbx_offset() as u64, arena.mbx_addr());
404 assert!(arena.mbx_offset() + size_of::<u64>() <= arena.ring_span_len());
405 }
406
407 #[test]
408 fn minimum_scratch_includes_transport() {
409 let fixed = arch::min_scratch_size().unwrap();
410
411 for transport_len in [0, (1 + 8 + 4) * crate::vmem::PAGE_SIZE, 5 << 20] {
412 assert_eq!(fixed + transport_len, min_scratch_size(transport_len));
413 }
414 }
415
416 #[test]
417 fn minimum_scratch_saturates_on_overflow() {
418 assert_eq!(usize::MAX, min_scratch_size(usize::MAX));
419 }
420}