1#![allow(unsafe_code)]
24
25use std::sync::atomic::{AtomicU64, Ordering};
26
27use crate::cpu;
28use crate::host_slot;
29
30pub const GUEST_TLS_MAGIC: u64 = 0x4B48_544C_5301;
32pub const GUEST_TLS_SIZE: usize = 64;
34pub const GUEST_TLS_ERRNO_OFF: usize = 8;
36pub const GUEST_TLS_PTHREAD_OFF: usize = 16;
38pub const GUEST_TLS_HOST_TPIDR_OFF: usize = 24;
40pub const GUEST_TLS_ALT_TOP_OFF: usize = 32;
42pub const GUEST_TLS_TSD_VALS_OFF: usize = 40;
44
45static MAIN_GUEST_TLS: AtomicU64 = AtomicU64::new(0);
47
48#[repr(C)]
50#[derive(Clone, Copy, Debug, Default)]
51pub struct HyperEnterRet {
52 pub alt_top: u64,
54 pub guest_tpidr: u64,
56}
57
58pub fn prepare_host_meta() {
62 let host = cpu::read_tpidr_el0();
63 host_slot::with_tls_init(|m| {
65 m.host_tpidr = host;
66 m.active = true;
67 });
68}
69
70#[must_use]
80pub fn install_main_guest_tls() -> u64 {
81 prepare_host_meta();
82 let alt = crate::thread::ensure_host_alt_stack();
84 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
85 {
86 let Some(base) = map_guest_tls_page() else {
87 return 0;
88 };
89 init_guest_tls_block(base, 0);
90 let va = crate::host::ptr_addr_u64(base);
91 MAIN_GUEST_TLS.store(va, Ordering::Release);
92 let host = cpu::read_tpidr_el0();
93 host_slot::with_tls_init(|m| {
95 m.guest_tpidr = va;
96 m.host_tpidr = host;
97 m.active = true;
98 });
99 publish_boundary_to_guest_tls(va, host, alt);
100 va
101 }
102 #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
103 {
104 let _ = alt;
105 0
106 }
107}
108
109pub fn enter_guest_tls(guest_tpidr: u64) {
116 if guest_tpidr == 0 {
117 return;
118 }
119 let host = cpu::read_tpidr_el0();
120 let alt = host_slot::with_alt_mut(|cell| cell.map_or(0, |s| s.top));
121 host_slot::with_tls_init(|m| {
123 if !m.active {
124 m.host_tpidr = host;
125 m.active = true;
126 } else if m.host_tpidr == 0 {
127 m.host_tpidr = host;
128 }
129 m.guest_tpidr = guest_tpidr;
130 });
131 let host_pub = host_slot::tls_get().host_tpidr;
132 publish_boundary_to_guest_tls(guest_tpidr, host_pub, alt);
133 unsafe {
135 cpu::write_tpidr_el0(guest_tpidr);
136 }
137}
138
139pub fn publish_boundary_to_guest_tls(guest_va: u64, host_tpidr: u64, alt_top: u64) {
144 if guest_va == 0 || host_tpidr == 0 {
145 return;
146 }
147 let Ok(addr) = usize::try_from(guest_va) else {
148 return;
149 };
150 let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
152 if magic != GUEST_TLS_MAGIC {
153 return;
154 }
155 unsafe {
156 core::ptr::with_exposed_provenance_mut::<u64>(
157 addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF),
158 )
159 .write(host_tpidr);
160 core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
161 .write(alt_top);
162 }
163}
164
165pub fn publish_alt_top_to_current_guest(alt_top: u64) {
167 let guest = host_slot::tls_get().guest_tpidr;
168 let host = host_slot::tls_get().host_tpidr;
169 if guest != 0 && host != 0 {
170 publish_boundary_to_guest_tls(guest, host, alt_top);
171 }
172}
173
174#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
182#[unsafe(no_mangle)]
183pub unsafe extern "C" fn kh_tls_enter_host() -> HyperEnterRet {
184 enter_host_tls_inner()
185}
186
187#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
196#[unsafe(no_mangle)]
197pub unsafe extern "C" fn kh_tls_leave_host(guest_tpidr: u64) {
198 leave_host_tls_inner(guest_tpidr);
199}
200
201#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
205#[allow(dead_code)]
206pub unsafe extern "C" fn kh_tls_enter_host() -> HyperEnterRet {
207 HyperEnterRet::default()
208}
209
210#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
214#[allow(dead_code)]
215pub unsafe extern "C" fn kh_tls_leave_host(_guest_tpidr: u64) {}
216
217#[inline]
219pub fn enter_host_tls() {
220 let _ = enter_host_tls_inner();
221}
222
223#[inline]
225pub fn leave_host_tls() {
226 leave_host_tls_inner(0);
227}
228
229fn try_enter_from_guest_tls(cur: u64) -> Option<(u64 , u64 )> {
234 if cur == 0 {
235 return None;
236 }
237 let Ok(addr) = usize::try_from(cur) else {
238 return None;
239 };
240 let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
242 if magic != GUEST_TLS_MAGIC {
243 return None;
244 }
245 let host = unsafe {
246 core::ptr::with_exposed_provenance::<u64>(addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF))
247 .read_volatile()
248 };
249 if host == 0 || host == cur {
250 return None;
252 }
253 let alt = unsafe {
254 core::ptr::with_exposed_provenance::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
255 .read_volatile()
256 };
257 Some((host, alt))
258}
259
260fn enter_host_tls_inner() -> HyperEnterRet {
261 let cur = cpu::read_tpidr_el0();
262
263 if let Some((host, alt)) = try_enter_from_guest_tls(cur) {
265 if cur != host {
266 unsafe {
268 cpu::write_tpidr_el0(host);
269 }
270 }
271 let alt_top = if alt != 0 {
272 alt
273 } else {
274 let top = crate::thread::ensure_host_alt_stack();
277 publish_boundary_to_guest_tls(cur, host, top);
278 top
279 };
280 return HyperEnterRet {
282 alt_top,
283 guest_tpidr: cur,
284 };
285 }
286
287 let Some(prep) = host_slot::prepare_enter_under_guest(cur) else {
289 return HyperEnterRet {
290 alt_top: 0,
291 guest_tpidr: cur,
292 };
293 };
294 if cur != prep.host_tpidr {
295 unsafe {
297 cpu::write_tpidr_el0(prep.host_tpidr);
298 }
299 }
300 let alt_top = if prep.alt_top != 0 {
301 prep.alt_top
302 } else {
303 crate::thread::ensure_host_alt_stack()
304 };
305 if is_guest_tls(cur) {
307 publish_boundary_to_guest_tls(cur, prep.host_tpidr, alt_top);
308 }
309 HyperEnterRet {
310 alt_top,
311 guest_tpidr: if is_guest_tls(cur) {
312 cur
313 } else {
314 host_slot::tls_get().guest_tpidr
315 },
316 }
317}
318
319fn leave_host_tls_inner(guest_hint: u64) {
320 let guest = if guest_hint != 0 {
321 guest_hint
322 } else {
323 host_slot::with_tls_existing_mut(|m| if m.active { m.guest_tpidr } else { 0 }).unwrap_or(0)
324 };
325 if guest == 0 {
326 return;
327 }
328 unsafe {
330 cpu::write_tpidr_el0(guest);
331 }
332}
333
334pub fn clear_guest_tls_on_exit() {
336 let ret = enter_host_tls_inner();
337 let _ = host_slot::with_tls_existing_mut(|m| {
338 m.guest_tpidr = 0;
339 });
340 if ret.guest_tpidr != 0 {
341 publish_boundary_to_guest_tls(ret.guest_tpidr, 0, 0);
343 clear_guest_tls_mirrors(ret.guest_tpidr);
345 }
346}
347
348fn clear_guest_tls_mirrors(guest_va: u64) {
349 let Ok(addr) = usize::try_from(guest_va) else {
350 return;
351 };
352 let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
353 if magic != GUEST_TLS_MAGIC {
354 return;
355 }
356 unsafe {
357 core::ptr::with_exposed_provenance_mut::<u64>(
358 addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF),
359 )
360 .write(0);
361 core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
362 .write(0);
363 }
364}
365
366#[must_use]
368pub fn host_tpidr() -> u64 {
369 host_slot::tls_get().host_tpidr
370}
371
372#[must_use]
374pub fn guest_tpidr() -> u64 {
375 host_slot::tls_get().guest_tpidr
376}
377
378#[must_use]
380pub fn main_guest_tls() -> u64 {
381 MAIN_GUEST_TLS.load(Ordering::Acquire)
382}
383
384#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
385fn map_guest_tls_page() -> Option<*mut u8> {
386 let len = GUEST_TLS_SIZE.max(64);
387 let page = crate::host::page_size().unwrap_or(4096);
388 let map_len = len.saturating_add(page.saturating_sub(1)) & !(page.saturating_sub(1));
389 let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
390 let prot = libc::PROT_READ | libc::PROT_WRITE;
391 crate::host::mmap(None, map_len, prot, flags, -1, 0)
392}
393
394pub fn init_guest_tls_block(base: *mut u8, pthread_va: u64) {
400 if base.is_null() {
401 return;
402 }
403 unsafe {
405 let addr = base.addr();
406 core::ptr::with_exposed_provenance_mut::<u64>(addr).write(GUEST_TLS_MAGIC);
407 core::ptr::with_exposed_provenance_mut::<i32>(addr.saturating_add(GUEST_TLS_ERRNO_OFF))
408 .write(0);
409 core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_PTHREAD_OFF))
410 .write(pthread_va);
411 core::ptr::with_exposed_provenance_mut::<u64>(
412 addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF),
413 )
414 .write(0);
415 core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
416 .write(0);
417 core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_TSD_VALS_OFF))
419 .write(0);
420 }
421}
422
423#[must_use]
425pub fn is_guest_tls(tpidr: u64) -> bool {
426 if tpidr == 0 {
427 return false;
428 }
429 let Ok(addr) = usize::try_from(tpidr) else {
430 return false;
431 };
432 let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
434 magic == GUEST_TLS_MAGIC
435}
436
437#[cfg(test)]
438mod tests {
439 use super::*;
440
441 #[test]
442 fn prepare_captures_host() {
443 prepare_host_meta();
444 let h = host_tpidr();
445 let _ = h;
446 assert!(host_slot::tls_get().active);
447 }
448
449 #[test]
450 fn guest_tls_fast_enter_without_gettid_map() {
451 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
452 {
453 prepare_host_meta();
454 let host = host_tpidr();
455 let Some(base) = map_guest_tls_page() else {
456 return;
457 };
458 init_guest_tls_block(base, 0);
459 let guest = crate::host::ptr_addr_u64(base);
460 let fake_alt = 0x_aaa0_u64;
461 publish_boundary_to_guest_tls(guest, host, fake_alt);
462
463 let ret = unsafe {
464 cpu::write_tpidr_el0(guest);
465 enter_host_tls_inner()
466 };
467 assert_eq!(cpu::read_tpidr_el0(), host);
468 assert_eq!(ret.alt_top, fake_alt);
469 assert_eq!(ret.guest_tpidr, guest);
470
471 leave_host_tls_inner(ret.guest_tpidr);
472 assert_eq!(cpu::read_tpidr_el0(), guest);
473
474 clear_guest_tls_on_exit();
475 assert_eq!(cpu::read_tpidr_el0(), host);
476 let _ = crate::host::munmap(base, crate::host::page_size().unwrap_or(4096));
477 }
478 }
479
480 #[test]
481 fn enter_leave_restores_host() {
482 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
483 {
484 prepare_host_meta();
485 let host = host_tpidr();
486 let Some(base) = map_guest_tls_page() else {
487 return;
488 };
489 init_guest_tls_block(base, 0);
490 let fake_guest = crate::host::ptr_addr_u64(base);
491 host_slot::with_tls_init(|m| {
492 m.guest_tpidr = fake_guest;
493 });
494 publish_boundary_to_guest_tls(fake_guest, host, 0);
495 let after_enter = unsafe {
497 cpu::write_tpidr_el0(fake_guest);
498 enter_host_tls();
499 cpu::read_tpidr_el0()
500 };
501 assert_eq!(after_enter, host);
502
503 let after_leave = {
504 leave_host_tls();
505 let v = cpu::read_tpidr_el0();
506 enter_host_tls();
507 v
508 };
509 assert_eq!(after_leave, fake_guest);
510
511 clear_guest_tls_on_exit();
512 assert_eq!(cpu::read_tpidr_el0(), host);
513 let _ = crate::host::munmap(base, crate::host::page_size().unwrap_or(4096));
514 }
515 }
516}