1#![allow(unsafe_code)]
22
23use std::sync::atomic::{AtomicU64, Ordering};
24
25use crate::cpu;
26use crate::host_slot;
27
28pub const GUEST_TLS_MAGIC: u64 = 0x4B48_544C_5301;
30pub const GUEST_TLS_SIZE: usize = 64;
32pub const GUEST_TLS_ERRNO_OFF: usize = 8;
34pub const GUEST_TLS_PTHREAD_OFF: usize = 16;
36pub const GUEST_TLS_HOST_TPIDR_OFF: usize = 24;
38pub const GUEST_TLS_ALT_TOP_OFF: usize = 32;
40
41static MAIN_GUEST_TLS: AtomicU64 = AtomicU64::new(0);
43
44#[repr(C)]
46#[derive(Clone, Copy, Debug, Default)]
47pub struct HyperEnterRet {
48 pub alt_top: u64,
50 pub guest_tpidr: u64,
52}
53
54pub fn prepare_host_meta() {
58 let host = cpu::read_tpidr_el0();
59 host_slot::with_tls_init(|m| {
61 m.host_tpidr = host;
62 m.active = true;
63 });
64}
65
66#[must_use]
76pub fn install_main_guest_tls() -> u64 {
77 prepare_host_meta();
78 let alt = crate::thread::ensure_host_alt_stack();
80 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
81 {
82 let Some(base) = map_guest_tls_page() else {
83 return 0;
84 };
85 init_guest_tls_block(base, 0);
86 let va = crate::host::ptr_addr_u64(base);
87 MAIN_GUEST_TLS.store(va, Ordering::Release);
88 let host = cpu::read_tpidr_el0();
89 host_slot::with_tls_init(|m| {
91 m.guest_tpidr = va;
92 m.host_tpidr = host;
93 m.active = true;
94 });
95 publish_boundary_to_guest_tls(va, host, alt);
96 va
97 }
98 #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
99 {
100 let _ = alt;
101 0
102 }
103}
104
105pub fn enter_guest_tls(guest_tpidr: u64) {
112 if guest_tpidr == 0 {
113 return;
114 }
115 let host = cpu::read_tpidr_el0();
116 let alt = host_slot::with_alt_mut(|cell| cell.map_or(0, |s| s.top));
117 host_slot::with_tls_init(|m| {
119 if !m.active {
120 m.host_tpidr = host;
121 m.active = true;
122 } else if m.host_tpidr == 0 {
123 m.host_tpidr = host;
124 }
125 m.guest_tpidr = guest_tpidr;
126 });
127 let host_pub = host_slot::tls_get().host_tpidr;
128 publish_boundary_to_guest_tls(guest_tpidr, host_pub, alt);
129 unsafe {
131 cpu::write_tpidr_el0(guest_tpidr);
132 }
133}
134
135pub fn publish_boundary_to_guest_tls(guest_va: u64, host_tpidr: u64, alt_top: u64) {
140 if guest_va == 0 || host_tpidr == 0 {
141 return;
142 }
143 let Ok(addr) = usize::try_from(guest_va) else {
144 return;
145 };
146 let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
148 if magic != GUEST_TLS_MAGIC {
149 return;
150 }
151 unsafe {
152 core::ptr::with_exposed_provenance_mut::<u64>(
153 addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF),
154 )
155 .write(host_tpidr);
156 core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
157 .write(alt_top);
158 }
159}
160
161pub fn publish_alt_top_to_current_guest(alt_top: u64) {
163 let guest = host_slot::tls_get().guest_tpidr;
164 let host = host_slot::tls_get().host_tpidr;
165 if guest != 0 && host != 0 {
166 publish_boundary_to_guest_tls(guest, host, alt_top);
167 }
168}
169
170#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
178#[unsafe(no_mangle)]
179pub unsafe extern "C" fn kh_tls_enter_host() -> HyperEnterRet {
180 enter_host_tls_inner()
181}
182
183#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
192#[unsafe(no_mangle)]
193pub unsafe extern "C" fn kh_tls_leave_host(guest_tpidr: u64) {
194 leave_host_tls_inner(guest_tpidr);
195}
196
197#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
201#[allow(dead_code)]
202pub unsafe extern "C" fn kh_tls_enter_host() -> HyperEnterRet {
203 HyperEnterRet::default()
204}
205
206#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
210#[allow(dead_code)]
211pub unsafe extern "C" fn kh_tls_leave_host(_guest_tpidr: u64) {}
212
213#[inline]
215pub fn enter_host_tls() {
216 let _ = enter_host_tls_inner();
217}
218
219#[inline]
221pub fn leave_host_tls() {
222 leave_host_tls_inner(0);
223}
224
225fn try_enter_from_guest_tls(cur: u64) -> Option<(u64 , u64 )> {
230 if cur == 0 {
231 return None;
232 }
233 let Ok(addr) = usize::try_from(cur) else {
234 return None;
235 };
236 let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
238 if magic != GUEST_TLS_MAGIC {
239 return None;
240 }
241 let host = unsafe {
242 core::ptr::with_exposed_provenance::<u64>(addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF))
243 .read_volatile()
244 };
245 if host == 0 || host == cur {
246 return None;
248 }
249 let alt = unsafe {
250 core::ptr::with_exposed_provenance::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
251 .read_volatile()
252 };
253 Some((host, alt))
254}
255
256fn enter_host_tls_inner() -> HyperEnterRet {
257 let cur = cpu::read_tpidr_el0();
258
259 if let Some((host, alt)) = try_enter_from_guest_tls(cur) {
261 if cur != host {
262 unsafe {
264 cpu::write_tpidr_el0(host);
265 }
266 }
267 let alt_top = if alt != 0 {
268 alt
269 } else {
270 let top = crate::thread::ensure_host_alt_stack();
273 publish_boundary_to_guest_tls(cur, host, top);
274 top
275 };
276 return HyperEnterRet {
278 alt_top,
279 guest_tpidr: cur,
280 };
281 }
282
283 let Some(prep) = host_slot::prepare_enter_under_guest(cur) else {
285 return HyperEnterRet {
286 alt_top: 0,
287 guest_tpidr: cur,
288 };
289 };
290 if cur != prep.host_tpidr {
291 unsafe {
293 cpu::write_tpidr_el0(prep.host_tpidr);
294 }
295 }
296 let alt_top = if prep.alt_top != 0 {
297 prep.alt_top
298 } else {
299 crate::thread::ensure_host_alt_stack()
300 };
301 if is_guest_tls(cur) {
303 publish_boundary_to_guest_tls(cur, prep.host_tpidr, alt_top);
304 }
305 HyperEnterRet {
306 alt_top,
307 guest_tpidr: if is_guest_tls(cur) {
308 cur
309 } else {
310 host_slot::tls_get().guest_tpidr
311 },
312 }
313}
314
315fn leave_host_tls_inner(guest_hint: u64) {
316 let guest = if guest_hint != 0 {
317 guest_hint
318 } else {
319 host_slot::with_tls_existing_mut(|m| if m.active { m.guest_tpidr } else { 0 }).unwrap_or(0)
320 };
321 if guest == 0 {
322 return;
323 }
324 unsafe {
326 cpu::write_tpidr_el0(guest);
327 }
328}
329
330pub fn clear_guest_tls_on_exit() {
332 let ret = enter_host_tls_inner();
333 let _ = host_slot::with_tls_existing_mut(|m| {
334 m.guest_tpidr = 0;
335 });
336 if ret.guest_tpidr != 0 {
337 publish_boundary_to_guest_tls(ret.guest_tpidr, 0, 0);
339 clear_guest_tls_mirrors(ret.guest_tpidr);
341 }
342}
343
344fn clear_guest_tls_mirrors(guest_va: u64) {
345 let Ok(addr) = usize::try_from(guest_va) else {
346 return;
347 };
348 let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
349 if magic != GUEST_TLS_MAGIC {
350 return;
351 }
352 unsafe {
353 core::ptr::with_exposed_provenance_mut::<u64>(
354 addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF),
355 )
356 .write(0);
357 core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
358 .write(0);
359 }
360}
361
362#[must_use]
364pub fn host_tpidr() -> u64 {
365 host_slot::tls_get().host_tpidr
366}
367
368#[must_use]
370pub fn guest_tpidr() -> u64 {
371 host_slot::tls_get().guest_tpidr
372}
373
374#[must_use]
376pub fn main_guest_tls() -> u64 {
377 MAIN_GUEST_TLS.load(Ordering::Acquire)
378}
379
380#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
381fn map_guest_tls_page() -> Option<*mut u8> {
382 let len = GUEST_TLS_SIZE.max(64);
383 let page = crate::host::page_size().unwrap_or(4096);
384 let map_len = len.saturating_add(page.saturating_sub(1)) & !(page.saturating_sub(1));
385 let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
386 let prot = libc::PROT_READ | libc::PROT_WRITE;
387 crate::host::mmap(None, map_len, prot, flags, -1, 0)
388}
389
390pub fn init_guest_tls_block(base: *mut u8, pthread_va: u64) {
396 if base.is_null() {
397 return;
398 }
399 unsafe {
401 let addr = base.addr();
402 core::ptr::with_exposed_provenance_mut::<u64>(addr).write(GUEST_TLS_MAGIC);
403 core::ptr::with_exposed_provenance_mut::<i32>(addr.saturating_add(GUEST_TLS_ERRNO_OFF))
404 .write(0);
405 core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_PTHREAD_OFF))
406 .write(pthread_va);
407 core::ptr::with_exposed_provenance_mut::<u64>(
408 addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF),
409 )
410 .write(0);
411 core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
412 .write(0);
413 }
414}
415
416#[must_use]
418pub fn is_guest_tls(tpidr: u64) -> bool {
419 if tpidr == 0 {
420 return false;
421 }
422 let Ok(addr) = usize::try_from(tpidr) else {
423 return false;
424 };
425 let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
427 magic == GUEST_TLS_MAGIC
428}
429
430#[cfg(test)]
431mod tests {
432 use super::*;
433
434 #[test]
435 fn prepare_captures_host() {
436 prepare_host_meta();
437 let h = host_tpidr();
438 let _ = h;
439 assert!(host_slot::tls_get().active);
440 }
441
442 #[test]
443 fn guest_tls_fast_enter_without_gettid_map() {
444 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
445 {
446 prepare_host_meta();
447 let host = host_tpidr();
448 let Some(base) = map_guest_tls_page() else {
449 return;
450 };
451 init_guest_tls_block(base, 0);
452 let guest = crate::host::ptr_addr_u64(base);
453 let fake_alt = 0x_aaa0_u64;
454 publish_boundary_to_guest_tls(guest, host, fake_alt);
455
456 let ret = unsafe {
457 cpu::write_tpidr_el0(guest);
458 enter_host_tls_inner()
459 };
460 assert_eq!(cpu::read_tpidr_el0(), host);
461 assert_eq!(ret.alt_top, fake_alt);
462 assert_eq!(ret.guest_tpidr, guest);
463
464 leave_host_tls_inner(ret.guest_tpidr);
465 assert_eq!(cpu::read_tpidr_el0(), guest);
466
467 clear_guest_tls_on_exit();
468 assert_eq!(cpu::read_tpidr_el0(), host);
469 let _ = crate::host::munmap(base, crate::host::page_size().unwrap_or(4096));
470 }
471 }
472
473 #[test]
474 fn enter_leave_restores_host() {
475 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
476 {
477 prepare_host_meta();
478 let host = host_tpidr();
479 let Some(base) = map_guest_tls_page() else {
480 return;
481 };
482 init_guest_tls_block(base, 0);
483 let fake_guest = crate::host::ptr_addr_u64(base);
484 host_slot::with_tls_init(|m| {
485 m.guest_tpidr = fake_guest;
486 });
487 publish_boundary_to_guest_tls(fake_guest, host, 0);
488 let after_enter = unsafe {
490 cpu::write_tpidr_el0(fake_guest);
491 enter_host_tls();
492 cpu::read_tpidr_el0()
493 };
494 assert_eq!(after_enter, host);
495
496 let after_leave = {
497 leave_host_tls();
498 let v = cpu::read_tpidr_el0();
499 enter_host_tls();
500 v
501 };
502 assert_eq!(after_leave, fake_guest);
503
504 clear_guest_tls_on_exit();
505 assert_eq!(cpu::read_tpidr_el0(), host);
506 let _ = crate::host::munmap(base, crate::host::page_size().unwrap_or(4096));
507 }
508 }
509}