1#![allow(unsafe_code)]
14
15use std::sync::atomic::{AtomicU64, Ordering};
16
17use crate::cpu;
18use crate::host_slot;
19
20pub const GUEST_TLS_MAGIC: u64 = 0x4B48_544C_5301;
22pub const GUEST_TLS_SIZE: usize = 64;
24pub const GUEST_TLS_ERRNO_OFF: usize = 8;
26pub const GUEST_TLS_PTHREAD_OFF: usize = 16;
28pub const GUEST_TLS_HOST_TPIDR_OFF: usize = 24;
30pub const GUEST_TLS_ALT_TOP_OFF: usize = 32;
32pub const GUEST_TLS_TSD_VALS_OFF: usize = 40;
34
35static MAIN_GUEST_TLS: AtomicU64 = AtomicU64::new(0);
37
38#[repr(C)]
40#[derive(Clone, Copy, Debug, Default)]
41pub struct HyperEnterRet {
42 pub alt_top: u64,
44 pub guest_tpidr: u64,
46}
47
48pub fn prepare_host_meta() {
52 let host = cpu::read_tpidr_el0();
53 host_slot::with_tls_init(|m| {
55 m.host_tpidr = host;
56 m.active = true;
57 });
58}
59
60#[must_use]
70pub fn install_main_guest_tls() -> u64 {
71 prepare_host_meta();
72 let alt = crate::thread::ensure_host_alt_stack();
74 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
75 {
76 let Some(base) = map_guest_tls_page() else {
77 return 0;
78 };
79 init_guest_tls_block(base, 0);
80 let va = crate::host::ptr_addr_u64(base);
81 MAIN_GUEST_TLS.store(va, Ordering::Release);
82 let host = cpu::read_tpidr_el0();
83 host_slot::with_tls_init(|m| {
85 m.guest_tpidr = va;
86 m.host_tpidr = host;
87 m.active = true;
88 });
89 publish_boundary_to_guest_tls(va, host, alt);
90 va
91 }
92 #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
93 {
94 let _ = alt;
95 0
96 }
97}
98
99pub fn enter_guest_tls(guest_tpidr: u64) {
106 if guest_tpidr == 0 {
107 return;
108 }
109 let host = cpu::read_tpidr_el0();
110 let alt = host_slot::with_alt_mut(|cell| cell.map_or(0, |s| s.top));
111 host_slot::with_tls_init(|m| {
113 if !m.active {
114 m.host_tpidr = host;
115 m.active = true;
116 } else if m.host_tpidr == 0 {
117 m.host_tpidr = host;
118 }
119 m.guest_tpidr = guest_tpidr;
120 });
121 let host_pub = host_slot::tls_get().host_tpidr;
122 publish_boundary_to_guest_tls(guest_tpidr, host_pub, alt);
123 unsafe {
125 cpu::write_tpidr_el0(guest_tpidr);
126 }
127}
128
129pub fn publish_boundary_to_guest_tls(guest_va: u64, host_tpidr: u64, alt_top: u64) {
134 if guest_va == 0 || host_tpidr == 0 {
135 return;
136 }
137 let Ok(addr) = usize::try_from(guest_va) else {
138 return;
139 };
140 let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
142 if magic != GUEST_TLS_MAGIC {
143 return;
144 }
145 unsafe {
146 core::ptr::with_exposed_provenance_mut::<u64>(
147 addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF),
148 )
149 .write(host_tpidr);
150 core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
151 .write(alt_top);
152 }
153}
154
155pub fn publish_alt_top_to_current_guest(alt_top: u64) {
157 let guest = host_slot::tls_get().guest_tpidr;
158 let host = host_slot::tls_get().host_tpidr;
159 if guest != 0 && host != 0 {
160 publish_boundary_to_guest_tls(guest, host, alt_top);
161 }
162}
163
164#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
172#[unsafe(no_mangle)]
173pub unsafe extern "C" fn kh_tls_enter_host() -> HyperEnterRet {
174 enter_host_tls_inner()
175}
176
177#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
186#[unsafe(no_mangle)]
187pub unsafe extern "C" fn kh_tls_leave_host(guest_tpidr: u64) {
188 leave_host_tls_inner(guest_tpidr);
189}
190
191#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
195#[allow(dead_code)]
196pub unsafe extern "C" fn kh_tls_enter_host() -> HyperEnterRet {
197 HyperEnterRet::default()
198}
199
200#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
204#[allow(dead_code)]
205pub unsafe extern "C" fn kh_tls_leave_host(_guest_tpidr: u64) {}
206
207#[inline]
209pub fn enter_host_tls() {
210 let _ = enter_host_tls_inner();
211}
212
213#[inline]
215pub fn leave_host_tls() {
216 leave_host_tls_inner(0);
217}
218
219fn try_enter_from_guest_tls(cur: u64) -> Option<(u64 , u64 )> {
224 if cur == 0 {
225 return None;
226 }
227 let Ok(addr) = usize::try_from(cur) else {
228 return None;
229 };
230 let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
232 if magic != GUEST_TLS_MAGIC {
233 return None;
234 }
235 let host = unsafe {
236 core::ptr::with_exposed_provenance::<u64>(addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF))
237 .read_volatile()
238 };
239 if host == 0 || host == cur {
240 return None;
242 }
243 let alt = unsafe {
244 core::ptr::with_exposed_provenance::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
245 .read_volatile()
246 };
247 Some((host, alt))
248}
249
250fn enter_host_tls_inner() -> HyperEnterRet {
251 let cur = cpu::read_tpidr_el0();
252
253 if let Some((host, alt)) = try_enter_from_guest_tls(cur) {
255 if cur != host {
256 unsafe {
258 cpu::write_tpidr_el0(host);
259 }
260 }
261 let alt_top = if alt != 0 {
262 alt
263 } else {
264 let top = crate::thread::ensure_host_alt_stack();
267 publish_boundary_to_guest_tls(cur, host, top);
268 top
269 };
270 return HyperEnterRet {
272 alt_top,
273 guest_tpidr: cur,
274 };
275 }
276
277 let Some(prep) = host_slot::prepare_enter_under_guest(cur) else {
279 return HyperEnterRet {
280 alt_top: 0,
281 guest_tpidr: cur,
282 };
283 };
284 if cur != prep.host_tpidr {
285 unsafe {
287 cpu::write_tpidr_el0(prep.host_tpidr);
288 }
289 }
290 let alt_top = if prep.alt_top != 0 {
291 prep.alt_top
292 } else {
293 crate::thread::ensure_host_alt_stack()
294 };
295 if is_guest_tls(cur) {
297 publish_boundary_to_guest_tls(cur, prep.host_tpidr, alt_top);
298 }
299 HyperEnterRet {
300 alt_top,
301 guest_tpidr: if is_guest_tls(cur) {
302 cur
303 } else {
304 host_slot::tls_get().guest_tpidr
305 },
306 }
307}
308
309fn leave_host_tls_inner(guest_hint: u64) {
310 let guest = if guest_hint != 0 {
311 guest_hint
312 } else {
313 host_slot::with_tls_existing_mut(|m| if m.active { m.guest_tpidr } else { 0 }).unwrap_or(0)
314 };
315 if guest == 0 {
316 return;
317 }
318 unsafe {
320 cpu::write_tpidr_el0(guest);
321 }
322}
323
324pub fn clear_guest_tls_on_exit() {
326 let ret = enter_host_tls_inner();
327 let _ = host_slot::with_tls_existing_mut(|m| {
328 m.guest_tpidr = 0;
329 });
330 if ret.guest_tpidr != 0 {
331 publish_boundary_to_guest_tls(ret.guest_tpidr, 0, 0);
333 clear_guest_tls_mirrors(ret.guest_tpidr);
335 }
336}
337
338fn clear_guest_tls_mirrors(guest_va: u64) {
339 let Ok(addr) = usize::try_from(guest_va) else {
340 return;
341 };
342 let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
343 if magic != GUEST_TLS_MAGIC {
344 return;
345 }
346 unsafe {
347 core::ptr::with_exposed_provenance_mut::<u64>(
348 addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF),
349 )
350 .write(0);
351 core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
352 .write(0);
353 }
354}
355
356#[must_use]
358pub fn host_tpidr() -> u64 {
359 host_slot::tls_get().host_tpidr
360}
361
362#[must_use]
364pub fn guest_tpidr() -> u64 {
365 host_slot::tls_get().guest_tpidr
366}
367
368#[must_use]
370pub fn main_guest_tls() -> u64 {
371 MAIN_GUEST_TLS.load(Ordering::Acquire)
372}
373
374#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
375fn map_guest_tls_page() -> Option<*mut u8> {
376 let len = GUEST_TLS_SIZE.max(64);
377 let page = crate::host::page_size().unwrap_or(4096);
378 let map_len = len.saturating_add(page.saturating_sub(1)) & !(page.saturating_sub(1));
379 let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
380 let prot = libc::PROT_READ | libc::PROT_WRITE;
381 crate::host::mmap(None, map_len, prot, flags, -1, 0)
382}
383
384pub fn init_guest_tls_block(base: *mut u8, pthread_va: u64) {
390 if base.is_null() {
391 return;
392 }
393 unsafe {
395 let addr = base.addr();
396 core::ptr::with_exposed_provenance_mut::<u64>(addr).write(GUEST_TLS_MAGIC);
397 core::ptr::with_exposed_provenance_mut::<i32>(addr.saturating_add(GUEST_TLS_ERRNO_OFF))
398 .write(0);
399 core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_PTHREAD_OFF))
400 .write(pthread_va);
401 core::ptr::with_exposed_provenance_mut::<u64>(
402 addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF),
403 )
404 .write(0);
405 core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
406 .write(0);
407 core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_TSD_VALS_OFF))
409 .write(0);
410 }
411}
412
413#[must_use]
415pub fn is_guest_tls(tpidr: u64) -> bool {
416 if tpidr == 0 {
417 return false;
418 }
419 let Ok(addr) = usize::try_from(tpidr) else {
420 return false;
421 };
422 let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
424 magic == GUEST_TLS_MAGIC
425}
426
427#[cfg(test)]
428mod tests {
429 use super::*;
430
431 #[test]
432 fn prepare_captures_host() {
433 prepare_host_meta();
434 let h = host_tpidr();
435 let _ = h;
436 assert!(host_slot::tls_get().active);
437 }
438
439 #[test]
440 fn guest_tls_fast_enter_without_gettid_map() {
441 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
442 {
443 prepare_host_meta();
444 let host = host_tpidr();
445 let Some(base) = map_guest_tls_page() else {
446 return;
447 };
448 init_guest_tls_block(base, 0);
449 let guest = crate::host::ptr_addr_u64(base);
450 let fake_alt = 0x_aaa0_u64;
451 publish_boundary_to_guest_tls(guest, host, fake_alt);
452
453 let ret = unsafe {
454 cpu::write_tpidr_el0(guest);
455 enter_host_tls_inner()
456 };
457 assert_eq!(cpu::read_tpidr_el0(), host);
458 assert_eq!(ret.alt_top, fake_alt);
459 assert_eq!(ret.guest_tpidr, guest);
460
461 leave_host_tls_inner(ret.guest_tpidr);
462 assert_eq!(cpu::read_tpidr_el0(), guest);
463
464 clear_guest_tls_on_exit();
465 assert_eq!(cpu::read_tpidr_el0(), host);
466 let _ = crate::host::munmap(base, crate::host::page_size().unwrap_or(4096));
467 }
468 }
469
470 #[test]
471 fn enter_leave_restores_host() {
472 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
473 {
474 prepare_host_meta();
475 let host = host_tpidr();
476 let Some(base) = map_guest_tls_page() else {
477 return;
478 };
479 init_guest_tls_block(base, 0);
480 let fake_guest = crate::host::ptr_addr_u64(base);
481 host_slot::with_tls_init(|m| {
482 m.guest_tpidr = fake_guest;
483 });
484 publish_boundary_to_guest_tls(fake_guest, host, 0);
485 let after_enter = unsafe {
487 cpu::write_tpidr_el0(fake_guest);
488 enter_host_tls();
489 cpu::read_tpidr_el0()
490 };
491 assert_eq!(after_enter, host);
492
493 let after_leave = {
494 leave_host_tls();
495 let v = cpu::read_tpidr_el0();
496 enter_host_tls();
497 v
498 };
499 assert_eq!(after_leave, fake_guest);
500
501 clear_guest_tls_on_exit();
502 assert_eq!(cpu::read_tpidr_el0(), host);
503 let _ = crate::host::munmap(base, crate::host::page_size().unwrap_or(4096));
504 }
505 }
506}