Skip to main content

plat/
linux_thread.rs

1// linux_thread -- Linux threads (clone + futex; linked against glibc: pthread_create, at the end of this file), semaphores, the wake-up of the UI thread,
2// system information, the date                       [src/plat/linux/lx_thread.s, plat_linux.s]
3//
4// A thread's stack is an mmap with a PROT_NONE guard page below it; the thread record sits
5// at its top.  The kernel clears the record's tid word and wakes it (a shared futex wake)
6// when the thread ends, which is what join waits for.  Semaphores are process-private
7// futex counters.  The wake-up is an eventfd: the display backends poll it next to their
8// socket, wait_events polls it alone (headless).
9
10use alloc::boxed::Box;
11use core::sync::atomic::{AtomicI32, AtomicU32, Ordering};
12
13use rt::os::*;
14
15use crate::{Date, E_WAKE, Event};
16
17const FUTEX_WAIT: usize = 0;
18const FUTEX_WAKE: usize = 1;
19const FUTEX_PRIVATE: usize = 128;
20const EINTR: isize = 4;
21// CLONE_VM | FS | FILES | SIGHAND | THREAD | SYSVSEM | PARENT_SETTID | CHILD_CLEARTID
22const CLONE_THREAD_FLAGS: usize = 0x100 | 0x200 | 0x400 | 0x800 | 0x10000 | 0x40000 | 0x100000 | 0x200000;
23#[cfg(target_arch = "x86_64")] const GUARD_SIZE: usize = 4096;
24/// each thread's alternate signal stack (the crash handler runs on it: a stack overflow of the
25/// thread still gets its report), between two guard pages below the thread's stack
26const ALT_SIZE: usize = 65536;
27const THREAD_STACK_DEF: usize = 0x10_0000; // 1 MB
28const THREAD_STACK_MAX: usize = 0x1000_0000; // 256 MB
29
30// ---------------------------------------------------------------- threads
31
32type Start = Box<dyn FnOnce() + Send + 'static>;
33
34/// the thread record at the top of the stack mapping (64 bytes)
35#[repr(C)] #[cfg_attr(jfgui_glibc, allow(dead_code))]
36struct ThreadRec {
37    tid: AtomicU32, // set by clone, cleared by the kernel when the thread ends
38    _pad: u32,
39    base: usize,
40    size: usize,
41    start: *mut Start,
42    _rsv: [usize; 4],
43}
44
45// clone(flags, stack = rec, parent_tid = &rec.tid, child_tid = &rec.tid, tls = 0; rdx, r10, r8 in
46// x86-64's order, rt::os::clone_args): the child starts on its own stack (rsp = rec,
47// 16-aligned) with the record in r9 and calls thread_main(rec)
48#[cfg(all(target_arch = "x86_64", not(jfgui_glibc)))] core::arch::global_asm!(
49    ".globl jfgui_rs_clone",
50    "jfgui_rs_clone:",
51    "    mov r9, rsi",
52    "    mov rdx, rsi",
53    "    mov r10, rsi",
54    "    xor r8d, r8d",
55    "    mov eax, {nr}",
56    "    syscall",
57    "    test rax, rax",
58    "    jz 2f",
59    "    ret",
60    "2:",
61    "    xor ebp, ebp",
62    "    mov rdi, r9",
63    "    call {main}",
64    "    ud2",
65    main = sym thread_main,
66    nr = const SYS_CLONE,
67);
68
69#[cfg(not(jfgui_glibc))] unsafe extern "C" {
70    fn jfgui_rs_clone(flags: usize, rec: *mut u8) -> isize;
71}
72
73#[cfg(not(jfgui_glibc))] extern "C" fn thread_main(rec: *mut ThreadRec) -> ! {
74    // SAFETY: the alternate signal stack thread_spawn reserved in this thread's mapping (the
75    // kernel gives a new thread none); rec is the record thread_spawn wrote, its closure is
76    // taken over once.
77    let f = unsafe {
78        let ss = [(*rec).base + GUARD_SIZE, 0, ALT_SIZE];
79        syscall3(SYS_SIGALTSTACK, ss.as_ptr() as usize, 0, 0);
80        Box::from_raw((*rec).start)
81    };
82    rt::abort_on_unwind(f); // feature unwind: a panic that leaves the thread ends the program
83    // SAFETY: ends this thread only (the kernel clears and wakes rec.tid).
84    unsafe { syscall3(SYS_EXIT, 0, 0, 0) };
85    #[allow(clippy::empty_loop)] // not reached: exit does not return
86    loop {}
87}
88
89/// a thread started by thread_spawn; join it to wait for its end (and free its stack)
90pub struct Thread(*mut ThreadRec);
91
92// SAFETY: the record is only read by join, after the thread ended or through atomics.
93unsafe impl Send for Thread {}
94
95/// run f on a new thread with stack bytes of stack (0 = 1 MB)  [plat_thread_create]
96pub fn thread_spawn(stack: usize, f: Start) -> Option<Thread> {
97    let size = if stack == 0 { THREAD_STACK_DEF } else { stack.min(THREAD_STACK_MAX) }; #[cfg(all(jfgui_glibc, target_arch = "aarch64"))] let size = size.max(PTHREAD_STACK_MIN);
98    let size = ((size + 4095) & !4095) + 2 * GUARD_SIZE + ALT_SIZE; #[cfg(jfgui_glibc)] let size = size + TLS_ROOM;
99    // SAFETY: a fresh anonymous mapping (PROT_READ | PROT_WRITE; MAP_PRIVATE | MAP_ANONYMOUS |
100    // MAP_NORESERVE | MAP_STACK): from the bottom a guard page, the alternate signal stack,
101    // a guard page, the stack.
102    unsafe {
103        let base = syscall6(SYS_MMAP, 0, size, 3, 0x02 | 0x20 | 0x4000 | 0x20000, usize::MAX, 0);
104        if base < 0 {
105            return None;
106        }
107        let base = base as usize;
108        if syscall3(SYS_MPROTECT, base, GUARD_SIZE, 0) < 0
109            || syscall3(SYS_MPROTECT, base + GUARD_SIZE + ALT_SIZE, GUARD_SIZE, 0) < 0 {
110            syscall3(SYS_MUNMAP, base, size, 0);
111            return None;
112        }
113        let rec = (base + size - 64) as *mut ThreadRec;
114        let start = Box::into_raw(Box::new(f));
115        rec.write(ThreadRec { tid: AtomicU32::new(0), _pad: 0, base, size, start, _rsv: [0; 4] });
116        if jfgui_rs_clone(CLONE_THREAD_FLAGS, rec as *mut u8) < 0 {
117            drop(Box::from_raw(start));
118            syscall3(SYS_MUNMAP, base, size, 0);
119            return None;
120        }
121        Some(Thread(rec))
122    }
123}
124
125#[cfg(not(jfgui_glibc))] impl Thread {
126    /// wait until the thread has ended, then free its stack  [plat_thread_join]
127    pub fn join(self) {
128        // SAFETY: the record stays mapped until the munmap below, which runs after the
129        // kernel cleared the tid (the thread no longer uses its stack).
130        unsafe {
131            let rec = &*self.0;
132            loop {
133                let t = rec.tid.load(Ordering::SeqCst);
134                if t == 0 {
135                    break;
136                }
137                // shared wait: matches the kernel's exit wake
138                syscall6(SYS_FUTEX, &rec.tid as *const AtomicU32 as usize, FUTEX_WAIT, t as usize, 0, 0, 0);
139            }
140            let (base, size) = (rec.base, rec.size);
141            syscall3(SYS_MUNMAP, base, size, 0);
142        }
143    }
144}
145
146/// give the rest of the time slice away  [plat_thread_yield]
147pub fn thread_yield() {
148    rt::os::yield_now()
149}
150
151// ---------------------------------------------------------------- semaphores
152
153struct SemInner {
154    count: AtomicI32, // the futex word
155    waiters: AtomicU32,
156}
157
158/// a counting semaphore
159pub struct Sem(Box<SemInner>);
160
161impl Sem {
162    pub fn new(initial: u32) -> Option<Sem> {
163        Some(Sem(Box::new(SemInner { count: AtomicI32::new(initial as i32), waiters: AtomicU32::new(0) })))
164    }
165
166    /// add n units, wake sleepers (0: nothing)
167    pub fn post(&self, n: u32) {
168        if n == 0 {
169            return;
170        }
171        // saturating: an i32 overflow would turn the count negative and block waiters for good
172        let add = n.min(i32::MAX as u32) as i32;
173        let _ = self.0.count.try_update(Ordering::SeqCst, Ordering::SeqCst, |c| Some(c.saturating_add(add)));
174        if self.0.waiters.load(Ordering::SeqCst) != 0 {
175            // SAFETY: the futex word lives as long as self.
176            unsafe {
177                syscall6(SYS_FUTEX, self.0.count.as_ptr() as usize, FUTEX_WAKE | FUTEX_PRIVATE, n as usize, 0, 0, 0)
178            };
179        }
180    }
181
182    /// take one unit: true taken, false timeout (ms -1 = forever, 0 = only try)
183    pub fn wait(&self, ms: i32) -> bool {
184        let deadline = if ms > 0 { Some(super::time_us() + ms as u64 * 1000) } else { None };
185        loop {
186            let mut c = self.0.count.load(Ordering::SeqCst);
187            while c > 0 {
188                match self.0.count.compare_exchange(c, c - 1, Ordering::SeqCst, Ordering::SeqCst) {
189                    Ok(_) => return true,
190                    Err(v) => c = v,
191                }
192            }
193            if ms == 0 {
194                return false;
195            }
196            let mut ts = [0i64; 2];
197            let tsp = match deadline {
198                Some(d) => {
199                    let now = super::time_us();
200                    if now >= d {
201                        return false;
202                    }
203                    let left = d - now;
204                    ts[0] = (left / 1_000_000) as i64;
205                    ts[1] = (left % 1_000_000 * 1000) as i64;
206                    ts.as_ptr() as usize
207                }
208                None => 0,
209            };
210            self.0.waiters.fetch_add(1, Ordering::SeqCst);
211            // SAFETY: sleeps only while the count is still 0; ts lives during the call.
212            unsafe { syscall6(SYS_FUTEX, self.0.count.as_ptr() as usize, FUTEX_WAIT | FUTEX_PRIVATE, 0, tsp, 0, 0) };
213            self.0.waiters.fetch_sub(1, Ordering::SeqCst);
214        }
215    }
216}
217
218// ---------------------------------------------------------------- UI wake-up
219
220static WAKE_FD: AtomicI32 = AtomicI32::new(-1);
221static WAKE_FLAG: AtomicU32 = AtomicU32::new(0);
222
223/// the non-blocking wake eventfd (process start)
224pub(crate) fn wake_init() {
225    // SAFETY: eventfd2(0, EFD_CLOEXEC | EFD_NONBLOCK).
226    let fd = unsafe { syscall3(SYS_EVENTFD2, 0, 0x80000 | 0x800, 0) };
227    WAKE_FD.store(if fd < 0 { -1 } else { fd as i32 }, Ordering::SeqCst);
228}
229
230pub(crate) fn wake_fd() -> i32 {
231    WAKE_FD.load(Ordering::SeqCst)
232}
233
234/// UI thread: reset the eventfd, then the flag (in this order: a wake() between the two is
235/// answered by the E_WAKE delivered now -- its work was queued before it; the other order
236/// could drain the write of a wake() that already set the flag again, and every later wake()
237/// would then find the flag set and stay silent); the E_WAKE to deliver
238pub(crate) fn wake_consume() -> Event {
239    let fd = WAKE_FD.load(Ordering::SeqCst);
240    if fd >= 0 {
241        let mut v = 0u64;
242        // SAFETY: non-blocking read of the 8-byte counter (EAGAIN is fine).
243        unsafe { syscall3(SYS_READ, fd as usize, &mut v as *mut u64 as usize, 8) };
244    }
245    WAKE_FLAG.store(0, Ordering::SeqCst);
246    Event { typ: E_WAKE, ..Default::default() }
247}
248
249/// any thread: make the UI thread's wait return with E_WAKE (coalesced)  [plat_wake]
250pub fn wake() {
251    if WAKE_FLAG.swap(1, Ordering::SeqCst) != 0 {
252        return; // already signalled, not consumed yet
253    }
254    let fd = WAKE_FD.load(Ordering::SeqCst);
255    if fd < 0 {
256        return;
257    }
258    let one = 1u64;
259    loop {
260        // SAFETY: writes the 8-byte counter increment.
261        let r = unsafe { syscall3(SYS_WRITE, fd as usize, &one as *const u64 as usize, 8) };
262        if r != -EINTR {
263            break;
264        }
265    }
266}
267
268/// headless wait (no display): until a wake-up (E_WAKE to on_event, returns 1) or the
269/// deadline in time_us() (0 = only poll, u64::MAX = forever; returns 0)  [lx_wait_nodisp]
270pub fn wait_events(deadline_us: u64, on_event: &mut dyn FnMut(&Event)) -> u32 {
271    loop {
272        let ms: i32 = if deadline_us == u64::MAX {
273            -1
274        } else if deadline_us == 0 {
275            0
276        } else {
277            let now = super::time_us();
278            if now >= deadline_us { 0 } else { ((deadline_us - now + 999) / 1000).min(0x7FFF_FFFF) as i32 }
279        };
280        // one pollfd: fd, POLLIN (a negative fd is ignored: a plain sleep)
281        let mut pfd = [(WAKE_FD.load(Ordering::SeqCst) as u32 as i64) | (1i64 << 32)];
282        // SAFETY: one pollfd.
283        let r = unsafe { rt::os::poll(pfd.as_mut_ptr() as usize, 1, ms) };
284        if r == -EINTR {
285            continue;
286        }
287        if r > 0 && (pfd[0] >> 48) & 1 != 0 {
288            on_event(&wake_consume());
289            return 1;
290        }
291        return 0;
292    }
293}
294
295// ---------------------------------------------------------------- system information
296
297/// CPUs this process may run on (sched_getaffinity), >= 1  [plat_cpu_count]
298pub fn cpu_count() -> u32 {
299    let mut mask = [0u64; 16]; // 1024 CPUs
300    // SAFETY: the mask buffer holds 128 bytes.
301    let r = unsafe { syscall3(SYS_SCHED_GETAFFINITY, 0, 128, mask.as_mut_ptr() as usize) };
302    let mut n = 0;
303    if r > 0 {
304        for m in &mask[..(r as usize).min(128) / 8] {
305            n += m.count_ones();
306        }
307    }
308    n.max(1)
309}
310
311/// physical memory in bytes (sysinfo), 0 unknown  [plat_mem_total]
312pub fn mem_total() -> u64 {
313    let mut si = [0u64; 14]; // struct sysinfo (112 bytes)
314    // SAFETY: the buffer holds a struct sysinfo.
315    if unsafe { syscall3(SYS_SYSINFO, si.as_mut_ptr() as usize, 0, 0) } != 0 {
316        return 0;
317    }
318    let unit = si[13] as u32; // mem_unit at 104
319    if unit == 0 { si[4] } else { si[4] * unit as u64 } // totalram at 32
320}
321
322/// the date and time in UTC (no time zone database); the flag is true (UTC)  [plat_date]
323pub fn date() -> (Date, bool) {
324    let mut ts = [0i64; 2];
325    // SAFETY: CLOCK_REALTIME into a timespec.
326    unsafe { syscall3(SYS_CLOCK_GETTIME, 0, ts.as_mut_ptr() as usize, 0) };
327    let secs = ts[0].max(0) as u64;
328    let days = secs / 86400;
329    let sod = secs % 86400;
330    // civil from days (H. Hinnant)
331    let z = days + 719468;
332    let era = z / 146097;
333    let doe = z - era * 146097;
334    let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365;
335    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
336    let mp = (5 * doy + 2) / 153;
337    let day = doy - (153 * mp + 2) / 5 + 1;
338    let month = if mp < 10 { mp + 3 } else { mp - 9 };
339    let year = yoe + era * 400 + (month <= 2) as u64;
340    let d = Date {
341        year: year as u16,
342        month: month as u16,
343        day: day as u16,
344        hour: (sod / 3600) as u16,
345        min: (sod % 3600 / 60) as u16,
346        sec: (sod % 60) as u16,
347        ms: (ts[1] / 1_000_000) as u16,
348        wday: ((days + 4) % 7) as u16,
349    };
350    (d, true)
351}
352
353// ---------------------------------------------------------------- threads of the glibc build
354//
355// (build_rs.py section "glibc", linux_glibc.rs) Drivers assume every calling thread is a glibc
356// thread: thread-local storage, a thread control block.  So pthread_create makes the thread, on
357// the mapping thread_spawn made: glibc takes its control block and the static TLS from the top of
358// the stack, which TLS_ROOM pays for (about 5 KB with libc alone; libraries opened later take
359// dynamic TLS, which glibc allocates elsewhere), so the stack that is left is the size asked for.
360// The record stays above it; _rsv[0] holds the pthread_t.  pthread_join is what makes the munmap
361// safe; glibc caches no stack of ours.
362
363#[cfg(jfgui_glibc)]
364const TLS_ROOM: usize = 16384;
365
366#[cfg(jfgui_glibc)]
367extern "C" fn thread_main(rec: *mut u8) -> *mut u8 {
368    let rec = rec as *mut ThreadRec;
369    // SAFETY: as in the other thread_main: the alternate signal stack thread_spawn reserved, the
370    // closure taken over once.
371    let f = unsafe {
372        let ss = [(*rec).base + GUARD_SIZE, 0, ALT_SIZE];
373        syscall3(SYS_SIGALTSTACK, ss.as_ptr() as usize, 0, 0);
374        Box::from_raw((*rec).start)
375    };
376    rt::abort_on_unwind(f); // feature unwind: a panic that leaves the thread ends the program
377    core::ptr::null_mut()
378}
379
380/// thread_spawn's call: start the thread on rec's mapping -> < 0 when it did not start
381#[cfg(jfgui_glibc)]
382unsafe fn jfgui_rs_clone(_flags: usize, rec: *mut u8) -> isize {
383    use super::glibc::*;
384    let r = rec as *mut ThreadRec;
385    let mut attr = PthreadAttr([0; PTHREAD_ATTR]);
386    // SAFETY: attr is a pthread_attr_t for the time of the call; the stack lies between the upper
387    // guard page and the record, thread_spawn wrote the record before this call.
388    unsafe {
389        let lo = (*r).base + 2 * GUARD_SIZE + ALT_SIZE;
390        if pthread_attr_init(&mut attr) != 0 {
391            return -1;
392        }
393        let e = pthread_attr_setstack(&mut attr, lo as *mut u8, rec as usize - lo);
394        let e = if e != 0 { e } else { pthread_create(&mut (*r)._rsv[0], &attr, thread_main, rec) };
395        pthread_attr_destroy(&mut attr);
396        if e != 0 { -1 } else { 0 }
397    }
398}
399
400#[cfg(jfgui_glibc)]
401impl Thread {
402    /// wait until the thread has ended, then free its stack  [plat_thread_join]
403    /// (a thread joining itself returns at once and keeps its stack: it still runs on it)
404    pub fn join(self) {
405        // SAFETY: the thread was created joinable and only this call joins it; after a join that
406        // succeeded nothing uses its mapping any more.  Any error (EDEADLK: the thread joins
407        // itself) leaves the mapping alone, since the thread may still run on it.
408        unsafe {
409            let rec = &*self.0;
410            if super::glibc::pthread_join(rec._rsv[0], core::ptr::null_mut()) != 0 {
411                return;
412            }
413            let (base, size) = (rec.base, rec.size);
414            syscall3(SYS_MUNMAP, base, size, 0);
415        }
416    }
417}
418
419// ARM64 (docs/ARM64.md): jfgui_rs_clone as above.  clone(flags, stack = rec, parent_tid = &rec.tid,
420// tls = 0, child_tid = &rec.tid) in ARM64's order (rt::os::clone_args); the kernel keeps every register
421// but x0, so the child finds the record in x1 and starts on its own stack (sp = rec, 16-aligned) with
422// frame pointer and link register 0
423#[cfg(all(target_arch = "aarch64", not(jfgui_glibc)))]
424core::arch::global_asm!(
425    ".globl jfgui_rs_clone",
426    "jfgui_rs_clone:",
427    "    mov x2, x1",
428    "    mov x3, xzr",
429    "    mov x4, x1",
430    "    mov x8, #{nr}",
431    "    svc 0",
432    "    cbz x0, 2f",
433    "    ret",
434    "2:",
435    "    mov x29, xzr",
436    "    mov x30, xzr",
437    "    mov x0, x1",
438    "    bl {main}",
439    "    brk #1",
440    main = sym thread_main,
441    nr = const SYS_CLONE,
442);
443
444// The guard pages on ARM64: 64 KB, the largest page size an ARM64 kernel uses (4, 16 or 64 KB), so both
445// guards and the alternate stack between them start on a page boundary whatever the kernel's page size
446// is (mprotect needs that); they are reserved, not committed.  glibc on ARM64 refuses stacks below its
447// PTHREAD_STACK_MIN (128 KB, for 64 KB pages): thread_spawn asks for at least that much there.
448#[cfg(target_arch = "aarch64")]
449const GUARD_SIZE: usize = 65536;
450#[cfg(all(jfgui_glibc, target_arch = "aarch64"))]
451const PTHREAD_STACK_MIN: usize = 131072;