Skip to main content

rt/
os.rs

1// os -- the OS calls the runtime itself needs: pages, console output, exit; plus the
2// symbols a no_std binary must define (MSVC float marker, stack probe, the unwinding
3// personality routines the precompiled `core` names -- with panic=abort none of them runs).
4// Everything else of the platform is in the `plat` crate.
5
6pub use imp::*;
7
8#[cfg(windows)]
9mod imp {
10    #[link(name = "kernel32", kind = "raw-dylib")]
11    unsafe extern "system" {
12        fn GetStdHandle(n: u32) -> isize;
13        fn WriteFile(h: isize, buf: *const u8, n: u32, done: *mut u32, ov: usize) -> i32;
14        fn ExitProcess(code: u32) -> !;
15        fn VirtualAlloc(addr: usize, size: usize, typ: u32, prot: u32) -> *mut u8;
16        fn VirtualFree(addr: *mut u8, size: usize, typ: u32) -> i32;
17        fn SwitchToThread() -> i32;
18    }
19
20    /// give the rest of the time slice to another ready thread
21    pub fn yield_now() {
22        // SAFETY: no arguments.
23        unsafe { SwitchToThread() };
24    }
25
26    pub fn exit(code: i32) -> ! {
27        // SAFETY: ends the process.
28        unsafe { ExitProcess(code as u32) }
29    }
30
31    fn write_handle(n: u32, b: &[u8]) {
32        let mut off = 0;
33        while off < b.len() {
34            let chunk = (b.len() - off).min(1 << 30);
35            let mut done = 0u32;
36            // SAFETY: b[off..] is a valid buffer; a missing console handle makes WriteFile fail harmlessly.
37            let ok = unsafe { WriteFile(GetStdHandle(n), b.as_ptr().add(off), chunk as u32, &mut done, 0) };
38            if ok == 0 || done == 0 {
39                break;
40            }
41            off += done as usize;
42        }
43    }
44
45    pub fn write_stdout(b: &[u8]) {
46        write_handle(-11i32 as u32, b)
47    }
48
49    pub fn write_stderr(b: &[u8]) {
50        write_handle(-12i32 as u32, b)
51    }
52
53    /// stderr for crash paths (Windows: write_stderr; a pipe without room in PIPE_NOWAIT mode
54    /// writes nothing, which ends the write)
55    pub fn write_stderr_crash(b: &[u8]) {
56        write_stderr(b)
57    }
58
59    /// zeroed pages, or null
60    ///
61    /// # Safety
62    /// Free them with pages_free.
63    pub unsafe fn pages_alloc(n: usize) -> *mut u8 {
64        // MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE
65        unsafe { VirtualAlloc(0, n, 0x3000, 4) }
66    }
67
68    /// # Safety
69    /// p must come from pages_alloc.
70    pub unsafe fn pages_free(p: *mut u8, _n: usize) {
71        unsafe { VirtualFree(p, 0, 0x8000) };
72    }
73
74    // what the C runtime defines, for a program without it (with std the C runtime is there)
75    #[cfg(not(feature = "std"))]
76    #[unsafe(no_mangle)]
77    pub static _fltused: i32 = 0;
78
79    #[cfg(not(feature = "std"))]
80    #[unsafe(no_mangle)]
81    pub extern "C" fn __CxxFrameHandler3() -> ! {
82        exit(4)
83    }
84
85    // stack probe: frames over 4 KB touch every page from the top down (ARM64: at the end of the file)
86    #[cfg(all(not(feature = "std"), target_arch = "x86_64"))]
87    core::arch::global_asm!(
88        ".globl __chkstk",
89        "__chkstk:",
90        "    push rcx",
91        "    push rax",
92        "    cmp rax, 4096",
93        "    lea rcx, [rsp + 24]",
94        "    jb 2f",
95        "1:",
96        "    sub rcx, 4096",
97        "    test [rcx], rcx",
98        "    sub rax, 4096",
99        "    cmp rax, 4096",
100        "    ja 1b",
101        "2:",
102        "    sub rcx, rax",
103        "    test [rcx], rcx",
104        "    pop rax",
105        "    pop rcx",
106        "    ret",
107    );
108}
109
110#[cfg(target_os = "linux")]
111mod imp {
112    use core::arch::asm;
113
114    pub use crate::kernel::*;
115
116    /// # Safety
117    /// Raw system call: the arguments must be valid for the call.
118    #[cfg(target_arch = "x86_64")]
119    #[inline]
120    pub unsafe fn syscall6(n: usize, a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) -> isize {
121        let r: isize;
122        unsafe {
123            asm!("syscall", inlateout("rax") n as isize => r, in("rdi") a, in("rsi") b, in("rdx") c,
124                 in("r10") d, in("r8") e, in("r9") f, lateout("rcx") _, lateout("r11") _, options(nostack));
125        }
126        r
127    }
128
129    /// # Safety
130    /// Raw system call: the arguments must be valid for the call.  ARM64: the number in x8, the
131    /// result in x0; the kernel keeps every other register.
132    #[cfg(target_arch = "aarch64")]
133    #[inline]
134    pub unsafe fn syscall6(n: usize, a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) -> isize {
135        let r: isize;
136        unsafe {
137            asm!("svc 0", in("x8") n, inlateout("x0") a as isize => r, in("x1") b, in("x2") c,
138                 in("x3") d, in("x4") e, in("x5") f, options(nostack));
139        }
140        r
141    }
142
143    /// # Safety
144    /// See syscall6.
145    #[inline]
146    pub unsafe fn syscall3(n: usize, a: usize, b: usize, c: usize) -> isize {
147        unsafe { syscall6(n, a, b, c, 0, 0, 0) }
148    }
149
150    /// poll with the timeout in milliseconds (negative: none), by ppoll: ARM64 has no poll
151    ///
152    /// # Safety
153    /// fds must point to n struct pollfd.
154    #[inline]
155    pub unsafe fn poll(fds: usize, n: usize, ms: i32) -> isize {
156        // ppoll writes the time left back
157        let mut ts = [(ms / 1000) as i64, (ms % 1000) as i64 * 1_000_000];
158        let timeout = if ms < 0 { 0 } else { ts.as_mut_ptr() as usize };
159        // SAFETY: fds as the caller promises, ts lives during the call; no signal mask.
160        unsafe { syscall6(SYS_PPOLL, fds, n, timeout, 0, 8, 0) }
161    }
162
163    /// clone with its arguments in the kernel's order
164    ///
165    /// # Safety
166    /// See syscall6; with a new stack or CLONE_VM the child must not return from here (a thread's
167    /// start is assembly: linux_thread.rs).
168    #[inline]
169    pub unsafe fn clone(flags: usize, stack: usize, parent_tid: usize, child_tid: usize, tls: usize) -> isize {
170        let [a, b, c] = clone_args(parent_tid, child_tid, tls);
171        unsafe { syscall6(SYS_CLONE, flags, stack, a, b, c, 0) }
172    }
173
174    /// fork, by clone with only the child's exit signal (SIGCHLD): ARM64 has no fork
175    ///
176    /// # Safety
177    /// As fork: in the child of a process with threads only what is safe there.
178    #[inline]
179    pub unsafe fn fork() -> isize {
180        unsafe { clone(17, 0, 0, 0, 0) }
181    }
182
183    /// dup2, by dup3: ARM64 has no dup2.  dup3 refuses old == new, dup2 only looks whether old
184    /// is open then
185    ///
186    /// # Safety
187    /// new is closed first if it is open: it must not be in use elsewhere.
188    #[inline]
189    pub unsafe fn dup2(old: usize, new: usize) -> isize {
190        if old == new {
191            // SAFETY: fcntl(F_GETFD) changes nothing.
192            let r = unsafe { syscall3(SYS_FCNTL, old, 1, 0) };
193            return if r < 0 { r } else { old as isize };
194        }
195        unsafe { syscall3(SYS_DUP3, old, new, 0) }
196    }
197
198    /// rename, replacing new, by renameat2 (renameat before Linux 3.15): ARM64 has no rename
199    ///
200    /// # Safety
201    /// old and new must be NUL-terminated paths.
202    #[inline]
203    pub unsafe fn rename(old: usize, new: usize) -> isize {
204        let r = unsafe { syscall6(SYS_RENAMEAT2, AT_FDCWD, old, AT_FDCWD, new, 0, 0) };
205        if r != -38 {
206            return r; // not ENOSYS
207        }
208        unsafe { syscall6(SYS_RENAMEAT, AT_FDCWD, old, AT_FDCWD, new, 0, 0) }
209    }
210
211    pub fn exit(code: i32) -> ! {
212        // SAFETY: ends the process.
213        unsafe { syscall3(SYS_EXIT_GROUP, code as usize, 0, 0) };
214        #[allow(clippy::empty_loop)] // not reached: exit_group does not return
215        loop {}
216    }
217
218    /// give the rest of the time slice to another ready thread
219    pub fn yield_now() {
220        // SAFETY: no arguments.
221        unsafe { syscall3(SYS_SCHED_YIELD, 0, 0, 0) };
222    }
223
224    pub fn write_fd(fd: usize, b: &[u8]) {
225        let mut off = 0;
226        while off < b.len() {
227            // SAFETY: b is a valid buffer.
228            let r = unsafe { syscall3(SYS_WRITE, fd, b.as_ptr() as usize + off, b.len() - off) };
229            if r == -4 {
230                continue; // EINTR
231            }
232            if r == -11 {
233                // EAGAIN on a non-blocking descriptor (a pipe the reader does not empty): wait
234                // until it takes more instead of spinning on sched_yield
235                let mut pfd = [fd as i32, 4]; // struct pollfd: fd, events POLLOUT, revents 0
236                // SAFETY: one pollfd record, no timeout.
237                unsafe { poll(pfd.as_mut_ptr() as usize, 1, -1) };
238                continue;
239            }
240            if r <= 0 {
241                break;
242            }
243            off += r as usize;
244        }
245    }
246
247    /// write_fd for crash paths: a non-blocking descriptor that takes nothing for wait_ms is
248    /// given up (false) instead of waited for without end; true: all written or the descriptor
249    /// failed
250    pub fn write_fd_within(fd: usize, b: &[u8], wait_ms: i32) -> bool {
251        let mut off = 0;
252        while off < b.len() {
253            // SAFETY: b is a valid buffer.
254            let r = unsafe { syscall3(SYS_WRITE, fd, b.as_ptr() as usize + off, b.len() - off) };
255            if r == -4 {
256                continue; // EINTR
257            }
258            if r == -11 {
259                // EAGAIN: wait at most wait_ms for room (a closed reader wakes the poll, then
260                // the write fails)
261                let mut pfd = [fd as i32, 4]; // struct pollfd: fd, events POLLOUT, revents 0
262                // SAFETY: one pollfd record.
263                let p = unsafe { poll(pfd.as_mut_ptr() as usize, 1, wait_ms) };
264                if p == 0 || (p < 0 && p != -4) {
265                    return false;
266                }
267                continue;
268            }
269            if r <= 0 {
270                break;
271            }
272            off += r as usize;
273        }
274        true
275    }
276
277    pub fn write_stdout(b: &[u8]) {
278        write_fd(1, b)
279    }
280
281    pub fn write_stderr(b: &[u8]) {
282        write_fd(2, b)
283    }
284
285    /// the crash paths' stderr gave up (it took nothing for ERR_WAIT_MS)
286    static ERR_GONE: core::sync::atomic::AtomicBool = core::sync::atomic::AtomicBool::new(false);
287    const ERR_WAIT_MS: i32 = 500;
288
289    /// stderr for crash paths, which neither allocate nor wait without end: a non-blocking
290    /// stderr that takes nothing for half a second is given up for the rest of the crash.  (A
291    /// blocking one whose reader stopped reading still blocks, as for any program.)
292    pub fn write_stderr_crash(b: &[u8]) {
293        use core::sync::atomic::Ordering::Relaxed;
294        if !ERR_GONE.load(Relaxed) && !write_fd_within(2, b, ERR_WAIT_MS) {
295            ERR_GONE.store(true, Relaxed);
296        }
297    }
298
299    /// zeroed pages, or null
300    ///
301    /// # Safety
302    /// Free them with pages_free.
303    pub unsafe fn pages_alloc(n: usize) -> *mut u8 {
304        // PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS
305        let r = unsafe { syscall6(SYS_MMAP, 0, n, 3, 0x22, usize::MAX, 0) };
306        if r < 0 {
307            return core::ptr::null_mut();
308        }
309        if n >= 4 << 20 {
310            // MADV_HUGEPAGE for large runs (images, planes): ~10x fewer page faults; JPEG / PNG
311            // loads 15-25 % faster in WSL (perf-load).  A no-op where THP is off; result ignored.
312            // SAFETY: r, n is the mapping just created.
313            unsafe { syscall3(SYS_MADVISE, r as usize, n, 14) };
314        }
315        r as *mut u8
316    }
317
318    /// # Safety
319    /// p, n must come from pages_alloc.
320    pub unsafe fn pages_free(p: *mut u8, n: usize) {
321        unsafe { syscall3(SYS_MUNMAP, p as usize, n, 0) };
322    }
323
324    // std defines these (and links libgcc_s)
325    #[cfg(not(feature = "std"))]
326    #[unsafe(no_mangle)]
327    pub extern "C" fn rust_eh_personality() {}
328
329    #[cfg(not(feature = "std"))]
330    #[unsafe(no_mangle)]
331    pub extern "C" fn _Unwind_Resume() -> ! {
332        exit(4)
333    }
334
335    // ---- a static ARM64 program (docs/ARM64.md): what the C runtime would do at the start.
336    // compiler-builtins brings compiler-rt's outline atomics for this target, whose constructor in
337    // .init_array asks getauxval(AT_HWCAP) whether the CPU has LSE (the atomics of ARMv8.1) and only
338    // then uses them, LL/SC otherwise; their object needs getauxval to link at all.  rt answers it from
339    // the auxiliary vector, process_start runs the constructors.  With glibc (or std) libc does both.
340    #[cfg(all(target_arch = "aarch64", not(jfgui_glibc), not(feature = "std")))]
341    static AUXV: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0);
342
343    /// The start of a static ARM64 program, before anything else (plat's _start): sp is the stack
344    /// the kernel started the process with (argc, argv, 0, envp, 0, then the auxiliary vector),
345    /// which answers getauxval from now on; then the constructors of .init_array run.
346    ///
347    /// # Safety
348    /// Once, on the only thread, with the kernel's initial stack pointer.
349    #[cfg(all(target_arch = "aarch64", not(jfgui_glibc), not(feature = "std")))]
350    #[doc(hidden)]
351    pub unsafe fn process_start(sp: *const usize) {
352        unsafe extern "C" {
353            static __init_array_start: usize; // the linker's bounds of .init_array
354            static __init_array_end: usize;
355        }
356        // SAFETY: the layout the kernel built (the caller's promise); the linker sorted the
357        // constructors' addresses between the two symbols (0 and -1 are no function).
358        unsafe {
359            let mut p = sp.add(*sp + 2); // envp
360            while *p != 0 {
361                p = p.add(1);
362            }
363            AUXV.store(p.add(1) as usize, core::sync::atomic::Ordering::Relaxed);
364            let mut f = &raw const __init_array_start;
365            while f < &raw const __init_array_end {
366                if *f != 0 && *f != usize::MAX {
367                    core::mem::transmute::<usize, extern "C" fn()>(*f)();
368                }
369                f = f.add(1);
370            }
371        }
372    }
373
374    /// getauxval(type) for compiler-builtins' C code: the value of that entry of the auxiliary
375    /// vector, 0 when it has none (or before process_start)
376    #[cfg(all(target_arch = "aarch64", not(jfgui_glibc), not(feature = "std")))]
377    #[unsafe(no_mangle)]
378    pub extern "C" fn getauxval(typ: usize) -> usize {
379        let mut p = AUXV.load(core::sync::atomic::Ordering::Relaxed) as *const usize;
380        if p.is_null() {
381            return 0;
382        }
383        // SAFETY: the vector process_start found: pairs of type and value, ended by AT_NULL (0).
384        unsafe {
385            while *p != 0 {
386                if *p == typ {
387                    return *p.add(1);
388                }
389                p = p.add(2);
390            }
391        }
392        0
393    }
394
395    // ---- ARM64: the page size (docs/ARM64.md, decision 7).  ARM64 kernels run with 4 KB, 16 KB (Asahi
396    // Linux) or 64 KB pages; x86-64 has 4 KB only, its code keeps the constant.
397    #[cfg(target_arch = "aarch64")]
398    pub const AT_HWCAP: usize = 16;
399    #[cfg(target_arch = "aarch64")]
400    pub const AT_PAGESZ: usize = 6;
401    #[cfg(target_arch = "aarch64")]
402    static PAGE_SIZE: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0);
403
404    /// an entry of the auxiliary vector (AT_HWCAP, AT_PAGESZ), 0 when it has none: a static program
405    /// reads the vector process_start found, one against glibc (or with std) asks glibc
406    #[cfg(target_arch = "aarch64")]
407    pub fn auxv(typ: usize) -> usize {
408        #[cfg(not(any(jfgui_glibc, feature = "std")))]
409        return getauxval(typ);
410        #[cfg(any(jfgui_glibc, feature = "std"))]
411        {
412            unsafe extern "C" {
413                fn getauxval(typ: usize) -> usize;
414            }
415            // SAFETY: glibc's getauxval (since 2.16) has no preconditions.
416            unsafe { getauxval(typ) }
417        }
418    }
419
420    /// the kernel's page size (AT_PAGESZ, cached): a power of two from 4096 on, 4096 when the vector
421    /// has none
422    #[cfg(target_arch = "aarch64")]
423    pub fn page_size() -> usize {
424        use core::sync::atomic::Ordering::Relaxed;
425        let p = PAGE_SIZE.load(Relaxed);
426        if p != 0 {
427            return p;
428        }
429        let v = auxv(AT_PAGESZ);
430        let v = if v.is_power_of_two() && v >= 4096 { v } else { 4096 };
431        PAGE_SIZE.store(v, Relaxed);
432        v
433    }
434}
435
436// Windows on ARM64 (docs/ARM64.md, ARM5): the stack probe gets the frame size in x15 in units of 16
437// bytes and touches every page from sp down; the caller moves sp itself.  Only x16, x17 and the flags
438// may change (what the compiler assumes of a call to __chkstk); x18 is the TEB and stays untouched.
439// The same steps as compiler-rt and modules/modrt/mem.rs.
440#[cfg(all(windows, target_arch = "aarch64", not(feature = "std")))]
441core::arch::global_asm!(
442    ".globl __chkstk",
443    ".p2align 2",
444    "__chkstk:",
445    "    lsl x16, x15, #4",
446    "    mov x17, sp",
447    "1:",
448    "    sub x17, x17, #4096",
449    "    subs x16, x16, #4096",
450    "    ldr xzr, [x17]",
451    "    b.gt 1b",
452    "    ret",
453);