rt/os.rs
1// os -- the OS calls the runtime itself needs: pages, console output, exit; plus the
2// symbols a no_std binary must define (MSVC float marker, stack probe, the unwinding
3// personality routines the precompiled `core` names -- with panic=abort none of them runs).
4// Everything else of the platform is in the `plat` crate.
5
6pub use imp::*;
7
8#[cfg(windows)]
9mod imp {
10 #[link(name = "kernel32", kind = "raw-dylib")]
11 unsafe extern "system" {
12 fn GetStdHandle(n: u32) -> isize;
13 fn WriteFile(h: isize, buf: *const u8, n: u32, done: *mut u32, ov: usize) -> i32;
14 fn ExitProcess(code: u32) -> !;
15 fn VirtualAlloc(addr: usize, size: usize, typ: u32, prot: u32) -> *mut u8;
16 fn VirtualFree(addr: *mut u8, size: usize, typ: u32) -> i32;
17 fn SwitchToThread() -> i32;
18 }
19
20 /// give the rest of the time slice to another ready thread
21 pub fn yield_now() {
22 // SAFETY: no arguments.
23 unsafe { SwitchToThread() };
24 }
25
26 pub fn exit(code: i32) -> ! {
27 // SAFETY: ends the process.
28 unsafe { ExitProcess(code as u32) }
29 }
30
31 fn write_handle(n: u32, b: &[u8]) {
32 let mut off = 0;
33 while off < b.len() {
34 let chunk = (b.len() - off).min(1 << 30);
35 let mut done = 0u32;
36 // SAFETY: b[off..] is a valid buffer; a missing console handle makes WriteFile fail harmlessly.
37 let ok = unsafe { WriteFile(GetStdHandle(n), b.as_ptr().add(off), chunk as u32, &mut done, 0) };
38 if ok == 0 || done == 0 {
39 break;
40 }
41 off += done as usize;
42 }
43 }
44
45 pub fn write_stdout(b: &[u8]) {
46 write_handle(-11i32 as u32, b)
47 }
48
49 pub fn write_stderr(b: &[u8]) {
50 write_handle(-12i32 as u32, b)
51 }
52
53 /// stderr for crash paths (Windows: write_stderr; a pipe without room in PIPE_NOWAIT mode
54 /// writes nothing, which ends the write)
55 pub fn write_stderr_crash(b: &[u8]) {
56 write_stderr(b)
57 }
58
59 /// zeroed pages, or null
60 ///
61 /// # Safety
62 /// Free them with pages_free.
63 pub unsafe fn pages_alloc(n: usize) -> *mut u8 {
64 // MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE
65 unsafe { VirtualAlloc(0, n, 0x3000, 4) }
66 }
67
68 /// # Safety
69 /// p must come from pages_alloc.
70 pub unsafe fn pages_free(p: *mut u8, _n: usize) {
71 unsafe { VirtualFree(p, 0, 0x8000) };
72 }
73
74 // what the C runtime defines, for a program without it (with std the C runtime is there)
75 #[cfg(not(feature = "std"))]
76 #[unsafe(no_mangle)]
77 pub static _fltused: i32 = 0;
78
79 #[cfg(not(feature = "std"))]
80 #[unsafe(no_mangle)]
81 pub extern "C" fn __CxxFrameHandler3() -> ! {
82 exit(4)
83 }
84
85 // stack probe: frames over 4 KB touch every page from the top down (ARM64: at the end of the file)
86 #[cfg(all(not(feature = "std"), target_arch = "x86_64"))]
87 core::arch::global_asm!(
88 ".globl __chkstk",
89 "__chkstk:",
90 " push rcx",
91 " push rax",
92 " cmp rax, 4096",
93 " lea rcx, [rsp + 24]",
94 " jb 2f",
95 "1:",
96 " sub rcx, 4096",
97 " test [rcx], rcx",
98 " sub rax, 4096",
99 " cmp rax, 4096",
100 " ja 1b",
101 "2:",
102 " sub rcx, rax",
103 " test [rcx], rcx",
104 " pop rax",
105 " pop rcx",
106 " ret",
107 );
108}
109
110#[cfg(target_os = "linux")]
111mod imp {
112 use core::arch::asm;
113
114 pub use crate::kernel::*;
115
116 /// # Safety
117 /// Raw system call: the arguments must be valid for the call.
118 #[cfg(target_arch = "x86_64")]
119 #[inline]
120 pub unsafe fn syscall6(n: usize, a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) -> isize {
121 let r: isize;
122 unsafe {
123 asm!("syscall", inlateout("rax") n as isize => r, in("rdi") a, in("rsi") b, in("rdx") c,
124 in("r10") d, in("r8") e, in("r9") f, lateout("rcx") _, lateout("r11") _, options(nostack));
125 }
126 r
127 }
128
129 /// # Safety
130 /// Raw system call: the arguments must be valid for the call. ARM64: the number in x8, the
131 /// result in x0; the kernel keeps every other register.
132 #[cfg(target_arch = "aarch64")]
133 #[inline]
134 pub unsafe fn syscall6(n: usize, a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) -> isize {
135 let r: isize;
136 unsafe {
137 asm!("svc 0", in("x8") n, inlateout("x0") a as isize => r, in("x1") b, in("x2") c,
138 in("x3") d, in("x4") e, in("x5") f, options(nostack));
139 }
140 r
141 }
142
143 /// # Safety
144 /// See syscall6.
145 #[inline]
146 pub unsafe fn syscall3(n: usize, a: usize, b: usize, c: usize) -> isize {
147 unsafe { syscall6(n, a, b, c, 0, 0, 0) }
148 }
149
150 /// poll with the timeout in milliseconds (negative: none), by ppoll: ARM64 has no poll
151 ///
152 /// # Safety
153 /// fds must point to n struct pollfd.
154 #[inline]
155 pub unsafe fn poll(fds: usize, n: usize, ms: i32) -> isize {
156 // ppoll writes the time left back
157 let mut ts = [(ms / 1000) as i64, (ms % 1000) as i64 * 1_000_000];
158 let timeout = if ms < 0 { 0 } else { ts.as_mut_ptr() as usize };
159 // SAFETY: fds as the caller promises, ts lives during the call; no signal mask.
160 unsafe { syscall6(SYS_PPOLL, fds, n, timeout, 0, 8, 0) }
161 }
162
163 /// clone with its arguments in the kernel's order
164 ///
165 /// # Safety
166 /// See syscall6; with a new stack or CLONE_VM the child must not return from here (a thread's
167 /// start is assembly: linux_thread.rs).
168 #[inline]
169 pub unsafe fn clone(flags: usize, stack: usize, parent_tid: usize, child_tid: usize, tls: usize) -> isize {
170 let [a, b, c] = clone_args(parent_tid, child_tid, tls);
171 unsafe { syscall6(SYS_CLONE, flags, stack, a, b, c, 0) }
172 }
173
174 /// fork, by clone with only the child's exit signal (SIGCHLD): ARM64 has no fork
175 ///
176 /// # Safety
177 /// As fork: in the child of a process with threads only what is safe there.
178 #[inline]
179 pub unsafe fn fork() -> isize {
180 unsafe { clone(17, 0, 0, 0, 0) }
181 }
182
183 /// dup2, by dup3: ARM64 has no dup2. dup3 refuses old == new, dup2 only looks whether old
184 /// is open then
185 ///
186 /// # Safety
187 /// new is closed first if it is open: it must not be in use elsewhere.
188 #[inline]
189 pub unsafe fn dup2(old: usize, new: usize) -> isize {
190 if old == new {
191 // SAFETY: fcntl(F_GETFD) changes nothing.
192 let r = unsafe { syscall3(SYS_FCNTL, old, 1, 0) };
193 return if r < 0 { r } else { old as isize };
194 }
195 unsafe { syscall3(SYS_DUP3, old, new, 0) }
196 }
197
198 /// rename, replacing new, by renameat2 (renameat before Linux 3.15): ARM64 has no rename
199 ///
200 /// # Safety
201 /// old and new must be NUL-terminated paths.
202 #[inline]
203 pub unsafe fn rename(old: usize, new: usize) -> isize {
204 let r = unsafe { syscall6(SYS_RENAMEAT2, AT_FDCWD, old, AT_FDCWD, new, 0, 0) };
205 if r != -38 {
206 return r; // not ENOSYS
207 }
208 unsafe { syscall6(SYS_RENAMEAT, AT_FDCWD, old, AT_FDCWD, new, 0, 0) }
209 }
210
211 pub fn exit(code: i32) -> ! {
212 // SAFETY: ends the process.
213 unsafe { syscall3(SYS_EXIT_GROUP, code as usize, 0, 0) };
214 #[allow(clippy::empty_loop)] // not reached: exit_group does not return
215 loop {}
216 }
217
218 /// give the rest of the time slice to another ready thread
219 pub fn yield_now() {
220 // SAFETY: no arguments.
221 unsafe { syscall3(SYS_SCHED_YIELD, 0, 0, 0) };
222 }
223
224 pub fn write_fd(fd: usize, b: &[u8]) {
225 let mut off = 0;
226 while off < b.len() {
227 // SAFETY: b is a valid buffer.
228 let r = unsafe { syscall3(SYS_WRITE, fd, b.as_ptr() as usize + off, b.len() - off) };
229 if r == -4 {
230 continue; // EINTR
231 }
232 if r == -11 {
233 // EAGAIN on a non-blocking descriptor (a pipe the reader does not empty): wait
234 // until it takes more instead of spinning on sched_yield
235 let mut pfd = [fd as i32, 4]; // struct pollfd: fd, events POLLOUT, revents 0
236 // SAFETY: one pollfd record, no timeout.
237 unsafe { poll(pfd.as_mut_ptr() as usize, 1, -1) };
238 continue;
239 }
240 if r <= 0 {
241 break;
242 }
243 off += r as usize;
244 }
245 }
246
247 /// write_fd for crash paths: a non-blocking descriptor that takes nothing for wait_ms is
248 /// given up (false) instead of waited for without end; true: all written or the descriptor
249 /// failed
250 pub fn write_fd_within(fd: usize, b: &[u8], wait_ms: i32) -> bool {
251 let mut off = 0;
252 while off < b.len() {
253 // SAFETY: b is a valid buffer.
254 let r = unsafe { syscall3(SYS_WRITE, fd, b.as_ptr() as usize + off, b.len() - off) };
255 if r == -4 {
256 continue; // EINTR
257 }
258 if r == -11 {
259 // EAGAIN: wait at most wait_ms for room (a closed reader wakes the poll, then
260 // the write fails)
261 let mut pfd = [fd as i32, 4]; // struct pollfd: fd, events POLLOUT, revents 0
262 // SAFETY: one pollfd record.
263 let p = unsafe { poll(pfd.as_mut_ptr() as usize, 1, wait_ms) };
264 if p == 0 || (p < 0 && p != -4) {
265 return false;
266 }
267 continue;
268 }
269 if r <= 0 {
270 break;
271 }
272 off += r as usize;
273 }
274 true
275 }
276
277 pub fn write_stdout(b: &[u8]) {
278 write_fd(1, b)
279 }
280
281 pub fn write_stderr(b: &[u8]) {
282 write_fd(2, b)
283 }
284
285 /// the crash paths' stderr gave up (it took nothing for ERR_WAIT_MS)
286 static ERR_GONE: core::sync::atomic::AtomicBool = core::sync::atomic::AtomicBool::new(false);
287 const ERR_WAIT_MS: i32 = 500;
288
289 /// stderr for crash paths, which neither allocate nor wait without end: a non-blocking
290 /// stderr that takes nothing for half a second is given up for the rest of the crash. (A
291 /// blocking one whose reader stopped reading still blocks, as for any program.)
292 pub fn write_stderr_crash(b: &[u8]) {
293 use core::sync::atomic::Ordering::Relaxed;
294 if !ERR_GONE.load(Relaxed) && !write_fd_within(2, b, ERR_WAIT_MS) {
295 ERR_GONE.store(true, Relaxed);
296 }
297 }
298
299 /// zeroed pages, or null
300 ///
301 /// # Safety
302 /// Free them with pages_free.
303 pub unsafe fn pages_alloc(n: usize) -> *mut u8 {
304 // PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS
305 let r = unsafe { syscall6(SYS_MMAP, 0, n, 3, 0x22, usize::MAX, 0) };
306 if r < 0 {
307 return core::ptr::null_mut();
308 }
309 if n >= 4 << 20 {
310 // MADV_HUGEPAGE for large runs (images, planes): ~10x fewer page faults; JPEG / PNG
311 // loads 15-25 % faster in WSL (perf-load). A no-op where THP is off; result ignored.
312 // SAFETY: r, n is the mapping just created.
313 unsafe { syscall3(SYS_MADVISE, r as usize, n, 14) };
314 }
315 r as *mut u8
316 }
317
318 /// # Safety
319 /// p, n must come from pages_alloc.
320 pub unsafe fn pages_free(p: *mut u8, n: usize) {
321 unsafe { syscall3(SYS_MUNMAP, p as usize, n, 0) };
322 }
323
324 // std defines these (and links libgcc_s)
325 #[cfg(not(feature = "std"))]
326 #[unsafe(no_mangle)]
327 pub extern "C" fn rust_eh_personality() {}
328
329 #[cfg(not(feature = "std"))]
330 #[unsafe(no_mangle)]
331 pub extern "C" fn _Unwind_Resume() -> ! {
332 exit(4)
333 }
334
335 // ---- a static ARM64 program (docs/ARM64.md): what the C runtime would do at the start.
336 // compiler-builtins brings compiler-rt's outline atomics for this target, whose constructor in
337 // .init_array asks getauxval(AT_HWCAP) whether the CPU has LSE (the atomics of ARMv8.1) and only
338 // then uses them, LL/SC otherwise; their object needs getauxval to link at all. rt answers it from
339 // the auxiliary vector, process_start runs the constructors. With glibc (or std) libc does both.
340 #[cfg(all(target_arch = "aarch64", not(jfgui_glibc), not(feature = "std")))]
341 static AUXV: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0);
342
343 /// The start of a static ARM64 program, before anything else (plat's _start): sp is the stack
344 /// the kernel started the process with (argc, argv, 0, envp, 0, then the auxiliary vector),
345 /// which answers getauxval from now on; then the constructors of .init_array run.
346 ///
347 /// # Safety
348 /// Once, on the only thread, with the kernel's initial stack pointer.
349 #[cfg(all(target_arch = "aarch64", not(jfgui_glibc), not(feature = "std")))]
350 #[doc(hidden)]
351 pub unsafe fn process_start(sp: *const usize) {
352 unsafe extern "C" {
353 static __init_array_start: usize; // the linker's bounds of .init_array
354 static __init_array_end: usize;
355 }
356 // SAFETY: the layout the kernel built (the caller's promise); the linker sorted the
357 // constructors' addresses between the two symbols (0 and -1 are no function).
358 unsafe {
359 let mut p = sp.add(*sp + 2); // envp
360 while *p != 0 {
361 p = p.add(1);
362 }
363 AUXV.store(p.add(1) as usize, core::sync::atomic::Ordering::Relaxed);
364 let mut f = &raw const __init_array_start;
365 while f < &raw const __init_array_end {
366 if *f != 0 && *f != usize::MAX {
367 core::mem::transmute::<usize, extern "C" fn()>(*f)();
368 }
369 f = f.add(1);
370 }
371 }
372 }
373
374 /// getauxval(type) for compiler-builtins' C code: the value of that entry of the auxiliary
375 /// vector, 0 when it has none (or before process_start)
376 #[cfg(all(target_arch = "aarch64", not(jfgui_glibc), not(feature = "std")))]
377 #[unsafe(no_mangle)]
378 pub extern "C" fn getauxval(typ: usize) -> usize {
379 let mut p = AUXV.load(core::sync::atomic::Ordering::Relaxed) as *const usize;
380 if p.is_null() {
381 return 0;
382 }
383 // SAFETY: the vector process_start found: pairs of type and value, ended by AT_NULL (0).
384 unsafe {
385 while *p != 0 {
386 if *p == typ {
387 return *p.add(1);
388 }
389 p = p.add(2);
390 }
391 }
392 0
393 }
394
395 // ---- ARM64: the page size (docs/ARM64.md, decision 7). ARM64 kernels run with 4 KB, 16 KB (Asahi
396 // Linux) or 64 KB pages; x86-64 has 4 KB only, its code keeps the constant.
397 #[cfg(target_arch = "aarch64")]
398 pub const AT_HWCAP: usize = 16;
399 #[cfg(target_arch = "aarch64")]
400 pub const AT_PAGESZ: usize = 6;
401 #[cfg(target_arch = "aarch64")]
402 static PAGE_SIZE: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0);
403
404 /// an entry of the auxiliary vector (AT_HWCAP, AT_PAGESZ), 0 when it has none: a static program
405 /// reads the vector process_start found, one against glibc (or with std) asks glibc
406 #[cfg(target_arch = "aarch64")]
407 pub fn auxv(typ: usize) -> usize {
408 #[cfg(not(any(jfgui_glibc, feature = "std")))]
409 return getauxval(typ);
410 #[cfg(any(jfgui_glibc, feature = "std"))]
411 {
412 unsafe extern "C" {
413 fn getauxval(typ: usize) -> usize;
414 }
415 // SAFETY: glibc's getauxval (since 2.16) has no preconditions.
416 unsafe { getauxval(typ) }
417 }
418 }
419
420 /// the kernel's page size (AT_PAGESZ, cached): a power of two from 4096 on, 4096 when the vector
421 /// has none
422 #[cfg(target_arch = "aarch64")]
423 pub fn page_size() -> usize {
424 use core::sync::atomic::Ordering::Relaxed;
425 let p = PAGE_SIZE.load(Relaxed);
426 if p != 0 {
427 return p;
428 }
429 let v = auxv(AT_PAGESZ);
430 let v = if v.is_power_of_two() && v >= 4096 { v } else { 4096 };
431 PAGE_SIZE.store(v, Relaxed);
432 v
433 }
434}
435
436// Windows on ARM64 (docs/ARM64.md, ARM5): the stack probe gets the frame size in x15 in units of 16
437// bytes and touches every page from sp down; the caller moves sp itself. Only x16, x17 and the flags
438// may change (what the compiler assumes of a call to __chkstk); x18 is the TEB and stays untouched.
439// The same steps as compiler-rt and modules/modrt/mem.rs.
440#[cfg(all(windows, target_arch = "aarch64", not(feature = "std")))]
441core::arch::global_asm!(
442 ".globl __chkstk",
443 ".p2align 2",
444 "__chkstk:",
445 " lsl x16, x15, #4",
446 " mov x17, sp",
447 "1:",
448 " sub x17, x17, #4096",
449 " subs x16, x16, #4096",
450 " ldr xzr, [x17]",
451 " b.gt 1b",
452 " ret",
453);