rt/
os.rs

1// os -- the OS calls the runtime itself needs: pages, console output, exit; plus the
2// symbols a no_std binary must define (MSVC float marker, stack probe, the unwinding
3// personality routines the precompiled `core` names -- with panic=abort none of them runs).
4// Everything else of the platform is in the `plat` crate.
5
6pub use imp::*;
7
8#[cfg(windows)]
9mod imp {
10    #[link(name = "kernel32", kind = "raw-dylib")]
11    unsafe extern "system" {
12        fn GetStdHandle(n: u32) -> isize;
13        fn WriteFile(h: isize, buf: *const u8, n: u32, done: *mut u32, ov: usize) -> i32;
14        fn ExitProcess(code: u32) -> !;
15        fn VirtualAlloc(addr: usize, size: usize, typ: u32, prot: u32) -> *mut u8;
16        fn VirtualFree(addr: *mut u8, size: usize, typ: u32) -> i32;
17        fn SwitchToThread() -> i32;
18    }
19
20    /// give the rest of the time slice to another ready thread
21    pub fn yield_now() {
22        // SAFETY: no arguments.
23        unsafe { SwitchToThread() };
24    }
25
26    pub fn exit(code: i32) -> ! {
27        // SAFETY: ends the process.
28        unsafe { ExitProcess(code as u32) }
29    }
30
31    fn write_handle(n: u32, b: &[u8]) {
32        let mut off = 0;
33        while off < b.len() {
34            let chunk = (b.len() - off).min(1 << 30);
35            let mut done = 0u32;
36            // SAFETY: b[off..] is a valid buffer; a missing console handle makes WriteFile fail harmlessly.
37            let ok = unsafe { WriteFile(GetStdHandle(n), b.as_ptr().add(off), chunk as u32, &mut done, 0) };
38            if ok == 0 || done == 0 {
39                break;
40            }
41            off += done as usize;
42        }
43    }
44
45    pub fn write_stdout(b: &[u8]) {
46        write_handle(-11i32 as u32, b)
47    }
48
49    pub fn write_stderr(b: &[u8]) {
50        write_handle(-12i32 as u32, b)
51    }
52
53    /// zeroed pages, or null
54    ///
55    /// # Safety
56    /// Free them with pages_free.
57    pub unsafe fn pages_alloc(n: usize) -> *mut u8 {
58        // MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE
59        unsafe { VirtualAlloc(0, n, 0x3000, 4) }
60    }
61
62    /// # Safety
63    /// p must come from pages_alloc.
64    pub unsafe fn pages_free(p: *mut u8, _n: usize) {
65        unsafe { VirtualFree(p, 0, 0x8000) };
66    }
67
68    // what the C runtime defines, for a program without it (with std the C runtime is there)
69    #[cfg(not(feature = "std"))]
70    #[unsafe(no_mangle)]
71    pub static _fltused: i32 = 0;
72
73    #[cfg(not(feature = "std"))]
74    #[unsafe(no_mangle)]
75    pub extern "C" fn __CxxFrameHandler3() -> ! {
76        exit(4)
77    }
78
79    // stack probe: frames over 4 KB touch every page from the top down
80    #[cfg(not(feature = "std"))]
81    core::arch::global_asm!(
82        ".globl __chkstk",
83        "__chkstk:",
84        "    push rcx",
85        "    push rax",
86        "    cmp rax, 4096",
87        "    lea rcx, [rsp + 24]",
88        "    jb 2f",
89        "1:",
90        "    sub rcx, 4096",
91        "    test [rcx], rcx",
92        "    sub rax, 4096",
93        "    cmp rax, 4096",
94        "    ja 1b",
95        "2:",
96        "    sub rcx, rax",
97        "    test [rcx], rcx",
98        "    pop rax",
99        "    pop rcx",
100        "    ret",
101    );
102}
103
104#[cfg(target_os = "linux")]
105mod imp {
106    use core::arch::asm;
107
108    pub use crate::kernel::*;
109
110    /// # Safety
111    /// Raw system call: the arguments must be valid for the call.
112    #[inline]
113    pub unsafe fn syscall6(n: usize, a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) -> isize {
114        let r: isize;
115        unsafe {
116            asm!("syscall", inlateout("rax") n as isize => r, in("rdi") a, in("rsi") b, in("rdx") c,
117                 in("r10") d, in("r8") e, in("r9") f, lateout("rcx") _, lateout("r11") _, options(nostack));
118        }
119        r
120    }
121
122    /// # Safety
123    /// See syscall6.
124    #[inline]
125    pub unsafe fn syscall3(n: usize, a: usize, b: usize, c: usize) -> isize {
126        unsafe { syscall6(n, a, b, c, 0, 0, 0) }
127    }
128
129    /// poll with the timeout in milliseconds (negative: none), by ppoll: ARM64 has no poll
130    ///
131    /// # Safety
132    /// fds must point to n struct pollfd.
133    #[inline]
134    pub unsafe fn poll(fds: usize, n: usize, ms: i32) -> isize {
135        // ppoll writes the time left back
136        let mut ts = [(ms / 1000) as i64, (ms % 1000) as i64 * 1_000_000];
137        let timeout = if ms < 0 { 0 } else { ts.as_mut_ptr() as usize };
138        // SAFETY: fds as the caller promises, ts lives during the call; no signal mask.
139        unsafe { syscall6(SYS_PPOLL, fds, n, timeout, 0, 8, 0) }
140    }
141
142    /// clone with its arguments in the kernel's order
143    ///
144    /// # Safety
145    /// See syscall6; with a new stack or CLONE_VM the child must not return from here (a thread's
146    /// start is assembly: linux_thread.rs).
147    #[inline]
148    pub unsafe fn clone(flags: usize, stack: usize, parent_tid: usize, child_tid: usize, tls: usize) -> isize {
149        let [a, b, c] = clone_args(parent_tid, child_tid, tls);
150        unsafe { syscall6(SYS_CLONE, flags, stack, a, b, c, 0) }
151    }
152
153    /// fork, by clone with only the child's exit signal (SIGCHLD): ARM64 has no fork
154    ///
155    /// # Safety
156    /// As fork: in the child of a process with threads only what is safe there.
157    #[inline]
158    pub unsafe fn fork() -> isize {
159        unsafe { clone(17, 0, 0, 0, 0) }
160    }
161
162    /// dup2, by dup3: ARM64 has no dup2.  dup3 refuses old == new, dup2 only looks whether old
163    /// is open then
164    ///
165    /// # Safety
166    /// new is closed first if it is open: it must not be in use elsewhere.
167    #[inline]
168    pub unsafe fn dup2(old: usize, new: usize) -> isize {
169        if old == new {
170            // SAFETY: fcntl(F_GETFD) changes nothing.
171            let r = unsafe { syscall3(SYS_FCNTL, old, 1, 0) };
172            return if r < 0 { r } else { old as isize };
173        }
174        unsafe { syscall3(SYS_DUP3, old, new, 0) }
175    }
176
177    /// rename, replacing new, by renameat2 (renameat before Linux 3.15): ARM64 has no rename
178    ///
179    /// # Safety
180    /// old and new must be NUL-terminated paths.
181    #[inline]
182    pub unsafe fn rename(old: usize, new: usize) -> isize {
183        let r = unsafe { syscall6(SYS_RENAMEAT2, AT_FDCWD, old, AT_FDCWD, new, 0, 0) };
184        if r != -38 {
185            return r; // not ENOSYS
186        }
187        unsafe { syscall6(SYS_RENAMEAT, AT_FDCWD, old, AT_FDCWD, new, 0, 0) }
188    }
189
190    pub fn exit(code: i32) -> ! {
191        // SAFETY: ends the process.
192        unsafe { syscall3(SYS_EXIT_GROUP, code as usize, 0, 0) };
193        loop {}
194    }
195
196    /// give the rest of the time slice to another ready thread
197    pub fn yield_now() {
198        // SAFETY: no arguments.
199        unsafe { syscall3(SYS_SCHED_YIELD, 0, 0, 0) };
200    }
201
202    pub fn write_fd(fd: usize, b: &[u8]) {
203        let mut off = 0;
204        while off < b.len() {
205            // SAFETY: b is a valid buffer.
206            let r = unsafe { syscall3(SYS_WRITE, fd, b.as_ptr() as usize + off, b.len() - off) };
207            if r == -4 {
208                continue; // EINTR
209            }
210            if r == -11 {
211                // EAGAIN on a non-blocking descriptor (a pipe the reader does not empty): wait
212                // until it takes more instead of spinning on sched_yield
213                let mut pfd = [fd as i32, 4]; // struct pollfd: fd, events POLLOUT, revents 0
214                // SAFETY: one pollfd record, no timeout.
215                unsafe { poll(pfd.as_mut_ptr() as usize, 1, -1) };
216                continue;
217            }
218            if r <= 0 {
219                break;
220            }
221            off += r as usize;
222        }
223    }
224
225    pub fn write_stdout(b: &[u8]) {
226        write_fd(1, b)
227    }
228
229    pub fn write_stderr(b: &[u8]) {
230        write_fd(2, b)
231    }
232
233    /// zeroed pages, or null
234    ///
235    /// # Safety
236    /// Free them with pages_free.
237    pub unsafe fn pages_alloc(n: usize) -> *mut u8 {
238        // PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS
239        let r = unsafe { syscall6(SYS_MMAP, 0, n, 3, 0x22, usize::MAX, 0) };
240        if r < 0 {
241            return core::ptr::null_mut();
242        }
243        if n >= 4 << 20 {
244            // MADV_HUGEPAGE for large runs (images, planes): ~10x fewer page faults; JPEG / PNG
245            // loads 15-25 % faster in WSL (perf-load).  A no-op where THP is off; result ignored.
246            // SAFETY: r, n is the mapping just created.
247            unsafe { syscall3(SYS_MADVISE, r as usize, n, 14) };
248        }
249        r as *mut u8
250    }
251
252    /// # Safety
253    /// p, n must come from pages_alloc.
254    pub unsafe fn pages_free(p: *mut u8, n: usize) {
255        unsafe { syscall3(SYS_MUNMAP, p as usize, n, 0) };
256    }
257
258    // std defines these (and links libgcc_s)
259    #[cfg(not(feature = "std"))]
260    #[unsafe(no_mangle)]
261    pub extern "C" fn rust_eh_personality() {}
262
263    #[cfg(not(feature = "std"))]
264    #[unsafe(no_mangle)]
265    pub extern "C" fn _Unwind_Resume() -> ! {
266        exit(4)
267    }
268}
rt/os.rs

rt/
os.rs
