rustld 0.1.55

A modern ELF loader (static & dynamic linker + compatible glibc & musl) written in Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160

//! Syscall trampoline: routes all syscalls through an anonymous RX page so
//! the `syscall` opcode (`0x0F 0x05`) never appears in the loader image.

use core::{
    arch::asm,
    sync::atomic::{AtomicBool, AtomicUsize, Ordering},
};

static TRAMPOLINE: AtomicUsize = AtomicUsize::new(0);
static USE_INDIRECT: AtomicBool = AtomicBool::new(true);

/// Enable (`true`) or disable (`false`) the trampoline. Called by `ElfLoader`.
pub fn set_use_indirect(enabled: bool) {
    USE_INDIRECT.store(enabled, Ordering::Relaxed);
}

// `syscall ; ret` — split so the byte pair 0x0F 0x05 is data, not code.
const STUB: [u8; 3] = [0x0F, 0x05, 0xC3];

// Bootstrap mmap/mprotect use `.byte` directives so 0x0F 0x05 only appears
// inside STUB (data) and these two cold functions, nowhere else.
#[inline(never)]
unsafe fn bootstrap_mmap(size: usize) -> *mut u8 {
    let result: isize;
    unsafe {
        asm!(
            ".byte 0x0F", ".byte 0x05",
            inlateout("rax") 9usize => result,  // SYS_mmap
            in("rdi") 0usize,
            in("rsi") size,
            in("rdx") 3usize,                   // PROT_READ | PROT_WRITE
            in("r10") 0x22usize,                // MAP_PRIVATE | MAP_ANONYMOUS
            in("r8")  -1isize as usize,
            in("r9")  0usize,
            out("rcx") _, out("r11") _,
            options(nostack),
        );
    }
    result as *mut u8
}

#[inline(never)]
unsafe fn bootstrap_mprotect(addr: *mut u8, len: usize, prot: usize) {
    let _rc: isize;
    unsafe {
        asm!(
            ".byte 0x0F", ".byte 0x05",
            inlateout("rax") 10usize => _rc,    // SYS_mprotect
            in("rdi") addr, in("rsi") len, in("rdx") prot,
            out("rcx") _, out("r11") _,
            options(nostack),
        );
    }
}

pub fn init_trampoline() {
    if TRAMPOLINE.load(Ordering::Acquire) != 0 {
        return;
    }
    unsafe {
        let page = bootstrap_mmap(4096);
        assert!(
            !page.is_null() && (page as isize) > 0,
            "rustld: trampoline mmap failed"
        );
        core::ptr::copy_nonoverlapping(STUB.as_ptr(), page, STUB.len());
        bootstrap_mprotect(page, 4096, 0x1 | 0x4); // PROT_READ | PROT_EXEC
        let _ = TRAMPOLINE.compare_exchange(0, page as usize, Ordering::Release, Ordering::Relaxed);
    }
}

#[inline(always)]
pub fn trampoline() -> usize {
    let addr = TRAMPOLINE.load(Ordering::Acquire);
    if addr == 0 {
        init_trampoline();
        TRAMPOLINE.load(Ordering::Acquire)
    } else {
        addr
    }
}

// `#[inline(never)]` keeps the `syscall` opcode confined to one copy in the
// binary instead of being duplicated at every inlined call site.
#[inline(never)]
unsafe fn direct_syscall6(
    nr: usize,
    a1: usize,
    a2: usize,
    a3: usize,
    a4: usize,
    a5: usize,
    a6: usize,
) -> isize {
    let result: isize;
    unsafe {
        asm!(
            "syscall",
            inlateout("rax") nr => result,
            in("rdi") a1, in("rsi") a2, in("rdx") a3,
            in("r10") a4, in("r8") a5, in("r9") a6,
            out("rcx") _, out("r11") _,
            options(nostack),
        );
    }
    result
}

#[inline(never)]
unsafe fn direct_syscall_noreturn(nr: usize, a1: usize) -> ! {
    unsafe {
        asm!("syscall", in("rax") nr, in("rdi") a1, options(noreturn));
    }
}

#[inline(always)]
pub unsafe fn indirect_syscall6(
    nr: usize,
    a1: usize,
    a2: usize,
    a3: usize,
    a4: usize,
    a5: usize,
    a6: usize,
) -> isize {
    if USE_INDIRECT.load(Ordering::Relaxed) {
        let stub = trampoline();
        let result: isize;
        unsafe {
            asm!(
                "call {stub}",
                stub = in(reg) stub,
                inlateout("rax") nr => result,
                in("rdi") a1, in("rsi") a2, in("rdx") a3,
                in("r10") a4, in("r8") a5, in("r9") a6,
                out("rcx") _, out("r11") _,
            );
        }
        result
    } else {
        unsafe { direct_syscall6(nr, a1, a2, a3, a4, a5, a6) }
    }
}

#[inline(always)]
pub unsafe fn indirect_syscall_noreturn(nr: usize, a1: usize) -> ! {
    if USE_INDIRECT.load(Ordering::Relaxed) {
        let stub = trampoline();
        unsafe {
            asm!(
                "jmp {stub}",
                stub = in(reg) stub,
                in("rax") nr, in("rdi") a1,
                options(noreturn),
            );
        }
    } else {
        unsafe { direct_syscall_noreturn(nr, a1) }
    }
}