1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
/*
* Copyright 2016, Data61
* Commonwealth Scientific and Industrial Research Organisation (CSIRO)
* ABN 41 687 119 230.
*
* This software may be distributed and modified according to the terms of
* the GNU General Public License version 2. Note that NO WARRANTY is provided.
* See "LICENSE_GPLv2.txt" for details.
*
* @TAG(DATA61_GPL)
*/
#include <config.h>
#include <model/statedata.h>
#include <machine/fpu.h>
#include <kernel/traps.h>
#include <arch/machine/debug.h>
#include <api/syscall.h>
#ifdef CONFIG_VTX
static void NORETURN vmlaunch_failed(word_t failInvalid, word_t failValid)
{
NODE_LOCK_SYS;
c_entry_hook();
if (failInvalid) {
userError("current VMCS pointer is not valid");
}
if (failValid) {
userError("vmlaunch/vmresume error %d", (int)vmread(VMX_DATA_INSTRUCTION_ERROR));
}
handleVmEntryFail();
restore_user_context();
}
static void NORETURN restore_vmx(void)
{
restoreVMCS();
tcb_t *cur_thread = NODE_STATE(ksCurThread);
#ifdef CONFIG_HARDWARE_DEBUG_API
/* Do not support breakpoints in VMs, so just disable all breakpoints */
loadAllDisabledBreakpointState(&cur_thread->tcbArch);
#endif
if (cur_thread->tcbArch.vcpu->launched) {
/* attempt to do a vmresume */
asm volatile(
// Set our stack pointer to the top of the tcb so we can efficiently pop
"movq %[reg], %%rsp\n"
"popq %%rax\n"
"popq %%rbx\n"
"popq %%rcx\n"
"popq %%rdx\n"
"popq %%rsi\n"
"popq %%rdi\n"
"popq %%rbp\n"
#if CONFIG_MAX_NUM_NODES > 1
"swapgs\n"
#endif
// Now do the vmresume
"vmresume\n"
"setb %%al\n"
"sete %%bl\n"
"movzx %%al, %%rdi\n"
"movzx %%bl, %%rsi\n"
// if we get here we failed
#if CONFIG_MAX_NUM_NODES > 1
"swapgs\n"
"movq %%gs:%c[stack_offset], %%rsp\n"
#else
"leaq kernel_stack_alloc + %c[stack_size], %%rsp\n"
#endif
"leaq %[failed], %%rax\n"
"jmp *%%rax\n"
:
: [reg]"r"(&cur_thread->tcbArch.vcpu->gp_registers[VCPU_EAX]),
[failed]"m"(vmlaunch_failed),
[stack_size]"i"(BIT(CONFIG_KERNEL_STACK_BITS))
#if CONFIG_MAX_NUM_NODES > 1
, [stack_offset]"i"(OFFSETOF(nodeInfo_t, stackTop))
#endif
// Clobber memory so the compiler is forced to complete all stores
// before running this assembler
: "memory"
);
} else {
/* attempt to do a vmlaunch */
asm volatile(
// Set our stack pointer to the top of the tcb so we can efficiently pop
"movq %[reg], %%rsp\n"
"popq %%rax\n"
"popq %%rbx\n"
"popq %%rcx\n"
"popq %%rdx\n"
"popq %%rsi\n"
"popq %%rdi\n"
"popq %%rbp\n"
#if CONFIG_MAX_NUM_NODES > 1
"swapgs\n"
#endif
// Now do the vmresume
"vmlaunch\n"
// if we get here we failed
"setb %%al\n"
"sete %%bl\n"
"movzx %%al, %%rdi\n"
"movzx %%bl, %%rsi\n"
#if CONFIG_MAX_NUM_NODES > 1
"swapgs\n"
"movq %%gs:%c[stack_offset], %%rsp\n"
#else
"leaq kernel_stack_alloc + %c[stack_size], %%rsp\n"
#endif
"leaq %[failed], %%rax\n"
"jmp *%%rax\n"
:
: [reg]"r"(&cur_thread->tcbArch.vcpu->gp_registers[VCPU_EAX]),
[failed]"m"(vmlaunch_failed),
[stack_size]"i"(BIT(CONFIG_KERNEL_STACK_BITS))
#if CONFIG_MAX_NUM_NODES > 1
, [stack_offset]"i"(OFFSETOF(nodeInfo_t, stackTop))
#endif
// Clobber memory so the compiler is forced to complete all stores
// before running this assembler
: "memory"
);
}
UNREACHABLE();
}
#endif
void VISIBLE NORETURN restore_user_context(void)
{
NODE_UNLOCK_IF_HELD;
c_exit_hook();
/* we've now 'exited' the kernel. If we have a pending interrupt
* we should 'enter' it again */
if (ARCH_NODE_STATE(x86KSPendingInterrupt) != int_invalid) {
irq_t irq = servicePendingIRQ();
/* reset our stack and jmp to the IRQ entry point */
asm volatile(
/* round our stack back to the top to reset it */
"andq %[stack_mask], %%rsp\n"
"addq %[stack_size], %%rsp\n"
"movq %[syscall], %%rsi\n"
"movq %[irq], %%rdi\n"
"call c_handle_interrupt"
:
: [stack_mask] "i"(~MASK(CONFIG_KERNEL_STACK_BITS)),
[stack_size] "i"(BIT(CONFIG_KERNEL_STACK_BITS)),
[syscall] "i"(0), /* syscall is unused for irq path */
[irq] "r"((seL4_Word)irq)
: "memory");
UNREACHABLE();
}
tcb_t *cur_thread = NODE_STATE(ksCurThread);
word_t *irqstack = MODE_NODE_STATE(x64KSIRQStack);
#ifdef CONFIG_VTX
if (thread_state_ptr_get_tsType(&cur_thread->tcbState) == ThreadState_RunningVM) {
restore_vmx();
}
#endif
lazyFPURestore(cur_thread);
#ifdef CONFIG_HARDWARE_DEBUG_API
restore_user_debug_context(cur_thread);
#endif
#if CONFIG_MAX_NUM_NODES > 1
cpu_id_t cpu = getCurrentCPUIndex();
swapgs();
#endif
/* Now that we have swapped back to the user gs we can safely
* update the GS base. We must *not* use any kernel functions
* that rely on having a kernel GS though. Most notably uses
* of NODE_STATE etc cannot be used beyond this point */
word_t base = getRegister(cur_thread, TLS_BASE);
x86_write_fs_base(base, SMP_TERNARY(cpu, 0));
base = cur_thread->tcbIPCBuffer;
x86_write_gs_base(base, SMP_TERNARY(cpu, 0));
// Check if we are returning from a syscall/sysenter or from an interrupt
// There is a special case where if we would be returning from a sysenter,
// but are current singlestepping, do a full return like an interrupt
if (likely(cur_thread->tcbArch.tcbContext.registers[Error] == -1) &&
(!config_set(CONFIG_SYSENTER) || !config_set(CONFIG_HARDWARE_DEBUG_API) || ((cur_thread->tcbArch.tcbContext.registers[FLAGS] & FLAGS_TF) == 0))) {
if (config_set(CONFIG_SYSENTER)) {
cur_thread->tcbArch.tcbContext.registers[FLAGS] &= ~FLAGS_IF;
asm volatile(
// Set our stack pointer to the top of the tcb so we can efficiently pop
"movq %0, %%rsp\n"
"popq %%rdi\n"
"popq %%rsi\n"
"popq %%rax\n"
"popq %%rbx\n"
"popq %%rbp\n"
"popq %%r12\n"
"popq %%r13\n"
"popq %%r14\n"
// skip RDX
"addq $8, %%rsp\n"
"popq %%r10\n"
"popq %%r8\n"
"popq %%r9\n"
"popq %%r15\n"
//restore RFLAGS
"popfq\n"
// reset interrupt bit
"orq %[IF], -8(%%rsp)\n"
// Restore NextIP
"popq %%rdx\n"
// Skip ERROR
"addq $8, %%rsp\n"
// Restore RSP
"popq %%rcx\n"
// Skip TLS_BASE, FaultIP
"addq $16, %%rsp\n"
"popq %%r11\n"
// More register but we can ignore and are done restoring
// enable interrupt disabled by sysenter
"sti\n"
/* return to user
* sysexit with rex.w user code = cs + 32, user data = cs + 40.
* without rex.w user code = cs + 16, user data = cs + 24
* */
"rex.w sysexit\n"
:
: "r"(&cur_thread->tcbArch.tcbContext.registers[RDI]),
[IF] "i" (FLAGS_IF)
// Clobber memory so the compiler is forced to complete all stores
// before running this assembler
: "memory"
);
} else {
asm volatile(
// Set our stack pointer to the top of the tcb so we can efficiently pop
"movq %0, %%rsp\n"
"popq %%rdi\n"
"popq %%rsi\n"
"popq %%rax\n"
"popq %%rbx\n"
"popq %%rbp\n"
"popq %%r12\n"
"popq %%r13\n"
"popq %%r14\n"
"popq %%rdx\n"
"popq %%r10\n"
"popq %%r8\n"
"popq %%r9\n"
"popq %%r15\n"
//restore RFLAGS
"popq %%r11\n"
// Restore NextIP
"popq %%rcx\n"
// clear RSP to not leak information to the user
"xor %%rsp, %%rsp\n"
// More register but we can ignore and are done restoring
// enable interrupt disabled by sysenter
"rex.w sysret\n"
:
: "r"(&cur_thread->tcbArch.tcbContext.registers[RDI])
// Clobber memory so the compiler is forced to complete all stores
// before running this assembler
: "memory"
);
}
} else {
/* construct our return from interrupt frame */
irqstack[1] = getRegister(cur_thread, NextIP);
irqstack[2] = getRegister(cur_thread, CS);
irqstack[3] = getRegister(cur_thread, FLAGS);
irqstack[4] = getRegister(cur_thread, RSP);
irqstack[5] = getRegister(cur_thread, SS);
asm volatile(
// Set our stack pointer to the top of the tcb so we can efficiently pop
"movq %0, %%rsp\n"
"popq %%rdi\n"
"popq %%rsi\n"
"popq %%rax\n"
"popq %%rbx\n"
"popq %%rbp\n"
"popq %%r12\n"
"popq %%r13\n"
"popq %%r14\n"
"popq %%rdx\n"
"popq %%r10\n"
"popq %%r8\n"
"popq %%r9\n"
"popq %%r15\n"
/* skip RFLAGS, Error NextIP RSP, TLS_BASE, FaultIP */
"addq $48, %%rsp\n"
"popq %%r11\n"
"popq %%rcx\n"
#if CONFIG_MAX_NUM_NODES > 1
// Swapping gs twice here is worth it as it allows us to efficiently
// set the user gs base previously
"swapgs\n"
"movq %%gs:8, %%rsp\n"
"addq $8, %%rsp\n"
// Switch to the user GS value
"swapgs\n"
#else
"leaq x64KSIRQStack + 8, %%rsp\n"
#endif
"iretq\n"
:
: "r"(&cur_thread->tcbArch.tcbContext.registers[RDI])
// Clobber memory so the compiler is forced to complete all stores
// before running this assembler
: "memory"
);
}
UNREACHABLE();
}
void VISIBLE NORETURN c_x64_handle_interrupt(int irq, int syscall);
void VISIBLE NORETURN c_x64_handle_interrupt(int irq, int syscall)
{
setRegister(NODE_STATE(ksCurThread), Error, MODE_NODE_STATE(x64KSIRQStack)[0]);
/* In the case of an interrupt the NextIP and the FaultIP should be the same value,
* i.e. the address of the instruction the CPU was about to execute before the
* interrupt. This is the 5th value pushed on by the hardware, so indexing from
* the bottom is x64KSIRQStack[1] */
setRegister(NODE_STATE(ksCurThread), NextIP, MODE_NODE_STATE(x64KSIRQStack)[1]);
setRegister(NODE_STATE(ksCurThread), FaultIP, MODE_NODE_STATE(x64KSIRQStack)[1]);
setRegister(NODE_STATE(ksCurThread), FLAGS, MODE_NODE_STATE(x64KSIRQStack)[3]);
setRegister(NODE_STATE(ksCurThread), RSP, MODE_NODE_STATE(x64KSIRQStack)[4]);
c_handle_interrupt(irq, syscall);
UNREACHABLE();
}