1#![allow(clippy::arithmetic_side_effects)]
2use crate::{
16 ebpf,
17 elf::Executable,
18 error::{EbpfError, ProgramResult},
19 interpreter::Interpreter,
20 memory_region::MemoryMapping,
21 program::{BuiltinFunction, BuiltinProgram, FunctionRegistry, SBPFVersion},
22 static_analysis::{Analysis, DummyContextObject, RegisterTraceEntry},
23};
24pub use defaults::get_stack_frame_size;
26use std::{collections::BTreeMap, fmt::Debug, marker::PhantomData, mem::offset_of, ptr};
27
28#[cfg(feature = "shuttle-test")]
29use shuttle::sync::Arc;
30#[cfg(not(feature = "shuttle-test"))]
31use std::sync::Arc;
32
33pub fn get_runtime_environment_key() -> i32 {
35 cfg_select! {
36 feature = "jit" => {
37 cfg_select! {
38 feature = "shuttle-test" => { use shuttle::rand::{thread_rng, Rng}; }
39 _ => { use rand::{thread_rng, Rng}; }
40 }
41 static RUNTIME_ENVIRONMENT_KEY: std::sync::OnceLock<i32> = std::sync::OnceLock::new();
42 *RUNTIME_ENVIRONMENT_KEY.get_or_init(|| thread_rng().gen::<i32>() >> 1)
43 }
44 _ => 0
45 }
46}
47
48pub(crate) mod defaults {
50 const DEFAULT_STACK_FRAME_SIZE: usize = 4_096;
51
52 #[cfg(feature = "conf-stack-frame-size")]
63 #[inline(always)]
64 pub fn get_stack_frame_size() -> usize {
65 static STACK_FRAME_SIZE_CACHE: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
66 *STACK_FRAME_SIZE_CACHE.get_or_init(|| {
67 let size = std::env::var("VM_STACK_FRAME_SIZE")
68 .ok()
69 .and_then(|v| {
70 v.parse::<usize>().ok().filter(|sfz| *sfz > 0).or_else(|| {
71 log::warn!(
72 "Invalid VM_STACK_FRAME_SIZE={}, falling back to {}.",
73 v,
74 DEFAULT_STACK_FRAME_SIZE
75 );
76 None
77 })
78 })
79 .unwrap_or(DEFAULT_STACK_FRAME_SIZE);
80 if size != DEFAULT_STACK_FRAME_SIZE {
81 log::warn!(
82 "VM_STACK_FRAME_SIZE is set to {} (default: {}).",
83 size,
84 DEFAULT_STACK_FRAME_SIZE
85 );
86 }
87 size
88 })
89 }
90
91 #[cfg(not(feature = "conf-stack-frame-size"))]
93 pub const fn get_stack_frame_size() -> usize {
94 DEFAULT_STACK_FRAME_SIZE
95 }
96}
97
98pub enum ExecutionMode {
100 Interpreted,
102 Jit,
106 PreferJit,
108}
109
110#[derive(Debug, Clone, PartialEq, Eq)]
112pub struct Config {
113 pub max_call_depth: usize,
115 pub stack_frame_size: usize,
117 pub enable_address_translation: bool,
119 pub enable_stack_frame_gaps: bool,
121 pub instruction_meter_checkpoint_distance: usize,
123 pub enable_instruction_meter: bool,
125 pub enable_symbol_and_section_labels: bool,
127 pub reject_broken_elfs: bool,
129 #[cfg(feature = "jit")]
130 pub noop_instruction_rate: u32,
132 #[cfg(feature = "jit")]
133 pub sanitize_user_provided_values: bool,
135 pub optimize_rodata: bool,
137 pub aligned_memory_mapping: bool,
139 pub enabled_sbpf_versions: std::ops::RangeInclusive<SBPFVersion>,
141}
142
143impl Config {
144 pub fn stack_size(&self) -> usize {
146 self.stack_frame_size * self.max_call_depth
147 }
148}
149
150impl Default for Config {
151 fn default() -> Self {
152 Self {
153 max_call_depth: 64,
154 stack_frame_size: defaults::get_stack_frame_size(),
155 enable_address_translation: true,
156 enable_stack_frame_gaps: true,
157 instruction_meter_checkpoint_distance: 1000,
158 enable_instruction_meter: true,
159 enable_symbol_and_section_labels: false,
160 reject_broken_elfs: false,
161 #[cfg(feature = "jit")]
162 noop_instruction_rate: 256,
163 #[cfg(feature = "jit")]
164 sanitize_user_provided_values: true,
165 optimize_rodata: true,
166 aligned_memory_mapping: false,
167 enabled_sbpf_versions: SBPFVersion::V0..=SBPFVersion::V4,
168 }
169 }
170}
171
172impl<C: ContextObject> Executable<C> {
174 pub fn from_elf(elf_bytes: &[u8], loader: Arc<BuiltinProgram<C>>) -> Result<Self, EbpfError> {
176 let executable = Executable::load(elf_bytes, loader)?;
177 Ok(executable)
178 }
179 pub fn from_text_bytes(
181 text_bytes: &[u8],
182 loader: Arc<BuiltinProgram<C>>,
183 sbpf_version: SBPFVersion,
184 function_registry: FunctionRegistry<usize>,
185 ) -> Result<Self, EbpfError> {
186 Executable::new_from_text_bytes(text_bytes, loader, sbpf_version, function_registry)
187 .map_err(EbpfError::ElfError)
188 }
189}
190
191pub trait ContextObject {
193 fn consume(&mut self, amount: u64);
195 fn get_remaining(&self) -> u64;
197 fn active_mapping_ptr(&mut self) -> ptr::NonNull<MemoryMapping>;
199}
200
201pub struct DynamicAnalysis {
203 pub edge_counter_max: usize,
205 pub edges: BTreeMap<usize, BTreeMap<usize, usize>>,
207}
208
209impl DynamicAnalysis {
210 pub fn new(register_trace: &[[u64; 12]], analysis: &Analysis) -> Self {
212 let mut result = Self {
213 edge_counter_max: 0,
214 edges: BTreeMap::new(),
215 };
216 let mut last_basic_block = usize::MAX;
217 for traced_instruction in register_trace.iter() {
218 let pc = traced_instruction[11] as usize;
219 if analysis.cfg_nodes.contains_key(&pc) {
220 let counter = result
221 .edges
222 .entry(last_basic_block)
223 .or_default()
224 .entry(pc)
225 .or_insert(0);
226 *counter += 1;
227 result.edge_counter_max = result.edge_counter_max.max(*counter);
228 last_basic_block = pc;
229 }
230 }
231 result
232 }
233}
234
235#[derive(Clone, Default)]
237pub struct CallFrame {
238 pub caller_saved_registers: [u64; ebpf::SCRATCH_REGS],
240 pub frame_pointer: u64,
242 pub target_pc: u64,
244}
245
246pub enum RuntimeEnvironmentSlot {
248 HostStackPointer = offset_of!(EbpfVm<DummyContextObject>, host_stack_pointer) as isize,
250 CallDepth = offset_of!(EbpfVm<DummyContextObject>, call_depth) as isize,
252 ContextObjectPointer = offset_of!(EbpfVm<DummyContextObject>, context_object_pointer) as isize,
254 PreviousInstructionMeter =
256 offset_of!(EbpfVm<DummyContextObject>, previous_instruction_meter) as isize,
257 DueInsnCount = offset_of!(EbpfVm<DummyContextObject>, due_insn_count) as isize,
259 StopwatchNumerator = offset_of!(EbpfVm<DummyContextObject>, stopwatch_numerator) as isize,
261 StopwatchDenominator = offset_of!(EbpfVm<DummyContextObject>, stopwatch_denominator) as isize,
263 Registers = offset_of!(EbpfVm<DummyContextObject>, registers) as isize,
265 ProgramResult = offset_of!(EbpfVm<DummyContextObject>, program_result) as isize,
267 MemoryMapping = offset_of!(EbpfVm<DummyContextObject>, memory_mapping) as isize,
269 RegisterTrace = offset_of!(EbpfVm<DummyContextObject>, register_trace) as isize,
271}
272
273#[repr(C)]
329pub struct EbpfVm<'a, C: ContextObject> {
330 pub host_stack_pointer: *mut u64,
332 pub call_depth: u64,
337 pub(crate) context_object_pointer: ptr::NonNull<C>,
339 context_object_lifetime: PhantomData<&'a mut C>,
341 pub previous_instruction_meter: u64,
343 pub due_insn_count: u64,
345 pub stopwatch_numerator: u64,
347 pub stopwatch_denominator: u64,
349 pub registers: [u64; 12],
351 pub program_result: ProgramResult,
353 pub(crate) memory_mapping: ptr::NonNull<MemoryMapping>,
355 pub loader: Arc<BuiltinProgram<C>>,
357 pub register_trace: Vec<RegisterTraceEntry>,
359 #[cfg(feature = "debugger")]
361 pub debug_port: Option<u16>,
362 #[cfg(feature = "debugger")]
364 pub debug_metadata: Option<String>,
365}
366
367impl<'a, C: ContextObject> EbpfVm<'a, C> {
368 pub fn new(
370 loader: Arc<BuiltinProgram<C>>,
371 sbpf_version: SBPFVersion,
372 context_object: &'a mut C,
373 stack_len: usize,
374 ) -> Self {
375 let config = loader.get_config();
376 let mut registers = [0u64; 12];
377 registers[ebpf::FRAME_PTR_REG] =
378 ebpf::MM_STACK_START.saturating_add(if !sbpf_version.manual_stack_frame_bump() {
379 config.stack_frame_size
380 } else {
381 stack_len
382 } as u64);
383
384 let memory_mapping = context_object.active_mapping_ptr();
385 EbpfVm {
386 host_stack_pointer: std::ptr::null_mut(),
387 call_depth: 0,
388 context_object_pointer: ptr::NonNull::from_mut(context_object),
389 context_object_lifetime: PhantomData,
390 previous_instruction_meter: 0,
391 due_insn_count: 0,
392 stopwatch_numerator: 0,
393 stopwatch_denominator: 0,
394 registers,
395 program_result: ProgramResult::Ok(0),
396 memory_mapping,
397 loader,
398 #[cfg(feature = "debugger")]
399 debug_port: std::env::var("VM_DEBUG_PORT")
400 .ok()
401 .and_then(|v| v.parse::<u16>().ok()),
402 #[cfg(feature = "debugger")]
403 debug_metadata: None,
404 register_trace: Vec::default(),
405 }
406 }
407
408 pub fn execute_program(
419 &mut self,
420 executable: &Executable<C>,
421 mode: &mut ExecutionMode,
422 call_frames: &mut [CallFrame],
423 ) -> (u64, ProgramResult) {
424 debug_assert!(Arc::ptr_eq(&self.loader, executable.get_loader()));
425 self.registers[11] = executable.get_entrypoint_instruction_offset() as u64;
426 let config = executable.get_config();
427 let initial_insn_count = self.context().get_remaining();
428 self.previous_instruction_meter = initial_insn_count;
429 self.due_insn_count = 0;
430 self.program_result = ProgramResult::Ok(0);
431
432 'execute: {
433 match *mode {
434 ExecutionMode::Interpreted => {}
435
436 #[cfg(all(feature = "jit", not(target_os = "windows"), target_arch = "x86_64"))]
437 ExecutionMode::PreferJit => {
438 if let Some(compiled_program) = executable.get_compiled_program() {
439 *mode = ExecutionMode::Jit;
440 break 'execute compiled_program.invoke(config, self, self.registers);
441 }
442 }
443 #[cfg(not(all(
444 feature = "jit",
445 not(target_os = "windows"),
446 target_arch = "x86_64"
447 )))]
448 ExecutionMode::PreferJit => {}
449
450 #[cfg(all(feature = "jit", not(target_os = "windows"), target_arch = "x86_64"))]
451 ExecutionMode::Jit => {
452 let Some(compiled_program) = executable.get_compiled_program() else {
453 return (0, ProgramResult::Err(EbpfError::JitNotCompiled));
454 };
455 *mode = ExecutionMode::Jit;
456 break 'execute compiled_program.invoke(config, self, self.registers);
457 }
458 #[cfg(not(all(
459 feature = "jit",
460 not(target_os = "windows"),
461 target_arch = "x86_64"
462 )))]
463 ExecutionMode::Jit => return (0, ProgramResult::Err(EbpfError::JitNotCompiled)),
464 }
465
466 *mode = ExecutionMode::Interpreted;
467 let interpreter = Interpreter::new(self, executable, self.registers, call_frames);
468 break 'execute run_interpreter(interpreter);
469 }
470
471 let instruction_count = if config.enable_instruction_meter {
472 let due_insn_count = self.due_insn_count;
473 let context = self.context();
474 context.consume(due_insn_count);
475 initial_insn_count.saturating_sub(context.get_remaining())
476 } else {
477 0
478 };
479 let mut result = ProgramResult::Ok(0);
480 std::mem::swap(&mut result, &mut self.program_result);
481 (instruction_count, result)
482 }
483
484 pub fn invoke_function(&mut self, function: BuiltinFunction<C>) {
486 function(
487 self.encrypted_host_address(),
488 self.registers[1],
489 self.registers[2],
490 self.registers[3],
491 self.registers[4],
492 self.registers[5],
493 );
494 }
495
496 pub(crate) fn encrypted_host_address(&mut self) -> EncryptedHostAddressToEbpfVm<C> {
502 let addr = (&raw mut *self).expose_provenance() as isize;
503 EncryptedHostAddressToEbpfVm(
504 addr.wrapping_add(get_runtime_environment_key() as isize) as usize as u64,
505 PhantomData,
506 )
507 }
508
509 pub fn context(&mut self) -> &mut C {
511 unsafe { self.context_object_pointer.as_mut() }
521 }
522
523 pub(crate) fn memory(&mut self) -> &mut MemoryMapping {
526 unsafe { self.memory_mapping.as_mut() }
532 }
533}
534
535#[repr(transparent)]
537pub struct EncryptedHostAddressToEbpfVm<C>(
538 pub(crate) u64,
541 PhantomData<C>,
542);
543
544impl<C: ContextObject> EncryptedHostAddressToEbpfVm<C> {
545 pub unsafe fn with_vm<R>(&mut self, cb: impl FnOnce(&mut EbpfVm<'_, C>) -> R) -> R {
553 let addr = (self.0 as usize as isize)
554 .wrapping_sub(crate::vm::get_runtime_environment_key() as isize);
555 let vm = unsafe {
559 std::ptr::with_exposed_provenance_mut::<crate::vm::EbpfVm<C>>(addr as usize)
560 .as_mut()
561 .unwrap()
562 };
563 cb(vm)
564 }
565}
566
567#[cold]
568#[inline(never)]
569fn run_interpreter<C: ContextObject>(mut interpreter: Interpreter<C>) {
570 #[cfg(feature = "debugger")]
571 if let Some(debug_port) = interpreter.vm.debug_port {
572 return crate::debugger::execute(&mut interpreter, debug_port);
573 }
574
575 while interpreter.step() {}
576 interpreter.vm.registers[11] = interpreter.reg[11];
577}
578
579#[cfg(test)]
580mod tests {
581 use crate::{
582 memory_region::MemoryMapping,
583 program::{BuiltinProgram, SBPFVersion},
584 vm::{Config, ContextObject, RuntimeEnvironmentSlot},
585 };
586 use std::{ptr::NonNull, sync::Arc};
587
588 #[test]
589 fn test_runtime_environment_slots() {
590 struct DummyContextObject(MemoryMapping);
591 impl ContextObject for DummyContextObject {
592 fn consume(&mut self, _: u64) {
593 todo!()
594 }
595 fn get_remaining(&self) -> u64 {
596 todo!()
597 }
598 fn active_mapping_ptr(&mut self) -> NonNull<MemoryMapping> {
599 NonNull::from_mut(&mut self.0)
600 }
601 }
602 let version = SBPFVersion::V4;
603 let config = Config::default();
604 let mut context_object =
605 unsafe { DummyContextObject(MemoryMapping::new(vec![], &config, version).unwrap()) };
606 let env = super::EbpfVm::new(
607 Arc::new(BuiltinProgram::new_mock()),
608 version,
609 &mut context_object,
610 4096,
611 );
612
613 macro_rules! check_slot {
614 ($env:expr, $entry:ident, $slot:ident) => {
615 assert_eq!(
616 unsafe {
617 std::ptr::addr_of!($env.$entry)
618 .cast::<u8>()
619 .offset_from(std::ptr::addr_of!($env).cast::<u8>()) as usize
620 },
621 RuntimeEnvironmentSlot::$slot as usize,
622 );
623 };
624 }
625
626 check_slot!(env, host_stack_pointer, HostStackPointer);
627 check_slot!(env, call_depth, CallDepth);
628 check_slot!(env, context_object_pointer, ContextObjectPointer);
629 check_slot!(env, previous_instruction_meter, PreviousInstructionMeter);
630 check_slot!(env, due_insn_count, DueInsnCount);
631 check_slot!(env, stopwatch_numerator, StopwatchNumerator);
632 check_slot!(env, stopwatch_denominator, StopwatchDenominator);
633 check_slot!(env, registers, Registers);
634 check_slot!(env, program_result, ProgramResult);
635 check_slot!(env, memory_mapping, MemoryMapping);
636 check_slot!(env, register_trace, RegisterTrace);
637 }
638}