cbe_rbpf/
vm.rs

1#![allow(clippy::integer_arithmetic)]
2// Derived from uBPF <https://github.com/iovisor/ubpf>
3// Copyright 2015 Big Switch Networks, Inc
4//      (uBPF: VM architecture, parts of the interpreter, originally in C)
5// Copyright 2016 6WIND S.A. <quentin.monnet@6wind.com>
6//      (Translation to Rust, MetaBuff/multiple classes addition, hashmaps for syscalls)
7// Copyright 2022 Cartallum CBE Maintainers <maintainers@cartallum.com>
8//
9// Licensed under the Apache License, Version 2.0 <http://www.apache.org/licenses/LICENSE-2.0> or
10// the MIT license <http://opensource.org/licenses/MIT>, at your option. This file may not be
11// copied, modified, or distributed except according to those terms.
12
13//! Virtual machine for eBPF programs.
14
15use crate::{
16    aligned_memory::AlignedMemory,
17    ebpf,
18    elf::Executable,
19    error::EbpfError,
20    interpreter::Interpreter,
21    memory_region::{MemoryMapping, MemoryRegion},
22    static_analysis::{Analysis, TraceLogEntry},
23    verifier::Verifier,
24};
25use rand::Rng;
26use std::{
27    collections::{BTreeMap, HashMap},
28    fmt::Debug,
29    marker::PhantomData,
30    mem,
31    sync::Arc,
32};
33
34/// Same as `Result` but provides a stable memory layout
35#[derive(Debug)]
36#[repr(C, u64)]
37pub enum StableResult<T, E> {
38    /// Success
39    Ok(T),
40    /// Failure
41    Err(E),
42}
43
44impl<T: Debug, E: Debug> StableResult<T, E> {
45    /// `true` if `Ok`
46    pub fn is_ok(&self) -> bool {
47        match self {
48            Self::Ok(_) => true,
49            Self::Err(_) => false,
50        }
51    }
52
53    /// `true` if `Err`
54    pub fn is_err(&self) -> bool {
55        match self {
56            Self::Ok(_) => false,
57            Self::Err(_) => true,
58        }
59    }
60
61    /// Returns the inner value if `Ok`, panics otherwise
62    pub fn unwrap(self) -> T {
63        match self {
64            Self::Ok(value) => value,
65            Self::Err(error) => panic!("unwrap {:?}", error),
66        }
67    }
68
69    /// Returns the inner error if `Err`, panics otherwise
70    pub fn unwrap_err(self) -> E {
71        match self {
72            Self::Ok(value) => panic!("unwrap_err {:?}", value),
73            Self::Err(error) => error,
74        }
75    }
76}
77
78impl<T, E> From<StableResult<T, E>> for Result<T, E> {
79    fn from(result: StableResult<T, E>) -> Self {
80        match result {
81            StableResult::Ok(value) => Ok(value),
82            StableResult::Err(value) => Err(value),
83        }
84    }
85}
86
87impl<T, E> From<Result<T, E>> for StableResult<T, E> {
88    fn from(result: Result<T, E>) -> Self {
89        match result {
90            Ok(value) => Self::Ok(value),
91            Err(value) => Self::Err(value),
92        }
93    }
94}
95
96/// Return value of programs and syscalls
97pub type ProgramResult = StableResult<u64, EbpfError>;
98
99/// Holds the function symbols of an Executable
100pub type FunctionRegistry = BTreeMap<u32, (usize, String)>;
101
102/// Syscall function without context
103pub type BuiltInFunction<C> =
104    fn(&mut C, u64, u64, u64, u64, u64, &mut MemoryMapping, &mut ProgramResult);
105
106/// Represents the interface to a fixed functionality program
107pub struct BuiltInProgram<C: ContextObject> {
108    /// Holds the Config if this is a loader program
109    config: Option<Box<Config>>,
110    /// Function pointers by symbol
111    functions: HashMap<u32, (&'static str, BuiltInFunction<C>)>,
112}
113
114impl<C: ContextObject> BuiltInProgram<C> {
115    /// Constructs a loader built-in program
116    pub fn new_loader(config: Config) -> Self {
117        Self {
118            config: Some(Box::new(config)),
119            functions: HashMap::new(),
120        }
121    }
122
123    /// Get the configuration settings assuming this is a loader program
124    pub fn get_config(&self) -> &Config {
125        self.config.as_ref().unwrap()
126    }
127
128    /// Register a built-in function
129    pub fn register_function_by_name(
130        &mut self,
131        name: &'static str,
132        function: BuiltInFunction<C>,
133    ) -> Result<(), EbpfError> {
134        let key = ebpf::hash_symbol_name(name.as_bytes());
135        if self.functions.insert(key, (name, function)).is_some() {
136            Err(EbpfError::FunctionAlreadyRegistered(key as usize))
137        } else {
138            Ok(())
139        }
140    }
141
142    /// Get a symbol's function pointer
143    pub fn lookup_function(&self, key: u32) -> Option<(&'static str, BuiltInFunction<C>)> {
144        self.functions.get(&key).cloned()
145    }
146
147    /// Calculate memory size
148    pub fn mem_size(&self) -> usize {
149        mem::size_of::<Self>()
150            + if self.config.is_some() {
151                mem::size_of::<Config>()
152            } else {
153                0
154            }
155            + self.functions.capacity()
156                * mem::size_of::<(u32, (&'static str, BuiltInFunction<C>))>()
157    }
158}
159
160impl<C: ContextObject> Default for BuiltInProgram<C> {
161    fn default() -> Self {
162        Self {
163            config: None,
164            functions: HashMap::new(),
165        }
166    }
167}
168
169impl<C: ContextObject> Debug for BuiltInProgram<C> {
170    fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> {
171        writeln!(f, "{:?}", unsafe {
172            std::mem::transmute::<_, &HashMap<u32, *const u8>>(&self.functions)
173        })?;
174        Ok(())
175    }
176}
177
178impl<C: ContextObject> PartialEq for BuiltInProgram<C> {
179    fn eq(&self, other: &Self) -> bool {
180        for ((a_key, a_function), (b_key, b_function)) in
181            self.functions.iter().zip(other.functions.iter())
182        {
183            if a_key != b_key || a_function as *const _ as usize != b_function as *const _ as usize
184            {
185                return false;
186            }
187        }
188        true
189    }
190}
191
192/// Shift the Config::runtime_environment_key by this many bits to the LSB
193///
194/// 3 bits for 8 Byte alignment, and 1 bit to have encoding space for the RuntimeEnvironment.
195pub const PROGRAM_ENVIRONMENT_KEY_SHIFT: u32 = 4;
196
197/// VM configuration settings
198#[derive(Debug, Clone, Copy, PartialEq, Eq)]
199pub struct Config {
200    /// Maximum call depth
201    pub max_call_depth: usize,
202    /// Size of a stack frame in bytes, must match the size specified in the LLVM BPF backend
203    pub stack_frame_size: usize,
204    /// Enables gaps in VM address space between the stack frames
205    pub enable_stack_frame_gaps: bool,
206    /// Maximal pc distance after which a new instruction meter validation is emitted by the JIT
207    pub instruction_meter_checkpoint_distance: usize,
208    /// Enable instruction meter and limiting
209    pub enable_instruction_meter: bool,
210    /// Enable instruction tracing
211    pub enable_instruction_tracing: bool,
212    /// Enable dynamic string allocation for labels
213    pub enable_symbol_and_section_labels: bool,
214    /// Reject ELF files containing issues that the verifier did not catch before (up to v0.2.21)
215    pub reject_broken_elfs: bool,
216    /// Ratio of native host instructions per random no-op in JIT (0 = OFF)
217    pub noop_instruction_rate: u32,
218    /// Enable disinfection of immediate values and offsets provided by the user in JIT
219    pub sanitize_user_provided_values: bool,
220    /// Encrypt the runtime environment in JIT
221    ///
222    /// Use 0 to disable encryption. Otherwise only leave PROGRAM_ENVIRONMENT_KEY_SHIFT MSBs 0.
223    pub runtime_environment_key: i32,
224    /// Throw ElfError::SymbolHashCollision when a BPF function collides with a registered syscall
225    pub external_internal_function_hash_collision: bool,
226    /// Have the verifier reject "callx r10"
227    pub reject_callx_r10: bool,
228    /// Use dynamic stack frame sizes
229    pub dynamic_stack_frames: bool,
230    /// Enable native signed division
231    pub enable_sdiv: bool,
232    /// Avoid copying read only sections when possible
233    pub optimize_rodata: bool,
234    /// Support syscalls via pseudo calls (insn.src = 0)
235    pub static_syscalls: bool,
236    /// Allow sh_addr != sh_offset in elf sections. Used in SBFv2 to align
237    /// section vaddrs to MM_PROGRAM_START.
238    pub enable_elf_vaddr: bool,
239    /// Use the new ELF parser
240    pub new_elf_parser: bool,
241    /// Ensure that rodata sections don't exceed their maximum allowed size and
242    /// overlap with the stack
243    pub reject_rodata_stack_overlap: bool,
244    /// Use aligned memory mapping
245    pub aligned_memory_mapping: bool,
246}
247
248impl Config {
249    /// Returns the size of the stack memory region
250    pub fn stack_size(&self) -> usize {
251        self.stack_frame_size * self.max_call_depth
252    }
253}
254
255impl Default for Config {
256    fn default() -> Self {
257        Self {
258            max_call_depth: 20,
259            stack_frame_size: 4_096,
260            enable_stack_frame_gaps: true,
261            instruction_meter_checkpoint_distance: 10000,
262            enable_instruction_meter: true,
263            enable_instruction_tracing: false,
264            enable_symbol_and_section_labels: false,
265            reject_broken_elfs: false,
266            noop_instruction_rate: 256,
267            sanitize_user_provided_values: true,
268            runtime_environment_key: rand::thread_rng().gen::<i32>()
269                >> PROGRAM_ENVIRONMENT_KEY_SHIFT,
270            external_internal_function_hash_collision: true,
271            reject_callx_r10: true,
272            dynamic_stack_frames: true,
273            enable_sdiv: true,
274            optimize_rodata: true,
275            static_syscalls: true,
276            enable_elf_vaddr: true,
277            new_elf_parser: true,
278            reject_rodata_stack_overlap: true,
279            aligned_memory_mapping: true,
280        }
281    }
282}
283
284/// Static constructors for Executable
285impl<C: ContextObject> Executable<C> {
286    /// Creates an executable from an ELF file
287    pub fn from_elf(elf_bytes: &[u8], loader: Arc<BuiltInProgram<C>>) -> Result<Self, EbpfError> {
288        let executable = Executable::load(elf_bytes, loader)?;
289        Ok(executable)
290    }
291    /// Creates an executable from machine code
292    pub fn from_text_bytes(
293        text_bytes: &[u8],
294        loader: Arc<BuiltInProgram<C>>,
295        function_registry: FunctionRegistry,
296    ) -> Result<Self, EbpfError> {
297        Executable::new_from_text_bytes(text_bytes, loader, function_registry)
298            .map_err(EbpfError::ElfError)
299    }
300}
301
302/// Verified executable
303#[derive(Debug, PartialEq)]
304#[repr(transparent)]
305pub struct VerifiedExecutable<V: Verifier, C: ContextObject> {
306    executable: Executable<C>,
307    _verifier: PhantomData<V>,
308}
309
310impl<V: Verifier, C: ContextObject> VerifiedExecutable<V, C> {
311    /// Verify an executable
312    pub fn from_executable(executable: Executable<C>) -> Result<Self, EbpfError> {
313        <V as Verifier>::verify(
314            executable.get_text_bytes().1,
315            executable.get_config(),
316            executable.get_function_registry(),
317        )?;
318        Ok(VerifiedExecutable {
319            executable,
320            _verifier: PhantomData,
321        })
322    }
323
324    /// JIT compile the executable
325    #[cfg(all(feature = "jit", not(target_os = "windows"), target_arch = "x86_64"))]
326    pub fn jit_compile(&mut self) -> Result<(), EbpfError> {
327        Executable::<C>::jit_compile(&mut self.executable)
328    }
329
330    /// Get a reference to the underlying executable
331    pub fn get_executable(&self) -> &Executable<C> {
332        &self.executable
333    }
334}
335
336/// Runtime context
337pub trait ContextObject {
338    /// Called for every instruction executed when tracing is enabled
339    fn trace(&mut self, state: [u64; 12]);
340    /// Consume instructions from meter
341    fn consume(&mut self, amount: u64);
342    /// Get the number of remaining instructions allowed
343    fn get_remaining(&self) -> u64;
344}
345
346/// Simple instruction meter for testing
347#[derive(Debug, Clone, Default, PartialEq, Eq)]
348pub struct TestContextObject {
349    /// Contains the register state at every instruction in order of execution
350    pub trace_log: Vec<TraceLogEntry>,
351    /// Maximal amount of instructions which still can be executed
352    pub remaining: u64,
353}
354
355impl ContextObject for TestContextObject {
356    fn trace(&mut self, state: [u64; 12]) {
357        self.trace_log.push(state);
358    }
359
360    fn consume(&mut self, amount: u64) {
361        debug_assert!(amount <= self.remaining, "Execution count exceeded");
362        self.remaining = self.remaining.saturating_sub(amount);
363    }
364
365    fn get_remaining(&self) -> u64 {
366        self.remaining
367    }
368}
369
370impl TestContextObject {
371    /// Initialize with instruction meter
372    pub fn new(remaining: u64) -> Self {
373        Self {
374            trace_log: Vec::new(),
375            remaining,
376        }
377    }
378
379    /// Compares an interpreter trace and a JIT trace.
380    ///
381    /// The log of the JIT can be longer because it only validates the instruction meter at branches.
382    pub fn compare_trace_log(interpreter: &Self, jit: &Self) -> bool {
383        let interpreter = interpreter.trace_log.as_slice();
384        let mut jit = jit.trace_log.as_slice();
385        if jit.len() > interpreter.len() {
386            jit = &jit[0..interpreter.len()];
387        }
388        interpreter == jit
389    }
390}
391
392/// Statistic of taken branches (from a recorded trace)
393pub struct DynamicAnalysis {
394    /// Maximal edge counter value
395    pub edge_counter_max: usize,
396    /// src_node, dst_node, edge_counter
397    pub edges: BTreeMap<usize, BTreeMap<usize, usize>>,
398}
399
400impl DynamicAnalysis {
401    /// Accumulates a trace
402    pub fn new(trace_log: &[[u64; 12]], analysis: &Analysis) -> Self {
403        let mut result = Self {
404            edge_counter_max: 0,
405            edges: BTreeMap::new(),
406        };
407        let mut last_basic_block = usize::MAX;
408        for traced_instruction in trace_log.iter() {
409            let pc = traced_instruction[11] as usize;
410            if analysis.cfg_nodes.contains_key(&pc) {
411                let counter = result
412                    .edges
413                    .entry(last_basic_block)
414                    .or_insert_with(BTreeMap::new)
415                    .entry(pc)
416                    .or_insert(0);
417                *counter += 1;
418                result.edge_counter_max = result.edge_counter_max.max(*counter);
419                last_basic_block = pc;
420            }
421        }
422        result
423    }
424}
425
426/// A call frame used for function calls inside the Interpreter
427#[derive(Clone, Default)]
428pub struct CallFrame {
429    /// The caller saved registers
430    pub caller_saved_registers: [u64; ebpf::SCRATCH_REGS],
431    /// The callers frame pointer
432    pub frame_pointer: u64,
433    /// The target_pc of the exit instruction which returns back to the caller
434    pub target_pc: usize,
435}
436
437/// Runtime state
438// Keep changes here in sync with RuntimeEnvironmentSlot
439#[repr(C)]
440pub struct RuntimeEnvironment<'a, C: ContextObject> {
441    /// Needed to exit from the guest back into the host
442    pub host_stack_pointer: *mut u64,
443    /// The current call depth.
444    ///
445    /// Incremented on calls and decremented on exits. It's used to enforce
446    /// config.max_call_depth and to know when to terminate execution.
447    pub call_depth: u64,
448    /// Guest stack pointer (r11).
449    ///
450    /// The stack pointer isn't exposed as an actual register. Only sub and add
451    /// instructions (typically generated by the LLVM backend) are allowed to
452    /// access it when config.dynamic_stack_frames=true. Its value is only
453    /// stored here and therefore the register is not tracked in REGISTER_MAP.
454    pub stack_pointer: u64,
455    /// Pointer to ContextObject
456    pub context_object_pointer: &'a mut C,
457    /// Last return value of instruction_meter.get_remaining()
458    pub previous_instruction_meter: u64,
459    /// CPU cycles accumulated by the stop watch
460    pub stopwatch_numerator: u64,
461    /// Number of times the stop watch was used
462    pub stopwatch_denominator: u64,
463    /// ProgramResult inlined
464    pub program_result: ProgramResult,
465    /// MemoryMapping inlined
466    pub memory_mapping: MemoryMapping<'a>,
467    /// Stack of CallFrames used by the Interpreter
468    pub call_frames: Vec<CallFrame>,
469}
470
471/// A virtual machine to run eBPF programs.
472///
473/// # Examples
474///
475/// ```
476/// use cbe_rbpf::{ebpf, elf::Executable, memory_region::MemoryRegion, vm::{Config, EbpfVm, TestContextObject, FunctionRegistry, BuiltInProgram, VerifiedExecutable}, verifier::RequisiteVerifier};
477///
478/// let prog = &[
479///     0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00  // exit
480/// ];
481/// let mem = &mut [
482///     0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd
483/// ];
484///
485/// let loader = std::sync::Arc::new(BuiltInProgram::new_loader(Config::default()));
486/// let function_registry = FunctionRegistry::default();
487/// let mut executable = Executable::<TestContextObject>::from_text_bytes(prog, loader, function_registry).unwrap();
488/// let mem_region = MemoryRegion::new_writable(mem, ebpf::MM_INPUT_START);
489/// let verified_executable = VerifiedExecutable::<RequisiteVerifier, TestContextObject>::from_executable(executable).unwrap();
490/// let mut context_object = TestContextObject::new(1);
491/// let mut vm = EbpfVm::new(&verified_executable, &mut context_object, &mut [], vec![mem_region]).unwrap();
492///
493/// let (instruction_count, result) = vm.execute_program(true);
494/// assert_eq!(instruction_count, 1);
495/// assert_eq!(result.unwrap(), 0);
496/// ```
497pub struct EbpfVm<'a, V: Verifier, C: ContextObject> {
498    pub(crate) verified_executable: &'a VerifiedExecutable<V, C>,
499    _stack: AlignedMemory<{ ebpf::HOST_ALIGN }>,
500    /// TCP port for the debugger interface
501    #[cfg(feature = "debugger")]
502    pub debug_port: Option<u16>,
503    /// Runtime state
504    pub env: RuntimeEnvironment<'a, C>,
505}
506
507impl<'a, V: Verifier, C: ContextObject> EbpfVm<'a, V, C> {
508    /// Create a new virtual machine instance
509    pub fn new(
510        verified_executable: &'a VerifiedExecutable<V, C>,
511        context_object: &'a mut C,
512        heap_region: &mut [u8],
513        additional_regions: Vec<MemoryRegion>,
514    ) -> Result<EbpfVm<'a, V, C>, EbpfError> {
515        let executable = verified_executable.get_executable();
516        let config = executable.get_config();
517        let mut stack = AlignedMemory::zero_filled(config.stack_size());
518        let stack_pointer = ebpf::MM_STACK_START.saturating_add(if config.dynamic_stack_frames {
519            // the stack is fully descending, frames start as empty and change size anytime r11 is modified
520            stack.len()
521        } else {
522            // within a frame the stack grows down, but frames are ascending
523            config.stack_frame_size
524        } as u64);
525        let regions: Vec<MemoryRegion> = vec![
526            verified_executable.get_executable().get_ro_region(),
527            MemoryRegion::new_writable_gapped(
528                stack.as_slice_mut(),
529                ebpf::MM_STACK_START,
530                if !config.dynamic_stack_frames && config.enable_stack_frame_gaps {
531                    config.stack_frame_size as u64
532                } else {
533                    0
534                },
535            ),
536            MemoryRegion::new_writable(heap_region, ebpf::MM_HEAP_START),
537        ]
538        .into_iter()
539        .chain(additional_regions.into_iter())
540        .collect();
541        let vm = EbpfVm {
542            verified_executable,
543            _stack: stack,
544            #[cfg(feature = "debugger")]
545            debug_port: None,
546            env: RuntimeEnvironment {
547                host_stack_pointer: std::ptr::null_mut(),
548                call_depth: 0,
549                stack_pointer,
550                context_object_pointer: context_object,
551                previous_instruction_meter: 0,
552                stopwatch_numerator: 0,
553                stopwatch_denominator: 0,
554                program_result: ProgramResult::Ok(0),
555                memory_mapping: MemoryMapping::new(regions, config)?,
556                call_frames: vec![CallFrame::default(); config.max_call_depth],
557            },
558        };
559        Ok(vm)
560    }
561
562    /// Execute the program
563    ///
564    /// If interpreted = `false` then the JIT compiled executable is used.
565    pub fn execute_program(&mut self, interpreted: bool) -> (u64, ProgramResult) {
566        let mut registers = [0u64; 11];
567        // R1 points to beginning of input memory, R10 to the stack of the first frame
568        registers[1] = ebpf::MM_INPUT_START;
569        registers[ebpf::FRAME_PTR_REG] = self.env.stack_pointer;
570        let executable = self.verified_executable.get_executable();
571        let target_pc = executable.get_entrypoint_instruction_offset();
572        let config = executable.get_config();
573        let initial_insn_count = if config.enable_instruction_meter {
574            self.env.context_object_pointer.get_remaining()
575        } else {
576            0
577        };
578        self.env.previous_instruction_meter = initial_insn_count;
579        self.env.program_result = ProgramResult::Ok(0);
580        let due_insn_count = if interpreted {
581            #[cfg(feature = "debugger")]
582            let debug_port = self.debug_port.clone();
583            let mut interpreter = match Interpreter::new(self, registers, target_pc) {
584                Ok(interpreter) => interpreter,
585                Err(error) => return (0, ProgramResult::Err(error)),
586            };
587            #[cfg(feature = "debugger")]
588            if let Some(debug_port) = debug_port {
589                crate::debugger::execute(&mut interpreter, debug_port);
590            } else {
591                while interpreter.step() {}
592            }
593            #[cfg(not(feature = "debugger"))]
594            while interpreter.step() {}
595            interpreter.due_insn_count
596        } else {
597            #[cfg(all(feature = "jit", not(target_os = "windows"), target_arch = "x86_64"))]
598            {
599                let compiled_program = match executable
600                    .get_compiled_program()
601                    .ok_or(EbpfError::JitNotCompiled)
602                {
603                    Ok(compiled_program) => compiled_program,
604                    Err(error) => return (0, ProgramResult::Err(error)),
605                };
606                let instruction_meter_final = compiled_program
607                    .invoke(config, &mut self.env, registers, target_pc)
608                    .max(0) as u64;
609                self.env
610                    .context_object_pointer
611                    .get_remaining()
612                    .saturating_sub(instruction_meter_final)
613            }
614            #[cfg(not(all(feature = "jit", not(target_os = "windows"), target_arch = "x86_64")))]
615            {
616                return (0, ProgramResult::Err(EbpfError::JitNotCompiled));
617            }
618        };
619        let instruction_count = if config.enable_instruction_meter {
620            self.env.context_object_pointer.consume(due_insn_count);
621            initial_insn_count.saturating_sub(self.env.context_object_pointer.get_remaining())
622        } else {
623            0
624        };
625        if let ProgramResult::Err(EbpfError::ExceededMaxInstructions(pc, _)) =
626            self.env.program_result
627        {
628            self.env.program_result =
629                ProgramResult::Err(EbpfError::ExceededMaxInstructions(pc, initial_insn_count));
630        }
631        let mut result = ProgramResult::Ok(0);
632        std::mem::swap(&mut result, &mut self.env.program_result);
633        (instruction_count, result)
634    }
635}
636
637#[cfg(test)]
638mod tests {
639    use super::*;
640
641    #[test]
642    fn test_program_result_is_stable() {
643        let ok = ProgramResult::Ok(42);
644        assert_eq!(unsafe { *(&ok as *const _ as *const u64) }, 0);
645        let err = ProgramResult::Err(EbpfError::JitNotCompiled);
646        assert_eq!(unsafe { *(&err as *const _ as *const u64) }, 1);
647    }
648}
cbe_rbpf/vm.rs

cbe_rbpf/
vm.rs
