#![doc(
html_logo_url = "https://raw.githubusercontent.com/qmonnet/rbpf/main/misc/rbpf.png",
html_favicon_url = "https://raw.githubusercontent.com/qmonnet/rbpf/main/misc/rbpf.ico"
)]
#![doc = include_str!("../README.md")]
#![cfg_attr(not(feature = "std"), no_std)]
extern crate byteorder;
extern crate combine;
extern crate log;
#[cfg(not(feature = "std"))]
extern crate alloc;
#[cfg(feature = "cranelift")]
extern crate cranelift_codegen;
#[cfg(feature = "cranelift")]
extern crate cranelift_frontend;
#[cfg(feature = "cranelift")]
extern crate cranelift_jit;
#[cfg(feature = "cranelift")]
extern crate cranelift_module;
#[cfg(feature = "cranelift")]
extern crate cranelift_native;
use crate::lib::*;
use byteorder::{ByteOrder, LittleEndian};
use core::ops::Range;
use stack::{StackUsage, StackVerifier};
mod asm_parser;
pub mod assembler;
#[cfg(feature = "cranelift")]
mod cranelift;
pub mod disassembler;
pub mod ebpf;
pub mod helpers;
pub mod insn_builder;
mod interpreter;
#[cfg(not(windows))]
mod jit;
#[cfg(not(feature = "std"))]
mod no_std_error;
mod stack;
mod verifier;
pub mod lib {
mod core {
#[cfg(not(feature = "std"))]
pub use core::*;
#[cfg(feature = "std")]
pub use std::*;
}
pub use self::core::any::Any;
pub use self::core::convert::TryInto;
pub use self::core::f64;
pub use self::core::mem;
pub use self::core::mem::ManuallyDrop;
pub use self::core::ptr;
pub use hashbrown::{HashMap, HashSet};
#[cfg(feature = "std")]
pub use std::println;
#[cfg(not(feature = "std"))]
pub use alloc::vec;
#[cfg(not(feature = "std"))]
pub use alloc::vec::Vec;
#[cfg(feature = "std")]
pub use std::vec;
#[cfg(feature = "std")]
pub use std::vec::Vec;
#[cfg(not(feature = "std"))]
pub use alloc::boxed::Box;
#[cfg(feature = "std")]
pub use std::boxed::Box;
#[cfg(not(feature = "std"))]
pub use alloc::string::{String, ToString};
#[cfg(feature = "std")]
pub use std::string::{String, ToString};
#[cfg(not(feature = "std"))]
pub use alloc::collections::BTreeMap;
#[cfg(feature = "std")]
pub use std::collections::BTreeMap;
#[cfg(not(feature = "std"))]
pub use crate::no_std_error::{Error, ErrorKind};
#[cfg(feature = "std")]
pub use std::io::{Error, ErrorKind};
#[cfg(not(feature = "std"))]
pub use alloc::format;
#[cfg(feature = "std")]
pub use std::format;
}
pub type Verifier = fn(prog: &[u8]) -> Result<(), Error>;
pub type Helper = fn(u64, u64, u64, u64, u64) -> u64;
pub type StackUsageCalculator = fn(prog: &[u8], pc: usize, data: &mut dyn Any) -> u16;
struct MetaBuff {
data_offset: usize,
data_end_offset: usize,
buffer: Vec<u8>,
}
pub struct EbpfVmMbuff<'a> {
prog: Option<&'a [u8]>,
verifier: Verifier,
#[cfg(not(windows))]
jit: Option<jit::JitMemory<'a>>,
#[cfg(all(not(windows), not(feature = "std")))]
custom_exec_memory: Option<&'a mut [u8]>,
#[cfg(feature = "cranelift")]
cranelift_prog: Option<cranelift::CraneliftProgram>,
helpers: HashMap<u32, ebpf::Helper>,
allowed_memory: HashSet<Range<u64>>,
stack_usage: Option<StackUsage>,
stack_verifier: StackVerifier,
}
impl<'a> EbpfVmMbuff<'a> {
pub fn new(prog: Option<&'a [u8]>) -> Result<EbpfVmMbuff<'a>, Error> {
let mut stack_verifier = StackVerifier::new(None, None);
let stack_usage = if let Some(prog) = prog {
verifier::check(prog)?;
Some(stack_verifier.stack_validate(prog)?)
} else {
None
};
Ok(EbpfVmMbuff {
prog,
verifier: verifier::check,
#[cfg(not(windows))]
jit: None,
#[cfg(all(not(windows), not(feature = "std")))]
custom_exec_memory: None,
#[cfg(feature = "cranelift")]
cranelift_prog: None,
helpers: HashMap::new(),
allowed_memory: HashSet::new(),
stack_usage,
stack_verifier,
})
}
pub fn set_program(&mut self, prog: &'a [u8]) -> Result<(), Error> {
(self.verifier)(prog)?;
let stack_usage = self.stack_verifier.stack_validate(prog)?;
self.prog = Some(prog);
self.stack_usage = Some(stack_usage);
Ok(())
}
pub fn set_verifier(&mut self, verifier: Verifier) -> Result<(), Error> {
if let Some(prog) = self.prog {
verifier(prog)?;
}
self.verifier = verifier;
Ok(())
}
pub fn set_stack_usage_calculator(
&mut self,
calculator: StackUsageCalculator,
data: Box<dyn Any>,
) -> Result<(), Error> {
let mut stack_verifier = StackVerifier::new(Some(calculator), Some(data));
if let Some(prog) = self.prog {
self.stack_usage = Some(stack_verifier.stack_validate(prog)?);
}
self.stack_verifier = stack_verifier;
Ok(())
}
#[cfg(all(not(windows), not(feature = "std")))]
pub fn set_jit_exec_memory(&mut self, memory: &'a mut [u8]) -> Result<(), Error> {
self.custom_exec_memory = Some(memory);
Ok(())
}
pub fn register_helper(&mut self, key: u32, function: Helper) -> Result<(), Error> {
self.helpers.insert(key, function);
Ok(())
}
pub fn register_allowed_memory(&mut self, addrs_range: Range<u64>) {
self.allowed_memory.insert(addrs_range);
}
pub fn execute_program(&self, mem: &[u8], mbuff: &[u8]) -> Result<u64, Error> {
let stack_usage = self.stack_usage.as_ref();
interpreter::execute_program(
self.prog,
stack_usage,
mem,
mbuff,
&self.helpers,
&self.allowed_memory,
)
}
#[cfg(not(windows))]
pub fn jit_compile(&mut self) -> Result<(), Error> {
let prog = match self.prog {
Some(prog) => prog,
None => Err(Error::other(
"Error: No program set, call prog_set() to load one",
))?,
};
#[cfg(feature = "std")]
{
self.jit = Some(jit::JitMemory::new(prog, &self.helpers, true, false)?);
}
#[cfg(not(feature = "std"))]
{
let exec_memory = match self.custom_exec_memory.take() {
Some(memory) => memory,
None => return Err(Error::new(
ErrorKind::Other,
"Error: No custom executable memory set, call set_jit_exec_memory() to set one",
))?,
};
self.jit = Some(jit::JitMemory::new(
prog,
exec_memory,
&self.helpers,
true,
false,
)?);
}
Ok(())
}
#[cfg(not(windows))]
pub unsafe fn execute_program_jit(
&self,
mem: &mut [u8],
mbuff: &'a mut [u8],
) -> Result<u64, Error> {
let mem_ptr = match mem.len() {
0 => core::ptr::null_mut(),
_ => mem.as_ptr() as *mut u8,
};
unsafe {
match &self.jit {
Some(jit) => Ok(jit.get_prog()(
mbuff.as_ptr() as *mut u8,
mbuff.len(),
mem_ptr,
mem.len(),
0,
0,
)),
None => Err(Error::other("Error: program has not been JIT-compiled")),
}
}
}
#[cfg(feature = "cranelift")]
pub fn cranelift_compile(&mut self) -> Result<(), Error> {
use crate::cranelift::CraneliftCompiler;
let prog = match self.prog {
Some(prog) => prog,
None => Err(Error::new(
ErrorKind::Other,
"Error: No program set, call prog_set() to load one",
))?,
};
let compiler = CraneliftCompiler::new(self.helpers.clone());
let program = compiler.compile_function(prog)?;
self.cranelift_prog = Some(program);
Ok(())
}
#[cfg(feature = "cranelift")]
pub fn execute_program_cranelift(
&self,
mem: &mut [u8],
mbuff: &'a mut [u8],
) -> Result<u64, Error> {
let mem_ptr = match mem.len() {
0 => ptr::null_mut(),
_ => mem.as_ptr() as *mut u8,
};
match &self.cranelift_prog {
Some(prog) => {
Ok(prog.execute(mem_ptr, mem.len(), mbuff.as_ptr() as *mut u8, mbuff.len()))
}
None => Err(Error::new(
ErrorKind::Other,
"Error: program has not been compiled with cranelift",
)),
}
}
}
pub struct EbpfVmFixedMbuff<'a> {
parent: EbpfVmMbuff<'a>,
mbuff: MetaBuff,
}
impl<'a> EbpfVmFixedMbuff<'a> {
pub fn new(
prog: Option<&'a [u8]>,
data_offset: usize,
data_end_offset: usize,
) -> Result<EbpfVmFixedMbuff<'a>, Error> {
let parent = EbpfVmMbuff::new(prog)?;
let get_buff_len = |x: usize, y: usize| if x >= y { x + 8 } else { y + 8 };
let buffer = vec![0u8; get_buff_len(data_offset, data_end_offset)];
let mbuff = MetaBuff {
data_offset,
data_end_offset,
buffer,
};
Ok(EbpfVmFixedMbuff { parent, mbuff })
}
pub fn set_program(
&mut self,
prog: &'a [u8],
data_offset: usize,
data_end_offset: usize,
) -> Result<(), Error> {
let get_buff_len = |x: usize, y: usize| if x >= y { x + 8 } else { y + 8 };
let buffer = vec![0u8; get_buff_len(data_offset, data_end_offset)];
self.mbuff.buffer = buffer;
self.mbuff.data_offset = data_offset;
self.mbuff.data_end_offset = data_end_offset;
self.parent.set_program(prog)?;
Ok(())
}
pub fn set_verifier(&mut self, verifier: Verifier) -> Result<(), Error> {
self.parent.set_verifier(verifier)
}
pub fn set_stack_usage_calculator(
&mut self,
calculator: StackUsageCalculator,
data: Box<dyn Any>,
) -> Result<(), Error> {
self.parent.set_stack_usage_calculator(calculator, data)
}
#[cfg(all(not(windows), not(feature = "std")))]
pub fn set_jit_exec_memory(&mut self, memory: &'a mut [u8]) -> Result<(), Error> {
self.parent.custom_exec_memory = Some(memory);
Ok(())
}
pub fn register_helper(
&mut self,
key: u32,
function: fn(u64, u64, u64, u64, u64) -> u64,
) -> Result<(), Error> {
self.parent.register_helper(key, function)
}
pub fn register_allowed_memory(&mut self, addrs_range: Range<u64>) {
self.parent.register_allowed_memory(addrs_range)
}
pub fn execute_program(&mut self, mem: &'a mut [u8]) -> Result<u64, Error> {
let l = self.mbuff.buffer.len();
if self.mbuff.data_offset + 8 > l || self.mbuff.data_end_offset + 8 > l {
Err(Error::other(format!("Error: buffer too small ({:?}), cannot use data_offset {:?} and data_end_offset {:?}",
l, self.mbuff.data_offset, self.mbuff.data_end_offset)))?;
}
LittleEndian::write_u64(
&mut self.mbuff.buffer[(self.mbuff.data_offset)..],
mem.as_ptr() as u64,
);
LittleEndian::write_u64(
&mut self.mbuff.buffer[(self.mbuff.data_end_offset)..],
mem.as_ptr() as u64 + mem.len() as u64,
);
self.parent.execute_program(mem, &self.mbuff.buffer)
}
#[cfg(not(windows))]
pub fn jit_compile(&mut self) -> Result<(), Error> {
let prog = match self.parent.prog {
Some(prog) => prog,
None => Err(Error::other(
"Error: No program set, call prog_set() to load one",
))?,
};
#[cfg(feature = "std")]
{
self.parent.jit = Some(jit::JitMemory::new(prog, &self.parent.helpers, true, true)?);
}
#[cfg(not(feature = "std"))]
{
let exec_memory = match self.parent.custom_exec_memory.take() {
Some(memory) => memory,
None => return Err(Error::new(
ErrorKind::Other,
"Error: No custom executable memory set, call set_jit_exec_memory() to set one",
))?,
};
self.parent.jit = Some(jit::JitMemory::new(
prog,
exec_memory,
&self.parent.helpers,
true,
true,
)?);
}
Ok(())
}
#[cfg(not(windows))]
pub unsafe fn execute_program_jit(&mut self, mem: &'a mut [u8]) -> Result<u64, Error> {
let mem_ptr = match mem.len() {
0 => ptr::null_mut(),
_ => mem.as_ptr() as *mut u8,
};
unsafe {
match &self.parent.jit {
Some(jit) => Ok(jit.get_prog()(
self.mbuff.buffer.as_ptr() as *mut u8,
self.mbuff.buffer.len(),
mem_ptr,
mem.len(),
self.mbuff.data_offset,
self.mbuff.data_end_offset,
)),
None => Err(Error::other("Error: program has not been JIT-compiled")),
}
}
}
#[cfg(feature = "cranelift")]
pub fn cranelift_compile(&mut self) -> Result<(), Error> {
use crate::cranelift::CraneliftCompiler;
let prog = match self.parent.prog {
Some(prog) => prog,
None => Err(Error::new(
ErrorKind::Other,
"Error: No program set, call prog_set() to load one",
))?,
};
let compiler = CraneliftCompiler::new(self.parent.helpers.clone());
let program = compiler.compile_function(prog)?;
self.parent.cranelift_prog = Some(program);
Ok(())
}
#[cfg(feature = "cranelift")]
pub fn execute_program_cranelift(&mut self, mem: &'a mut [u8]) -> Result<u64, Error> {
let mem_ptr = match mem.len() {
0 => ptr::null_mut(),
_ => mem.as_ptr() as *mut u8,
};
let l = self.mbuff.buffer.len();
if self.mbuff.data_offset + 8 > l || self.mbuff.data_end_offset + 8 > l {
Err(Error::new(ErrorKind::Other, format!("Error: buffer too small ({:?}), cannot use data_offset {:?} and data_end_offset {:?}",
l, self.mbuff.data_offset, self.mbuff.data_end_offset)))?;
}
LittleEndian::write_u64(
&mut self.mbuff.buffer[(self.mbuff.data_offset)..],
mem.as_ptr() as u64,
);
LittleEndian::write_u64(
&mut self.mbuff.buffer[(self.mbuff.data_end_offset)..],
mem.as_ptr() as u64 + mem.len() as u64,
);
match &self.parent.cranelift_prog {
Some(prog) => Ok(prog.execute(
mem_ptr,
mem.len(),
self.mbuff.buffer.as_ptr() as *mut u8,
self.mbuff.buffer.len(),
)),
None => Err(Error::new(
ErrorKind::Other,
"Error: program has not been compiled with cranelift",
)),
}
}
}
pub struct EbpfVmRaw<'a> {
parent: EbpfVmMbuff<'a>,
}
impl<'a> EbpfVmRaw<'a> {
pub fn new(prog: Option<&'a [u8]>) -> Result<EbpfVmRaw<'a>, Error> {
let parent = EbpfVmMbuff::new(prog)?;
Ok(EbpfVmRaw { parent })
}
pub fn set_program(&mut self, prog: &'a [u8]) -> Result<(), Error> {
self.parent.set_program(prog)?;
Ok(())
}
pub fn set_verifier(&mut self, verifier: Verifier) -> Result<(), Error> {
self.parent.set_verifier(verifier)
}
pub fn set_stack_usage_calculator(
&mut self,
calculator: StackUsageCalculator,
data: Box<dyn Any>,
) -> Result<(), Error> {
self.parent.set_stack_usage_calculator(calculator, data)
}
#[cfg(all(not(windows), not(feature = "std")))]
pub fn set_jit_exec_memory(&mut self, memory: &'a mut [u8]) -> Result<(), Error> {
self.parent.custom_exec_memory = Some(memory);
Ok(())
}
pub fn register_helper(
&mut self,
key: u32,
function: fn(u64, u64, u64, u64, u64) -> u64,
) -> Result<(), Error> {
self.parent.register_helper(key, function)
}
pub fn register_allowed_memory(&mut self, addrs_range: Range<u64>) {
self.parent.register_allowed_memory(addrs_range)
}
pub fn execute_program(&self, mem: &'a mut [u8]) -> Result<u64, Error> {
self.parent.execute_program(mem, &[])
}
#[cfg(not(windows))]
pub fn jit_compile(&mut self) -> Result<(), Error> {
let prog = match self.parent.prog {
Some(prog) => prog,
None => Err(Error::other(
"Error: No program set, call prog_set() to load one",
))?,
};
#[cfg(feature = "std")]
{
self.parent.jit = Some(jit::JitMemory::new(
prog,
&self.parent.helpers,
false,
false,
)?);
}
#[cfg(not(feature = "std"))]
{
let exec_memory = match self.parent.custom_exec_memory.take() {
Some(memory) => memory,
None => return Err(Error::new(
ErrorKind::Other,
"Error: No custom executable memory set, call set_jit_exec_memory() to set one",
))?,
};
self.parent.jit = Some(jit::JitMemory::new(
prog,
exec_memory,
&self.parent.helpers,
false,
false,
)?);
}
Ok(())
}
#[cfg(not(windows))]
pub unsafe fn execute_program_jit(&self, mem: &'a mut [u8]) -> Result<u64, Error> {
let mut mbuff = vec![];
unsafe { self.parent.execute_program_jit(mem, &mut mbuff) }
}
#[cfg(feature = "cranelift")]
pub fn cranelift_compile(&mut self) -> Result<(), Error> {
use crate::cranelift::CraneliftCompiler;
let prog = match self.parent.prog {
Some(prog) => prog,
None => Err(Error::new(
ErrorKind::Other,
"Error: No program set, call prog_set() to load one",
))?,
};
let compiler = CraneliftCompiler::new(self.parent.helpers.clone());
let program = compiler.compile_function(prog)?;
self.parent.cranelift_prog = Some(program);
Ok(())
}
#[cfg(feature = "cranelift")]
pub fn execute_program_cranelift(&self, mem: &'a mut [u8]) -> Result<u64, Error> {
let mut mbuff = vec![];
self.parent.execute_program_cranelift(mem, &mut mbuff)
}
}
pub struct EbpfVmNoData<'a> {
parent: EbpfVmRaw<'a>,
}
impl<'a> EbpfVmNoData<'a> {
pub fn new(prog: Option<&'a [u8]>) -> Result<EbpfVmNoData<'a>, Error> {
let parent = EbpfVmRaw::new(prog)?;
Ok(EbpfVmNoData { parent })
}
pub fn set_program(&mut self, prog: &'a [u8]) -> Result<(), Error> {
self.parent.set_program(prog)?;
Ok(())
}
pub fn set_verifier(&mut self, verifier: Verifier) -> Result<(), Error> {
self.parent.set_verifier(verifier)
}
pub fn set_stack_usage_calculator(
&mut self,
calculator: StackUsageCalculator,
data: Box<dyn Any>,
) -> Result<(), Error> {
self.parent.set_stack_usage_calculator(calculator, data)
}
#[cfg(all(not(windows), not(feature = "std")))]
pub fn set_jit_exec_memory(&mut self, memory: &'a mut [u8]) -> Result<(), Error> {
self.parent.set_jit_exec_memory(memory)
}
pub fn register_helper(
&mut self,
key: u32,
function: fn(u64, u64, u64, u64, u64) -> u64,
) -> Result<(), Error> {
self.parent.register_helper(key, function)
}
pub fn register_allowed_memory(&mut self, addrs_range: Range<u64>) {
self.parent.register_allowed_memory(addrs_range)
}
#[cfg(not(windows))]
pub fn jit_compile(&mut self) -> Result<(), Error> {
self.parent.jit_compile()
}
pub fn execute_program(&self) -> Result<u64, Error> {
self.parent.execute_program(&mut [])
}
#[cfg(not(windows))]
pub unsafe fn execute_program_jit(&self) -> Result<u64, Error> {
unsafe { self.parent.execute_program_jit(&mut []) }
}
#[cfg(feature = "cranelift")]
pub fn cranelift_compile(&mut self) -> Result<(), Error> {
self.parent.cranelift_compile()
}
#[cfg(feature = "cranelift")]
pub fn execute_program_cranelift(&self) -> Result<u64, Error> {
self.parent.execute_program_cranelift(&mut [])
}
}