pub mod cgroup_skb;
pub mod cgroup_sock_addr;
pub mod cgroup_sockopt;
pub mod cgroup_sysctl;
pub mod extension;
pub mod fentry;
pub mod fexit;
pub mod kprobe;
pub mod links;
pub mod lirc_mode2;
pub mod lsm;
pub mod perf_attach;
pub mod perf_event;
mod probe;
mod raw_trace_point;
mod sk_lookup;
mod sk_msg;
mod sk_skb;
mod sock_ops;
mod socket_filter;
pub mod tc;
pub mod tp_btf;
pub mod trace_point;
pub mod uprobe;
mod utils;
pub mod xdp;
use libc::ENOSPC;
use std::{
convert::TryFrom,
ffi::CString,
io,
os::unix::io::{AsRawFd, RawFd},
path::Path,
};
use thiserror::Error;
pub use cgroup_skb::{CgroupSkb, CgroupSkbAttachType};
pub use cgroup_sock_addr::{CgroupSockAddr, CgroupSockAddrAttachType};
pub use cgroup_sockopt::{CgroupSockopt, CgroupSockoptAttachType};
pub use cgroup_sysctl::CgroupSysctl;
pub use extension::{Extension, ExtensionError};
pub use fentry::FEntry;
pub use fexit::FExit;
pub use kprobe::{KProbe, KProbeError};
use links::*;
pub use links::{Link, OwnedLink};
pub use lirc_mode2::LircMode2;
pub use lsm::Lsm;
use perf_attach::*;
pub use perf_event::{PerfEvent, PerfEventScope, PerfTypeId, SamplePolicy};
pub use probe::ProbeKind;
pub use raw_trace_point::RawTracePoint;
pub use sk_lookup::SkLookup;
pub use sk_msg::SkMsg;
pub use sk_skb::{SkSkb, SkSkbKind};
pub use sock_ops::SockOps;
pub use socket_filter::{SocketFilter, SocketFilterError};
pub use tc::{SchedClassifier, TcAttachType, TcError};
pub use tp_btf::BtfTracePoint;
pub use trace_point::{TracePoint, TracePointError};
pub use uprobe::{UProbe, UProbeError};
pub use xdp::{Xdp, XdpError, XdpFlags};
use crate::{
generated::{bpf_attach_type, bpf_prog_info, bpf_prog_type},
maps::MapError,
obj::{self, btf::BtfError, Function, KernelVersion},
sys::{
bpf_get_object, bpf_load_program, bpf_obj_get_info_by_fd, bpf_pin_object,
bpf_prog_get_fd_by_id, bpf_prog_query, retry_with_verifier_logs, BpfLoadProgramAttrs,
},
util::VerifierLog,
};
#[derive(Debug, Error)]
pub enum ProgramError {
#[error("the program is already loaded")]
AlreadyLoaded,
#[error("the program is not loaded")]
NotLoaded,
#[error("the program was already attached")]
AlreadyAttached,
#[error("the program is not attached")]
NotAttached,
#[error("the BPF_PROG_LOAD syscall failed. Verifier output: {verifier_log}")]
LoadError {
#[source]
io_error: io::Error,
verifier_log: String,
},
#[error("`{call}` failed")]
SyscallError {
call: String,
#[source]
io_error: io::Error,
},
#[error("unknown network interface {name}")]
UnknownInterface {
name: String,
},
#[error("unexpected program type")]
UnexpectedProgramType,
#[error("invalid pin path `{error}`")]
InvalidPinPath {
error: String,
},
#[error(transparent)]
MapError(#[from] MapError),
#[error(transparent)]
KProbeError(#[from] KProbeError),
#[error(transparent)]
UProbeError(#[from] UProbeError),
#[error(transparent)]
TracePointError(#[from] TracePointError),
#[error(transparent)]
SocketFilterError(#[from] SocketFilterError),
#[error(transparent)]
XdpError(#[from] XdpError),
#[error(transparent)]
TcError(#[from] TcError),
#[error(transparent)]
ExtensionError(#[from] ExtensionError),
#[error(transparent)]
Btf(#[from] BtfError),
#[error("the program name `{name}` is invalid")]
InvalidName {
name: String,
},
}
pub trait ProgramFd {
fn fd(&self) -> Option<RawFd>;
}
#[derive(Debug)]
pub enum Program {
KProbe(KProbe),
UProbe(UProbe),
TracePoint(TracePoint),
SocketFilter(SocketFilter),
Xdp(Xdp),
SkMsg(SkMsg),
SkSkb(SkSkb),
CgroupSockAddr(CgroupSockAddr),
SockOps(SockOps),
SchedClassifier(SchedClassifier),
CgroupSkb(CgroupSkb),
CgroupSysctl(CgroupSysctl),
CgroupSockopt(CgroupSockopt),
LircMode2(LircMode2),
PerfEvent(PerfEvent),
RawTracePoint(RawTracePoint),
Lsm(Lsm),
BtfTracePoint(BtfTracePoint),
FEntry(FEntry),
FExit(FExit),
Extension(Extension),
SkLookup(SkLookup),
}
impl Program {
pub fn prog_type(&self) -> bpf_prog_type {
use crate::generated::bpf_prog_type::*;
match self {
Program::KProbe(_) => BPF_PROG_TYPE_KPROBE,
Program::UProbe(_) => BPF_PROG_TYPE_KPROBE,
Program::TracePoint(_) => BPF_PROG_TYPE_TRACEPOINT,
Program::SocketFilter(_) => BPF_PROG_TYPE_SOCKET_FILTER,
Program::Xdp(_) => BPF_PROG_TYPE_XDP,
Program::SkMsg(_) => BPF_PROG_TYPE_SK_MSG,
Program::SkSkb(_) => BPF_PROG_TYPE_SK_SKB,
Program::SockOps(_) => BPF_PROG_TYPE_SOCK_OPS,
Program::SchedClassifier(_) => BPF_PROG_TYPE_SCHED_CLS,
Program::CgroupSkb(_) => BPF_PROG_TYPE_CGROUP_SKB,
Program::CgroupSysctl(_) => BPF_PROG_TYPE_CGROUP_SYSCTL,
Program::CgroupSockopt(_) => BPF_PROG_TYPE_CGROUP_SOCKOPT,
Program::LircMode2(_) => BPF_PROG_TYPE_LIRC_MODE2,
Program::PerfEvent(_) => BPF_PROG_TYPE_PERF_EVENT,
Program::RawTracePoint(_) => BPF_PROG_TYPE_RAW_TRACEPOINT,
Program::Lsm(_) => BPF_PROG_TYPE_LSM,
Program::BtfTracePoint(_) => BPF_PROG_TYPE_TRACING,
Program::FEntry(_) => BPF_PROG_TYPE_TRACING,
Program::FExit(_) => BPF_PROG_TYPE_TRACING,
Program::Extension(_) => BPF_PROG_TYPE_EXT,
Program::CgroupSockAddr(_) => BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
Program::SkLookup(_) => BPF_PROG_TYPE_SK_LOOKUP,
}
}
pub fn pin<P: AsRef<Path>>(&mut self, path: P) -> Result<(), ProgramError> {
match self {
Program::KProbe(p) => p.data.pin(path),
Program::UProbe(p) => p.data.pin(path),
Program::TracePoint(p) => p.data.pin(path),
Program::SocketFilter(p) => p.data.pin(path),
Program::Xdp(p) => p.data.pin(path),
Program::SkMsg(p) => p.data.pin(path),
Program::SkSkb(p) => p.data.pin(path),
Program::SockOps(p) => p.data.pin(path),
Program::SchedClassifier(p) => p.data.pin(path),
Program::CgroupSkb(p) => p.data.pin(path),
Program::CgroupSysctl(p) => p.data.pin(path),
Program::CgroupSockopt(p) => p.data.pin(path),
Program::LircMode2(p) => p.data.pin(path),
Program::PerfEvent(p) => p.data.pin(path),
Program::RawTracePoint(p) => p.data.pin(path),
Program::Lsm(p) => p.data.pin(path),
Program::BtfTracePoint(p) => p.data.pin(path),
Program::FEntry(p) => p.data.pin(path),
Program::FExit(p) => p.data.pin(path),
Program::Extension(p) => p.data.pin(path),
Program::CgroupSockAddr(p) => p.data.pin(path),
Program::SkLookup(p) => p.data.pin(path),
}
}
}
#[derive(Debug)]
pub(crate) struct ProgramData<T: Link> {
pub(crate) name: Option<String>,
pub(crate) obj: obj::Program,
pub(crate) fd: Option<RawFd>,
pub(crate) links: LinkMap<T>,
pub(crate) expected_attach_type: Option<bpf_attach_type>,
pub(crate) attach_btf_obj_fd: Option<u32>,
pub(crate) attach_btf_id: Option<u32>,
pub(crate) attach_prog_fd: Option<RawFd>,
pub(crate) btf_fd: Option<RawFd>,
}
impl<T: Link> ProgramData<T> {
pub(crate) fn new(
name: Option<String>,
obj: obj::Program,
btf_fd: Option<RawFd>,
) -> ProgramData<T> {
ProgramData {
name,
obj,
fd: None,
links: LinkMap::new(),
expected_attach_type: None,
attach_btf_obj_fd: None,
attach_btf_id: None,
attach_prog_fd: None,
btf_fd,
}
}
}
impl<T: Link> ProgramData<T> {
fn fd_or_err(&self) -> Result<RawFd, ProgramError> {
self.fd.ok_or(ProgramError::NotLoaded)
}
pub fn pin<P: AsRef<Path>>(&mut self, path: P) -> Result<(), ProgramError> {
let fd = self.fd_or_err()?;
let path_string =
CString::new(path.as_ref().to_string_lossy().into_owned()).map_err(|e| {
MapError::InvalidPinPath {
error: e.to_string(),
}
})?;
bpf_pin_object(fd, &path_string).map_err(|(_code, io_error)| {
ProgramError::SyscallError {
call: "BPF_OBJ_PIN".to_string(),
io_error,
}
})?;
Ok(())
}
pub(crate) fn take_link(&mut self, link_id: T::Id) -> Result<T, ProgramError> {
self.links.forget(link_id)
}
}
fn unload_program<T: Link>(data: &mut ProgramData<T>) -> Result<(), ProgramError> {
data.links.remove_all()?;
let fd = data.fd.take().ok_or(ProgramError::NotLoaded)?;
unsafe {
libc::close(fd);
}
Ok(())
}
fn load_program<T: Link>(
prog_type: bpf_prog_type,
data: &mut ProgramData<T>,
) -> Result<(), ProgramError> {
let ProgramData { obj, fd, .. } = data;
if fd.is_some() {
return Err(ProgramError::AlreadyLoaded);
}
let crate::obj::Program {
function:
Function {
instructions,
func_info,
line_info,
func_info_rec_size,
line_info_rec_size,
..
},
license,
kernel_version,
..
} = obj;
let target_kernel_version = match *kernel_version {
KernelVersion::Any => {
let (major, minor, patch) = crate::sys::kernel_version().unwrap();
(major << 16) + (minor << 8) + patch
}
_ => (*kernel_version).into(),
};
let mut logger = VerifierLog::new();
let prog_name = if let Some(name) = &data.name {
let mut name = name.clone();
if name.len() > 15 {
name.truncate(15);
}
let prog_name = CString::new(name.clone())
.map_err(|_| ProgramError::InvalidName { name: name.clone() })?;
Some(prog_name)
} else {
None
};
let attr = BpfLoadProgramAttrs {
name: prog_name,
ty: prog_type,
insns: instructions,
license,
kernel_version: target_kernel_version,
expected_attach_type: data.expected_attach_type,
prog_btf_fd: data.btf_fd,
attach_btf_obj_fd: data.attach_btf_obj_fd,
attach_btf_id: data.attach_btf_id,
attach_prog_fd: data.attach_prog_fd,
func_info_rec_size: *func_info_rec_size,
func_info: func_info.clone(),
line_info_rec_size: *line_info_rec_size,
line_info: line_info.clone(),
};
let ret = retry_with_verifier_logs(10, &mut logger, |logger| bpf_load_program(&attr, logger));
match ret {
Ok(prog_fd) => {
*fd = Some(prog_fd as RawFd);
Ok(())
}
Err((_, io_error)) => {
logger.truncate();
return Err(ProgramError::LoadError {
io_error,
verifier_log: logger
.as_c_str()
.map(|s| s.to_string_lossy().to_string())
.unwrap_or_else(|| "[none]".to_owned()),
});
}
}
}
pub(crate) fn query<T: AsRawFd>(
target_fd: T,
attach_type: bpf_attach_type,
query_flags: u32,
attach_flags: &mut Option<u32>,
) -> Result<Vec<u32>, ProgramError> {
let mut prog_ids = vec![0u32; 64];
let mut prog_cnt = prog_ids.len() as u32;
let mut retries = 0;
loop {
match bpf_prog_query(
target_fd.as_raw_fd(),
attach_type,
query_flags,
attach_flags.as_mut(),
&mut prog_ids,
&mut prog_cnt,
) {
Ok(_) => {
prog_ids.resize(prog_cnt as usize, 0);
return Ok(prog_ids);
}
Err((_, io_error)) if retries == 0 && io_error.raw_os_error() == Some(ENOSPC) => {
prog_ids.resize(prog_cnt as usize, 0);
retries += 1;
}
Err((_, io_error)) => {
return Err(ProgramError::SyscallError {
call: "bpf_prog_query".to_owned(),
io_error,
});
}
}
}
}
impl ProgramFd for Program {
fn fd(&self) -> Option<RawFd> {
match self {
Program::KProbe(p) => p.data.fd,
Program::UProbe(p) => p.data.fd,
Program::TracePoint(p) => p.data.fd,
Program::SocketFilter(p) => p.data.fd,
Program::Xdp(p) => p.data.fd,
Program::SkMsg(p) => p.data.fd,
Program::SkSkb(p) => p.data.fd,
Program::SockOps(p) => p.data.fd,
Program::SchedClassifier(p) => p.data.fd,
Program::CgroupSkb(p) => p.data.fd,
Program::CgroupSysctl(p) => p.data.fd,
Program::CgroupSockopt(p) => p.data.fd,
Program::LircMode2(p) => p.data.fd,
Program::PerfEvent(p) => p.data.fd,
Program::RawTracePoint(p) => p.data.fd,
Program::Lsm(p) => p.data.fd,
Program::BtfTracePoint(p) => p.data.fd,
Program::FEntry(p) => p.data.fd,
Program::FExit(p) => p.data.fd,
Program::Extension(p) => p.data.fd,
Program::CgroupSockAddr(p) => p.data.fd,
Program::SkLookup(p) => p.data.fd,
}
}
}
impl<'a, P: ProgramFd> ProgramFd for &'a P {
fn fd(&self) -> Option<RawFd> {
(*self).fd()
}
}
macro_rules! impl_program_unload {
($($struct_name:ident),+ $(,)?) => {
$(
impl $struct_name {
pub fn unload(&mut self) -> Result<(), ProgramError> {
unload_program(&mut self.data)
}
}
)+
}
}
impl_program_unload!(
KProbe,
UProbe,
TracePoint,
SocketFilter,
Xdp,
SkMsg,
SkSkb,
SchedClassifier,
CgroupSkb,
CgroupSysctl,
CgroupSockopt,
LircMode2,
PerfEvent,
Lsm,
RawTracePoint,
BtfTracePoint,
FEntry,
FExit,
Extension,
CgroupSockAddr,
SkLookup,
SockOps
);
#[cfg(test)]
mod tests {
use super::Program;
#[allow(dead_code)]
fn program_implements_unload(a: Program) {
let _ = match a {
Program::KProbe(mut p) => p.unload(),
Program::UProbe(mut p) => p.unload(),
Program::TracePoint(mut p) => p.unload(),
Program::SocketFilter(mut p) => p.unload(),
Program::Xdp(mut p) => p.unload(),
Program::SkMsg(mut p) => p.unload(),
Program::SkSkb(mut p) => p.unload(),
Program::SockOps(mut p) => p.unload(),
Program::SchedClassifier(mut p) => p.unload(),
Program::CgroupSkb(mut p) => p.unload(),
Program::CgroupSysctl(mut p) => p.unload(),
Program::CgroupSockopt(mut p) => p.unload(),
Program::LircMode2(mut p) => p.unload(),
Program::PerfEvent(mut p) => p.unload(),
Program::RawTracePoint(mut p) => p.unload(),
Program::Lsm(mut p) => p.unload(),
Program::BtfTracePoint(mut p) => p.unload(),
Program::FEntry(mut p) => p.unload(),
Program::FExit(mut p) => p.unload(),
Program::Extension(mut p) => p.unload(),
Program::CgroupSockAddr(mut p) => p.unload(),
Program::SkLookup(mut p) => p.unload(),
};
}
}
macro_rules! impl_program_fd {
($($struct_name:ident),+ $(,)?) => {
$(
impl ProgramFd for $struct_name {
fn fd(&self) -> Option<RawFd> {
self.data.fd
}
}
impl ProgramFd for &mut $struct_name {
fn fd(&self) -> Option<RawFd> {
self.data.fd
}
}
)+
}
}
impl_program_fd!(
KProbe,
UProbe,
TracePoint,
SocketFilter,
Xdp,
SkMsg,
SkSkb,
SchedClassifier,
CgroupSkb,
CgroupSysctl,
CgroupSockopt,
LircMode2,
PerfEvent,
Lsm,
RawTracePoint,
BtfTracePoint,
FEntry,
FExit,
Extension,
CgroupSockAddr,
SkLookup,
);
macro_rules! impl_try_from_program {
($($ty:ident),+ $(,)?) => {
$(
impl<'a> TryFrom<&'a Program> for &'a $ty {
type Error = ProgramError;
fn try_from(program: &'a Program) -> Result<&'a $ty, ProgramError> {
match program {
Program::$ty(p) => Ok(p),
_ => Err(ProgramError::UnexpectedProgramType),
}
}
}
impl<'a> TryFrom<&'a mut Program> for &'a mut $ty {
type Error = ProgramError;
fn try_from(program: &'a mut Program) -> Result<&'a mut $ty, ProgramError> {
match program {
Program::$ty(p) => Ok(p),
_ => Err(ProgramError::UnexpectedProgramType),
}
}
}
)+
}
}
impl_try_from_program!(
KProbe,
UProbe,
TracePoint,
SocketFilter,
Xdp,
SkMsg,
SkSkb,
SockOps,
SchedClassifier,
CgroupSkb,
CgroupSysctl,
CgroupSockopt,
LircMode2,
PerfEvent,
Lsm,
RawTracePoint,
BtfTracePoint,
FEntry,
FExit,
Extension,
CgroupSockAddr,
SkLookup,
);
pub struct ProgramInfo(bpf_prog_info);
impl ProgramInfo {
pub fn name(&self) -> &[u8] {
let length = self
.0
.name
.iter()
.rposition(|ch| *ch != 0)
.map(|pos| pos + 1)
.unwrap_or(0);
unsafe { std::slice::from_raw_parts(self.0.name.as_ptr() as *const _, length) }
}
pub fn name_as_str(&self) -> Option<&str> {
std::str::from_utf8(self.name()).ok()
}
pub fn id(&self) -> u32 {
self.0.id
}
pub fn fd(&self) -> Result<RawFd, ProgramError> {
let fd =
bpf_prog_get_fd_by_id(self.0.id).map_err(|io_error| ProgramError::SyscallError {
call: "bpf_prog_get_fd_by_id".to_owned(),
io_error,
})?;
Ok(fd as RawFd)
}
pub fn from_pinned<P: AsRef<Path>>(path: P) -> Result<ProgramInfo, ProgramError> {
let path_string = match CString::new(path.as_ref().to_str().unwrap()) {
Ok(s) => s,
Err(e) => {
return Err(ProgramError::InvalidPinPath {
error: e.to_string(),
})
}
};
let fd =
bpf_get_object(&path_string).map_err(|(_, io_error)| ProgramError::SyscallError {
call: "bpf_obj_get".to_owned(),
io_error,
})? as RawFd;
let info = bpf_obj_get_info_by_fd(fd).map_err(|io_error| ProgramError::SyscallError {
call: "bpf_obj_get_info_by_fd".to_owned(),
io_error,
})?;
unsafe {
libc::close(fd);
}
Ok(ProgramInfo(info))
}
}