use aya::{
maps::{
perf::{PerfEvent, PerfEventArray},
Array, HashMap as AyaHashMap, MapData, PerCpuArray, ProgramArray, RingBuf,
},
programs::{
uprobe::{UProbeLinkId, UProbeScope},
ProgramError, UProbe,
},
Ebpf, EbpfLoader, VerifierLogLevel,
};
use ghostscope_protocol::{
BacktraceModuleRowRange, BacktraceUnwindRow, ParsedTraceEvent, StreamingTraceParser,
TraceContext, BACKTRACE_UNWIND_ROW_SIZE,
};
use log::log_enabled;
use log::Level as LogLevel;
use std::borrow::Borrow;
use std::collections::HashSet;
use std::convert::TryInto;
use std::future::poll_fn;
use std::num::NonZeroU32;
use std::os::unix::io::AsRawFd;
use std::os::unix::io::RawFd;
use std::path::Path;
use std::task::Poll;
use std::time::Instant;
use std::{io, ops::ControlFlow};
use tokio::io::unix::AsyncFd;
use tokio::io::Interest;
use tracing::{debug, error, info, warn};
const MAX_EVENTS_PER_WAIT: usize = 128;
const MAX_RINGBUF_RECORDS_PER_WAIT: usize = 256;
const PERF_READ_BATCH_SIZE: usize = 64;
const EVENT_LOSS_OUTPUT_FAILURES_KEY: u32 = 0;
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct EventLossStats {
pub output_failures: u64,
}
impl EventLossStats {
pub fn is_empty(self) -> bool {
self.output_failures == 0
}
pub fn saturating_sub(self, previous: Self) -> Self {
Self {
output_failures: self
.output_failures
.saturating_sub(previous.output_failures),
}
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct BacktraceUnwindRowsAppendStats {
pub modules: usize,
pub rows: usize,
}
mod kernel_caps;
pub use kernel_caps::{KernelCapabilities, KernelCapabilityError};
mod error;
pub use error::{LoaderError, Result};
mod uprobe;
use uprobe::UprobeAttachmentParams;
use ghostscope_process::pinned_bpf_maps::{
bpffs_mount_hint_for_pin_path, bt_module_row_ranges_pin_path, bt_unwind_rows_pin_path,
pid_aliases_pin_path, proc_module_range_meta_pin_path, proc_module_ranges_pin_path,
proc_offsets_pin_dir, proc_offsets_pin_path, BT_MODULE_ROW_RANGES_MAP_NAME,
BT_UNWIND_ROWS_MAP_NAME, PID_ALIASES_MAP_NAME, PROC_MODULE_RANGES_MAP_NAME,
PROC_MODULE_RANGE_META_MAP_NAME, PROC_OFFSETS_MAP_NAME,
};
enum EventMap {
RingBuf(RingBuf<MapData>),
PerfEventArray {
_map: PerfEventArray<MapData>,
cpu_buffers: Vec<PerfEventCpuBuffer>,
},
}
#[derive(Clone, Copy, Debug)]
struct PerfBufferFd(RawFd);
impl AsRawFd for PerfBufferFd {
fn as_raw_fd(&self) -> RawFd {
self.0
}
}
fn log_backtrace_unwind_row_samples<T: Borrow<MapData>>(
array: &Array<T, BacktraceUnwindRow>,
rows: &[BacktraceUnwindRow],
) -> Result<()> {
fn read_row<T: Borrow<MapData>>(
array: &Array<T, BacktraceUnwindRow>,
row_index: usize,
) -> Result<BacktraceUnwindRow> {
let key = row_index as u32;
array.get(&key, 0).map_err(|e| {
LoaderError::Generic(format!("Failed to read back unwind row {row_index}: {e}"))
})
}
let mut sample_indices = vec![0usize, rows.len() / 2, rows.len().saturating_sub(1)];
sample_indices.sort_unstable();
sample_indices.dedup();
for index in sample_indices {
let stored = read_row(array, index)?;
if stored == rows[index] {
debug!(index, row = ?stored, "bt unwind row readback sample");
} else {
warn!(
index,
expected = ?rows[index],
stored = ?stored,
"bt unwind row readback mismatch"
);
}
}
Ok(())
}
struct PerfEventCpuBuffer {
cpu_id: u32,
buffer: aya::maps::perf::PerfEventArrayBuffer<MapData>,
readiness: AsyncFd<PerfBufferFd>,
}
fn drain_perf_cpu_buffer(
entry: &mut PerfEventCpuBuffer,
parser: &mut StreamingTraceParser,
trace_context: &TraceContext,
events: &mut Vec<ParsedTraceEvent>,
) -> Result<bool> {
let mut produced = false;
if events.len() >= MAX_EVENTS_PER_WAIT {
return Ok(false);
}
let cpu = entry.cpu_id;
let drain_result = entry.buffer.try_fold(
(0usize, 0u64),
|(mut read_count, mut lost_count), event| {
if events.len() >= MAX_EVENTS_PER_WAIT || read_count >= PERF_READ_BATCH_SIZE {
return ControlFlow::Break(Ok((read_count, lost_count)));
}
match event {
PerfEvent::Sample { head, tail } => {
read_count += 1;
produced = true;
debug!(
"PerfEvent {}: {} bytes - {:02x?}",
read_count - 1,
head.len() + tail.len(),
&head[..head.len().min(32)]
);
for segment in [head, tail] {
if segment.is_empty() {
continue;
}
match parser.process_segment(segment, trace_context) {
Ok(Some(parsed_event)) => events.push(parsed_event),
Ok(None) => {}
Err(e) => {
return ControlFlow::Break(Err(LoaderError::Generic(format!(
"Fatal: Failed to parse trace event from PerfEventArray CPU {cpu}: {e}"
))));
}
}
}
}
PerfEvent::Lost { count } => {
lost_count = lost_count.saturating_add(count);
}
}
ControlFlow::Continue((read_count, lost_count))
},
);
let (read_count, lost_count) = match drain_result {
ControlFlow::Continue(counts) => counts,
ControlFlow::Break(result) => result?,
};
if read_count > 0 {
info!(
"Read {} events from CPU {} buffer",
read_count, entry.cpu_id
);
}
if lost_count > 0 {
warn!(
"Lost {} events from CPU {} buffer",
lost_count, entry.cpu_id
);
}
Ok(produced)
}
enum UProbeAttachLocation<'a> {
AbsoluteOffset(u64),
Function(&'a str),
}
impl<'a> UProbeAttachLocation<'a> {
fn attach<T: AsRef<Path>>(
self,
program: &mut UProbe,
target: T,
pid: Option<i32>,
) -> std::result::Result<UProbeLinkId, ProgramError> {
let scope = uprobe_scope(pid)?;
match self {
Self::AbsoluteOffset(offset) => program.attach(offset, target, scope),
Self::Function(fn_name) => program.attach(fn_name, target, scope),
}
}
}
fn uprobe_scope(pid: Option<i32>) -> std::result::Result<UProbeScope, ProgramError> {
match pid {
None => Ok(UProbeScope::AllProcesses),
Some(pid) => {
let pid = u32::try_from(pid)
.ok()
.and_then(NonZeroU32::new)
.ok_or_else(|| {
ProgramError::IOError(io::Error::new(
io::ErrorKind::InvalidInput,
format!("invalid uprobe PID scope: {pid}"),
))
})?;
Ok(UProbeScope::OneProcess(pid))
}
}
}
pub fn hello() -> String {
format!("Loader: {}", ghostscope_compiler::hello())
}
pub struct GhostScopeLoader {
bpf: Ebpf,
event_map: Option<EventMap>,
event_loss_counters: Option<PerCpuArray<MapData, u64>>,
bt_prog_array: Option<ProgramArray<MapData>>,
uprobe_link: Option<UProbeLinkId>,
attachment_params: Option<UprobeAttachmentParams>,
parser: StreamingTraceParser,
trace_context: Option<TraceContext>,
perf_page_count: Option<usize>,
backtrace_unwind_row_count: u32,
backtrace_module_row_cookies: HashSet<u64>,
shared_backtrace_maps: bool,
}
impl std::fmt::Debug for GhostScopeLoader {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("GhostScopeLoader")
.field("bpf", &"<eBPF object>")
.field("event_map", &self.event_map.is_some())
.field("event_loss_counters", &self.event_loss_counters.is_some())
.field("bt_prog_array", &self.bt_prog_array.is_some())
.field("uprobe_attached", &self.uprobe_link.is_some())
.field("attachment_params", &self.attachment_params.is_some())
.field(
"backtrace_unwind_row_count",
&self.backtrace_unwind_row_count,
)
.field(
"backtrace_module_row_cookies",
&self.backtrace_module_row_cookies.len(),
)
.field("shared_backtrace_maps", &self.shared_backtrace_maps)
.finish()
}
}
impl GhostScopeLoader {
pub fn new(bytecode: &[u8]) -> Result<Self> {
Self::new_with_shared_backtrace_maps(bytecode, false)
}
pub fn new_with_shared_backtrace_maps(
bytecode: &[u8],
shared_backtrace_maps: bool,
) -> Result<Self> {
info!(
"Loading eBPF program from bytecode ({} bytes)",
bytecode.len()
);
let pin_path = proc_offsets_pin_path()
.map_err(|e| LoaderError::Generic(format!("Failed to resolve pinned map path: {e}")))?;
if !pin_path.exists() {
let hint = bpffs_mount_hint_for_pin_path(&pin_path)
.map(|hint| format!(" {hint}"))
.unwrap_or_default();
return Err(LoaderError::Generic(format!(
"Pinned map '{}' not found. Please call ghostscope-process to create it first.{}",
pin_path.display(),
hint
)));
}
let alias_pin_path = pid_aliases_pin_path().map_err(|e| {
LoaderError::Generic(format!("Failed to resolve pinned alias map path: {e}"))
})?;
if !alias_pin_path.exists() {
let hint = bpffs_mount_hint_for_pin_path(&alias_pin_path)
.map(|hint| format!(" {hint}"))
.unwrap_or_default();
return Err(LoaderError::Generic(format!(
"Pinned map '{}' not found. Please call ghostscope-process to create it first.{}",
alias_pin_path.display(),
hint
)));
}
let range_meta_pin_path = proc_module_range_meta_pin_path().map_err(|e| {
LoaderError::Generic(format!("Failed to resolve pinned range meta map path: {e}"))
})?;
if !range_meta_pin_path.exists() {
let hint = bpffs_mount_hint_for_pin_path(&range_meta_pin_path)
.map(|hint| format!(" {hint}"))
.unwrap_or_default();
return Err(LoaderError::Generic(format!(
"Pinned map '{}' not found. Please call ghostscope-process to create it first.{}",
range_meta_pin_path.display(),
hint
)));
}
let ranges_pin_path = proc_module_ranges_pin_path().map_err(|e| {
LoaderError::Generic(format!("Failed to resolve pinned ranges map path: {e}"))
})?;
if !ranges_pin_path.exists() {
let hint = bpffs_mount_hint_for_pin_path(&ranges_pin_path)
.map(|hint| format!(" {hint}"))
.unwrap_or_default();
return Err(LoaderError::Generic(format!(
"Pinned map '{}' not found. Please call ghostscope-process to create it first.{}",
ranges_pin_path.display(),
hint
)));
}
let bt_rows_pin_path = if shared_backtrace_maps {
let path = bt_unwind_rows_pin_path().map_err(|e| {
LoaderError::Generic(format!(
"Failed to resolve pinned bt_unwind_rows map path: {e}"
))
})?;
if !path.exists() {
let hint = bpffs_mount_hint_for_pin_path(&path)
.map(|hint| format!(" {hint}"))
.unwrap_or_default();
return Err(LoaderError::Generic(format!(
"Pinned map '{}' not found. Please call ghostscope-process to create it first.{}",
path.display(),
hint
)));
}
Some(path)
} else {
None
};
let bt_ranges_pin_path = if shared_backtrace_maps {
let path = bt_module_row_ranges_pin_path().map_err(|e| {
LoaderError::Generic(format!(
"Failed to resolve pinned bt_module_row_ranges map path: {e}"
))
})?;
if !path.exists() {
let hint = bpffs_mount_hint_for_pin_path(&path)
.map(|hint| format!(" {hint}"))
.unwrap_or_default();
return Err(LoaderError::Generic(format!(
"Pinned map '{}' not found. Please call ghostscope-process to create it first.{}",
path.display(),
hint
)));
}
Some(path)
} else {
None
};
let mut loader = EbpfLoader::new();
let use_verbose = cfg!(debug_assertions)
|| log_enabled!(LogLevel::Trace)
|| log_enabled!(LogLevel::Debug);
if use_verbose {
loader.verifier_log_level(VerifierLogLevel::VERBOSE | VerifierLogLevel::STATS);
tracing::info!("BPF verifier logs: VERBOSE (debug build/log)");
} else {
loader.verifier_log_level(VerifierLogLevel::DEBUG | VerifierLogLevel::STATS);
tracing::info!("BPF verifier logs: DEBUG (release/info)");
}
let pin_dir = proc_offsets_pin_dir().map_err(|e| {
LoaderError::Generic(format!("Failed to resolve pinned map directory: {e}"))
})?;
if pin_dir.exists() {
loader.map_pin_path(PROC_OFFSETS_MAP_NAME, pin_path);
loader.map_pin_path(PID_ALIASES_MAP_NAME, alias_pin_path);
loader.map_pin_path(PROC_MODULE_RANGE_META_MAP_NAME, range_meta_pin_path);
loader.map_pin_path(PROC_MODULE_RANGES_MAP_NAME, ranges_pin_path);
if let (Some(rows_path), Some(ranges_path)) =
(bt_rows_pin_path.as_ref(), bt_ranges_pin_path.as_ref())
{
loader.map_pin_path(BT_UNWIND_ROWS_MAP_NAME, rows_path);
loader.map_pin_path(BT_MODULE_ROW_RANGES_MAP_NAME, ranges_path);
}
tracing::info!(
"Configured map pin directory for reuse: {}",
pin_dir.display()
);
}
match loader.load(bytecode) {
Ok(bpf) => {
info!("Successfully loaded eBPF program");
Ok(Self {
bpf,
event_map: None,
event_loss_counters: None,
bt_prog_array: None,
uprobe_link: None,
attachment_params: None,
parser: StreamingTraceParser::new(),
trace_context: None,
perf_page_count: None,
backtrace_unwind_row_count: 0,
backtrace_module_row_cookies: HashSet::new(),
shared_backtrace_maps,
})
}
Err(e) => {
error!("Failed to load BPF program: {:?}", e);
match &e {
aya::EbpfError::ParseError(parse_err) => {
error!("Parse error details: {:?}", parse_err);
}
aya::EbpfError::BtfError(btf_err) => {
error!("BTF error details: {:?}", btf_err);
}
_ => {
error!("Other BPF error: {:?}", e);
}
}
Err(LoaderError::Aya(e))
}
}
}
pub fn attach_uprobe(
&mut self,
target_binary: &str,
function_name: &str,
offset: Option<u64>,
pid: Option<i32>,
) -> Result<()> {
self.attach_uprobe_with_program_name(target_binary, function_name, offset, pid, None)
}
pub fn set_perf_page_count(&mut self, pages: u32) {
self.perf_page_count = Some(pages as usize);
}
pub fn register_backtrace_tail_call_program(
&mut self,
program_name: Option<&str>,
) -> Result<()> {
let Some(program_name) = program_name else {
return Ok(());
};
info!("Registering bt tail-call step program: {}", program_name);
let program_ref = self.bpf.program_mut(program_name).ok_or_else(|| {
LoaderError::Generic(format!("bt tail-call program '{program_name}' not found"))
})?;
let program: &mut UProbe = program_ref.try_into().map_err(|e| {
LoaderError::Generic(format!(
"bt tail-call program '{program_name}' is not a UProbe: {e:?}"
))
})?;
program.load().map_err(LoaderError::Program)?;
let step_fd = program
.fd()
.map_err(LoaderError::Program)?
.try_clone()
.map_err(|e| {
LoaderError::Generic(format!(
"Failed to clone bt tail-call program fd for '{program_name}': {e}"
))
})?;
let map = self
.bpf
.take_map("bt_prog_array")
.ok_or_else(|| LoaderError::MapNotFound("bt_prog_array".to_string()))?;
let mut prog_array: ProgramArray<_> = map.try_into().map_err(|e| {
LoaderError::Generic(format!("Failed to convert bt_prog_array map: {e}"))
})?;
prog_array.set(0, &step_fd, 0).map_err(|e| {
LoaderError::Generic(format!("Failed to set bt tail-call program fd: {e}"))
})?;
self.bt_prog_array = Some(prog_array);
info!("Registered bt tail-call step program at bt_prog_array[0]");
Ok(())
}
pub fn attach_uprobe_with_program_name(
&mut self,
target_binary: &str,
function_name: &str,
offset: Option<u64>,
pid: Option<i32>,
program_name: Option<&str>,
) -> Result<()> {
info!("attach_uprobe called with offset: {:?}", offset);
if let Some(offset) = offset {
info!(
"Using offset-based attachment: {} at 0x{:x} ({}) (pid: {:?})",
target_binary, offset, function_name, pid
);
} else {
info!(
"Using function name-based attachment: {}:{} (pid: {:?})",
target_binary, function_name, pid
);
}
let available_programs: Vec<String> = self
.bpf
.programs()
.map(|(name, _)| name.to_string())
.collect();
info!("Available programs:");
for name in &available_programs {
info!(" - {}", name);
}
let program_name: String = if let Some(name) = program_name {
info!("Using specified program name: {}", name);
if available_programs.contains(&name.to_string()) {
name.to_string()
} else {
return Err(LoaderError::Generic(format!(
"Specified program '{name}' not found in eBPF object"
)));
}
} else {
let program_names = ["uprobe", "main"];
let mut found_program_name: Option<String> = None;
for name in &program_names {
info!("Checking if program exists: {}", name);
if available_programs.contains(&name.to_string()) {
info!("Found program: {}", name);
found_program_name = Some(name.to_string());
break;
}
}
if found_program_name.is_none() {
if let Some(first_name) = available_programs.first() {
info!(
"No standard program names found, using first available: {}",
first_name
);
found_program_name = Some(first_name.clone());
}
}
found_program_name
.ok_or_else(|| LoaderError::Generic("No suitable program found".to_string()))?
};
info!("Attempting to load program: {}", program_name);
let program_ref = self
.bpf
.program_mut(&program_name)
.ok_or_else(|| LoaderError::Generic(format!("Program '{program_name}' not found")))?;
info!("Found program, attempting to convert to UProbe");
info!("Program type: {:?}", program_ref.prog_type());
match program_ref {
aya::programs::Program::UProbe(_) => {
info!("Program is correctly recognized as UProbe");
}
aya::programs::Program::KProbe(_) => {
error!("Program is incorrectly recognized as KProbe, should be UProbe");
}
ref _other => {
error!("Program is unexpected type (not UProbe or KProbe)");
}
}
let program: &mut UProbe = program_ref.try_into().map_err(|e| {
LoaderError::Generic(format!("Program '{program_name}' is not a UProbe: {e:?}"))
})?;
info!("About to load eBPF program");
match program.load() {
Ok(()) => {
info!("Program loaded successfully");
}
Err(e) => {
error!("eBPF program load failed: {}", e);
error!("This typically indicates eBPF verifier rejection");
if let ProgramError::SyscallError(syscall_error) = &e {
error!(
"Syscall '{}' failed: {}",
syscall_error.call, syscall_error.io_error
);
if let Some(errno) = syscall_error.io_error.raw_os_error() {
match errno {
22 => error!(
"EINVAL (22): Invalid argument - likely eBPF verifier rejection"
),
7 => error!("E2BIG (7): Program too large"),
13 => error!("EACCES (13): Permission denied"),
95 => error!("EOPNOTSUPP (95): Operation not supported"),
_ => error!("Unknown errno: {}", errno),
}
}
}
error!("Program name: {}", program_name);
error!("Program type: {:?}", program_ref.prog_type());
return Err(LoaderError::Program(e));
}
}
let attach_location = match offset {
Some(offset) => UProbeAttachLocation::AbsoluteOffset(offset),
None => UProbeAttachLocation::Function(function_name),
};
let attach_result = attach_location.attach(program, target_binary, pid);
match attach_result {
Ok(link) => {
if let Some(offset) = offset {
info!(
"Uprobe attached successfully to {} at offset 0x{:x}",
target_binary, offset
);
} else {
info!(
"Uprobe attached successfully to {}:{}",
target_binary, function_name
);
}
self.uprobe_link = Some(link);
self.attachment_params = Some(UprobeAttachmentParams {
target_binary: target_binary.to_string(),
function_name: function_name.to_string(),
offset,
pid,
program_name,
});
}
Err(e) => {
if let Some(offset) = offset {
error!(
"Failed to attach uprobe to {} at offset 0x{:x}: {}",
target_binary, offset, e
);
error!("Detailed error: {:#?}", e);
} else {
error!(
"Failed to attach uprobe to {}:{}: {}",
target_binary, function_name, e
);
error!("Detailed error: {:#?}", e);
}
if let ProgramError::SyscallError(syscall_error) = &e {
error!(
"Syscall '{}' failed: {}",
syscall_error.call, syscall_error.io_error
);
if let Some(13) = syscall_error.io_error.raw_os_error() {
error!("Permission denied - make sure to run with sudo");
}
}
return Err(LoaderError::Program(e));
}
}
let event_map = if let Some(map) = self.bpf.take_map("ringbuf") {
info!("Initializing RingBuf event map");
let ringbuf: RingBuf<_> = map
.try_into()
.map_err(|e| LoaderError::Generic(format!("Failed to convert ringbuf map: {e}")))?;
EventMap::RingBuf(ringbuf)
} else if let Some(map) = self.bpf.take_map("events") {
info!("Initializing PerfEventArray event map");
let mut perf_array: PerfEventArray<_> = map.try_into().map_err(|e| {
LoaderError::Generic(format!("Failed to convert perf event array map: {e}"))
})?;
let online_cpus = aya::util::online_cpus().map_err(|(_, e)| {
LoaderError::Generic(format!("Failed to get online CPUs: {e}"))
})?;
info!(
"Opening PerfEventArray buffers for {} online CPUs",
online_cpus.len()
);
let mut cpu_buffers = Vec::new();
for cpu_id in online_cpus {
let pages = self.perf_page_count;
match perf_array.open(cpu_id, pages) {
Ok(buffer) => {
if let Some(p) = pages {
info!(
"Opened PerfEventArray buffer for CPU {} with {} pages",
cpu_id, p
);
} else {
info!(
"Opened PerfEventArray buffer for CPU {} (default pages)",
cpu_id
);
}
let fd = buffer.as_raw_fd();
let readiness =
AsyncFd::with_interest(PerfBufferFd(fd), Interest::READABLE).map_err(
|err| {
LoaderError::Generic(format!(
"Failed to register perf buffer fd for CPU {cpu_id}: {err}"
))
},
)?;
cpu_buffers.push(PerfEventCpuBuffer {
cpu_id,
buffer,
readiness,
});
}
Err(e) => {
warn!("Failed to open perf buffer for CPU {}: {}", cpu_id, e);
}
}
}
if cpu_buffers.is_empty() {
return Err(LoaderError::Generic(
"Failed to open any perf event buffers".to_string(),
));
}
EventMap::PerfEventArray {
_map: perf_array,
cpu_buffers,
}
} else {
return Err(LoaderError::MapNotFound(
"Neither 'ringbuf' nor 'events' map found".to_string(),
));
};
self.event_loss_counters = if let Some(map) = self.bpf.take_map("event_loss_counters") {
info!("Initializing eBPF event loss counter map");
Some(map.try_into().map_err(|e| {
LoaderError::Generic(format!("Failed to convert event_loss_counters map: {e}"))
})?)
} else {
warn!("No eBPF event loss counter map found; kernel output loss stats unavailable");
None
};
let event_source = match &event_map {
EventMap::RingBuf(_) => {
info!("Using RingBuf mode for parser");
ghostscope_protocol::EventSource::RingBuf
}
EventMap::PerfEventArray { .. } => {
info!("Using PerfEventArray mode for parser");
ghostscope_protocol::EventSource::PerfEventArray
}
};
self.parser = StreamingTraceParser::with_event_source(event_source);
self.event_map = Some(event_map);
info!("Event map initialized");
Ok(())
}
pub fn detach_uprobe(&mut self) -> Result<()> {
if let Some(link_id) = self.uprobe_link.take() {
if let Some(params) = &self.attachment_params {
info!("Detaching uprobe...");
let program_ref = self.bpf.program_mut(¶ms.program_name).ok_or_else(|| {
let program_name = ¶ms.program_name;
LoaderError::Generic(format!("Program '{program_name}' not found"))
})?;
let program: &mut UProbe = program_ref.try_into().map_err(|e| {
let program_name = ¶ms.program_name;
LoaderError::Generic(format!("Program '{program_name}' is not a UProbe: {e:?}"))
})?;
program.detach(link_id).map_err(LoaderError::Program)?;
info!("Uprobe detached successfully");
Ok(())
} else {
error!("No attachment parameters stored");
Err(LoaderError::Generic(
"No attachment parameters stored".to_string(),
))
}
} else {
warn!("No uprobe attached, nothing to detach");
Ok(())
}
}
pub fn reattach_uprobe(&mut self) -> Result<()> {
if self.uprobe_link.is_some() {
info!("Uprobe already attached");
return Ok(());
}
let params = self
.attachment_params
.as_ref()
.ok_or_else(|| {
LoaderError::Generic(
"No attachment parameters stored. Call attach_uprobe first.".to_string(),
)
})?
.clone();
info!("Reattaching uprobe with stored parameters...");
let program_ref = self.bpf.program_mut(¶ms.program_name).ok_or_else(|| {
LoaderError::Generic(format!("Program '{}' not found", params.program_name))
})?;
let program: &mut UProbe = program_ref.try_into().map_err(|e| {
LoaderError::Generic(format!(
"Program '{}' is not a UProbe: {:?}",
params.program_name, e
))
})?;
let attach_location = match params.offset {
Some(offset) => UProbeAttachLocation::AbsoluteOffset(offset),
None => UProbeAttachLocation::Function(params.function_name.as_str()),
};
let attach_result = attach_location.attach(program, ¶ms.target_binary, params.pid);
match attach_result {
Ok(link) => {
if let Some(offset) = params.offset {
info!(
"Uprobe reattached successfully to {} at offset 0x{:x}",
params.target_binary, offset
);
} else {
info!(
"Uprobe reattached successfully to {}:{}",
params.target_binary, params.function_name
);
}
self.uprobe_link = Some(link);
Ok(())
}
Err(e) => {
error!("Failed to reattach uprobe: {:?}", e);
Err(LoaderError::Program(e))
}
}
}
pub fn is_uprobe_attached(&self) -> bool {
self.uprobe_link.is_some()
}
pub fn destroy(&mut self) -> Result<()> {
info!("Destroying GhostScopeLoader and all associated resources");
if self.uprobe_link.is_some() {
if let Err(e) = self.detach_uprobe() {
warn!("Failed to detach uprobe during destroy: {}", e);
}
}
self.attachment_params = None;
self.event_map = None;
info!("GhostScopeLoader destroyed successfully");
Ok(())
}
pub fn get_attachment_info(&self) -> Option<String> {
if let Some(params) = &self.attachment_params {
if let Some(offset) = params.offset {
Some(format!(
"{}:{} (offset: 0x{:x}, pid: {:?}) - {}",
params.target_binary,
params.function_name,
offset,
params.pid,
if self.is_uprobe_attached() {
"attached"
} else {
"detached"
}
))
} else {
Some(format!(
"{}:{} (pid: {:?}) - {}",
params.target_binary,
params.function_name,
params.pid,
if self.is_uprobe_attached() {
"attached"
} else {
"detached"
}
))
}
} else {
None
}
}
pub async fn wait_for_events_async(&mut self) -> Result<Vec<ParsedTraceEvent>> {
let trace_context = self.trace_context.as_ref().ok_or_else(|| {
LoaderError::Generic(
"No trace context available - cannot parse trace events".to_string(),
)
})?;
let event_map = self.event_map.as_mut().ok_or_else(|| {
LoaderError::Generic("Event map not initialized. Call attach_uprobe first.".to_string())
})?;
let mut events = Vec::with_capacity(MAX_EVENTS_PER_WAIT.min(128));
match event_map {
EventMap::RingBuf(ringbuf) => {
let async_fd = AsyncFd::new(ringbuf.as_raw_fd())
.map_err(|e| LoaderError::Generic(format!("Failed to create AsyncFd: {e}")))?;
let mut guard = async_fd
.readable()
.await
.map_err(|e| LoaderError::Generic(format!("AsyncFd error: {e}")))?;
guard.clear_ready();
let mut records_read = 0;
while events.len() < MAX_EVENTS_PER_WAIT
&& records_read < MAX_RINGBUF_RECORDS_PER_WAIT
{
let Some(item) = ringbuf.next() else {
break;
};
records_read += 1;
match self.parser.process_segment(&item, trace_context) {
Ok(Some(parsed_event)) => events.push(parsed_event),
Ok(None) => {}
Err(e) => {
return Err(LoaderError::Generic(format!(
"Fatal: Failed to parse trace event from RingBuf (async): {e}"
)));
}
}
}
if events.len() == MAX_EVENTS_PER_WAIT
|| records_read == MAX_RINGBUF_RECORDS_PER_WAIT
{
debug!(
"RingBuf batch limit reached ({} events, {} records); yielding to caller",
events.len(),
records_read
);
}
}
EventMap::PerfEventArray { cpu_buffers, .. } => {
let parser = &mut self.parser;
loop {
let mut made_progress = false;
for entry in cpu_buffers.iter_mut() {
if events.len() >= MAX_EVENTS_PER_WAIT {
break;
}
if entry.buffer.readable() {
made_progress |=
drain_perf_cpu_buffer(entry, parser, trace_context, &mut events)?;
}
}
if made_progress {
break;
}
let ready_idx = poll_fn(|cx| {
for (idx, entry) in cpu_buffers.iter().enumerate() {
match entry.readiness.poll_read_ready(cx) {
Poll::Ready(Ok(mut guard)) => {
guard.clear_ready();
return Poll::Ready(Ok(idx));
}
Poll::Ready(Err(e)) => return Poll::Ready(Err(e)),
Poll::Pending => {}
}
}
Poll::Pending
})
.await
.map_err(|e| {
LoaderError::Generic(format!(
"AsyncFd error while waiting for perf events: {e}"
))
})?;
made_progress |= drain_perf_cpu_buffer(
cpu_buffers
.get_mut(ready_idx)
.expect("ready index should be valid"),
parser,
trace_context,
&mut events,
)?;
for (idx, entry) in cpu_buffers.iter_mut().enumerate() {
if events.len() >= MAX_EVENTS_PER_WAIT {
break;
}
if idx == ready_idx || !entry.buffer.readable() {
continue;
}
made_progress |=
drain_perf_cpu_buffer(entry, parser, trace_context, &mut events)?;
}
if made_progress {
if events.len() == MAX_EVENTS_PER_WAIT {
debug!(
"PerfEventArray event batch limit reached ({} events); yielding to caller",
MAX_EVENTS_PER_WAIT
);
}
break;
}
}
}
}
Ok(events)
}
pub fn read_event_loss_stats(&self) -> Result<Option<EventLossStats>> {
let Some(counters) = &self.event_loss_counters else {
return Ok(None);
};
let values = counters
.get(&EVENT_LOSS_OUTPUT_FAILURES_KEY, 0)
.map_err(|e| {
LoaderError::Generic(format!("Failed to read event_loss_counters map: {e}"))
})?;
Ok(Some(EventLossStats {
output_failures: values.iter().copied().sum(),
}))
}
pub fn set_trace_context(&mut self, trace_context: TraceContext) {
info!("Setting trace context for trace event parsing");
self.trace_context = Some(trace_context);
}
fn sync_shared_backtrace_row_state(&mut self) -> Result<()> {
if !self.shared_backtrace_maps {
return Ok(());
}
let Some(map) = self.bpf.map_mut(BT_MODULE_ROW_RANGES_MAP_NAME) else {
return Ok(());
};
let hash: AyaHashMap<_, u64, BacktraceModuleRowRange> = map.try_into().map_err(|e| {
LoaderError::Generic(format!(
"Failed to convert shared bt_module_row_ranges map: {e}"
))
})?;
let keys = hash
.keys()
.collect::<std::result::Result<Vec<_>, _>>()
.map_err(|e| {
LoaderError::Generic(format!(
"Failed to list shared bt_module_row_ranges keys: {e}"
))
})?;
let mut max_row_end = self.backtrace_unwind_row_count;
for cookie in keys {
match hash.get(&cookie, 0) {
Ok(range) => {
self.backtrace_module_row_cookies.insert(cookie);
max_row_end = max_row_end.max(range.row_end);
}
Err(e) => {
debug!(
cookie = format_args!("0x{cookie:016x}"),
"Skipped shared bt module row range during sync: {}", e
);
}
}
}
self.backtrace_unwind_row_count = max_row_end;
Ok(())
}
pub fn populate_backtrace_unwind_rows_and_module_row_ranges(
&mut self,
rows: &[BacktraceUnwindRow],
ranges: &[(u64, BacktraceModuleRowRange)],
) -> Result<()> {
if rows.is_empty() {
return Ok(());
}
if ranges.is_empty() || !self.shared_backtrace_maps {
self.populate_backtrace_unwind_rows(rows)?;
self.populate_backtrace_module_row_ranges(ranges)?;
return Ok(());
}
self.sync_shared_backtrace_row_state()?;
for (cookie, range) in ranges.iter().copied() {
if self.backtrace_module_row_cookies.contains(&cookie) {
continue;
}
let row_start = usize::try_from(range.row_start).map_err(|_| {
LoaderError::Generic(format!(
"Invalid row_start for module cookie 0x{cookie:016x}: {}",
range.row_start
))
})?;
let row_end = usize::try_from(range.row_end).map_err(|_| {
LoaderError::Generic(format!(
"Invalid row_end for module cookie 0x{cookie:016x}: {}",
range.row_end
))
})?;
if row_start > row_end || row_end > rows.len() {
return Err(LoaderError::Generic(format!(
"Invalid bt row range for module cookie 0x{cookie:016x}: \
{}..{} with {} rows",
range.row_start,
range.row_end,
rows.len()
)));
}
self.append_backtrace_unwind_rows_for_module_after_sync(
cookie,
&rows[row_start..row_end],
)?;
}
Ok(())
}
pub fn populate_backtrace_unwind_rows(&mut self, rows: &[BacktraceUnwindRow]) -> Result<()> {
if rows.is_empty() {
return Ok(());
}
let Some(map) = self.bpf.map_mut("bt_unwind_rows") else {
return Err(LoaderError::MapNotFound("bt_unwind_rows".to_string()));
};
let mut array: Array<_, BacktraceUnwindRow> = map.try_into().map_err(|e| {
LoaderError::Generic(format!("Failed to convert bt_unwind_rows map: {e}"))
})?;
let populate_started_at = Instant::now();
for (row_index, row) in rows.iter().copied().enumerate() {
array.set(row_index as u32, row, 0).map_err(|e| {
LoaderError::Generic(format!("Failed to set unwind row {row_index}: {e}"))
})?;
}
self.backtrace_unwind_row_count = self.backtrace_unwind_row_count.max(rows.len() as u32);
info!(
rows = rows.len(),
capacity = array.len(),
row_size = BACKTRACE_UNWIND_ROW_SIZE,
elapsed_ms = populate_started_at.elapsed().as_millis(),
"Loaded DWARF unwind rows for bt"
);
if log_enabled!(LogLevel::Debug) {
log_backtrace_unwind_row_samples(&array, rows)?;
}
Ok(())
}
pub fn populate_backtrace_module_row_ranges(
&mut self,
ranges: &[(u64, BacktraceModuleRowRange)],
) -> Result<()> {
if ranges.is_empty() {
return Ok(());
}
let Some(map) = self.bpf.map_mut("bt_module_row_ranges") else {
return Err(LoaderError::MapNotFound("bt_module_row_ranges".to_string()));
};
let mut hash: AyaHashMap<_, u64, BacktraceModuleRowRange> =
map.try_into().map_err(|e| {
LoaderError::Generic(format!("Failed to convert bt_module_row_ranges map: {e}"))
})?;
let populate_started_at = Instant::now();
for (cookie, range) in ranges.iter().copied() {
hash.insert(cookie, range, 0).map_err(|e| {
LoaderError::Generic(format!(
"Failed to set bt module row range for cookie 0x{cookie:016x}: {e}"
))
})?;
self.backtrace_module_row_cookies.insert(cookie);
self.backtrace_unwind_row_count = self.backtrace_unwind_row_count.max(range.row_end);
}
info!(
modules = ranges.len(),
elapsed_ms = populate_started_at.elapsed().as_millis(),
"Loaded DWARF unwind row ranges for bt"
);
Ok(())
}
pub fn append_backtrace_unwind_rows_for_module(
&mut self,
cookie: u64,
rows: &[BacktraceUnwindRow],
) -> Result<Option<BacktraceModuleRowRange>> {
if self.shared_backtrace_maps {
self.sync_shared_backtrace_row_state()?;
}
self.append_backtrace_unwind_rows_for_module_after_sync(cookie, rows)
}
fn append_backtrace_unwind_rows_for_module_after_sync(
&mut self,
cookie: u64,
rows: &[BacktraceUnwindRow],
) -> Result<Option<BacktraceModuleRowRange>> {
if rows.is_empty() || self.backtrace_module_row_cookies.contains(&cookie) {
return Ok(None);
}
if self.bpf.map("bt_unwind_rows").is_none()
|| self.bpf.map("bt_module_row_ranges").is_none()
{
return Ok(None);
}
let start = self.backtrace_unwind_row_count;
let row_count = u32::try_from(rows.len()).map_err(|_| {
LoaderError::Generic(format!(
"Too many unwind rows for module cookie 0x{cookie:016x}: {}",
rows.len()
))
})?;
let end = start.checked_add(row_count).ok_or_else(|| {
LoaderError::Generic(format!(
"Unwind row index overflow for module cookie 0x{cookie:016x}"
))
})?;
let Some(map) = self.bpf.map_mut("bt_unwind_rows") else {
return Ok(None);
};
let mut array: Array<_, BacktraceUnwindRow> = map.try_into().map_err(|e| {
LoaderError::Generic(format!("Failed to convert bt_unwind_rows map: {e}"))
})?;
if end > array.len() {
return Err(LoaderError::Generic(format!(
"bt_unwind_rows capacity exceeded while appending module \
0x{cookie:016x}: need end row {}, capacity {}",
end,
array.len()
)));
}
for (offset, row) in rows.iter().copied().enumerate() {
let row_index = start + offset as u32;
array.set(row_index, row, 0).map_err(|e| {
LoaderError::Generic(format!("Failed to append unwind row {row_index}: {e}"))
})?;
}
let range = BacktraceModuleRowRange {
row_start: start,
row_end: end,
};
let Some(map) = self.bpf.map_mut("bt_module_row_ranges") else {
return Ok(None);
};
let mut hash: AyaHashMap<_, u64, BacktraceModuleRowRange> =
map.try_into().map_err(|e| {
LoaderError::Generic(format!("Failed to convert bt_module_row_ranges map: {e}"))
})?;
hash.insert(cookie, range, 0).map_err(|e| {
LoaderError::Generic(format!(
"Failed to append bt module row range for cookie 0x{cookie:016x}: {e}"
))
})?;
self.backtrace_unwind_row_count = end;
self.backtrace_module_row_cookies.insert(cookie);
debug!(
cookie = format_args!("0x{cookie:016x}"),
rows = rows.len(),
row_start = range.row_start,
row_end = range.row_end,
"Appended DWARF unwind rows for bt module"
);
Ok(Some(range))
}
pub fn append_backtrace_unwind_rows_for_modules(
&mut self,
modules: &[(u64, Vec<BacktraceUnwindRow>)],
) -> Result<BacktraceUnwindRowsAppendStats> {
let mut stats = BacktraceUnwindRowsAppendStats::default();
if self.shared_backtrace_maps {
self.sync_shared_backtrace_row_state()?;
}
for (cookie, rows) in modules {
if self
.append_backtrace_unwind_rows_for_module_after_sync(*cookie, rows)?
.is_some()
{
stats.modules += 1;
stats.rows += rows.len();
}
}
Ok(stats)
}
pub fn get_map_info(&self) -> Vec<String> {
self.bpf
.maps()
.map(|(name, _map)| format!("Map: {name}"))
.collect()
}
pub fn get_program_info(&self) -> Vec<String> {
self.bpf
.programs()
.map(|(name, _prog)| format!("Program: {name}"))
.collect()
}
}