1use aya::{
19 maps::{
20 perf::{PerfEvent, PerfEventArray},
21 Array, HashMap as AyaHashMap, MapData, PerCpuArray, ProgramArray, RingBuf,
22 },
23 programs::{
24 uprobe::{UProbeLinkId, UProbeScope},
25 ProgramError, UProbe,
26 },
27 Ebpf, EbpfLoader, VerifierLogLevel,
28};
29use ghostscope_protocol::{
30 BacktraceModuleRowRange, BacktraceUnwindRow, ParsedTraceEvent, StreamingTraceParser,
31 TraceContext, BACKTRACE_UNWIND_ROW_SIZE,
32};
33use log::log_enabled;
34use log::Level as LogLevel;
35use std::borrow::Borrow;
36use std::collections::HashSet;
37use std::convert::TryInto;
38use std::future::poll_fn;
39use std::num::NonZeroU32;
40use std::os::unix::io::AsRawFd;
41use std::os::unix::io::RawFd;
42use std::path::Path;
43use std::task::Poll;
44use std::time::Instant;
45use std::{io, ops::ControlFlow};
46use tokio::io::unix::AsyncFd;
47use tokio::io::Interest;
48use tracing::{debug, error, info, warn};
49
50const MAX_EVENTS_PER_WAIT: usize = 128;
51const MAX_RINGBUF_RECORDS_PER_WAIT: usize = 256;
52const PERF_READ_BATCH_SIZE: usize = 64;
53const EVENT_LOSS_OUTPUT_FAILURES_KEY: u32 = 0;
54
55#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
56pub struct EventLossStats {
57 pub output_failures: u64,
58}
59
60impl EventLossStats {
61 pub fn is_empty(self) -> bool {
62 self.output_failures == 0
63 }
64
65 pub fn saturating_sub(self, previous: Self) -> Self {
66 Self {
67 output_failures: self
68 .output_failures
69 .saturating_sub(previous.output_failures),
70 }
71 }
72}
73
74#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
75pub struct BacktraceUnwindRowsAppendStats {
76 pub modules: usize,
77 pub rows: usize,
78}
79
80mod kernel_caps;
82pub use kernel_caps::{KernelCapabilities, KernelCapabilityError};
83
84mod error;
86pub use error::{LoaderError, Result};
87
88mod uprobe;
90use uprobe::UprobeAttachmentParams;
91
92use ghostscope_process::pinned_bpf_maps::{
94 bpffs_mount_hint_for_pin_path, bt_module_row_ranges_pin_path, bt_unwind_rows_pin_path,
95 pid_aliases_pin_path, proc_module_range_meta_pin_path, proc_module_ranges_pin_path,
96 proc_offsets_pin_dir, proc_offsets_pin_path, BT_MODULE_ROW_RANGES_MAP_NAME,
97 BT_UNWIND_ROWS_MAP_NAME, PID_ALIASES_MAP_NAME, PROC_MODULE_RANGES_MAP_NAME,
98 PROC_MODULE_RANGE_META_MAP_NAME, PROC_OFFSETS_MAP_NAME,
99};
100
101enum EventMap {
103 RingBuf(RingBuf<MapData>),
104 PerfEventArray {
105 _map: PerfEventArray<MapData>,
106 cpu_buffers: Vec<PerfEventCpuBuffer>,
107 },
108}
109
110#[derive(Clone, Copy, Debug)]
111struct PerfBufferFd(RawFd);
112
113impl AsRawFd for PerfBufferFd {
114 fn as_raw_fd(&self) -> RawFd {
115 self.0
116 }
117}
118
119fn log_backtrace_unwind_row_samples<T: Borrow<MapData>>(
120 array: &Array<T, BacktraceUnwindRow>,
121 rows: &[BacktraceUnwindRow],
122) -> Result<()> {
123 fn read_row<T: Borrow<MapData>>(
124 array: &Array<T, BacktraceUnwindRow>,
125 row_index: usize,
126 ) -> Result<BacktraceUnwindRow> {
127 let key = row_index as u32;
128 array.get(&key, 0).map_err(|e| {
129 LoaderError::Generic(format!("Failed to read back unwind row {row_index}: {e}"))
130 })
131 }
132
133 let mut sample_indices = vec![0usize, rows.len() / 2, rows.len().saturating_sub(1)];
134 sample_indices.sort_unstable();
135 sample_indices.dedup();
136
137 for index in sample_indices {
138 let stored = read_row(array, index)?;
139 if stored == rows[index] {
140 debug!(index, row = ?stored, "bt unwind row readback sample");
141 } else {
142 warn!(
143 index,
144 expected = ?rows[index],
145 stored = ?stored,
146 "bt unwind row readback mismatch"
147 );
148 }
149 }
150 Ok(())
151}
152
153struct PerfEventCpuBuffer {
154 cpu_id: u32,
155 buffer: aya::maps::perf::PerfEventArrayBuffer<MapData>,
156 readiness: AsyncFd<PerfBufferFd>,
157}
158
159fn drain_perf_cpu_buffer(
160 entry: &mut PerfEventCpuBuffer,
161 parser: &mut StreamingTraceParser,
162 trace_context: &TraceContext,
163 events: &mut Vec<ParsedTraceEvent>,
164) -> Result<bool> {
165 let mut produced = false;
166 if events.len() >= MAX_EVENTS_PER_WAIT {
167 return Ok(false);
168 }
169
170 let cpu = entry.cpu_id;
171 let drain_result = entry.buffer.try_fold(
172 (0usize, 0u64),
173 |(mut read_count, mut lost_count), event| {
174 if events.len() >= MAX_EVENTS_PER_WAIT || read_count >= PERF_READ_BATCH_SIZE {
175 return ControlFlow::Break(Ok((read_count, lost_count)));
176 }
177
178 match event {
179 PerfEvent::Sample { head, tail } => {
180 read_count += 1;
181 produced = true;
182 debug!(
183 "PerfEvent {}: {} bytes - {:02x?}",
184 read_count - 1,
185 head.len() + tail.len(),
186 &head[..head.len().min(32)]
187 );
188
189 for segment in [head, tail] {
190 if segment.is_empty() {
191 continue;
192 }
193 match parser.process_segment(segment, trace_context) {
194 Ok(Some(parsed_event)) => events.push(parsed_event),
195 Ok(None) => {}
196 Err(e) => {
197 return ControlFlow::Break(Err(LoaderError::Generic(format!(
198 "Fatal: Failed to parse trace event from PerfEventArray CPU {cpu}: {e}"
199 ))));
200 }
201 }
202 }
203 }
204 PerfEvent::Lost { count } => {
205 lost_count = lost_count.saturating_add(count);
206 }
207 }
208
209 ControlFlow::Continue((read_count, lost_count))
210 },
211 );
212
213 let (read_count, lost_count) = match drain_result {
214 ControlFlow::Continue(counts) => counts,
215 ControlFlow::Break(result) => result?,
216 };
217
218 if read_count > 0 {
219 info!(
220 "Read {} events from CPU {} buffer",
221 read_count, entry.cpu_id
222 );
223 }
224 if lost_count > 0 {
225 warn!(
226 "Lost {} events from CPU {} buffer",
227 lost_count, entry.cpu_id
228 );
229 }
230
231 Ok(produced)
232}
233
234enum UProbeAttachLocation<'a> {
237 AbsoluteOffset(u64),
238 Function(&'a str),
239}
240
241impl<'a> UProbeAttachLocation<'a> {
242 fn attach<T: AsRef<Path>>(
243 self,
244 program: &mut UProbe,
245 target: T,
246 pid: Option<i32>,
247 ) -> std::result::Result<UProbeLinkId, ProgramError> {
248 let scope = uprobe_scope(pid)?;
249 match self {
250 Self::AbsoluteOffset(offset) => program.attach(offset, target, scope),
251 Self::Function(fn_name) => program.attach(fn_name, target, scope),
252 }
253 }
254}
255
256fn uprobe_scope(pid: Option<i32>) -> std::result::Result<UProbeScope, ProgramError> {
257 match pid {
258 None => Ok(UProbeScope::AllProcesses),
259 Some(pid) => {
260 let pid = u32::try_from(pid)
261 .ok()
262 .and_then(NonZeroU32::new)
263 .ok_or_else(|| {
264 ProgramError::IOError(io::Error::new(
265 io::ErrorKind::InvalidInput,
266 format!("invalid uprobe PID scope: {pid}"),
267 ))
268 })?;
269 Ok(UProbeScope::OneProcess(pid))
270 }
271 }
272}
273
274pub fn hello() -> String {
275 format!("Loader: {}", ghostscope_compiler::hello())
276}
277
278pub struct GhostScopeLoader {
286 bpf: Ebpf,
288 event_map: Option<EventMap>,
290 event_loss_counters: Option<PerCpuArray<MapData, u64>>,
292 bt_prog_array: Option<ProgramArray<MapData>>,
294 uprobe_link: Option<UProbeLinkId>,
296 attachment_params: Option<UprobeAttachmentParams>,
298 parser: StreamingTraceParser,
300 trace_context: Option<TraceContext>,
302 perf_page_count: Option<usize>,
304 backtrace_unwind_row_count: u32,
306 backtrace_module_row_cookies: HashSet<u64>,
308 shared_backtrace_maps: bool,
310}
311
312impl std::fmt::Debug for GhostScopeLoader {
313 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
314 f.debug_struct("GhostScopeLoader")
315 .field("bpf", &"<eBPF object>")
316 .field("event_map", &self.event_map.is_some())
317 .field("event_loss_counters", &self.event_loss_counters.is_some())
318 .field("bt_prog_array", &self.bt_prog_array.is_some())
319 .field("uprobe_attached", &self.uprobe_link.is_some())
320 .field("attachment_params", &self.attachment_params.is_some())
321 .field(
322 "backtrace_unwind_row_count",
323 &self.backtrace_unwind_row_count,
324 )
325 .field(
326 "backtrace_module_row_cookies",
327 &self.backtrace_module_row_cookies.len(),
328 )
329 .field("shared_backtrace_maps", &self.shared_backtrace_maps)
330 .finish()
331 }
332}
333
334impl GhostScopeLoader {
335 pub fn new(bytecode: &[u8]) -> Result<Self> {
341 Self::new_with_shared_backtrace_maps(bytecode, false)
342 }
343
344 pub fn new_with_shared_backtrace_maps(
347 bytecode: &[u8],
348 shared_backtrace_maps: bool,
349 ) -> Result<Self> {
350 info!(
351 "Loading eBPF program from bytecode ({} bytes)",
352 bytecode.len()
353 );
354
355 let pin_path = proc_offsets_pin_path()
357 .map_err(|e| LoaderError::Generic(format!("Failed to resolve pinned map path: {e}")))?;
358 if !pin_path.exists() {
359 let hint = bpffs_mount_hint_for_pin_path(&pin_path)
360 .map(|hint| format!(" {hint}"))
361 .unwrap_or_default();
362 return Err(LoaderError::Generic(format!(
363 "Pinned map '{}' not found. Please call ghostscope-process to create it first.{}",
364 pin_path.display(),
365 hint
366 )));
367 }
368 let alias_pin_path = pid_aliases_pin_path().map_err(|e| {
369 LoaderError::Generic(format!("Failed to resolve pinned alias map path: {e}"))
370 })?;
371 if !alias_pin_path.exists() {
372 let hint = bpffs_mount_hint_for_pin_path(&alias_pin_path)
373 .map(|hint| format!(" {hint}"))
374 .unwrap_or_default();
375 return Err(LoaderError::Generic(format!(
376 "Pinned map '{}' not found. Please call ghostscope-process to create it first.{}",
377 alias_pin_path.display(),
378 hint
379 )));
380 }
381 let range_meta_pin_path = proc_module_range_meta_pin_path().map_err(|e| {
382 LoaderError::Generic(format!("Failed to resolve pinned range meta map path: {e}"))
383 })?;
384 if !range_meta_pin_path.exists() {
385 let hint = bpffs_mount_hint_for_pin_path(&range_meta_pin_path)
386 .map(|hint| format!(" {hint}"))
387 .unwrap_or_default();
388 return Err(LoaderError::Generic(format!(
389 "Pinned map '{}' not found. Please call ghostscope-process to create it first.{}",
390 range_meta_pin_path.display(),
391 hint
392 )));
393 }
394 let ranges_pin_path = proc_module_ranges_pin_path().map_err(|e| {
395 LoaderError::Generic(format!("Failed to resolve pinned ranges map path: {e}"))
396 })?;
397 if !ranges_pin_path.exists() {
398 let hint = bpffs_mount_hint_for_pin_path(&ranges_pin_path)
399 .map(|hint| format!(" {hint}"))
400 .unwrap_or_default();
401 return Err(LoaderError::Generic(format!(
402 "Pinned map '{}' not found. Please call ghostscope-process to create it first.{}",
403 ranges_pin_path.display(),
404 hint
405 )));
406 }
407 let bt_rows_pin_path = if shared_backtrace_maps {
408 let path = bt_unwind_rows_pin_path().map_err(|e| {
409 LoaderError::Generic(format!(
410 "Failed to resolve pinned bt_unwind_rows map path: {e}"
411 ))
412 })?;
413 if !path.exists() {
414 let hint = bpffs_mount_hint_for_pin_path(&path)
415 .map(|hint| format!(" {hint}"))
416 .unwrap_or_default();
417 return Err(LoaderError::Generic(format!(
418 "Pinned map '{}' not found. Please call ghostscope-process to create it first.{}",
419 path.display(),
420 hint
421 )));
422 }
423 Some(path)
424 } else {
425 None
426 };
427 let bt_ranges_pin_path = if shared_backtrace_maps {
428 let path = bt_module_row_ranges_pin_path().map_err(|e| {
429 LoaderError::Generic(format!(
430 "Failed to resolve pinned bt_module_row_ranges map path: {e}"
431 ))
432 })?;
433 if !path.exists() {
434 let hint = bpffs_mount_hint_for_pin_path(&path)
435 .map(|hint| format!(" {hint}"))
436 .unwrap_or_default();
437 return Err(LoaderError::Generic(format!(
438 "Pinned map '{}' not found. Please call ghostscope-process to create it first.{}",
439 path.display(),
440 hint
441 )));
442 }
443 Some(path)
444 } else {
445 None
446 };
447
448 let mut loader = EbpfLoader::new();
449 let use_verbose = cfg!(debug_assertions)
450 || log_enabled!(LogLevel::Trace)
451 || log_enabled!(LogLevel::Debug);
452 if use_verbose {
453 loader.verifier_log_level(VerifierLogLevel::VERBOSE | VerifierLogLevel::STATS);
454 tracing::info!("BPF verifier logs: VERBOSE (debug build/log)");
455 } else {
456 loader.verifier_log_level(VerifierLogLevel::DEBUG | VerifierLogLevel::STATS);
457 tracing::info!("BPF verifier logs: DEBUG (release/info)");
458 }
459 let pin_dir = proc_offsets_pin_dir().map_err(|e| {
463 LoaderError::Generic(format!("Failed to resolve pinned map directory: {e}"))
464 })?;
465 if pin_dir.exists() {
466 loader.map_pin_path(PROC_OFFSETS_MAP_NAME, pin_path);
467 loader.map_pin_path(PID_ALIASES_MAP_NAME, alias_pin_path);
468 loader.map_pin_path(PROC_MODULE_RANGE_META_MAP_NAME, range_meta_pin_path);
469 loader.map_pin_path(PROC_MODULE_RANGES_MAP_NAME, ranges_pin_path);
470 if let (Some(rows_path), Some(ranges_path)) =
471 (bt_rows_pin_path.as_ref(), bt_ranges_pin_path.as_ref())
472 {
473 loader.map_pin_path(BT_UNWIND_ROWS_MAP_NAME, rows_path);
474 loader.map_pin_path(BT_MODULE_ROW_RANGES_MAP_NAME, ranges_path);
475 }
476 tracing::info!(
477 "Configured map pin directory for reuse: {}",
478 pin_dir.display()
479 );
480 }
481 match loader.load(bytecode) {
482 Ok(bpf) => {
483 info!("Successfully loaded eBPF program");
484 Ok(Self {
485 bpf,
486 event_map: None,
487 event_loss_counters: None,
488 bt_prog_array: None,
489 uprobe_link: None,
490 attachment_params: None,
491 parser: StreamingTraceParser::new(),
492 trace_context: None,
493 perf_page_count: None,
494 backtrace_unwind_row_count: 0,
495 backtrace_module_row_cookies: HashSet::new(),
496 shared_backtrace_maps,
497 })
498 }
499 Err(e) => {
500 error!("Failed to load BPF program: {:?}", e);
501 match &e {
503 aya::EbpfError::ParseError(parse_err) => {
504 error!("Parse error details: {:?}", parse_err);
505 }
506 aya::EbpfError::BtfError(btf_err) => {
507 error!("BTF error details: {:?}", btf_err);
508 }
509 _ => {
510 error!("Other BPF error: {:?}", e);
511 }
512 }
513 Err(LoaderError::Aya(e))
514 }
515 }
516 }
517
518 pub fn attach_uprobe(
524 &mut self,
525 target_binary: &str,
526 function_name: &str,
527 offset: Option<u64>,
528 pid: Option<i32>,
529 ) -> Result<()> {
530 self.attach_uprobe_with_program_name(target_binary, function_name, offset, pid, None)
531 }
532
533 pub fn set_perf_page_count(&mut self, pages: u32) {
535 self.perf_page_count = Some(pages as usize);
536 }
537
538 pub fn register_backtrace_tail_call_program(
540 &mut self,
541 program_name: Option<&str>,
542 ) -> Result<()> {
543 let Some(program_name) = program_name else {
544 return Ok(());
545 };
546
547 info!("Registering bt tail-call step program: {}", program_name);
548 let program_ref = self.bpf.program_mut(program_name).ok_or_else(|| {
549 LoaderError::Generic(format!("bt tail-call program '{program_name}' not found"))
550 })?;
551 let program: &mut UProbe = program_ref.try_into().map_err(|e| {
552 LoaderError::Generic(format!(
553 "bt tail-call program '{program_name}' is not a UProbe: {e:?}"
554 ))
555 })?;
556 program.load().map_err(LoaderError::Program)?;
557 let step_fd = program
558 .fd()
559 .map_err(LoaderError::Program)?
560 .try_clone()
561 .map_err(|e| {
562 LoaderError::Generic(format!(
563 "Failed to clone bt tail-call program fd for '{program_name}': {e}"
564 ))
565 })?;
566
567 let map = self
568 .bpf
569 .take_map("bt_prog_array")
570 .ok_or_else(|| LoaderError::MapNotFound("bt_prog_array".to_string()))?;
571 let mut prog_array: ProgramArray<_> = map.try_into().map_err(|e| {
572 LoaderError::Generic(format!("Failed to convert bt_prog_array map: {e}"))
573 })?;
574 prog_array.set(0, &step_fd, 0).map_err(|e| {
575 LoaderError::Generic(format!("Failed to set bt tail-call program fd: {e}"))
576 })?;
577 self.bt_prog_array = Some(prog_array);
578 info!("Registered bt tail-call step program at bt_prog_array[0]");
579 Ok(())
580 }
581
582 pub fn attach_uprobe_with_program_name(
584 &mut self,
585 target_binary: &str,
586 function_name: &str,
587 offset: Option<u64>,
588 pid: Option<i32>,
589 program_name: Option<&str>,
590 ) -> Result<()> {
591 info!("attach_uprobe called with offset: {:?}", offset);
592 if let Some(offset) = offset {
593 info!(
594 "Using offset-based attachment: {} at 0x{:x} ({}) (pid: {:?})",
595 target_binary, offset, function_name, pid
596 );
597 } else {
598 info!(
599 "Using function name-based attachment: {}:{} (pid: {:?})",
600 target_binary, function_name, pid
601 );
602 }
603
604 let available_programs: Vec<String> = self
606 .bpf
607 .programs()
608 .map(|(name, _)| name.to_string())
609 .collect();
610
611 info!("Available programs:");
613 for name in &available_programs {
614 info!(" - {}", name);
615 }
616
617 let program_name: String = if let Some(name) = program_name {
619 info!("Using specified program name: {}", name);
621 if available_programs.contains(&name.to_string()) {
622 name.to_string()
623 } else {
624 return Err(LoaderError::Generic(format!(
625 "Specified program '{name}' not found in eBPF object"
626 )));
627 }
628 } else {
629 let program_names = ["uprobe", "main"];
631 let mut found_program_name: Option<String> = None;
632
633 for name in &program_names {
634 info!("Checking if program exists: {}", name);
635 if available_programs.contains(&name.to_string()) {
636 info!("Found program: {}", name);
637 found_program_name = Some(name.to_string());
638 break;
639 }
640 }
641
642 if found_program_name.is_none() {
644 if let Some(first_name) = available_programs.first() {
645 info!(
646 "No standard program names found, using first available: {}",
647 first_name
648 );
649 found_program_name = Some(first_name.clone());
650 }
651 }
652
653 found_program_name
654 .ok_or_else(|| LoaderError::Generic("No suitable program found".to_string()))?
655 };
656
657 info!("Attempting to load program: {}", program_name);
658
659 let program_ref = self
660 .bpf
661 .program_mut(&program_name)
662 .ok_or_else(|| LoaderError::Generic(format!("Program '{program_name}' not found")))?;
663
664 info!("Found program, attempting to convert to UProbe");
665 info!("Program type: {:?}", program_ref.prog_type());
666
667 match program_ref {
669 aya::programs::Program::UProbe(_) => {
670 info!("Program is correctly recognized as UProbe");
671 }
672 aya::programs::Program::KProbe(_) => {
673 error!("Program is incorrectly recognized as KProbe, should be UProbe");
674 }
675 ref _other => {
676 error!("Program is unexpected type (not UProbe or KProbe)");
677 }
678 }
679
680 let program: &mut UProbe = program_ref.try_into().map_err(|e| {
681 LoaderError::Generic(format!("Program '{program_name}' is not a UProbe: {e:?}"))
682 })?;
683
684 info!("About to load eBPF program");
686 match program.load() {
687 Ok(()) => {
688 info!("Program loaded successfully");
689 }
690 Err(e) => {
691 error!("eBPF program load failed: {}", e);
692 error!("This typically indicates eBPF verifier rejection");
693
694 if let ProgramError::SyscallError(syscall_error) = &e {
696 error!(
697 "Syscall '{}' failed: {}",
698 syscall_error.call, syscall_error.io_error
699 );
700
701 if let Some(errno) = syscall_error.io_error.raw_os_error() {
703 match errno {
704 22 => error!(
705 "EINVAL (22): Invalid argument - likely eBPF verifier rejection"
706 ),
707 7 => error!("E2BIG (7): Program too large"),
708 13 => error!("EACCES (13): Permission denied"),
709 95 => error!("EOPNOTSUPP (95): Operation not supported"),
710 _ => error!("Unknown errno: {}", errno),
711 }
712 }
713 }
714
715 error!("Program name: {}", program_name);
717 error!("Program type: {:?}", program_ref.prog_type());
718
719 return Err(LoaderError::Program(e));
720 }
721 }
722
723 let attach_location = match offset {
726 Some(offset) => UProbeAttachLocation::AbsoluteOffset(offset),
727 None => UProbeAttachLocation::Function(function_name),
728 };
729 let attach_result = attach_location.attach(program, target_binary, pid);
730
731 match attach_result {
732 Ok(link) => {
733 if let Some(offset) = offset {
734 info!(
735 "Uprobe attached successfully to {} at offset 0x{:x}",
736 target_binary, offset
737 );
738 } else {
739 info!(
740 "Uprobe attached successfully to {}:{}",
741 target_binary, function_name
742 );
743 }
744
745 self.uprobe_link = Some(link);
747 self.attachment_params = Some(UprobeAttachmentParams {
748 target_binary: target_binary.to_string(),
749 function_name: function_name.to_string(),
750 offset,
751 pid,
752 program_name,
753 });
754 }
755 Err(e) => {
756 if let Some(offset) = offset {
757 error!(
758 "Failed to attach uprobe to {} at offset 0x{:x}: {}",
759 target_binary, offset, e
760 );
761 error!("Detailed error: {:#?}", e);
762 } else {
763 error!(
764 "Failed to attach uprobe to {}:{}: {}",
765 target_binary, function_name, e
766 );
767 error!("Detailed error: {:#?}", e);
768 }
769
770 if let ProgramError::SyscallError(syscall_error) = &e {
772 error!(
773 "Syscall '{}' failed: {}",
774 syscall_error.call, syscall_error.io_error
775 );
776 if let Some(13) = syscall_error.io_error.raw_os_error() {
777 error!("Permission denied - make sure to run with sudo");
778 }
779 }
780
781 return Err(LoaderError::Program(e));
782 }
783 }
784
785 let event_map = if let Some(map) = self.bpf.take_map("ringbuf") {
788 info!("Initializing RingBuf event map");
789 let ringbuf: RingBuf<_> = map
790 .try_into()
791 .map_err(|e| LoaderError::Generic(format!("Failed to convert ringbuf map: {e}")))?;
792 EventMap::RingBuf(ringbuf)
793 } else if let Some(map) = self.bpf.take_map("events") {
794 info!("Initializing PerfEventArray event map");
795 let mut perf_array: PerfEventArray<_> = map.try_into().map_err(|e| {
796 LoaderError::Generic(format!("Failed to convert perf event array map: {e}"))
797 })?;
798
799 let online_cpus = aya::util::online_cpus().map_err(|(_, e)| {
801 LoaderError::Generic(format!("Failed to get online CPUs: {e}"))
802 })?;
803
804 info!(
805 "Opening PerfEventArray buffers for {} online CPUs",
806 online_cpus.len()
807 );
808
809 let mut cpu_buffers = Vec::new();
811
812 for cpu_id in online_cpus {
813 let pages = self.perf_page_count;
814 match perf_array.open(cpu_id, pages) {
815 Ok(buffer) => {
816 if let Some(p) = pages {
817 info!(
818 "Opened PerfEventArray buffer for CPU {} with {} pages",
819 cpu_id, p
820 );
821 } else {
822 info!(
823 "Opened PerfEventArray buffer for CPU {} (default pages)",
824 cpu_id
825 );
826 }
827 let fd = buffer.as_raw_fd();
828 let readiness =
829 AsyncFd::with_interest(PerfBufferFd(fd), Interest::READABLE).map_err(
830 |err| {
831 LoaderError::Generic(format!(
832 "Failed to register perf buffer fd for CPU {cpu_id}: {err}"
833 ))
834 },
835 )?;
836 cpu_buffers.push(PerfEventCpuBuffer {
837 cpu_id,
838 buffer,
839 readiness,
840 });
841 }
842 Err(e) => {
843 warn!("Failed to open perf buffer for CPU {}: {}", cpu_id, e);
844 }
845 }
846 }
847
848 if cpu_buffers.is_empty() {
849 return Err(LoaderError::Generic(
850 "Failed to open any perf event buffers".to_string(),
851 ));
852 }
853
854 EventMap::PerfEventArray {
855 _map: perf_array,
856 cpu_buffers,
857 }
858 } else {
859 return Err(LoaderError::MapNotFound(
860 "Neither 'ringbuf' nor 'events' map found".to_string(),
861 ));
862 };
863
864 self.event_loss_counters = if let Some(map) = self.bpf.take_map("event_loss_counters") {
865 info!("Initializing eBPF event loss counter map");
866 Some(map.try_into().map_err(|e| {
867 LoaderError::Generic(format!("Failed to convert event_loss_counters map: {e}"))
868 })?)
869 } else {
870 warn!("No eBPF event loss counter map found; kernel output loss stats unavailable");
871 None
872 };
873
874 let event_source = match &event_map {
876 EventMap::RingBuf(_) => {
877 info!("Using RingBuf mode for parser");
878 ghostscope_protocol::EventSource::RingBuf
879 }
880 EventMap::PerfEventArray { .. } => {
881 info!("Using PerfEventArray mode for parser");
882 ghostscope_protocol::EventSource::PerfEventArray
883 }
884 };
885 self.parser = StreamingTraceParser::with_event_source(event_source);
886
887 self.event_map = Some(event_map);
888 info!("Event map initialized");
889
890 Ok(())
891 }
892
893 pub fn detach_uprobe(&mut self) -> Result<()> {
896 if let Some(link_id) = self.uprobe_link.take() {
897 if let Some(params) = &self.attachment_params {
898 info!("Detaching uprobe...");
899
900 let program_ref = self.bpf.program_mut(¶ms.program_name).ok_or_else(|| {
902 let program_name = ¶ms.program_name;
903 LoaderError::Generic(format!("Program '{program_name}' not found"))
904 })?;
905
906 let program: &mut UProbe = program_ref.try_into().map_err(|e| {
907 let program_name = ¶ms.program_name;
908 LoaderError::Generic(format!("Program '{program_name}' is not a UProbe: {e:?}"))
909 })?;
910
911 program.detach(link_id).map_err(LoaderError::Program)?;
913
914 info!("Uprobe detached successfully");
915 Ok(())
916 } else {
917 error!("No attachment parameters stored");
918 Err(LoaderError::Generic(
919 "No attachment parameters stored".to_string(),
920 ))
921 }
922 } else {
923 warn!("No uprobe attached, nothing to detach");
924 Ok(())
925 }
926 }
927
928 pub fn reattach_uprobe(&mut self) -> Result<()> {
931 if self.uprobe_link.is_some() {
932 info!("Uprobe already attached");
933 return Ok(());
934 }
935
936 let params = self
937 .attachment_params
938 .as_ref()
939 .ok_or_else(|| {
940 LoaderError::Generic(
941 "No attachment parameters stored. Call attach_uprobe first.".to_string(),
942 )
943 })?
944 .clone();
945
946 info!("Reattaching uprobe with stored parameters...");
947
948 let program_ref = self.bpf.program_mut(¶ms.program_name).ok_or_else(|| {
950 LoaderError::Generic(format!("Program '{}' not found", params.program_name))
951 })?;
952
953 let program: &mut UProbe = program_ref.try_into().map_err(|e| {
954 LoaderError::Generic(format!(
955 "Program '{}' is not a UProbe: {:?}",
956 params.program_name, e
957 ))
958 })?;
959
960 let attach_location = match params.offset {
962 Some(offset) => UProbeAttachLocation::AbsoluteOffset(offset),
963 None => UProbeAttachLocation::Function(params.function_name.as_str()),
964 };
965 let attach_result = attach_location.attach(program, ¶ms.target_binary, params.pid);
966
967 match attach_result {
968 Ok(link) => {
969 if let Some(offset) = params.offset {
970 info!(
971 "Uprobe reattached successfully to {} at offset 0x{:x}",
972 params.target_binary, offset
973 );
974 } else {
975 info!(
976 "Uprobe reattached successfully to {}:{}",
977 params.target_binary, params.function_name
978 );
979 }
980
981 self.uprobe_link = Some(link);
983 Ok(())
984 }
985 Err(e) => {
986 error!("Failed to reattach uprobe: {:?}", e);
987 Err(LoaderError::Program(e))
988 }
989 }
990 }
991
992 pub fn is_uprobe_attached(&self) -> bool {
994 self.uprobe_link.is_some()
995 }
996
997 pub fn destroy(&mut self) -> Result<()> {
1001 info!("Destroying GhostScopeLoader and all associated resources");
1002
1003 if self.uprobe_link.is_some() {
1005 if let Err(e) = self.detach_uprobe() {
1006 warn!("Failed to detach uprobe during destroy: {}", e);
1007 }
1009 }
1010
1011 self.attachment_params = None;
1013
1014 self.event_map = None;
1017
1018 info!("GhostScopeLoader destroyed successfully");
1022 Ok(())
1023 }
1024
1025 pub fn get_attachment_info(&self) -> Option<String> {
1027 if let Some(params) = &self.attachment_params {
1028 if let Some(offset) = params.offset {
1029 Some(format!(
1030 "{}:{} (offset: 0x{:x}, pid: {:?}) - {}",
1031 params.target_binary,
1032 params.function_name,
1033 offset,
1034 params.pid,
1035 if self.is_uprobe_attached() {
1036 "attached"
1037 } else {
1038 "detached"
1039 }
1040 ))
1041 } else {
1042 Some(format!(
1043 "{}:{} (pid: {:?}) - {}",
1044 params.target_binary,
1045 params.function_name,
1046 params.pid,
1047 if self.is_uprobe_attached() {
1048 "attached"
1049 } else {
1050 "detached"
1051 }
1052 ))
1053 }
1054 } else {
1055 None
1056 }
1057 }
1058
1059 pub async fn wait_for_events_async(&mut self) -> Result<Vec<ParsedTraceEvent>> {
1065 let trace_context = self.trace_context.as_ref().ok_or_else(|| {
1066 LoaderError::Generic(
1067 "No trace context available - cannot parse trace events".to_string(),
1068 )
1069 })?;
1070
1071 let event_map = self.event_map.as_mut().ok_or_else(|| {
1072 LoaderError::Generic("Event map not initialized. Call attach_uprobe first.".to_string())
1073 })?;
1074
1075 let mut events = Vec::with_capacity(MAX_EVENTS_PER_WAIT.min(128));
1076
1077 match event_map {
1078 EventMap::RingBuf(ringbuf) => {
1079 let async_fd = AsyncFd::new(ringbuf.as_raw_fd())
1081 .map_err(|e| LoaderError::Generic(format!("Failed to create AsyncFd: {e}")))?;
1082 let mut guard = async_fd
1083 .readable()
1084 .await
1085 .map_err(|e| LoaderError::Generic(format!("AsyncFd error: {e}")))?;
1086 guard.clear_ready();
1087
1088 let mut records_read = 0;
1091 while events.len() < MAX_EVENTS_PER_WAIT
1092 && records_read < MAX_RINGBUF_RECORDS_PER_WAIT
1093 {
1094 let Some(item) = ringbuf.next() else {
1095 break;
1096 };
1097 records_read += 1;
1098 match self.parser.process_segment(&item, trace_context) {
1099 Ok(Some(parsed_event)) => events.push(parsed_event),
1100 Ok(None) => {}
1101 Err(e) => {
1102 return Err(LoaderError::Generic(format!(
1103 "Fatal: Failed to parse trace event from RingBuf (async): {e}"
1104 )));
1105 }
1106 }
1107 }
1108 if events.len() == MAX_EVENTS_PER_WAIT
1109 || records_read == MAX_RINGBUF_RECORDS_PER_WAIT
1110 {
1111 debug!(
1112 "RingBuf batch limit reached ({} events, {} records); yielding to caller",
1113 events.len(),
1114 records_read
1115 );
1116 }
1117 }
1118 EventMap::PerfEventArray { cpu_buffers, .. } => {
1119 let parser = &mut self.parser;
1120
1121 loop {
1122 let mut made_progress = false;
1124 for entry in cpu_buffers.iter_mut() {
1125 if events.len() >= MAX_EVENTS_PER_WAIT {
1126 break;
1127 }
1128 if entry.buffer.readable() {
1129 made_progress |=
1130 drain_perf_cpu_buffer(entry, parser, trace_context, &mut events)?;
1131 }
1132 }
1133
1134 if made_progress {
1135 break;
1136 }
1137
1138 let ready_idx = poll_fn(|cx| {
1140 for (idx, entry) in cpu_buffers.iter().enumerate() {
1141 match entry.readiness.poll_read_ready(cx) {
1142 Poll::Ready(Ok(mut guard)) => {
1143 guard.clear_ready();
1144 return Poll::Ready(Ok(idx));
1145 }
1146 Poll::Ready(Err(e)) => return Poll::Ready(Err(e)),
1147 Poll::Pending => {}
1148 }
1149 }
1150 Poll::Pending
1151 })
1152 .await
1153 .map_err(|e| {
1154 LoaderError::Generic(format!(
1155 "AsyncFd error while waiting for perf events: {e}"
1156 ))
1157 })?;
1158
1159 made_progress |= drain_perf_cpu_buffer(
1161 cpu_buffers
1162 .get_mut(ready_idx)
1163 .expect("ready index should be valid"),
1164 parser,
1165 trace_context,
1166 &mut events,
1167 )?;
1168
1169 for (idx, entry) in cpu_buffers.iter_mut().enumerate() {
1171 if events.len() >= MAX_EVENTS_PER_WAIT {
1172 break;
1173 }
1174 if idx == ready_idx || !entry.buffer.readable() {
1175 continue;
1176 }
1177 made_progress |=
1178 drain_perf_cpu_buffer(entry, parser, trace_context, &mut events)?;
1179 }
1180
1181 if made_progress {
1182 if events.len() == MAX_EVENTS_PER_WAIT {
1183 debug!(
1184 "PerfEventArray event batch limit reached ({} events); yielding to caller",
1185 MAX_EVENTS_PER_WAIT
1186 );
1187 }
1188 break;
1189 }
1190 }
1193 }
1194 }
1195
1196 Ok(events)
1197 }
1198
1199 pub fn read_event_loss_stats(&self) -> Result<Option<EventLossStats>> {
1200 let Some(counters) = &self.event_loss_counters else {
1201 return Ok(None);
1202 };
1203
1204 let values = counters
1205 .get(&EVENT_LOSS_OUTPUT_FAILURES_KEY, 0)
1206 .map_err(|e| {
1207 LoaderError::Generic(format!("Failed to read event_loss_counters map: {e}"))
1208 })?;
1209
1210 Ok(Some(EventLossStats {
1211 output_failures: values.iter().copied().sum(),
1212 }))
1213 }
1214
1215 pub fn set_trace_context(&mut self, trace_context: TraceContext) {
1217 info!("Setting trace context for trace event parsing");
1218 self.trace_context = Some(trace_context);
1219 }
1220
1221 fn sync_shared_backtrace_row_state(&mut self) -> Result<()> {
1222 if !self.shared_backtrace_maps {
1223 return Ok(());
1224 }
1225
1226 let Some(map) = self.bpf.map_mut(BT_MODULE_ROW_RANGES_MAP_NAME) else {
1227 return Ok(());
1228 };
1229 let hash: AyaHashMap<_, u64, BacktraceModuleRowRange> = map.try_into().map_err(|e| {
1230 LoaderError::Generic(format!(
1231 "Failed to convert shared bt_module_row_ranges map: {e}"
1232 ))
1233 })?;
1234 let keys = hash
1235 .keys()
1236 .collect::<std::result::Result<Vec<_>, _>>()
1237 .map_err(|e| {
1238 LoaderError::Generic(format!(
1239 "Failed to list shared bt_module_row_ranges keys: {e}"
1240 ))
1241 })?;
1242
1243 let mut max_row_end = self.backtrace_unwind_row_count;
1244 for cookie in keys {
1245 match hash.get(&cookie, 0) {
1246 Ok(range) => {
1247 self.backtrace_module_row_cookies.insert(cookie);
1248 max_row_end = max_row_end.max(range.row_end);
1249 }
1250 Err(e) => {
1251 debug!(
1252 cookie = format_args!("0x{cookie:016x}"),
1253 "Skipped shared bt module row range during sync: {}", e
1254 );
1255 }
1256 }
1257 }
1258 self.backtrace_unwind_row_count = max_row_end;
1259 Ok(())
1260 }
1261
1262 pub fn populate_backtrace_unwind_rows_and_module_row_ranges(
1263 &mut self,
1264 rows: &[BacktraceUnwindRow],
1265 ranges: &[(u64, BacktraceModuleRowRange)],
1266 ) -> Result<()> {
1267 if rows.is_empty() {
1268 return Ok(());
1269 }
1270
1271 if ranges.is_empty() || !self.shared_backtrace_maps {
1272 self.populate_backtrace_unwind_rows(rows)?;
1273 self.populate_backtrace_module_row_ranges(ranges)?;
1274 return Ok(());
1275 }
1276
1277 self.sync_shared_backtrace_row_state()?;
1278 for (cookie, range) in ranges.iter().copied() {
1279 if self.backtrace_module_row_cookies.contains(&cookie) {
1280 continue;
1281 }
1282
1283 let row_start = usize::try_from(range.row_start).map_err(|_| {
1284 LoaderError::Generic(format!(
1285 "Invalid row_start for module cookie 0x{cookie:016x}: {}",
1286 range.row_start
1287 ))
1288 })?;
1289 let row_end = usize::try_from(range.row_end).map_err(|_| {
1290 LoaderError::Generic(format!(
1291 "Invalid row_end for module cookie 0x{cookie:016x}: {}",
1292 range.row_end
1293 ))
1294 })?;
1295 if row_start > row_end || row_end > rows.len() {
1296 return Err(LoaderError::Generic(format!(
1297 "Invalid bt row range for module cookie 0x{cookie:016x}: \
1298 {}..{} with {} rows",
1299 range.row_start,
1300 range.row_end,
1301 rows.len()
1302 )));
1303 }
1304
1305 self.append_backtrace_unwind_rows_for_module_after_sync(
1306 cookie,
1307 &rows[row_start..row_end],
1308 )?;
1309 }
1310
1311 Ok(())
1312 }
1313
1314 pub fn populate_backtrace_unwind_rows(&mut self, rows: &[BacktraceUnwindRow]) -> Result<()> {
1315 if rows.is_empty() {
1316 return Ok(());
1317 }
1318
1319 let Some(map) = self.bpf.map_mut("bt_unwind_rows") else {
1320 return Err(LoaderError::MapNotFound("bt_unwind_rows".to_string()));
1321 };
1322 let mut array: Array<_, BacktraceUnwindRow> = map.try_into().map_err(|e| {
1323 LoaderError::Generic(format!("Failed to convert bt_unwind_rows map: {e}"))
1324 })?;
1325 let populate_started_at = Instant::now();
1326 for (row_index, row) in rows.iter().copied().enumerate() {
1327 array.set(row_index as u32, row, 0).map_err(|e| {
1328 LoaderError::Generic(format!("Failed to set unwind row {row_index}: {e}"))
1329 })?;
1330 }
1331 self.backtrace_unwind_row_count = self.backtrace_unwind_row_count.max(rows.len() as u32);
1332 info!(
1333 rows = rows.len(),
1334 capacity = array.len(),
1335 row_size = BACKTRACE_UNWIND_ROW_SIZE,
1336 elapsed_ms = populate_started_at.elapsed().as_millis(),
1337 "Loaded DWARF unwind rows for bt"
1338 );
1339 if log_enabled!(LogLevel::Debug) {
1340 log_backtrace_unwind_row_samples(&array, rows)?;
1341 }
1342 Ok(())
1343 }
1344
1345 pub fn populate_backtrace_module_row_ranges(
1346 &mut self,
1347 ranges: &[(u64, BacktraceModuleRowRange)],
1348 ) -> Result<()> {
1349 if ranges.is_empty() {
1350 return Ok(());
1351 }
1352
1353 let Some(map) = self.bpf.map_mut("bt_module_row_ranges") else {
1354 return Err(LoaderError::MapNotFound("bt_module_row_ranges".to_string()));
1355 };
1356 let mut hash: AyaHashMap<_, u64, BacktraceModuleRowRange> =
1357 map.try_into().map_err(|e| {
1358 LoaderError::Generic(format!("Failed to convert bt_module_row_ranges map: {e}"))
1359 })?;
1360 let populate_started_at = Instant::now();
1361 for (cookie, range) in ranges.iter().copied() {
1362 hash.insert(cookie, range, 0).map_err(|e| {
1363 LoaderError::Generic(format!(
1364 "Failed to set bt module row range for cookie 0x{cookie:016x}: {e}"
1365 ))
1366 })?;
1367 self.backtrace_module_row_cookies.insert(cookie);
1368 self.backtrace_unwind_row_count = self.backtrace_unwind_row_count.max(range.row_end);
1369 }
1370 info!(
1371 modules = ranges.len(),
1372 elapsed_ms = populate_started_at.elapsed().as_millis(),
1373 "Loaded DWARF unwind row ranges for bt"
1374 );
1375 Ok(())
1376 }
1377
1378 pub fn append_backtrace_unwind_rows_for_module(
1379 &mut self,
1380 cookie: u64,
1381 rows: &[BacktraceUnwindRow],
1382 ) -> Result<Option<BacktraceModuleRowRange>> {
1383 if self.shared_backtrace_maps {
1384 self.sync_shared_backtrace_row_state()?;
1385 }
1386 self.append_backtrace_unwind_rows_for_module_after_sync(cookie, rows)
1387 }
1388
1389 fn append_backtrace_unwind_rows_for_module_after_sync(
1390 &mut self,
1391 cookie: u64,
1392 rows: &[BacktraceUnwindRow],
1393 ) -> Result<Option<BacktraceModuleRowRange>> {
1394 if rows.is_empty() || self.backtrace_module_row_cookies.contains(&cookie) {
1395 return Ok(None);
1396 }
1397
1398 if self.bpf.map("bt_unwind_rows").is_none()
1399 || self.bpf.map("bt_module_row_ranges").is_none()
1400 {
1401 return Ok(None);
1402 }
1403
1404 let start = self.backtrace_unwind_row_count;
1405 let row_count = u32::try_from(rows.len()).map_err(|_| {
1406 LoaderError::Generic(format!(
1407 "Too many unwind rows for module cookie 0x{cookie:016x}: {}",
1408 rows.len()
1409 ))
1410 })?;
1411 let end = start.checked_add(row_count).ok_or_else(|| {
1412 LoaderError::Generic(format!(
1413 "Unwind row index overflow for module cookie 0x{cookie:016x}"
1414 ))
1415 })?;
1416
1417 let Some(map) = self.bpf.map_mut("bt_unwind_rows") else {
1418 return Ok(None);
1419 };
1420 let mut array: Array<_, BacktraceUnwindRow> = map.try_into().map_err(|e| {
1421 LoaderError::Generic(format!("Failed to convert bt_unwind_rows map: {e}"))
1422 })?;
1423 if end > array.len() {
1424 return Err(LoaderError::Generic(format!(
1425 "bt_unwind_rows capacity exceeded while appending module \
1426 0x{cookie:016x}: need end row {}, capacity {}",
1427 end,
1428 array.len()
1429 )));
1430 }
1431
1432 for (offset, row) in rows.iter().copied().enumerate() {
1433 let row_index = start + offset as u32;
1434 array.set(row_index, row, 0).map_err(|e| {
1435 LoaderError::Generic(format!("Failed to append unwind row {row_index}: {e}"))
1436 })?;
1437 }
1438
1439 let range = BacktraceModuleRowRange {
1440 row_start: start,
1441 row_end: end,
1442 };
1443 let Some(map) = self.bpf.map_mut("bt_module_row_ranges") else {
1444 return Ok(None);
1445 };
1446 let mut hash: AyaHashMap<_, u64, BacktraceModuleRowRange> =
1447 map.try_into().map_err(|e| {
1448 LoaderError::Generic(format!("Failed to convert bt_module_row_ranges map: {e}"))
1449 })?;
1450 hash.insert(cookie, range, 0).map_err(|e| {
1451 LoaderError::Generic(format!(
1452 "Failed to append bt module row range for cookie 0x{cookie:016x}: {e}"
1453 ))
1454 })?;
1455
1456 self.backtrace_unwind_row_count = end;
1457 self.backtrace_module_row_cookies.insert(cookie);
1458 debug!(
1459 cookie = format_args!("0x{cookie:016x}"),
1460 rows = rows.len(),
1461 row_start = range.row_start,
1462 row_end = range.row_end,
1463 "Appended DWARF unwind rows for bt module"
1464 );
1465
1466 Ok(Some(range))
1467 }
1468
1469 pub fn append_backtrace_unwind_rows_for_modules(
1470 &mut self,
1471 modules: &[(u64, Vec<BacktraceUnwindRow>)],
1472 ) -> Result<BacktraceUnwindRowsAppendStats> {
1473 let mut stats = BacktraceUnwindRowsAppendStats::default();
1474 if self.shared_backtrace_maps {
1475 self.sync_shared_backtrace_row_state()?;
1476 }
1477 for (cookie, rows) in modules {
1478 if self
1479 .append_backtrace_unwind_rows_for_module_after_sync(*cookie, rows)?
1480 .is_some()
1481 {
1482 stats.modules += 1;
1483 stats.rows += rows.len();
1484 }
1485 }
1486 Ok(stats)
1487 }
1488
1489 pub fn get_map_info(&self) -> Vec<String> {
1495 self.bpf
1496 .maps()
1497 .map(|(name, _map)| format!("Map: {name}"))
1498 .collect()
1499 }
1500
1501 pub fn get_program_info(&self) -> Vec<String> {
1503 self.bpf
1504 .programs()
1505 .map(|(name, _prog)| format!("Program: {name}"))
1506 .collect()
1507 }
1508}