Skip to main content

ghostscope_loader/
lib.rs

1//! GhostScope eBPF Loader
2//!
3//! This crate provides the `GhostScopeLoader` which manages the lifecycle of eBPF programs:
4//! - Loading eBPF bytecode into the kernel
5//! - Attaching/detaching uprobes to target binaries
6//! - Reading trace events from RingBuf or PerfEventArray
7//! - Managing BPF maps for process module offsets
8//!
9//! ## Architecture
10//!
11//! The loader supports two event output mechanisms:
12//! - **RingBuf**: Modern kernel (>= 5.8) continuous byte stream
13//! - **PerfEventArray**: Legacy kernel (< 5.8) per-CPU independent events
14//!
15//! Event parsing is handled by `ghostscope_protocol::StreamingTraceParser` which
16//! adapts to the event source type automatically.
17
18use aya::{
19    maps::{
20        perf::{PerfEvent, PerfEventArray},
21        Array, HashMap as AyaHashMap, MapData, PerCpuArray, ProgramArray, RingBuf,
22    },
23    programs::{
24        uprobe::{UProbeLinkId, UProbeScope},
25        ProgramError, UProbe,
26    },
27    Ebpf, EbpfLoader, VerifierLogLevel,
28};
29use ghostscope_protocol::{
30    BacktraceModuleRowRange, BacktraceUnwindRow, ParsedTraceEvent, StreamingTraceParser,
31    TraceContext, BACKTRACE_UNWIND_ROW_SIZE,
32};
33use log::log_enabled;
34use log::Level as LogLevel;
35use std::borrow::Borrow;
36use std::collections::HashSet;
37use std::convert::TryInto;
38use std::future::poll_fn;
39use std::num::NonZeroU32;
40use std::os::unix::io::AsRawFd;
41use std::os::unix::io::RawFd;
42use std::path::Path;
43use std::task::Poll;
44use std::time::Instant;
45use std::{io, ops::ControlFlow};
46use tokio::io::unix::AsyncFd;
47use tokio::io::Interest;
48use tracing::{debug, error, info, warn};
49
50const MAX_EVENTS_PER_WAIT: usize = 128;
51const MAX_RINGBUF_RECORDS_PER_WAIT: usize = 256;
52const PERF_READ_BATCH_SIZE: usize = 64;
53const EVENT_LOSS_OUTPUT_FAILURES_KEY: u32 = 0;
54
55#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
56pub struct EventLossStats {
57    pub output_failures: u64,
58}
59
60impl EventLossStats {
61    pub fn is_empty(self) -> bool {
62        self.output_failures == 0
63    }
64
65    pub fn saturating_sub(self, previous: Self) -> Self {
66        Self {
67            output_failures: self
68                .output_failures
69                .saturating_sub(previous.output_failures),
70        }
71    }
72}
73
74#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
75pub struct BacktraceUnwindRowsAppendStats {
76    pub modules: usize,
77    pub rows: usize,
78}
79
80// Export kernel capabilities detection
81mod kernel_caps;
82pub use kernel_caps::{KernelCapabilities, KernelCapabilityError};
83
84// Export error types
85mod error;
86pub use error::{LoaderError, Result};
87
88// Internal uprobe module
89mod uprobe;
90use uprobe::UprobeAttachmentParams;
91
92// Use shared map types from ghostscope-process
93use ghostscope_process::pinned_bpf_maps::{
94    bpffs_mount_hint_for_pin_path, bt_module_row_ranges_pin_path, bt_unwind_rows_pin_path,
95    pid_aliases_pin_path, proc_module_range_meta_pin_path, proc_module_ranges_pin_path,
96    proc_offsets_pin_dir, proc_offsets_pin_path, BT_MODULE_ROW_RANGES_MAP_NAME,
97    BT_UNWIND_ROWS_MAP_NAME, PID_ALIASES_MAP_NAME, PROC_MODULE_RANGES_MAP_NAME,
98    PROC_MODULE_RANGE_META_MAP_NAME, PROC_OFFSETS_MAP_NAME,
99};
100
101/// Event output map type wrapper
102enum EventMap {
103    RingBuf(RingBuf<MapData>),
104    PerfEventArray {
105        _map: PerfEventArray<MapData>,
106        cpu_buffers: Vec<PerfEventCpuBuffer>,
107    },
108}
109
110#[derive(Clone, Copy, Debug)]
111struct PerfBufferFd(RawFd);
112
113impl AsRawFd for PerfBufferFd {
114    fn as_raw_fd(&self) -> RawFd {
115        self.0
116    }
117}
118
119fn log_backtrace_unwind_row_samples<T: Borrow<MapData>>(
120    array: &Array<T, BacktraceUnwindRow>,
121    rows: &[BacktraceUnwindRow],
122) -> Result<()> {
123    fn read_row<T: Borrow<MapData>>(
124        array: &Array<T, BacktraceUnwindRow>,
125        row_index: usize,
126    ) -> Result<BacktraceUnwindRow> {
127        let key = row_index as u32;
128        array.get(&key, 0).map_err(|e| {
129            LoaderError::Generic(format!("Failed to read back unwind row {row_index}: {e}"))
130        })
131    }
132
133    let mut sample_indices = vec![0usize, rows.len() / 2, rows.len().saturating_sub(1)];
134    sample_indices.sort_unstable();
135    sample_indices.dedup();
136
137    for index in sample_indices {
138        let stored = read_row(array, index)?;
139        if stored == rows[index] {
140            debug!(index, row = ?stored, "bt unwind row readback sample");
141        } else {
142            warn!(
143                index,
144                expected = ?rows[index],
145                stored = ?stored,
146                "bt unwind row readback mismatch"
147            );
148        }
149    }
150    Ok(())
151}
152
153struct PerfEventCpuBuffer {
154    cpu_id: u32,
155    buffer: aya::maps::perf::PerfEventArrayBuffer<MapData>,
156    readiness: AsyncFd<PerfBufferFd>,
157}
158
159fn drain_perf_cpu_buffer(
160    entry: &mut PerfEventCpuBuffer,
161    parser: &mut StreamingTraceParser,
162    trace_context: &TraceContext,
163    events: &mut Vec<ParsedTraceEvent>,
164) -> Result<bool> {
165    let mut produced = false;
166    if events.len() >= MAX_EVENTS_PER_WAIT {
167        return Ok(false);
168    }
169
170    let cpu = entry.cpu_id;
171    let drain_result = entry.buffer.try_fold(
172        (0usize, 0u64),
173        |(mut read_count, mut lost_count), event| {
174            if events.len() >= MAX_EVENTS_PER_WAIT || read_count >= PERF_READ_BATCH_SIZE {
175                return ControlFlow::Break(Ok((read_count, lost_count)));
176            }
177
178            match event {
179                PerfEvent::Sample { head, tail } => {
180                    read_count += 1;
181                    produced = true;
182                    debug!(
183                        "PerfEvent {}: {} bytes - {:02x?}",
184                        read_count - 1,
185                        head.len() + tail.len(),
186                        &head[..head.len().min(32)]
187                    );
188
189                    for segment in [head, tail] {
190                        if segment.is_empty() {
191                            continue;
192                        }
193                        match parser.process_segment(segment, trace_context) {
194                            Ok(Some(parsed_event)) => events.push(parsed_event),
195                            Ok(None) => {}
196                            Err(e) => {
197                                return ControlFlow::Break(Err(LoaderError::Generic(format!(
198                                    "Fatal: Failed to parse trace event from PerfEventArray CPU {cpu}: {e}"
199                                ))));
200                            }
201                        }
202                    }
203                }
204                PerfEvent::Lost { count } => {
205                    lost_count = lost_count.saturating_add(count);
206                }
207            }
208
209            ControlFlow::Continue((read_count, lost_count))
210        },
211    );
212
213    let (read_count, lost_count) = match drain_result {
214        ControlFlow::Continue(counts) => counts,
215        ControlFlow::Break(result) => result?,
216    };
217
218    if read_count > 0 {
219        info!(
220            "Read {} events from CPU {} buffer",
221            read_count, entry.cpu_id
222        );
223    }
224    if lost_count > 0 {
225        warn!(
226            "Lost {} events from CPU {} buffer",
227            lost_count, entry.cpu_id
228        );
229    }
230
231    Ok(produced)
232}
233
234/// Compatibility shim that mimics Aya's newer attach location helper so we can keep
235/// a single call-site regardless of which `UProbe::attach` signature we compile against.
236enum UProbeAttachLocation<'a> {
237    AbsoluteOffset(u64),
238    Function(&'a str),
239}
240
241impl<'a> UProbeAttachLocation<'a> {
242    fn attach<T: AsRef<Path>>(
243        self,
244        program: &mut UProbe,
245        target: T,
246        pid: Option<i32>,
247    ) -> std::result::Result<UProbeLinkId, ProgramError> {
248        let scope = uprobe_scope(pid)?;
249        match self {
250            Self::AbsoluteOffset(offset) => program.attach(offset, target, scope),
251            Self::Function(fn_name) => program.attach(fn_name, target, scope),
252        }
253    }
254}
255
256fn uprobe_scope(pid: Option<i32>) -> std::result::Result<UProbeScope, ProgramError> {
257    match pid {
258        None => Ok(UProbeScope::AllProcesses),
259        Some(pid) => {
260            let pid = u32::try_from(pid)
261                .ok()
262                .and_then(NonZeroU32::new)
263                .ok_or_else(|| {
264                    ProgramError::IOError(io::Error::new(
265                        io::ErrorKind::InvalidInput,
266                        format!("invalid uprobe PID scope: {pid}"),
267                    ))
268                })?;
269            Ok(UProbeScope::OneProcess(pid))
270        }
271    }
272}
273
274pub fn hello() -> String {
275    format!("Loader: {}", ghostscope_compiler::hello())
276}
277
278/// Main eBPF program loader and manager
279///
280/// Manages the lifecycle of eBPF programs and provides methods for:
281/// - Loading eBPF bytecode
282/// - Attaching/detaching uprobes
283/// - Reading trace events
284/// - Managing BPF maps
285pub struct GhostScopeLoader {
286    /// Loaded eBPF program
287    bpf: Ebpf,
288    /// Event output map (RingBuf or PerfEventArray)
289    event_map: Option<EventMap>,
290    /// eBPF-side output helper failure counters.
291    event_loss_counters: Option<PerCpuArray<MapData, u64>>,
292    /// ProgramArray holding bt tail-call targets.
293    bt_prog_array: Option<ProgramArray<MapData>>,
294    /// Active uprobe link
295    uprobe_link: Option<UProbeLinkId>,
296    /// Stored parameters for re-attaching uprobe
297    attachment_params: Option<UprobeAttachmentParams>,
298    /// Streaming parser for trace events
299    parser: StreamingTraceParser,
300    /// String table and metadata for parsing trace events
301    trace_context: Option<TraceContext>,
302    /// Optional override for PerfEventArray page count (per CPU buffer size in pages)
303    perf_page_count: Option<usize>,
304    /// Number of compact unwind rows currently written to bt_unwind_rows.
305    backtrace_unwind_row_count: u32,
306    /// Module cookies already published in bt_module_row_ranges.
307    backtrace_module_row_cookies: HashSet<u64>,
308    /// Whether bt_unwind_rows and bt_module_row_ranges are shared pinned maps.
309    shared_backtrace_maps: bool,
310}
311
312impl std::fmt::Debug for GhostScopeLoader {
313    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
314        f.debug_struct("GhostScopeLoader")
315            .field("bpf", &"<eBPF object>")
316            .field("event_map", &self.event_map.is_some())
317            .field("event_loss_counters", &self.event_loss_counters.is_some())
318            .field("bt_prog_array", &self.bt_prog_array.is_some())
319            .field("uprobe_attached", &self.uprobe_link.is_some())
320            .field("attachment_params", &self.attachment_params.is_some())
321            .field(
322                "backtrace_unwind_row_count",
323                &self.backtrace_unwind_row_count,
324            )
325            .field(
326                "backtrace_module_row_cookies",
327                &self.backtrace_module_row_cookies.len(),
328            )
329            .field("shared_backtrace_maps", &self.shared_backtrace_maps)
330            .finish()
331    }
332}
333
334impl GhostScopeLoader {
335    // ============================================================================
336    // Lifecycle Management
337    // ============================================================================
338
339    /// Create a new loader instance from eBPF bytecode
340    pub fn new(bytecode: &[u8]) -> Result<Self> {
341        Self::new_with_shared_backtrace_maps(bytecode, false)
342    }
343
344    /// Create a new loader instance from eBPF bytecode, optionally binding
345    /// module-normalized backtrace CFI maps to the per-process shared pins.
346    pub fn new_with_shared_backtrace_maps(
347        bytecode: &[u8],
348        shared_backtrace_maps: bool,
349    ) -> Result<Self> {
350        info!(
351            "Loading eBPF program from bytecode ({} bytes)",
352            bytecode.len()
353        );
354
355        // Enforce: proc_module_offsets must be provided as a pinned global map by the process layer
356        let pin_path = proc_offsets_pin_path()
357            .map_err(|e| LoaderError::Generic(format!("Failed to resolve pinned map path: {e}")))?;
358        if !pin_path.exists() {
359            let hint = bpffs_mount_hint_for_pin_path(&pin_path)
360                .map(|hint| format!(" {hint}"))
361                .unwrap_or_default();
362            return Err(LoaderError::Generic(format!(
363                "Pinned map '{}' not found. Please call ghostscope-process to create it first.{}",
364                pin_path.display(),
365                hint
366            )));
367        }
368        let alias_pin_path = pid_aliases_pin_path().map_err(|e| {
369            LoaderError::Generic(format!("Failed to resolve pinned alias map path: {e}"))
370        })?;
371        if !alias_pin_path.exists() {
372            let hint = bpffs_mount_hint_for_pin_path(&alias_pin_path)
373                .map(|hint| format!(" {hint}"))
374                .unwrap_or_default();
375            return Err(LoaderError::Generic(format!(
376                "Pinned map '{}' not found. Please call ghostscope-process to create it first.{}",
377                alias_pin_path.display(),
378                hint
379            )));
380        }
381        let range_meta_pin_path = proc_module_range_meta_pin_path().map_err(|e| {
382            LoaderError::Generic(format!("Failed to resolve pinned range meta map path: {e}"))
383        })?;
384        if !range_meta_pin_path.exists() {
385            let hint = bpffs_mount_hint_for_pin_path(&range_meta_pin_path)
386                .map(|hint| format!(" {hint}"))
387                .unwrap_or_default();
388            return Err(LoaderError::Generic(format!(
389                "Pinned map '{}' not found. Please call ghostscope-process to create it first.{}",
390                range_meta_pin_path.display(),
391                hint
392            )));
393        }
394        let ranges_pin_path = proc_module_ranges_pin_path().map_err(|e| {
395            LoaderError::Generic(format!("Failed to resolve pinned ranges map path: {e}"))
396        })?;
397        if !ranges_pin_path.exists() {
398            let hint = bpffs_mount_hint_for_pin_path(&ranges_pin_path)
399                .map(|hint| format!(" {hint}"))
400                .unwrap_or_default();
401            return Err(LoaderError::Generic(format!(
402                "Pinned map '{}' not found. Please call ghostscope-process to create it first.{}",
403                ranges_pin_path.display(),
404                hint
405            )));
406        }
407        let bt_rows_pin_path = if shared_backtrace_maps {
408            let path = bt_unwind_rows_pin_path().map_err(|e| {
409                LoaderError::Generic(format!(
410                    "Failed to resolve pinned bt_unwind_rows map path: {e}"
411                ))
412            })?;
413            if !path.exists() {
414                let hint = bpffs_mount_hint_for_pin_path(&path)
415                    .map(|hint| format!(" {hint}"))
416                    .unwrap_or_default();
417                return Err(LoaderError::Generic(format!(
418                    "Pinned map '{}' not found. Please call ghostscope-process to create it first.{}",
419                    path.display(),
420                    hint
421                )));
422            }
423            Some(path)
424        } else {
425            None
426        };
427        let bt_ranges_pin_path = if shared_backtrace_maps {
428            let path = bt_module_row_ranges_pin_path().map_err(|e| {
429                LoaderError::Generic(format!(
430                    "Failed to resolve pinned bt_module_row_ranges map path: {e}"
431                ))
432            })?;
433            if !path.exists() {
434                let hint = bpffs_mount_hint_for_pin_path(&path)
435                    .map(|hint| format!(" {hint}"))
436                    .unwrap_or_default();
437                return Err(LoaderError::Generic(format!(
438                    "Pinned map '{}' not found. Please call ghostscope-process to create it first.{}",
439                    path.display(),
440                    hint
441                )));
442            }
443            Some(path)
444        } else {
445            None
446        };
447
448        let mut loader = EbpfLoader::new();
449        let use_verbose = cfg!(debug_assertions)
450            || log_enabled!(LogLevel::Trace)
451            || log_enabled!(LogLevel::Debug);
452        if use_verbose {
453            loader.verifier_log_level(VerifierLogLevel::VERBOSE | VerifierLogLevel::STATS);
454            tracing::info!("BPF verifier logs: VERBOSE (debug build/log)");
455        } else {
456            loader.verifier_log_level(VerifierLogLevel::DEBUG | VerifierLogLevel::STATS);
457            tracing::info!("BPF verifier logs: DEBUG (release/info)");
458        }
459        // Configure Aya loader to reuse pinned maps by name under our per-process pin directory.
460        // This makes @proc_module_offsets in the eBPF object bind to the already pinned map
461        // created by ghostscope-process instead of creating a new private map.
462        let pin_dir = proc_offsets_pin_dir().map_err(|e| {
463            LoaderError::Generic(format!("Failed to resolve pinned map directory: {e}"))
464        })?;
465        if pin_dir.exists() {
466            loader.map_pin_path(PROC_OFFSETS_MAP_NAME, pin_path);
467            loader.map_pin_path(PID_ALIASES_MAP_NAME, alias_pin_path);
468            loader.map_pin_path(PROC_MODULE_RANGE_META_MAP_NAME, range_meta_pin_path);
469            loader.map_pin_path(PROC_MODULE_RANGES_MAP_NAME, ranges_pin_path);
470            if let (Some(rows_path), Some(ranges_path)) =
471                (bt_rows_pin_path.as_ref(), bt_ranges_pin_path.as_ref())
472            {
473                loader.map_pin_path(BT_UNWIND_ROWS_MAP_NAME, rows_path);
474                loader.map_pin_path(BT_MODULE_ROW_RANGES_MAP_NAME, ranges_path);
475            }
476            tracing::info!(
477                "Configured map pin directory for reuse: {}",
478                pin_dir.display()
479            );
480        }
481        match loader.load(bytecode) {
482            Ok(bpf) => {
483                info!("Successfully loaded eBPF program");
484                Ok(Self {
485                    bpf,
486                    event_map: None,
487                    event_loss_counters: None,
488                    bt_prog_array: None,
489                    uprobe_link: None,
490                    attachment_params: None,
491                    parser: StreamingTraceParser::new(),
492                    trace_context: None,
493                    perf_page_count: None,
494                    backtrace_unwind_row_count: 0,
495                    backtrace_module_row_cookies: HashSet::new(),
496                    shared_backtrace_maps,
497                })
498            }
499            Err(e) => {
500                error!("Failed to load BPF program: {:?}", e);
501                // Try to provide more specific error information
502                match &e {
503                    aya::EbpfError::ParseError(parse_err) => {
504                        error!("Parse error details: {:?}", parse_err);
505                    }
506                    aya::EbpfError::BtfError(btf_err) => {
507                        error!("BTF error details: {:?}", btf_err);
508                    }
509                    _ => {
510                        error!("Other BPF error: {:?}", e);
511                    }
512                }
513                Err(LoaderError::Aya(e))
514            }
515        }
516    }
517
518    // ============================================================================
519    // Uprobe Management
520    // ============================================================================
521
522    /// Attach to a uprobe at the specified function offset
523    pub fn attach_uprobe(
524        &mut self,
525        target_binary: &str,
526        function_name: &str,
527        offset: Option<u64>,
528        pid: Option<i32>,
529    ) -> Result<()> {
530        self.attach_uprobe_with_program_name(target_binary, function_name, offset, pid, None)
531    }
532
533    /// Set PerfEventArray page count override (applies when using Perf backend)
534    pub fn set_perf_page_count(&mut self, pages: u32) {
535        self.perf_page_count = Some(pages as usize);
536    }
537
538    /// Load and register optional bt tail-call programs before the entry uprobe is attached.
539    pub fn register_backtrace_tail_call_program(
540        &mut self,
541        program_name: Option<&str>,
542    ) -> Result<()> {
543        let Some(program_name) = program_name else {
544            return Ok(());
545        };
546
547        info!("Registering bt tail-call step program: {}", program_name);
548        let program_ref = self.bpf.program_mut(program_name).ok_or_else(|| {
549            LoaderError::Generic(format!("bt tail-call program '{program_name}' not found"))
550        })?;
551        let program: &mut UProbe = program_ref.try_into().map_err(|e| {
552            LoaderError::Generic(format!(
553                "bt tail-call program '{program_name}' is not a UProbe: {e:?}"
554            ))
555        })?;
556        program.load().map_err(LoaderError::Program)?;
557        let step_fd = program
558            .fd()
559            .map_err(LoaderError::Program)?
560            .try_clone()
561            .map_err(|e| {
562                LoaderError::Generic(format!(
563                    "Failed to clone bt tail-call program fd for '{program_name}': {e}"
564                ))
565            })?;
566
567        let map = self
568            .bpf
569            .take_map("bt_prog_array")
570            .ok_or_else(|| LoaderError::MapNotFound("bt_prog_array".to_string()))?;
571        let mut prog_array: ProgramArray<_> = map.try_into().map_err(|e| {
572            LoaderError::Generic(format!("Failed to convert bt_prog_array map: {e}"))
573        })?;
574        prog_array.set(0, &step_fd, 0).map_err(|e| {
575            LoaderError::Generic(format!("Failed to set bt tail-call program fd: {e}"))
576        })?;
577        self.bt_prog_array = Some(prog_array);
578        info!("Registered bt tail-call step program at bt_prog_array[0]");
579        Ok(())
580    }
581
582    /// Attach to a uprobe with a specific eBPF program name
583    pub fn attach_uprobe_with_program_name(
584        &mut self,
585        target_binary: &str,
586        function_name: &str,
587        offset: Option<u64>,
588        pid: Option<i32>,
589        program_name: Option<&str>,
590    ) -> Result<()> {
591        info!("attach_uprobe called with offset: {:?}", offset);
592        if let Some(offset) = offset {
593            info!(
594                "Using offset-based attachment: {} at 0x{:x} ({}) (pid: {:?})",
595                target_binary, offset, function_name, pid
596            );
597        } else {
598            info!(
599                "Using function name-based attachment: {}:{} (pid: {:?})",
600                target_binary, function_name, pid
601            );
602        }
603
604        // Collect all available program names first to avoid borrowing conflicts
605        let available_programs: Vec<String> = self
606            .bpf
607            .programs()
608            .map(|(name, _)| name.to_string())
609            .collect();
610
611        // Debug: Print all available programs
612        info!("Available programs:");
613        for name in &available_programs {
614            info!("  - {}", name);
615        }
616
617        // Get the program from the BPF object
618        let program_name: String = if let Some(name) = program_name {
619            // Use the specified program name
620            info!("Using specified program name: {}", name);
621            if available_programs.contains(&name.to_string()) {
622                name.to_string()
623            } else {
624                return Err(LoaderError::Generic(format!(
625                    "Specified program '{name}' not found in eBPF object"
626                )));
627            }
628        } else {
629            // Try different program names: section name first, then function name, then any program
630            let program_names = ["uprobe", "main"];
631            let mut found_program_name: Option<String> = None;
632
633            for name in &program_names {
634                info!("Checking if program exists: {}", name);
635                if available_programs.contains(&name.to_string()) {
636                    info!("Found program: {}", name);
637                    found_program_name = Some(name.to_string());
638                    break;
639                }
640            }
641
642            // If no standard names found, use the first available program
643            if found_program_name.is_none() {
644                if let Some(first_name) = available_programs.first() {
645                    info!(
646                        "No standard program names found, using first available: {}",
647                        first_name
648                    );
649                    found_program_name = Some(first_name.clone());
650                }
651            }
652
653            found_program_name
654                .ok_or_else(|| LoaderError::Generic("No suitable program found".to_string()))?
655        };
656
657        info!("Attempting to load program: {}", program_name);
658
659        let program_ref = self
660            .bpf
661            .program_mut(&program_name)
662            .ok_or_else(|| LoaderError::Generic(format!("Program '{program_name}' not found")))?;
663
664        info!("Found program, attempting to convert to UProbe");
665        info!("Program type: {:?}", program_ref.prog_type());
666
667        // Check what type of program this actually is
668        match program_ref {
669            aya::programs::Program::UProbe(_) => {
670                info!("Program is correctly recognized as UProbe");
671            }
672            aya::programs::Program::KProbe(_) => {
673                error!("Program is incorrectly recognized as KProbe, should be UProbe");
674            }
675            ref _other => {
676                error!("Program is unexpected type (not UProbe or KProbe)");
677            }
678        }
679
680        let program: &mut UProbe = program_ref.try_into().map_err(|e| {
681            LoaderError::Generic(format!("Program '{program_name}' is not a UProbe: {e:?}"))
682        })?;
683
684        // Load the program
685        info!("About to load eBPF program");
686        match program.load() {
687            Ok(()) => {
688                info!("Program loaded successfully");
689            }
690            Err(e) => {
691                error!("eBPF program load failed: {}", e);
692                error!("This typically indicates eBPF verifier rejection");
693
694                // Check for specific verifier errors
695                if let ProgramError::SyscallError(syscall_error) = &e {
696                    error!(
697                        "Syscall '{}' failed: {}",
698                        syscall_error.call, syscall_error.io_error
699                    );
700
701                    // Check for common error codes
702                    if let Some(errno) = syscall_error.io_error.raw_os_error() {
703                        match errno {
704                            22 => error!(
705                                "EINVAL (22): Invalid argument - likely eBPF verifier rejection"
706                            ),
707                            7 => error!("E2BIG (7): Program too large"),
708                            13 => error!("EACCES (13): Permission denied"),
709                            95 => error!("EOPNOTSUPP (95): Operation not supported"),
710                            _ => error!("Unknown errno: {}", errno),
711                        }
712                    }
713                }
714
715                // Log additional debugging info
716                error!("Program name: {}", program_name);
717                error!("Program type: {:?}", program_ref.prog_type());
718
719                return Err(LoaderError::Program(e));
720            }
721        }
722
723        // Attach the uprobe using Aya API via a compatibility helper
724        // so argument ordering stays explicit regardless of Aya version.
725        let attach_location = match offset {
726            Some(offset) => UProbeAttachLocation::AbsoluteOffset(offset),
727            None => UProbeAttachLocation::Function(function_name),
728        };
729        let attach_result = attach_location.attach(program, target_binary, pid);
730
731        match attach_result {
732            Ok(link) => {
733                if let Some(offset) = offset {
734                    info!(
735                        "Uprobe attached successfully to {} at offset 0x{:x}",
736                        target_binary, offset
737                    );
738                } else {
739                    info!(
740                        "Uprobe attached successfully to {}:{}",
741                        target_binary, function_name
742                    );
743                }
744
745                // Store the link handle and attachment parameters for later use
746                self.uprobe_link = Some(link);
747                self.attachment_params = Some(UprobeAttachmentParams {
748                    target_binary: target_binary.to_string(),
749                    function_name: function_name.to_string(),
750                    offset,
751                    pid,
752                    program_name,
753                });
754            }
755            Err(e) => {
756                if let Some(offset) = offset {
757                    error!(
758                        "Failed to attach uprobe to {} at offset 0x{:x}: {}",
759                        target_binary, offset, e
760                    );
761                    error!("Detailed error: {:#?}", e);
762                } else {
763                    error!(
764                        "Failed to attach uprobe to {}:{}: {}",
765                        target_binary, function_name, e
766                    );
767                    error!("Detailed error: {:#?}", e);
768                }
769
770                // Try to provide more helpful error information
771                if let ProgramError::SyscallError(syscall_error) = &e {
772                    error!(
773                        "Syscall '{}' failed: {}",
774                        syscall_error.call, syscall_error.io_error
775                    );
776                    if let Some(13) = syscall_error.io_error.raw_os_error() {
777                        error!("Permission denied - make sure to run with sudo");
778                    }
779                }
780
781                return Err(LoaderError::Program(e));
782            }
783        }
784
785        // Initialize event map after successful attachment
786        // Try RingBuf first, fall back to PerfEventArray
787        let event_map = if let Some(map) = self.bpf.take_map("ringbuf") {
788            info!("Initializing RingBuf event map");
789            let ringbuf: RingBuf<_> = map
790                .try_into()
791                .map_err(|e| LoaderError::Generic(format!("Failed to convert ringbuf map: {e}")))?;
792            EventMap::RingBuf(ringbuf)
793        } else if let Some(map) = self.bpf.take_map("events") {
794            info!("Initializing PerfEventArray event map");
795            let mut perf_array: PerfEventArray<_> = map.try_into().map_err(|e| {
796                LoaderError::Generic(format!("Failed to convert perf event array map: {e}"))
797            })?;
798
799            // Get online CPUs
800            let online_cpus = aya::util::online_cpus().map_err(|(_, e)| {
801                LoaderError::Generic(format!("Failed to get online CPUs: {e}"))
802            })?;
803
804            info!(
805                "Opening PerfEventArray buffers for {} online CPUs",
806                online_cpus.len()
807            );
808
809            // Open buffers for all online CPUs
810            let mut cpu_buffers = Vec::new();
811
812            for cpu_id in online_cpus {
813                let pages = self.perf_page_count;
814                match perf_array.open(cpu_id, pages) {
815                    Ok(buffer) => {
816                        if let Some(p) = pages {
817                            info!(
818                                "Opened PerfEventArray buffer for CPU {} with {} pages",
819                                cpu_id, p
820                            );
821                        } else {
822                            info!(
823                                "Opened PerfEventArray buffer for CPU {} (default pages)",
824                                cpu_id
825                            );
826                        }
827                        let fd = buffer.as_raw_fd();
828                        let readiness =
829                            AsyncFd::with_interest(PerfBufferFd(fd), Interest::READABLE).map_err(
830                                |err| {
831                                    LoaderError::Generic(format!(
832                                        "Failed to register perf buffer fd for CPU {cpu_id}: {err}"
833                                    ))
834                                },
835                            )?;
836                        cpu_buffers.push(PerfEventCpuBuffer {
837                            cpu_id,
838                            buffer,
839                            readiness,
840                        });
841                    }
842                    Err(e) => {
843                        warn!("Failed to open perf buffer for CPU {}: {}", cpu_id, e);
844                    }
845                }
846            }
847
848            if cpu_buffers.is_empty() {
849                return Err(LoaderError::Generic(
850                    "Failed to open any perf event buffers".to_string(),
851                ));
852            }
853
854            EventMap::PerfEventArray {
855                _map: perf_array,
856                cpu_buffers,
857            }
858        } else {
859            return Err(LoaderError::MapNotFound(
860                "Neither 'ringbuf' nor 'events' map found".to_string(),
861            ));
862        };
863
864        self.event_loss_counters = if let Some(map) = self.bpf.take_map("event_loss_counters") {
865            info!("Initializing eBPF event loss counter map");
866            Some(map.try_into().map_err(|e| {
867                LoaderError::Generic(format!("Failed to convert event_loss_counters map: {e}"))
868            })?)
869        } else {
870            warn!("No eBPF event loss counter map found; kernel output loss stats unavailable");
871            None
872        };
873
874        // Set parser event source based on map type
875        let event_source = match &event_map {
876            EventMap::RingBuf(_) => {
877                info!("Using RingBuf mode for parser");
878                ghostscope_protocol::EventSource::RingBuf
879            }
880            EventMap::PerfEventArray { .. } => {
881                info!("Using PerfEventArray mode for parser");
882                ghostscope_protocol::EventSource::PerfEventArray
883            }
884        };
885        self.parser = StreamingTraceParser::with_event_source(event_source);
886
887        self.event_map = Some(event_map);
888        info!("Event map initialized");
889
890        Ok(())
891    }
892
893    /// Detach the uprobe (disable tracing) while keeping eBPF resources loaded
894    /// This allows the trace to be quickly re-enabled later
895    pub fn detach_uprobe(&mut self) -> Result<()> {
896        if let Some(link_id) = self.uprobe_link.take() {
897            if let Some(params) = &self.attachment_params {
898                info!("Detaching uprobe...");
899
900                // Get the program to detach the link
901                let program_ref = self.bpf.program_mut(&params.program_name).ok_or_else(|| {
902                    let program_name = &params.program_name;
903                    LoaderError::Generic(format!("Program '{program_name}' not found"))
904                })?;
905
906                let program: &mut UProbe = program_ref.try_into().map_err(|e| {
907                    let program_name = &params.program_name;
908                    LoaderError::Generic(format!("Program '{program_name}' is not a UProbe: {e:?}"))
909                })?;
910
911                // Detach the uprobe using the link ID
912                program.detach(link_id).map_err(LoaderError::Program)?;
913
914                info!("Uprobe detached successfully");
915                Ok(())
916            } else {
917                error!("No attachment parameters stored");
918                Err(LoaderError::Generic(
919                    "No attachment parameters stored".to_string(),
920                ))
921            }
922        } else {
923            warn!("No uprobe attached, nothing to detach");
924            Ok(())
925        }
926    }
927
928    /// Reattach the uprobe (re-enable tracing) using previously stored parameters
929    /// This requires that attach_uprobe was called previously to store the parameters
930    pub fn reattach_uprobe(&mut self) -> Result<()> {
931        if self.uprobe_link.is_some() {
932            info!("Uprobe already attached");
933            return Ok(());
934        }
935
936        let params = self
937            .attachment_params
938            .as_ref()
939            .ok_or_else(|| {
940                LoaderError::Generic(
941                    "No attachment parameters stored. Call attach_uprobe first.".to_string(),
942                )
943            })?
944            .clone();
945
946        info!("Reattaching uprobe with stored parameters...");
947
948        // Get the program directly (it's already loaded)
949        let program_ref = self.bpf.program_mut(&params.program_name).ok_or_else(|| {
950            LoaderError::Generic(format!("Program '{}' not found", params.program_name))
951        })?;
952
953        let program: &mut UProbe = program_ref.try_into().map_err(|e| {
954            LoaderError::Generic(format!(
955                "Program '{}' is not a UProbe: {:?}",
956                params.program_name, e
957            ))
958        })?;
959
960        // Attach the uprobe directly (don't load - it's already loaded)
961        let attach_location = match params.offset {
962            Some(offset) => UProbeAttachLocation::AbsoluteOffset(offset),
963            None => UProbeAttachLocation::Function(params.function_name.as_str()),
964        };
965        let attach_result = attach_location.attach(program, &params.target_binary, params.pid);
966
967        match attach_result {
968            Ok(link) => {
969                if let Some(offset) = params.offset {
970                    info!(
971                        "Uprobe reattached successfully to {} at offset 0x{:x}",
972                        params.target_binary, offset
973                    );
974                } else {
975                    info!(
976                        "Uprobe reattached successfully to {}:{}",
977                        params.target_binary, params.function_name
978                    );
979                }
980
981                // Store the new link handle
982                self.uprobe_link = Some(link);
983                Ok(())
984            }
985            Err(e) => {
986                error!("Failed to reattach uprobe: {:?}", e);
987                Err(LoaderError::Program(e))
988            }
989        }
990    }
991
992    /// Check if the uprobe is currently attached
993    pub fn is_uprobe_attached(&self) -> bool {
994        self.uprobe_link.is_some()
995    }
996
997    /// Completely destroy this loader and all associated resources
998    /// This detaches any attached uprobes and clears all eBPF resources
999    /// After calling this, the loader cannot be reused
1000    pub fn destroy(&mut self) -> Result<()> {
1001        info!("Destroying GhostScopeLoader and all associated resources");
1002
1003        // First detach uprobe if attached
1004        if self.uprobe_link.is_some() {
1005            if let Err(e) = self.detach_uprobe() {
1006                warn!("Failed to detach uprobe during destroy: {}", e);
1007                // Continue with destruction even if detach fails
1008            }
1009        }
1010
1011        // Clear attachment parameters
1012        self.attachment_params = None;
1013
1014        // Clear event map reference (this doesn't destroy the actual eBPF map,
1015        // but removes our handle to it)
1016        self.event_map = None;
1017
1018        // Note: The eBPF programs and maps will be automatically cleaned up
1019        // when the `bpf` field is dropped (when this struct is dropped)
1020
1021        info!("GhostScopeLoader destroyed successfully");
1022        Ok(())
1023    }
1024
1025    /// Get current attachment status information
1026    pub fn get_attachment_info(&self) -> Option<String> {
1027        if let Some(params) = &self.attachment_params {
1028            if let Some(offset) = params.offset {
1029                Some(format!(
1030                    "{}:{} (offset: 0x{:x}, pid: {:?}) - {}",
1031                    params.target_binary,
1032                    params.function_name,
1033                    offset,
1034                    params.pid,
1035                    if self.is_uprobe_attached() {
1036                        "attached"
1037                    } else {
1038                        "detached"
1039                    }
1040                ))
1041            } else {
1042                Some(format!(
1043                    "{}:{} (pid: {:?}) - {}",
1044                    params.target_binary,
1045                    params.function_name,
1046                    params.pid,
1047                    if self.is_uprobe_attached() {
1048                        "attached"
1049                    } else {
1050                        "detached"
1051                    }
1052                ))
1053            }
1054        } else {
1055            None
1056        }
1057    }
1058
1059    // ============================================================================
1060    // Event Reading
1061    // ============================================================================
1062
1063    /// Wait for events asynchronously using AsyncFd
1064    pub async fn wait_for_events_async(&mut self) -> Result<Vec<ParsedTraceEvent>> {
1065        let trace_context = self.trace_context.as_ref().ok_or_else(|| {
1066            LoaderError::Generic(
1067                "No trace context available - cannot parse trace events".to_string(),
1068            )
1069        })?;
1070
1071        let event_map = self.event_map.as_mut().ok_or_else(|| {
1072            LoaderError::Generic("Event map not initialized. Call attach_uprobe first.".to_string())
1073        })?;
1074
1075        let mut events = Vec::with_capacity(MAX_EVENTS_PER_WAIT.min(128));
1076
1077        match event_map {
1078            EventMap::RingBuf(ringbuf) => {
1079                // Create AsyncFd and wait for readable; clear readiness to avoid spin
1080                let async_fd = AsyncFd::new(ringbuf.as_raw_fd())
1081                    .map_err(|e| LoaderError::Generic(format!("Failed to create AsyncFd: {e}")))?;
1082                let mut guard = async_fd
1083                    .readable()
1084                    .await
1085                    .map_err(|e| LoaderError::Generic(format!("AsyncFd error: {e}")))?;
1086                guard.clear_ready();
1087
1088                // Drain a bounded batch. Under very hot probes the ringbuf may never
1089                // become empty, so an unbounded drain would starve output and signals.
1090                let mut records_read = 0;
1091                while events.len() < MAX_EVENTS_PER_WAIT
1092                    && records_read < MAX_RINGBUF_RECORDS_PER_WAIT
1093                {
1094                    let Some(item) = ringbuf.next() else {
1095                        break;
1096                    };
1097                    records_read += 1;
1098                    match self.parser.process_segment(&item, trace_context) {
1099                        Ok(Some(parsed_event)) => events.push(parsed_event),
1100                        Ok(None) => {}
1101                        Err(e) => {
1102                            return Err(LoaderError::Generic(format!(
1103                                "Fatal: Failed to parse trace event from RingBuf (async): {e}"
1104                            )));
1105                        }
1106                    }
1107                }
1108                if events.len() == MAX_EVENTS_PER_WAIT
1109                    || records_read == MAX_RINGBUF_RECORDS_PER_WAIT
1110                {
1111                    debug!(
1112                        "RingBuf batch limit reached ({} events, {} records); yielding to caller",
1113                        events.len(),
1114                        records_read
1115                    );
1116                }
1117            }
1118            EventMap::PerfEventArray { cpu_buffers, .. } => {
1119                let parser = &mut self.parser;
1120
1121                loop {
1122                    // Drain any buffers that already report data without waiting.
1123                    let mut made_progress = false;
1124                    for entry in cpu_buffers.iter_mut() {
1125                        if events.len() >= MAX_EVENTS_PER_WAIT {
1126                            break;
1127                        }
1128                        if entry.buffer.readable() {
1129                            made_progress |=
1130                                drain_perf_cpu_buffer(entry, parser, trace_context, &mut events)?;
1131                        }
1132                    }
1133
1134                    if made_progress {
1135                        break;
1136                    }
1137
1138                    // Wait for at least one buffer to become readable.
1139                    let ready_idx = poll_fn(|cx| {
1140                        for (idx, entry) in cpu_buffers.iter().enumerate() {
1141                            match entry.readiness.poll_read_ready(cx) {
1142                                Poll::Ready(Ok(mut guard)) => {
1143                                    guard.clear_ready();
1144                                    return Poll::Ready(Ok(idx));
1145                                }
1146                                Poll::Ready(Err(e)) => return Poll::Ready(Err(e)),
1147                                Poll::Pending => {}
1148                            }
1149                        }
1150                        Poll::Pending
1151                    })
1152                    .await
1153                    .map_err(|e| {
1154                        LoaderError::Generic(format!(
1155                            "AsyncFd error while waiting for perf events: {e}"
1156                        ))
1157                    })?;
1158
1159                    // Drain the buffer that triggered readiness.
1160                    made_progress |= drain_perf_cpu_buffer(
1161                        cpu_buffers
1162                            .get_mut(ready_idx)
1163                            .expect("ready index should be valid"),
1164                        parser,
1165                        trace_context,
1166                        &mut events,
1167                    )?;
1168
1169                    // Drain any other buffers now advertising data.
1170                    for (idx, entry) in cpu_buffers.iter_mut().enumerate() {
1171                        if events.len() >= MAX_EVENTS_PER_WAIT {
1172                            break;
1173                        }
1174                        if idx == ready_idx || !entry.buffer.readable() {
1175                            continue;
1176                        }
1177                        made_progress |=
1178                            drain_perf_cpu_buffer(entry, parser, trace_context, &mut events)?;
1179                    }
1180
1181                    if made_progress {
1182                        if events.len() == MAX_EVENTS_PER_WAIT {
1183                            debug!(
1184                                "PerfEventArray event batch limit reached ({} events); yielding to caller",
1185                                MAX_EVENTS_PER_WAIT
1186                            );
1187                        }
1188                        break;
1189                    }
1190                    // No events were produced despite readiness (eg. lost event markers).
1191                    // Loop back and wait again.
1192                }
1193            }
1194        }
1195
1196        Ok(events)
1197    }
1198
1199    pub fn read_event_loss_stats(&self) -> Result<Option<EventLossStats>> {
1200        let Some(counters) = &self.event_loss_counters else {
1201            return Ok(None);
1202        };
1203
1204        let values = counters
1205            .get(&EVENT_LOSS_OUTPUT_FAILURES_KEY, 0)
1206            .map_err(|e| {
1207                LoaderError::Generic(format!("Failed to read event_loss_counters map: {e}"))
1208            })?;
1209
1210        Ok(Some(EventLossStats {
1211            output_failures: values.iter().copied().sum(),
1212        }))
1213    }
1214
1215    /// Set the trace context for parsing trace events
1216    pub fn set_trace_context(&mut self, trace_context: TraceContext) {
1217        info!("Setting trace context for trace event parsing");
1218        self.trace_context = Some(trace_context);
1219    }
1220
1221    fn sync_shared_backtrace_row_state(&mut self) -> Result<()> {
1222        if !self.shared_backtrace_maps {
1223            return Ok(());
1224        }
1225
1226        let Some(map) = self.bpf.map_mut(BT_MODULE_ROW_RANGES_MAP_NAME) else {
1227            return Ok(());
1228        };
1229        let hash: AyaHashMap<_, u64, BacktraceModuleRowRange> = map.try_into().map_err(|e| {
1230            LoaderError::Generic(format!(
1231                "Failed to convert shared bt_module_row_ranges map: {e}"
1232            ))
1233        })?;
1234        let keys = hash
1235            .keys()
1236            .collect::<std::result::Result<Vec<_>, _>>()
1237            .map_err(|e| {
1238                LoaderError::Generic(format!(
1239                    "Failed to list shared bt_module_row_ranges keys: {e}"
1240                ))
1241            })?;
1242
1243        let mut max_row_end = self.backtrace_unwind_row_count;
1244        for cookie in keys {
1245            match hash.get(&cookie, 0) {
1246                Ok(range) => {
1247                    self.backtrace_module_row_cookies.insert(cookie);
1248                    max_row_end = max_row_end.max(range.row_end);
1249                }
1250                Err(e) => {
1251                    debug!(
1252                        cookie = format_args!("0x{cookie:016x}"),
1253                        "Skipped shared bt module row range during sync: {}", e
1254                    );
1255                }
1256            }
1257        }
1258        self.backtrace_unwind_row_count = max_row_end;
1259        Ok(())
1260    }
1261
1262    pub fn populate_backtrace_unwind_rows_and_module_row_ranges(
1263        &mut self,
1264        rows: &[BacktraceUnwindRow],
1265        ranges: &[(u64, BacktraceModuleRowRange)],
1266    ) -> Result<()> {
1267        if rows.is_empty() {
1268            return Ok(());
1269        }
1270
1271        if ranges.is_empty() || !self.shared_backtrace_maps {
1272            self.populate_backtrace_unwind_rows(rows)?;
1273            self.populate_backtrace_module_row_ranges(ranges)?;
1274            return Ok(());
1275        }
1276
1277        self.sync_shared_backtrace_row_state()?;
1278        for (cookie, range) in ranges.iter().copied() {
1279            if self.backtrace_module_row_cookies.contains(&cookie) {
1280                continue;
1281            }
1282
1283            let row_start = usize::try_from(range.row_start).map_err(|_| {
1284                LoaderError::Generic(format!(
1285                    "Invalid row_start for module cookie 0x{cookie:016x}: {}",
1286                    range.row_start
1287                ))
1288            })?;
1289            let row_end = usize::try_from(range.row_end).map_err(|_| {
1290                LoaderError::Generic(format!(
1291                    "Invalid row_end for module cookie 0x{cookie:016x}: {}",
1292                    range.row_end
1293                ))
1294            })?;
1295            if row_start > row_end || row_end > rows.len() {
1296                return Err(LoaderError::Generic(format!(
1297                    "Invalid bt row range for module cookie 0x{cookie:016x}: \
1298                     {}..{} with {} rows",
1299                    range.row_start,
1300                    range.row_end,
1301                    rows.len()
1302                )));
1303            }
1304
1305            self.append_backtrace_unwind_rows_for_module_after_sync(
1306                cookie,
1307                &rows[row_start..row_end],
1308            )?;
1309        }
1310
1311        Ok(())
1312    }
1313
1314    pub fn populate_backtrace_unwind_rows(&mut self, rows: &[BacktraceUnwindRow]) -> Result<()> {
1315        if rows.is_empty() {
1316            return Ok(());
1317        }
1318
1319        let Some(map) = self.bpf.map_mut("bt_unwind_rows") else {
1320            return Err(LoaderError::MapNotFound("bt_unwind_rows".to_string()));
1321        };
1322        let mut array: Array<_, BacktraceUnwindRow> = map.try_into().map_err(|e| {
1323            LoaderError::Generic(format!("Failed to convert bt_unwind_rows map: {e}"))
1324        })?;
1325        let populate_started_at = Instant::now();
1326        for (row_index, row) in rows.iter().copied().enumerate() {
1327            array.set(row_index as u32, row, 0).map_err(|e| {
1328                LoaderError::Generic(format!("Failed to set unwind row {row_index}: {e}"))
1329            })?;
1330        }
1331        self.backtrace_unwind_row_count = self.backtrace_unwind_row_count.max(rows.len() as u32);
1332        info!(
1333            rows = rows.len(),
1334            capacity = array.len(),
1335            row_size = BACKTRACE_UNWIND_ROW_SIZE,
1336            elapsed_ms = populate_started_at.elapsed().as_millis(),
1337            "Loaded DWARF unwind rows for bt"
1338        );
1339        if log_enabled!(LogLevel::Debug) {
1340            log_backtrace_unwind_row_samples(&array, rows)?;
1341        }
1342        Ok(())
1343    }
1344
1345    pub fn populate_backtrace_module_row_ranges(
1346        &mut self,
1347        ranges: &[(u64, BacktraceModuleRowRange)],
1348    ) -> Result<()> {
1349        if ranges.is_empty() {
1350            return Ok(());
1351        }
1352
1353        let Some(map) = self.bpf.map_mut("bt_module_row_ranges") else {
1354            return Err(LoaderError::MapNotFound("bt_module_row_ranges".to_string()));
1355        };
1356        let mut hash: AyaHashMap<_, u64, BacktraceModuleRowRange> =
1357            map.try_into().map_err(|e| {
1358                LoaderError::Generic(format!("Failed to convert bt_module_row_ranges map: {e}"))
1359            })?;
1360        let populate_started_at = Instant::now();
1361        for (cookie, range) in ranges.iter().copied() {
1362            hash.insert(cookie, range, 0).map_err(|e| {
1363                LoaderError::Generic(format!(
1364                    "Failed to set bt module row range for cookie 0x{cookie:016x}: {e}"
1365                ))
1366            })?;
1367            self.backtrace_module_row_cookies.insert(cookie);
1368            self.backtrace_unwind_row_count = self.backtrace_unwind_row_count.max(range.row_end);
1369        }
1370        info!(
1371            modules = ranges.len(),
1372            elapsed_ms = populate_started_at.elapsed().as_millis(),
1373            "Loaded DWARF unwind row ranges for bt"
1374        );
1375        Ok(())
1376    }
1377
1378    pub fn append_backtrace_unwind_rows_for_module(
1379        &mut self,
1380        cookie: u64,
1381        rows: &[BacktraceUnwindRow],
1382    ) -> Result<Option<BacktraceModuleRowRange>> {
1383        if self.shared_backtrace_maps {
1384            self.sync_shared_backtrace_row_state()?;
1385        }
1386        self.append_backtrace_unwind_rows_for_module_after_sync(cookie, rows)
1387    }
1388
1389    fn append_backtrace_unwind_rows_for_module_after_sync(
1390        &mut self,
1391        cookie: u64,
1392        rows: &[BacktraceUnwindRow],
1393    ) -> Result<Option<BacktraceModuleRowRange>> {
1394        if rows.is_empty() || self.backtrace_module_row_cookies.contains(&cookie) {
1395            return Ok(None);
1396        }
1397
1398        if self.bpf.map("bt_unwind_rows").is_none()
1399            || self.bpf.map("bt_module_row_ranges").is_none()
1400        {
1401            return Ok(None);
1402        }
1403
1404        let start = self.backtrace_unwind_row_count;
1405        let row_count = u32::try_from(rows.len()).map_err(|_| {
1406            LoaderError::Generic(format!(
1407                "Too many unwind rows for module cookie 0x{cookie:016x}: {}",
1408                rows.len()
1409            ))
1410        })?;
1411        let end = start.checked_add(row_count).ok_or_else(|| {
1412            LoaderError::Generic(format!(
1413                "Unwind row index overflow for module cookie 0x{cookie:016x}"
1414            ))
1415        })?;
1416
1417        let Some(map) = self.bpf.map_mut("bt_unwind_rows") else {
1418            return Ok(None);
1419        };
1420        let mut array: Array<_, BacktraceUnwindRow> = map.try_into().map_err(|e| {
1421            LoaderError::Generic(format!("Failed to convert bt_unwind_rows map: {e}"))
1422        })?;
1423        if end > array.len() {
1424            return Err(LoaderError::Generic(format!(
1425                "bt_unwind_rows capacity exceeded while appending module \
1426                 0x{cookie:016x}: need end row {}, capacity {}",
1427                end,
1428                array.len()
1429            )));
1430        }
1431
1432        for (offset, row) in rows.iter().copied().enumerate() {
1433            let row_index = start + offset as u32;
1434            array.set(row_index, row, 0).map_err(|e| {
1435                LoaderError::Generic(format!("Failed to append unwind row {row_index}: {e}"))
1436            })?;
1437        }
1438
1439        let range = BacktraceModuleRowRange {
1440            row_start: start,
1441            row_end: end,
1442        };
1443        let Some(map) = self.bpf.map_mut("bt_module_row_ranges") else {
1444            return Ok(None);
1445        };
1446        let mut hash: AyaHashMap<_, u64, BacktraceModuleRowRange> =
1447            map.try_into().map_err(|e| {
1448                LoaderError::Generic(format!("Failed to convert bt_module_row_ranges map: {e}"))
1449            })?;
1450        hash.insert(cookie, range, 0).map_err(|e| {
1451            LoaderError::Generic(format!(
1452                "Failed to append bt module row range for cookie 0x{cookie:016x}: {e}"
1453            ))
1454        })?;
1455
1456        self.backtrace_unwind_row_count = end;
1457        self.backtrace_module_row_cookies.insert(cookie);
1458        debug!(
1459            cookie = format_args!("0x{cookie:016x}"),
1460            rows = rows.len(),
1461            row_start = range.row_start,
1462            row_end = range.row_end,
1463            "Appended DWARF unwind rows for bt module"
1464        );
1465
1466        Ok(Some(range))
1467    }
1468
1469    pub fn append_backtrace_unwind_rows_for_modules(
1470        &mut self,
1471        modules: &[(u64, Vec<BacktraceUnwindRow>)],
1472    ) -> Result<BacktraceUnwindRowsAppendStats> {
1473        let mut stats = BacktraceUnwindRowsAppendStats::default();
1474        if self.shared_backtrace_maps {
1475            self.sync_shared_backtrace_row_state()?;
1476        }
1477        for (cookie, rows) in modules {
1478            if self
1479                .append_backtrace_unwind_rows_for_module_after_sync(*cookie, rows)?
1480                .is_some()
1481            {
1482                stats.modules += 1;
1483                stats.rows += rows.len();
1484            }
1485        }
1486        Ok(stats)
1487    }
1488
1489    // ============================================================================
1490    // Information and Debugging
1491    // ============================================================================
1492
1493    /// Get information about loaded maps
1494    pub fn get_map_info(&self) -> Vec<String> {
1495        self.bpf
1496            .maps()
1497            .map(|(name, _map)| format!("Map: {name}"))
1498            .collect()
1499    }
1500
1501    /// Get information about loaded programs
1502    pub fn get_program_info(&self) -> Vec<String> {
1503        self.bpf
1504            .programs()
1505            .map(|(name, _prog)| format!("Program: {name}"))
1506            .collect()
1507    }
1508}