starry-kernel 0.10.2

//! perf side-band records (`PERF_RECORD_COMM` / `MMAP2` / `FORK` / `EXIT`).
//!
//! Sampling gives `perf` raw instruction pointers; to turn those into
//! `function@binary` `perf report` needs the monitored task's executable memory
//! map and name. The kernel supplies that out of band, interleaved into the same
//! mmap ring as the samples, gated by the event's `attr.{comm,mmap2,task}` bits:
//!
//! * `PERF_RECORD_COMM` — the process name, at `execve`.
//! * `PERF_RECORD_MMAP2` — one per executable mapping: the exec image + the
//!   dynamic loader at `execve`, then each shared library as the loader `mmap`s
//!   it. Carries `addr`/`len`/`pgoff`/`filename` so `perf` can map an IP back to
//!   `(binary, file offset)` and read that binary's symbol table.
//! * `PERF_RECORD_FORK` / `EXIT` — task lifetime, at `clone` / `exit`.
//!
//! These are written from *process context* (syscall time), not the IRQ handler,
//! via [`super::sampling::ring_write_process`]. The output's shared non-blocking
//! producer gate serializes them with overflow IRQs, including across CPUs.
//!
//! ## `sample_id_all`
//!
//! Real `perf record` sets `attr.sample_id_all`, which means *every* record —
//! including these side-band ones — carries a trailing "sample id" section with
//! the `attr.sample_type` subset `{TID, TIME, ID, STREAM_ID, CPU, IDENTIFIER}`.
//! The trailer is part of `header.size`; omitting it would desync `perf`'s parser.
//! [`push_trailer`] appends it when [`SidebandTarget::sample_id_all`] is set.

use alloc::{
    sync::{Arc, Weak},
    vec::Vec,
};
use core::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};

use ax_lazyinit::LazyInit;

use super::output::PerfRingOutput;
use crate::{
    sync::IrqMutex,
    task::{PidNamespaceId, TgidNumber, Thread, TidNumber},
};

/// `PERF_RECORD_COMM`.
const PERF_RECORD_COMM: u32 = 3;
/// `PERF_RECORD_EXIT`.
const PERF_RECORD_EXIT: u32 = 4;
/// `PERF_RECORD_FORK`.
const PERF_RECORD_FORK: u32 = 7;
/// `PERF_RECORD_MMAP2`.
const PERF_RECORD_MMAP2: u32 = 10;
/// `PERF_RECORD_MISC_USER`: the record describes user-space state.
const PERF_RECORD_MISC_USER: u16 = 2;
/// `PERF_RECORD_MISC_COMM_EXEC`: this `COMM` came from `execve` (not `prctl`).
const PERF_RECORD_MISC_COMM_EXEC: u16 = 1 << 13;

/// `comm` is capped at Linux's `TASK_COMM_LEN` (16, including the NUL).
const COMM_MAX: usize = 15;

static SYSTEM_SOURCES: LazyInit<IrqMutex<Vec<Weak<SystemSidebandSource>>>> = LazyInit::new();
static SYSTEM_SOURCE_COUNT: AtomicUsize = AtomicUsize::new(0);

pub fn initialize() {
    SYSTEM_SOURCES.init_once(IrqMutex::new(Vec::new()));
}

/// Side-band-only subscription for one fixed CPU perf context.
pub struct SystemSidebandSource {
    owner_cpu: usize,
    observer: PidNamespaceId,
    sample_type: u64,
    sample_id_all: bool,
    want_comm: bool,
    want_mmap2: bool,
    want_task: bool,
    sample_id: AtomicU64,
    enabled: AtomicBool,
    redirect: IrqMutex<Option<PerfRingOutput>>,
}

impl SystemSidebandSource {
    pub fn register(
        owner_cpu: usize,
        sample_type: u64,
        sample_id_all: bool,
        want_comm: bool,
        want_mmap2: bool,
        want_task: bool,
    ) -> Option<Arc<Self>> {
        if !(want_comm || want_mmap2 || want_task) {
            return None;
        }
        let source = Arc::new(Self {
            owner_cpu,
            observer: crate::task::current_user_task()
                .as_thread()
                .active_pid_namespace()
                .id(),
            sample_type,
            sample_id_all,
            want_comm,
            want_mmap2,
            want_task,
            sample_id: AtomicU64::new(0),
            enabled: AtomicBool::new(false),
            redirect: IrqMutex::new(None),
        });
        SYSTEM_SOURCES
            .get()
            .expect("perf side-band registry not initialized")
            .lock()
            .push(Arc::downgrade(&source));
        SYSTEM_SOURCE_COUNT.fetch_add(1, Ordering::AcqRel);
        Some(source)
    }

    pub fn set_sample_id(&self, id: u64) {
        self.sample_id.store(id, Ordering::Release);
    }

    pub fn set_enabled(&self, enabled: bool) {
        self.enabled.store(enabled, Ordering::Release);
    }

    pub fn set_redirect(&self, redirect: Option<PerfRingOutput>) {
        *self.redirect.lock() = redirect;
    }

    pub fn output_scope(&self) -> super::output::PerfOutputScope {
        super::output::PerfOutputScope::Cpu(self.owner_cpu)
    }

    fn target(&self, thread: &Thread) -> Option<SystemSidebandTarget> {
        if !self.enabled.load(Ordering::Acquire) {
            return None;
        }
        let ring = self.redirect.lock().clone()?;
        let pid = thread
            .proc_data
            .identity()
            .visible_number_in(self.observer)
            .map(TgidNumber::from)?;
        let tid = thread
            .pid_identity()
            .visible_number_in(self.observer)
            .map(TidNumber::from)?;
        Some(SystemSidebandTarget {
            observer: self.observer,
            target: SidebandTarget {
                ring,
                sample_type: self.sample_type,
                sample_id_all: self.sample_id_all,
                id: self.sample_id.load(Ordering::Acquire),
                stream_id: self.sample_id.load(Ordering::Acquire),
                pid,
                tid,
            },
            comm: self.want_comm,
            mmap2: self.want_mmap2,
            task: self.want_task,
        })
    }
}

impl Drop for SystemSidebandSource {
    fn drop(&mut self) {
        SYSTEM_SOURCE_COUNT.fetch_sub(1, Ordering::AcqRel);
    }
}

pub struct SystemSidebandTarget {
    pub observer: PidNamespaceId,
    pub target: SidebandTarget,
    pub comm: bool,
    pub mmap2: bool,
    pub task: bool,
}

/// Snapshots enabled fixed-CPU side-band sources for the executing CPU.
pub fn system_targets(thread: &Thread) -> Vec<SystemSidebandTarget> {
    if SYSTEM_SOURCE_COUNT.load(Ordering::Acquire) == 0 {
        return Vec::new();
    }
    let cpu = ax_hal::percpu::this_cpu_id();
    let Some(sources) = SYSTEM_SOURCES.get() else {
        return Vec::new();
    };
    let mut sources = sources.lock();
    let mut targets = Vec::new();
    sources.retain(|weak| {
        let Some(source) = weak.upgrade() else {
            return false;
        };
        if source.owner_cpu == cpu
            && let Some(target) = source.target(thread)
        {
            targets.push(target);
        }
        true
    });
    targets
}

/// Where a side-band record is written, plus the parameters of its
/// `sample_id_all` trailer. Built per monitored event from its `PerTaskCounter`.
pub struct SidebandTarget {
    /// Destination geometry coupled to the reference that pins its pages.
    pub(crate) ring: PerfRingOutput,
    /// `attr.sample_type` — selects which fields the trailer carries.
    pub sample_type: u64,
    /// Whether to append the `sample_id_all` trailer at all.
    pub sample_id_all: bool,
    /// Event id (for the trailer's `ID` / `IDENTIFIER` fields).
    pub id: u64,
    /// Concrete event identity for the trailer's STREAM_ID field.
    pub stream_id: u64,
    /// Process id of the monitored task in the event's captured view.
    pub pid: TgidNumber,
    /// Thread id of the monitored task in the event's captured view.
    pub tid: TidNumber,
}

/// One executable mapping, for [`emit_mmap2`].
pub struct Mmap2Info {
    /// Mapped virtual address.
    pub addr: u64,
    /// Mapping length in bytes.
    pub len: u64,
    /// File offset of the mapping (`pgoff`).
    pub pgoff: u64,
    /// Backing file device major/minor and inode (best-effort; 0 if unknown).
    pub maj: u32,
    pub min: u32,
    pub ino: u64,
    /// Protection + flags (`PROT_*` / `MAP_*`).
    pub prot: u32,
    pub flags: u32,
    /// Backing file path (what `perf` opens to read symbols).
    pub filename: alloc::string::String,
}

#[inline]
fn push_u32(b: &mut Vec<u8>, v: u32) {
    b.extend_from_slice(&v.to_ne_bytes());
}
#[inline]
fn push_u64(b: &mut Vec<u8>, v: u64) {
    b.extend_from_slice(&v.to_ne_bytes());
}

/// Append a NUL-terminated string padded to an 8-byte boundary (perf record
/// string fields are always 8-aligned).
fn push_cstr_padded(b: &mut Vec<u8>, s: &[u8]) {
    b.extend_from_slice(s);
    b.push(0);
    while !b.len().is_multiple_of(8) {
        b.push(0);
    }
}

/// Append the `sample_id_all` trailer in the canonical order.
fn push_trailer(b: &mut Vec<u8>, t: &SidebandTarget) {
    if !t.sample_id_all {
        return;
    }
    let identity = super::sample_id::SampleId {
        pid: t.pid.get(),
        tid: t.tid.get(),
        time: ax_runtime::hal::time::monotonic_time_nanos(),
        id: t.id,
        stream_id: t.stream_id,
        cpu: ax_hal::percpu::this_cpu_id() as u32,
    };
    let mut trailer = [0u8; super::sample_id::SAMPLE_ID_MAX_LEN];
    let length = identity.encode(t.sample_type, &mut trailer);
    b.extend_from_slice(&trailer[..length]);
}

/// Back-patch the 8-byte header (reserved at the front of `b`) once the full
/// record length (8-aligned) is known, then write it into the ring.
fn finish_and_write(mut b: Vec<u8>, t: &SidebandTarget, type_: u32, misc: u16) {
    while !b.len().is_multiple_of(8) {
        b.push(0);
    }
    let size = b.len() as u16;
    b[0..4].copy_from_slice(&type_.to_ne_bytes());
    b[4..6].copy_from_slice(&misc.to_ne_bytes());
    b[6..8].copy_from_slice(&size.to_ne_bytes());
    // SAFETY: `SidebandTarget` owns the ring pin for the full write, independent
    // of concurrent fd close, task exit, output redirect, or VMA teardown.
    unsafe { super::sampling::ring_write_process(&t.ring, &b) };
}

/// Emit a `PERF_RECORD_COMM` for `comm` (truncated to `TASK_COMM_LEN`).
pub fn emit_comm(t: &SidebandTarget, comm: &str, exec: bool) {
    let mut b = Vec::with_capacity(64);
    b.extend_from_slice(&[0u8; 8]); // header placeholder
    push_u32(&mut b, t.pid.get());
    push_u32(&mut b, t.tid.get());
    let name = comm.as_bytes();
    push_cstr_padded(&mut b, &name[..name.len().min(COMM_MAX)]);
    push_trailer(&mut b, t);
    let misc = PERF_RECORD_MISC_USER | if exec { PERF_RECORD_MISC_COMM_EXEC } else { 0 };
    finish_and_write(b, t, PERF_RECORD_COMM, misc);
}

/// Emit a `PERF_RECORD_FORK` (`type_` == [`PERF_RECORD_FORK`]) or
/// `PERF_RECORD_EXIT` task-lifetime record. Both carry the same body: the subject
/// task's `pid`/`tid`, its parent's `ppid`/`ptid`, then a `time` stamp.
///
/// The `sample_id_all` trailer reflects the task whose context emits the record
/// (the *parent* for `FORK`, the *exiting task* for `EXIT`) — encoded by the
/// caller in `t.pid`/`t.tid` — matching Linux's `perf_event_header__init_id`.
fn emit_task(
    t: &SidebandTarget,
    type_: u32,
    pid: TgidNumber,
    ppid: Option<TgidNumber>,
    tid: TidNumber,
    ptid: Option<TidNumber>,
) {
    let mut b = Vec::with_capacity(64);
    b.extend_from_slice(&[0u8; 8]); // header placeholder
    push_u32(&mut b, pid.get());
    push_u32(&mut b, ppid.map_or(0, TgidNumber::get));
    push_u32(&mut b, tid.get());
    push_u32(&mut b, ptid.map_or(0, TidNumber::get));
    push_u64(&mut b, ax_runtime::hal::time::monotonic_time_nanos());
    push_trailer(&mut b, t);
    // FORK/EXIT carry no cpu-mode misc bits (the task, not a sampled IP).
    finish_and_write(b, t, type_, 0);
}

/// Emit a `PERF_RECORD_FORK` describing a newly-cloned child (`pid`/`tid`) of the
/// monitored parent (`ppid`/`ptid`). `t` is built in the parent's context.
pub fn emit_fork(
    t: &SidebandTarget,
    pid: TgidNumber,
    ppid: TgidNumber,
    tid: TidNumber,
    ptid: TidNumber,
) {
    emit_task(t, PERF_RECORD_FORK, pid, Some(ppid), tid, Some(ptid));
}

/// Emit a `PERF_RECORD_EXIT` for the exiting task (`pid`/`tid`) and its parent
/// (`ppid`/`ptid`). `t` is built in the exiting task's context.
pub fn emit_exit(
    t: &SidebandTarget,
    pid: TgidNumber,
    ppid: Option<TgidNumber>,
    tid: TidNumber,
    ptid: Option<TidNumber>,
) {
    emit_task(t, PERF_RECORD_EXIT, pid, ppid, tid, ptid);
}

/// Emit a `PERF_RECORD_MMAP2` for one executable mapping.
pub fn emit_mmap2(t: &SidebandTarget, m: &Mmap2Info) {
    let mut b = Vec::with_capacity(128);
    b.extend_from_slice(&[0u8; 8]); // header placeholder
    push_u32(&mut b, t.pid.get());
    push_u32(&mut b, t.tid.get());
    push_u64(&mut b, m.addr);
    push_u64(&mut b, m.len);
    push_u64(&mut b, m.pgoff);
    push_u32(&mut b, m.maj);
    push_u32(&mut b, m.min);
    push_u64(&mut b, m.ino);
    push_u64(&mut b, 0); // ino_generation
    push_u32(&mut b, m.prot);
    push_u32(&mut b, m.flags);
    push_cstr_padded(&mut b, m.filename.as_bytes());
    push_trailer(&mut b, t);
    finish_and_write(b, t, PERF_RECORD_MMAP2, PERF_RECORD_MISC_USER);
}