starry-kernel 0.10.1

//! PMU overflow-IRQ sampling backend (`perf record`).
//!
//! This is the IRQ half of hardware-PMU sampling. A sampling perf event
//! ([`super::hw::HwPerfEvent`] with `sample_period > 0`) preloads a programmable
//! counter so it overflows after `period` events; the overflow raises the PMUv3
//! interrupt (PPI 7 / INTID 23). [`pmu_overflow_handler`] runs in hard-IRQ
//! context, reads the interrupted PC, builds one `PERF_RECORD_SAMPLE` per
//! overflowed counter, writes it into that event's mmap ring buffer, re-arms the
//! counter, and wakes a deferred worker (via [`crate::task::future::IrqNotify`]) that
//! delivers `POLLIN` to userspace pollers.
//!
//! The record emitted honours the event's `attr.sample_type`: [`build_sample`]
//! lays out every requested scalar field in the canonical `man perf_event_open`
//! order, so the real `perf` tool — which always sets `IP|TID|TIME|PERIOD` —
//! parses the stream and reports samples. The supported field set is
//! [`SUPPORTED_SAMPLE_TYPE`]; an unsupported bit is rejected at open in
//! [`super::hw`]. A `sample_type` of exactly `PERF_SAMPLE_IP` still yields the
//! original 16-byte IP-only record.
//!
//! IRQ-context discipline (enforced throughout this module's handler path):
//! no allocation, no sleeping locks, and the interrupted `ELR_EL1` / `SPSR_EL1`
//! are read *first* (before touching the PMU or memory) so a nested fault can
//! never clobber them.
//!
//! # Per-CPU registry
//!
//! The handler must locate the ring buffer for an overflowed counter `n` without
//! allocating or taking a lock. [`REGISTRY`] is one fixed generation-bearing
//! registry per CPU (index = programmable counter index). Each [`SampleSlot`]
//! owns strong output and notification references rather than borrowing raw
//! callback storage. `register` / `unregister` mutate the current CPU's registry
//! under a local-IRQ-off critical section ([`NoPreemptIrqSave`]) so removal is
//! also the local hard-IRQ grace period.

use alloc::sync::Arc;
use core::sync::atomic::{AtomicU64, Ordering};

use ax_hal::irq::{IrqContext, IrqId, IrqReturn};
use kbpf_basic::linux_bpf::perf_event_mmap_page;

use super::{
    output::PerfRingOutput,
    sampling_lifecycle::SampleRegistration,
    sampling_registry::{RegisterError, SamplingRegistry, UnregisterError},
    target::PerfCpuId,
};
use crate::{
    sync::NoPreemptIrqSave,
    task::{PidNamespaceId, TgidNumber, TidNumber, future::IrqNotify, try_current_user_irq_view},
};

fn pmu_irq() -> Result<IrqId, ax_hal::irq::IrqError> {
    ax_hal::pmu::irq()
}

/// Maximum programmable counter index (matches [`ax_cpu::pmu::CounterId`] /
/// [`ax_cpu::pmu::Pmu`]); the registry is sized one past this for indexing.
const MAX_COUNTER: usize = 30;

/// Minimum sampling period for frequency mode. Floors the adaptive control loop
/// so a rare event cannot drive the period to 0 (which would re-arm the counter
/// to overflow only after a full `2^32` wrap, i.e. effectively never). `1`
/// matches Linux's lower bound — a counter preloaded to overflow after a single
/// event.
const MIN_FREQ_PERIOD: u32 = 1;
/// Sampling policy caps periods at u32::MAX. The CPU preload operation uses
/// the owner CPU's actual 32-bit or 64-bit overflow width.
const MAX_SAMPLE_PERIOD: u32 = u32::MAX;
/// Upper bound on a frequency-mode target rate (Hz). Mirrors the advertised
/// `/proc/sys/kernel/perf_event_max_sample_rate`; a wild `sample_freq` is clamped
/// here rather than rejected so `perf` still records.
pub const MAX_TARGET_FREQ: u32 = 100_000;

/// Initial period estimate for a frequency-mode event targeting `freq` Hz.
///
/// Assumes a ~1 GHz event rate as the starting point (so e.g. `-F 4000` starts
/// at `250_000`); [`pmu_overflow_handler`] adapts from here within a few samples.
/// Clamped so a degenerate `freq` cannot produce a 0 period.
pub fn initial_period_for_freq(freq: u32) -> u32 {
    (1_000_000_000u64 / freq.max(1) as u64).clamp(MIN_FREQ_PERIOD as u64, MAX_SAMPLE_PERIOD as u64)
        as u32
}

/// Next adaptive period after a frequency-mode sample (Linux `perf_adjust_period`).
///
/// `cur` events elapsed over `delta_ns` ns produced exactly one sample; to hit
/// `target_freq` samples/sec the ideal period is `cur * 1e9 / (delta_ns *
/// target_freq)`. The move toward that ideal is damped by 1/8 to avoid
/// oscillation, then clamped to a valid 32-bit period. All integer math (IRQ
/// context): the `u128` intermediate cannot overflow for `cur,delta_ns <= u64`.
fn next_freq_period(cur: u32, target_freq: u32, delta_ns: u64) -> u32 {
    if delta_ns == 0 || target_freq == 0 {
        return cur;
    }
    let ideal = (cur as u128 * 1_000_000_000u128) / (delta_ns as u128 * target_freq as u128);
    let ideal = ideal.clamp(MIN_FREQ_PERIOD as u128, MAX_SAMPLE_PERIOD as u128) as i64;
    // Damp by 1/8 toward the ideal (the `+7` biases the truncating divide so a
    // small positive gap still nudges the period up; it converges either way).
    let delta = (ideal - cur as i64 + 7) / 8;
    (cur as i64 + delta).clamp(MIN_FREQ_PERIOD as i64, MAX_SAMPLE_PERIOD as i64) as u32
}

/// `PERF_RECORD_SAMPLE` discriminant (`perf_event_type::PERF_RECORD_SAMPLE`).
const PERF_RECORD_SAMPLE: u32 = 9;
/// `PERF_RECORD_MISC_KERNEL`: the sample landed in kernel (EL1) context.
const PERF_RECORD_MISC_KERNEL: u16 = 1;
/// `PERF_RECORD_MISC_USER`: the sample landed in user (EL0) context.
const PERF_RECORD_MISC_USER: u16 = 2;

/// Upper bound on a single `PERF_RECORD_SAMPLE` we emit: 8-byte header plus at
/// most nine 8-byte scalar fields (IDENTIFIER, IP, TID(pid+tid), TIME, ADDR, ID,
/// STREAM_ID, CPU(cpu+res), PERIOD). [`build_sample`] writes into a stack buffer
/// of this size and returns the actual length.
const SAMPLE_RECORD_MAX_LEN: usize = 8 + 9 * 8;

// `perf_event_sample_format` bits (see `man perf_event_open`). Only the scalar
// fields below are supported; every other bit (READ, CALLCHAIN, RAW,
// BRANCH_STACK, REGS_USER/INTR, STACK_USER, WEIGHT, DATA_SRC, TRANSACTION,
// PHYS_ADDR, …) is rejected at open time.
/// `PERF_SAMPLE_IP`: instruction pointer. Always set by real `perf` for samples.
const PERF_SAMPLE_IP: u64 = 1 << 0;
/// `PERF_SAMPLE_TID`: thread + process id (`u32 pid, u32 tid`).
const PERF_SAMPLE_TID: u64 = 1 << 1;
/// `PERF_SAMPLE_TIME`: monotonic timestamp (`u64`).
const PERF_SAMPLE_TIME: u64 = 1 << 2;
/// `PERF_SAMPLE_ADDR`: data address (`u64`); always 0 for our IP samples.
const PERF_SAMPLE_ADDR: u64 = 1 << 3;
/// `PERF_SAMPLE_ID`: event id (`u64`).
const PERF_SAMPLE_ID: u64 = 1 << 6;
/// `PERF_SAMPLE_CPU`: cpu number (`u32 cpu, u32 res`).
const PERF_SAMPLE_CPU: u64 = 1 << 7;
/// `PERF_SAMPLE_PERIOD`: sampling period (`u64`).
const PERF_SAMPLE_PERIOD: u64 = 1 << 8;
/// `PERF_SAMPLE_STREAM_ID`: stream id (`u64`).
const PERF_SAMPLE_STREAM_ID: u64 = 1 << 9;
/// `PERF_SAMPLE_IDENTIFIER`: leading event id (`u64`), emitted first.
const PERF_SAMPLE_IDENTIFIER: u64 = 1 << 16;

/// Every `sample_type` bit the sampling backend can emit a well-formed
/// `PERF_RECORD_SAMPLE` for. A sampling event whose `sample_type` sets any bit
/// outside this mask is rejected at open ([`super::hw`] reuses this constant);
/// real `perf record` sets `IP|TID|TIME|PERIOD`, all within the mask.
pub const SUPPORTED_SAMPLE_TYPE: u64 = PERF_SAMPLE_IP
    | PERF_SAMPLE_TID
    | PERF_SAMPLE_TIME
    | PERF_SAMPLE_ADDR
    | PERF_SAMPLE_ID
    | PERF_SAMPLE_CPU
    | PERF_SAMPLE_PERIOD
    | PERF_SAMPLE_STREAM_ID
    | PERF_SAMPLE_IDENTIFIER;

/// Owned ring and wake target used by one registered sampling generation.
#[derive(Clone)]
pub struct SampleOutput {
    ring: Option<PerfRingOutput>,
    notify: Option<Arc<IrqNotify>>,
}

impl core::fmt::Debug for SampleOutput {
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
        f.debug_struct("SampleOutput")
            .field(
                "ring",
                &self
                    .ring
                    .as_ref()
                    .map(|ring| (ring.ring_vaddr(), ring.ring_len())),
            )
            .field("notifies", &self.notify.is_some())
            .finish()
    }
}

impl SampleOutput {
    /// Creates an output whose ring geometry and lifetime are one value.
    pub fn new(ring: Option<PerfRingOutput>, notify: Option<Arc<IrqNotify>>) -> Self {
        Self { ring, notify }
    }
}

/// Everything the overflow handler needs for one counter.
///
/// Stored by value in the owner CPU's [`REGISTRY`]. Strong references in
/// [`SampleOutput`] remain live until generation-checked unregister completes
/// with local IRQs excluded.
pub struct SampleSlot {
    output: SampleOutput,
    /// Sampling period: the counter is re-armed to overflow after this many
    /// events via [`ax_cpu::pmu::Pmu::preload`]. Also emitted as the
    /// `PERF_SAMPLE_PERIOD` field of each record.
    pub period: u32,
    /// `attr.sample_type`: the set of scalar fields each record carries (see
    /// [`build_sample`]). Validated against [`SUPPORTED_SAMPLE_TYPE`] at open.
    pub sample_type: u64,
    /// Event id emitted for the `PERF_SAMPLE_ID` / `PERF_SAMPLE_IDENTIFIER`
    /// fields. `0` when the event was opened without per-event ids (the common
    /// case in this single-group implementation).
    pub id: u64,
    /// PID namespace view captured by the event owner.
    pub observer: PidNamespaceId,
    /// Frequency mode (`attr.freq`): after each sample re-derive [`period`](Self::period)
    /// to converge on [`target_freq`](Self::target_freq) samples/sec. Fixed
    /// `-c` period when false.
    pub freq: bool,
    /// Target sample rate in Hz for frequency mode; `0` in fixed-period mode.
    pub target_freq: u32,
    /// Monotonic ns of the previous sample, for the frequency-mode delta. `0`
    /// before the first sample, when the period is left at its initial estimate.
    /// Mutated in place by the handler as the period adapts.
    pub last_time: u64,
}

/// Immutable attributes copied into one owner-CPU sampling slot.
pub struct SampleSlotConfig {
    pub period: u32,
    pub sample_type: u64,
    pub id: u64,
    pub observer: PidNamespaceId,
    pub freq: bool,
    pub target_freq: u32,
    pub last_time: u64,
}

impl SampleSlot {
    /// Creates one owned per-CPU registry entry.
    pub fn new(output: SampleOutput, config: SampleSlotConfig) -> Self {
        Self {
            output,
            period: config.period,
            sample_type: config.sample_type,
            id: config.id,
            observer: config.observer,
            freq: config.freq,
            target_freq: config.target_freq,
            last_time: config.last_time,
        }
    }
}

/// Per-CPU map from programmable counter index to its registered sampling slot.
///
/// Index `n` (`0..=30`) holds the slot for `PMEVCNTRn_EL0`. `None` means no
/// sampling event currently owns that counter on this CPU.
#[ax_percpu::def_percpu]
static REGISTRY: SamplingRegistry<SampleSlot> = SamplingRegistry::new();

/// Globally unique registry generation. Counter slots may be reused, but an old
/// teardown token can never match the next event that occupies the same index.
static NEXT_REGISTRATION_GENERATION: AtomicU64 = AtomicU64::new(1);

/// Whether [`pmu_overflow_handler`] has been registered with the IRQ framework.
///
/// Registration is process-global and idempotent: the handler walks the per-CPU
/// registry, so a single action installed on all CPUs suffices.
static REGISTERED: core::sync::atomic::AtomicBool = core::sync::atomic::AtomicBool::new(false);

/// Mutates the current CPU's sampling registry without exposing a reference
/// beyond the CPU-local exclusive-access scope.
///
/// # Safety
///
/// The caller must prevent migration, local IRQ re-entry, and remote mutation
/// for the complete callback. Process-context callers use
/// [`NoPreemptIrqSave`]; the overflow handler already runs with local IRQs
/// masked on the CPU that owns the registry.
unsafe fn with_registry_mut<R>(
    operation: impl for<'value> FnOnce(&'value mut SamplingRegistry<SampleSlot>) -> R,
) -> R {
    // SAFETY: the caller establishes the migration and exclusion contract.
    unsafe {
        ax_percpu::with_cpu_pin(|pin| {
            ax_percpu::with_exclusive_cpu(pin, |exclusive| {
                REGISTRY.with_current_mut(exclusive, operation)
            })
        })
    }
    .unwrap_or_else(|error| panic!("perf sampling CPU-local state is invalid: {error}"))
}

/// Registers `slot` for programmable counter `n` on the current CPU.
///
/// Runs on the event's owner CPU. The mutation is performed under
/// [`NoPreemptIrqSave`] so the overflow handler — which reads the same per-CPU
/// array — can never observe a half-written entry.
pub fn register(n: usize, slot: SampleSlot) -> Result<SampleRegistration, RegisterError> {
    if n > MAX_COUNTER {
        return Err(RegisterError::InvalidCounter);
    }
    let owner = PerfCpuId::new(ax_hal::percpu::this_cpu_id());
    let generation = NEXT_REGISTRATION_GENERATION
        .try_update(Ordering::Relaxed, Ordering::Relaxed, |generation| {
            generation.checked_add(1)
        })
        .expect("PMU sampling registration generation exhausted");
    let _guard = NoPreemptIrqSave::new();
    // SAFETY: the guard prevents migration and local IRQ reentry.
    unsafe { with_registry_mut(|registry| registry.register(n, generation, slot)) }?;
    Ok(SampleRegistration::new(owner, n, generation))
}

/// Failure to remove an owner-CPU sampling registration.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum SamplingUnregisterError {
    /// Teardown ran on a CPU other than the registry owner.
    WrongCpu,
    /// The counter slot no longer carries this generation.
    Registry(UnregisterError),
}

/// Clears one exact sampling generation on its owner CPU.
///
/// Returning successfully proves both that the registry no longer reaches the
/// output and that any local hard-IRQ reader has completed.
pub fn unregister(registration: SampleRegistration) -> Result<(), SamplingUnregisterError> {
    if registration.owner().as_usize() != ax_hal::percpu::this_cpu_id() {
        return Err(SamplingUnregisterError::WrongCpu);
    }
    let removed = {
        let _guard = NoPreemptIrqSave::new();
        // SAFETY: the guard prevents migration and local IRQ reentry.
        unsafe {
            with_registry_mut(|registry| {
                registry.unregister(registration.counter(), registration.generation())
            })
        }
        .map_err(SamplingUnregisterError::Registry)?
    };
    drop(removed);
    Ok(())
}

/// Ensures [`pmu_overflow_handler`] is registered with the IRQ framework.
///
/// This process-context operation may allocate inside IRQ registration and must
/// run before scheduler hooks can arm a sampling event.
pub fn ensure_pmu_irq_registered() -> Result<(), ax_hal::irq::IrqError> {
    let pmu_irq = pmu_irq()?;
    if REGISTERED
        .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
        .is_ok()
    {
        let cpus = ax_hal::irq::CpuMask::first_n(ax_hal::cpu_num());
        // Mirror the timer's unit-data pattern: the handler does not use `data`.
        if let Err(err) = ax_hal::irq::request_percpu_irq(pmu_irq, cpus, pmu_overflow_handler) {
            // Roll back so a later open can retry registration.
            REGISTERED.store(false, Ordering::Release);
            return Err(err);
        }
    }
    Ok(())
}

/// Enables the already-registered PMU PPI on the current owner CPU.
///
/// This is bounded and allocation-free, so a scheduler hook may call it before
/// publishing the local sampling slot.
pub fn enable_local_pmu_irq() -> Result<(), ax_hal::irq::IrqError> {
    ax_hal::irq::set_enable(pmu_irq()?, true)
}

fn service_overflowed_slots(
    registry: &mut SamplingRegistry<SampleSlot>,
    overflow: u64,
    misc: u16,
    ip: u64,
) -> u64 {
    let current = try_current_user_irq_view();

    // Bits we have serviced; cleared (write-1-to-clear) only after every slot
    // has been inspected, so re-arming one counter cannot drop another event.
    let mut handled = 0;

    for n in 0..=MAX_COUNTER {
        if overflow & (1 << n) == 0 {
            continue;
        }
        handled |= 1 << n;

        let Some(slot) = registry.get_mut(n) else {
            // Counting-only events own their re-arm policy. The IRQ path only
            // acknowledges an overflow that has no registered sampling slot.
            continue;
        };

        let sample_type = slot.sample_type;
        let id = slot.id;
        let cur_period = slot.period;

        let time = ax_runtime::hal::time::monotonic_time_nanos();
        let cpu = ax_hal::percpu::this_cpu_id() as u32;
        let (pid, tid) = current.as_ref().map_or((None, None), |task| {
            (
                task.visible_tgid(slot.observer),
                task.visible_tid(slot.observer),
            )
        });
        let mut record = [0u8; SAMPLE_RECORD_MAX_LEN];
        let data = SampleData {
            ip,
            pid,
            tid,
            time,
            addr: 0,
            id,
            stream_id: 0,
            cpu,
            period: cur_period as u64,
        };
        let len = build_sample(&mut record, sample_type, misc, &data);

        if let Some(ring) = &slot.output.ring {
            // SAFETY: `ring` owns the reference that pins this kernel mapping
            // until generation-checked unregister removes the complete slot.
            unsafe { ring_write(ring, &record[..len]) };
        }

        let next_period = if slot.freq {
            let next = if slot.last_time != 0 {
                next_freq_period(
                    cur_period,
                    slot.target_freq,
                    time.saturating_sub(slot.last_time),
                )
            } else {
                cur_period
            };
            slot.period = next;
            slot.last_time = time;
            next
        } else {
            cur_period
        };

        crate::perf::hw_owner::on_counter(n, |pmu, id| pmu.preload(id, u64::from(next_period)));

        if let Some(notify) = &slot.output.notify {
            notify.notify_irq();
        }
    }

    handled
}

/// PMU overflow IRQ handler (hard-IRQ context).
///
/// Reads the interrupted PC and EL *first*, then services every overflowed
/// programmable counter that has a registered sampling slot: builds a
/// `PERF_RECORD_SAMPLE`, writes it into the event's ring, re-arms the counter,
/// and wakes the event's deferred worker. Clears only the overflow bits it
/// actually serviced (write-1-to-clear) at the end.
///
/// Returns [`IrqReturn::Handled`] if any counter overflowed (whether or not a
/// slot was registered for it), else [`IrqReturn::Unhandled`].
///
/// # Safety
///
/// Must only be invoked by the IRQ framework in hard-IRQ context on the core the
/// overflow fired on. Performs no allocation and takes no sleeping locks.
pub fn pmu_overflow_handler(_ctx: IrqContext) -> IrqReturn {
    // Capture the interrupted context before doing anything that could fault or
    // overwrite ELR_EL1 / SPSR_EL1.
    let context = ax_hal::irq::interrupted_context()
        .expect("PMU trap must supply its interrupted register image");
    let ip = context.pc as u64;
    let is_user = context.privilege == ax_cpu::trap::InterruptedPrivilege::User;

    let ovf = crate::perf::hw_owner::on_pmu(|pmu| pmu.overflow_status());
    if ovf == 0 {
        return IrqReturn::Unhandled;
    }

    let misc = if is_user {
        PERF_RECORD_MISC_USER
    } else {
        PERF_RECORD_MISC_KERNEL
    };

    // SAFETY: the handler runs with local IRQs masked on its current CPU, so
    // the registry cannot be re-entered or observed after migration.
    let handled =
        unsafe { with_registry_mut(|registry| service_overflowed_slots(registry, ovf, misc, ip)) };

    // Clear exactly the overflow bits we serviced.
    crate::perf::hw_owner::on_pmu(|pmu| pmu.clear_overflow(handled));
    IrqReturn::Handled
}

#[cfg(all(test, axtest))]
fn kernel_task_sample_ids_are_empty_for_test() -> bool {
    try_current_user_irq_view().is_none()
}

/// Lays out one `PERF_RECORD_SAMPLE` into `buf` per `sample_type`, returning its
/// total length in bytes.
///
/// The fields are written in the canonical order mandated by `man
/// perf_event_open` (`PERF_RECORD_SAMPLE`), each gated on its `sample_type` bit:
///
/// 1. header — `u32 type = PERF_RECORD_SAMPLE`, `u16 misc`, `u16 size`
///    (back-patched once the body length is known)
/// 2. `IDENTIFIER` → `u64 id`
/// 3. `IP` → `u64 ip`
/// 4. `TID` → `u32 pid`, `u32 tid`
/// 5. `TIME` → `u64 time`
/// 6. `ADDR` → `u64 addr`
/// 7. `ID` → `u64 id`
/// 8. `STREAM_ID` → `u64 stream_id`
/// 9. `CPU` → `u32 cpu`, `u32 res = 0`
/// 10. `PERIOD` → `u64 period`
///
/// `buf` must be at least [`SAMPLE_RECORD_MAX_LEN`] bytes. With
/// `sample_type == PERF_SAMPLE_IP` exactly, the result is the original 16-byte
/// IP-only record (8-byte header + `u64 ip`).
/// The per-sample scalar values [`build_sample`] may emit (those not implied by
/// `sample_type` alone). Gathered by the overflow handler at interrupt time.
struct SampleData {
    ip: u64,
    pid: Option<TgidNumber>,
    tid: Option<TidNumber>,
    time: u64,
    addr: u64,
    id: u64,
    stream_id: u64,
    cpu: u32,
    period: u64,
}

fn build_sample(buf: &mut [u8], sample_type: u64, misc: u16, d: &SampleData) -> usize {
    // Cursor into `buf`. All offsets stay within `SAMPLE_RECORD_MAX_LEN` because
    // at most the header + 9 u64-sized fields are written and the caller passes a
    // buffer of that size. `put!` appends a native-endian scalar and advances the
    // cursor (a macro, not a closure, so it never holds a borrow of `off`).
    let mut off = 0usize;
    macro_rules! put {
        ($v:expr) => {{
            let bytes = $v.to_ne_bytes();
            buf[off..off + bytes.len()].copy_from_slice(&bytes);
            off += bytes.len();
        }};
    }

    // Header: type, misc, and a placeholder size (back-patched below).
    put!(PERF_RECORD_SAMPLE); // u32
    put!(misc); // u16
    let size_off = off;
    put!(0u16); // size placeholder

    // Body, in canonical PERF_RECORD_SAMPLE order, each field gated by its bit.
    if sample_type & PERF_SAMPLE_IDENTIFIER != 0 {
        put!(d.id);
    }
    if sample_type & PERF_SAMPLE_IP != 0 {
        put!(d.ip);
    }
    if sample_type & PERF_SAMPLE_TID != 0 {
        // pid and tid are a packed `u32` pair in one 8-byte slot.
        put!(d.pid.map_or(0, TgidNumber::get));
        put!(d.tid.map_or(0, TidNumber::get));
    }
    if sample_type & PERF_SAMPLE_TIME != 0 {
        put!(d.time);
    }
    if sample_type & PERF_SAMPLE_ADDR != 0 {
        put!(d.addr);
    }
    if sample_type & PERF_SAMPLE_ID != 0 {
        put!(d.id);
    }
    if sample_type & PERF_SAMPLE_STREAM_ID != 0 {
        put!(d.stream_id);
    }
    if sample_type & PERF_SAMPLE_CPU != 0 {
        // cpu and a reserved zero, again a packed `u32` pair.
        put!(d.cpu);
        put!(0u32);
    }
    if sample_type & PERF_SAMPLE_PERIOD != 0 {
        put!(d.period);
    }

    // Back-patch the header's `size` field now that the total length is known.
    buf[size_off..size_off + 2].copy_from_slice(&(off as u16).to_ne_bytes());
    off
}

/// Writes one record into a perf ring buffer, IRQ-safe and self-contained.
///
/// Page 0 of the range described by `ring` is a
/// [`perf_event_mmap_page`]; the data region starts at `ring_vaddr + data_offset`
/// (`data_offset == PAGE_SIZE` for our buffers) and is `data_size` bytes. The
/// record is copied at `data_head % data_size` (split into two copies on wrap),
/// then `data_head` is published with a release fence so a userspace reader that
/// observes the new `data_head` also observes the bytes.
///
/// If the record would overwrite still-unread bytes
/// (`data_head - data_tail + len > data_size`) it is dropped: `data_head` is not
/// advanced. Lost-record accounting is intentionally omitted for M2.
///
/// # Safety
///
/// `ring` must describe a kernel-mapped ring whose header was initialized by
/// `HwPerfEvent::device_mmap`. Its owned lifetime anchor must keep that mapping
/// valid for this call.
unsafe fn ring_write(ring: &PerfRingOutput, record: &[u8]) {
    let Some(_writer) = ring.try_begin_write() else {
        ring.record_contention_drop();
        return;
    };
    let ring_vaddr = ring.ring_vaddr();
    let ring_len = ring.ring_len();
    // Guard the enable-before-mmap case (slot registered with a zero ring) and
    // any ring too small to even hold the header page: there is nowhere to
    // write, and the header pointer would be null/out of bounds.
    if ring_vaddr == 0 || ring_len < core::mem::size_of::<perf_event_mmap_page>() {
        return;
    }

    let header = ring_vaddr as *mut perf_event_mmap_page;

    // SAFETY: `header` points at the initialized header page.
    let data_offset =
        unsafe { core::ptr::addr_of!((*header).data_offset).read_volatile() } as usize;
    let data_size = unsafe { core::ptr::addr_of!((*header).data_size).read_volatile() } as usize;

    // Defensive: a malformed/zero header (no data region, or a data window that
    // does not fit in the buffer) means there is nowhere safe to write.
    if data_size == 0 || data_offset > ring_len || data_offset + data_size > ring_len {
        return;
    }

    let len = record.len();
    if len > data_size {
        return;
    }

    // SAFETY: header page is initialized; these are plain u64 fields.
    let head = unsafe { core::ptr::addr_of!((*header).data_head).read_volatile() };
    let tail = unsafe { core::ptr::addr_of!((*header).data_tail).read_volatile() };

    // Would this record overwrite bytes the reader has not consumed yet? Drop it
    // if so (back-pressure; no lost-record accounting in M2).
    if head.wrapping_sub(tail).wrapping_add(len as u64) > data_size as u64 {
        return;
    }

    let data_base = ring_vaddr + data_offset;
    let start = (head % data_size as u64) as usize;
    let first = core::cmp::min(len, data_size - start);

    // SAFETY: `data_base + start + first <= data_base + data_size`, within the
    // mapped data region; same for the wrapped remainder below.
    unsafe {
        core::ptr::copy_nonoverlapping(record.as_ptr(), (data_base + start) as *mut u8, first);
        if first < len {
            core::ptr::copy_nonoverlapping(
                record.as_ptr().add(first),
                data_base as *mut u8,
                len - first,
            );
        }
    }

    // Publish the bytes before the new head: a reader observing the updated
    // `data_head` must also observe the record contents.
    core::sync::atomic::fence(Ordering::Release);
    // SAFETY: header page is initialized.
    unsafe {
        core::ptr::addr_of_mut!((*header).data_head).write_volatile(head.wrapping_add(len as u64));
    }
}

/// Write one record into a sampling ring from **process context** (the side-band
/// path: `PERF_RECORD_MMAP2` / `COMM` / `FORK` / `EXIT` emitted at execve / mmap /
/// clone / exit), serialized against every producer sharing the output.
///
/// [`PerfRingOutput`] owns one shared, non-blocking producer gate. Both hard-IRQ
/// and process producers attempt one CAS and drop on contention, so redirected
/// or inherited events remain bounded even when writers run on different CPUs.
///
/// # Safety
///
/// Same contract as [`ring_write`]: `ring` must keep the initialized mapping
/// pinned for the duration of the call.
pub(crate) unsafe fn ring_write_process(ring: &PerfRingOutput, record: &[u8]) {
    // SAFETY: the caller upholds the mapping initialization contract; `ring`
    // owns the lifetime and cross-CPU producer gate.
    unsafe { ring_write(ring, record) };
}

#[cfg(all(test, axtest))]
mod tests {
    #[cfg(all(test, axtest, target_arch = "aarch64"))]
    #[axtest::axtest]
    fn kernel_task_sample_ids_are_empty() {
        assert!(super::kernel_task_sample_ids_are_empty_for_test());
    }
}