fips-core 0.3.90

//! Runtime perf profiler for the dataplane hot path and queue handoffs.
//!
//! Avoids external dependencies (`perf`, samply, etc.) by instrumenting
//! the key stages directly with `AtomicU64` ns counters, histograms,
//! and packet counts. A background task prints a per-stage breakdown
//! every `FIPS_PERF_INTERVAL_SECS` seconds when `FIPS_PERF=1`,
//! `FIPS_PIPELINE_TRACE=1`, or `NVPN_PIPELINE_TRACE=1` is set at
//! runtime.
//!
//! Enabling adds `Instant::now()` plus a few relaxed atomics per
//! measured stage, so the measured numbers are slightly pessimistic vs
//! production. The relative picture is the point: it shows whether a
//! run is spending time in crypto, syscalls, or scheduler/channel
//! waits.
//!
//! Live stages track UDP receive/send, TUN and endpoint output, bounded
//! transport/endpoint queue residence, dataplane stateless AEAD service, and dataplane
//! ordered retirement. Historical slots stay reserved so old logs remain
//! index-comparable without advertising retired paths.

use std::num::NonZeroU64;
use std::sync::OnceLock;
use std::sync::atomic::{
    AtomicU64,
    Ordering::{Acquire, Relaxed, Release},
};
use std::time::Instant;

mod format;

use format::{fmt_ns, fmt_rate_per_sec};

/// Number of measurement buckets. Indices match `Stage`.
const N_STAGES: usize = 75;
const N_EVENTS: usize = 269;
const HIST_BUCKETS: usize = 48;

/// Stage identifier. `as usize` indexes into the counter arrays.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
#[repr(usize)]
pub enum Stage {
    UdpRecv = 0,
    ReservedStage1 = 1,
    ReservedStage2 = 2,
    ReservedStage3 = 3,
    TunWrite = 4,
    ReservedStage5 = 5,
    ReservedStage6 = 6,
    UdpSend = 7,
    ReservedStage8 = 8,
    EndpointDeliver = 9,
    ReservedStage10 = 10,
    ReservedStage11 = 11,
    ReservedStage12 = 12,
    ReservedStage13 = 13,
    ReservedStage14 = 14,
    EndpointEventWait = 15,
    ReservedStage16 = 16,
    ReservedStage17 = 17,
    ReservedStage18 = 18,
    ReservedStage19 = 19,
    TransportChannelWait = 20,
    TransportPriorityChannelWait = 21,
    TransportBulkChannelWait = 22,
    TransportRxLoopOwnedWait = 23,
    ReservedStage24 = 24,
    ReservedStage25 = 25,
    ReservedStage26 = 26,
    ReservedStage27 = 27,
    ReservedStage28 = 28,
    ReservedStage29 = 29,
    ReservedStage30 = 30,
    ReservedStage31 = 31,
    ReservedStage32 = 32,
    ReservedStage33 = 33,
    ReservedStage34 = 34,
    ReservedStage35 = 35,
    ReservedStage36 = 36,
    ReservedStage37 = 37,
    ReservedStage38 = 38,
    ReservedStage39 = 39,
    ReservedStage40 = 40,
    ReservedStage41 = 41,
    ReservedStage42 = 42,
    ReservedStage43 = 43,
    ReservedStage44 = 44,
    ReservedStage45 = 45,
    ReservedStage46 = 46,
    ReservedStage47 = 47,
    ReservedStage48 = 48,
    ReservedStage49 = 49,
    ReservedStage50 = 50,
    ReservedStage51 = 51,
    ReservedStage52 = 52,
    DataplaneAeadOpen = 53,
    DataplaneAeadSeal = 54,
    DataplaneRetire = 55,
    DataplaneFspOwnerSync = 56,
    DataplaneLiveTurn = 57,
    DataplaneCompletionDrain = 58,
    DataplaneLiveAdmit = 59,
    DataplaneAeadDispatch = 60,
    DataplaneOutputSink = 61,
    DataplaneTransportSend = 62,
    DataplaneTransportSendBatch = 63,
    DataplaneOwnerDispatch = 64,
    DataplaneExecutorSubmit = 65,
    DataplaneCompletionQueue = 66,
    DataplaneAeadWorkerQueueWait = 67,
    ReservedStage68 = 68,
    SessionDatagramDecode = 69,
    CoordCacheWarm = 70,
    TransitRoute = 71,
    TransitEncode = 72,
    TransitSubmit = 73,
    TransitTotal = 74,
}

impl Stage {
    const fn name(self) -> &'static str {
        match self {
            Stage::UdpRecv => "udp_recv",
            Stage::ReservedStage1 => "reserved_stage_1",
            Stage::ReservedStage2 => "reserved_stage_2",
            Stage::ReservedStage3 => "reserved_stage_3",
            Stage::TunWrite => "tun_write",
            Stage::ReservedStage5 => "reserved_stage_5",
            Stage::ReservedStage6 => "reserved_stage_6",
            Stage::UdpSend => "udp_send",
            Stage::ReservedStage8 => "reserved_stage_8",
            Stage::EndpointDeliver => "endpoint_deliver",
            Stage::ReservedStage10 => "reserved_stage_10",
            Stage::ReservedStage11 => "reserved_stage_11",
            Stage::ReservedStage12 => "reserved_stage_12",
            Stage::ReservedStage13 => "reserved_stage_13",
            Stage::ReservedStage14 => "reserved_stage_14",
            Stage::EndpointEventWait => "endpoint_event_wait",
            Stage::ReservedStage16 => "reserved_stage_16",
            Stage::ReservedStage17 => "reserved_stage_17",
            Stage::ReservedStage18 => "reserved_stage_18",
            Stage::ReservedStage19 => "reserved_stage_19",
            Stage::TransportChannelWait => "transport_channel_wait",
            Stage::TransportPriorityChannelWait => "transport_priority_channel_wait",
            Stage::TransportBulkChannelWait => "transport_bulk_channel_wait",
            Stage::TransportRxLoopOwnedWait => "transport_rx_loop_owned_wait",
            Stage::ReservedStage24 => "reserved_stage_24",
            Stage::ReservedStage25 => "reserved_stage_25",
            Stage::ReservedStage26 => "reserved_stage_26",
            Stage::ReservedStage27 => "reserved_stage_27",
            Stage::ReservedStage28 => "reserved_stage_28",
            Stage::ReservedStage29 => "reserved_stage_29",
            Stage::ReservedStage30 => "reserved_stage_30",
            Stage::ReservedStage31 => "reserved_stage_31",
            Stage::ReservedStage32 => "reserved_stage_32",
            Stage::ReservedStage33 => "reserved_stage_33",
            Stage::ReservedStage34 => "reserved_stage_34",
            Stage::ReservedStage35 => "reserved_stage_35",
            Stage::ReservedStage36 => "reserved_stage_36",
            Stage::ReservedStage37 => "reserved_stage_37",
            Stage::ReservedStage38 => "reserved_stage_38",
            Stage::ReservedStage39 => "reserved_stage_39",
            Stage::ReservedStage40 => "reserved_stage_40",
            Stage::ReservedStage41 => "reserved_stage_41",
            Stage::ReservedStage42 => "reserved_stage_42",
            Stage::ReservedStage43 => "reserved_stage_43",
            Stage::ReservedStage44 => "reserved_stage_44",
            Stage::ReservedStage45 => "reserved_stage_45",
            Stage::ReservedStage46 => "reserved_stage_46",
            Stage::ReservedStage47 => "reserved_stage_47",
            Stage::ReservedStage48 => "reserved_stage_48",
            Stage::ReservedStage49 => "reserved_stage_49",
            Stage::ReservedStage50 => "reserved_stage_50",
            Stage::ReservedStage51 => "reserved_stage_51",
            Stage::ReservedStage52 => "reserved_stage_52",
            Stage::DataplaneAeadOpen => "dataplane_aead_open",
            Stage::DataplaneAeadSeal => "dataplane_aead_seal",
            Stage::DataplaneRetire => "dataplane_retire",
            Stage::DataplaneFspOwnerSync => "dataplane_fsp_owner_sync",
            Stage::DataplaneLiveTurn => "dataplane_live_turn",
            Stage::DataplaneCompletionDrain => "dataplane_completion_drain",
            Stage::DataplaneLiveAdmit => "dataplane_live_admit",
            Stage::DataplaneAeadDispatch => "dataplane_aead_dispatch",
            Stage::DataplaneOutputSink => "dataplane_output_sink",
            Stage::DataplaneTransportSend => "dataplane_transport_send",
            Stage::DataplaneTransportSendBatch => "dataplane_transport_send_batch",
            Stage::DataplaneOwnerDispatch => "dataplane_owner_dispatch",
            Stage::DataplaneExecutorSubmit => "dataplane_executor_submit",
            Stage::DataplaneCompletionQueue => "dataplane_completion_queue",
            Stage::DataplaneAeadWorkerQueueWait => "dataplane_aead_worker_queue_wait",
            Stage::ReservedStage68 => "reserved_stage_68",
            Stage::SessionDatagramDecode => "session_datagram_decode",
            Stage::CoordCacheWarm => "coord_cache_warm",
            Stage::TransitRoute => "transit_route",
            Stage::TransitEncode => "transit_encode",
            Stage::TransitSubmit => "transit_submit",
            Stage::TransitTotal => "transit_total",
        }
    }
}

fn stage_from_index(idx: usize) -> Stage {
    match idx {
        0 => Stage::UdpRecv,
        1 => Stage::ReservedStage1,
        2 => Stage::ReservedStage2,
        3 => Stage::ReservedStage3,
        4 => Stage::TunWrite,
        5 => Stage::ReservedStage5,
        6 => Stage::ReservedStage6,
        7 => Stage::UdpSend,
        8 => Stage::ReservedStage8,
        9 => Stage::EndpointDeliver,
        10 => Stage::ReservedStage10,
        11 => Stage::ReservedStage11,
        12 => Stage::ReservedStage12,
        13 => Stage::ReservedStage13,
        14 => Stage::ReservedStage14,
        15 => Stage::EndpointEventWait,
        16 => Stage::ReservedStage16,
        17 => Stage::ReservedStage17,
        18 => Stage::ReservedStage18,
        19 => Stage::ReservedStage19,
        20 => Stage::TransportChannelWait,
        21 => Stage::TransportPriorityChannelWait,
        22 => Stage::TransportBulkChannelWait,
        23 => Stage::TransportRxLoopOwnedWait,
        24 => Stage::ReservedStage24,
        25 => Stage::ReservedStage25,
        26 => Stage::ReservedStage26,
        27 => Stage::ReservedStage27,
        28 => Stage::ReservedStage28,
        29 => Stage::ReservedStage29,
        30 => Stage::ReservedStage30,
        31 => Stage::ReservedStage31,
        32 => Stage::ReservedStage32,
        33 => Stage::ReservedStage33,
        34 => Stage::ReservedStage34,
        35 => Stage::ReservedStage35,
        36 => Stage::ReservedStage36,
        37 => Stage::ReservedStage37,
        38 => Stage::ReservedStage38,
        39 => Stage::ReservedStage39,
        40 => Stage::ReservedStage40,
        41 => Stage::ReservedStage41,
        42 => Stage::ReservedStage42,
        43 => Stage::ReservedStage43,
        44 => Stage::ReservedStage44,
        45 => Stage::ReservedStage45,
        46 => Stage::ReservedStage46,
        47 => Stage::ReservedStage47,
        48 => Stage::ReservedStage48,
        49 => Stage::ReservedStage49,
        50 => Stage::ReservedStage50,
        51 => Stage::ReservedStage51,
        52 => Stage::ReservedStage52,
        53 => Stage::DataplaneAeadOpen,
        54 => Stage::DataplaneAeadSeal,
        55 => Stage::DataplaneRetire,
        56 => Stage::DataplaneFspOwnerSync,
        57 => Stage::DataplaneLiveTurn,
        58 => Stage::DataplaneCompletionDrain,
        59 => Stage::DataplaneLiveAdmit,
        60 => Stage::DataplaneAeadDispatch,
        61 => Stage::DataplaneOutputSink,
        62 => Stage::DataplaneTransportSend,
        63 => Stage::DataplaneTransportSendBatch,
        64 => Stage::DataplaneOwnerDispatch,
        65 => Stage::DataplaneExecutorSubmit,
        66 => Stage::DataplaneCompletionQueue,
        67 => Stage::DataplaneAeadWorkerQueueWait,
        68 => Stage::ReservedStage68,
        69 => Stage::SessionDatagramDecode,
        70 => Stage::CoordCacheWarm,
        71 => Stage::TransitRoute,
        72 => Stage::TransitEncode,
        73 => Stage::TransitSubmit,
        74 => Stage::TransitTotal,
        _ => unreachable!(),
    }
}

include!("perf_events.rs");
include!("perf_runtime.rs");