Struct BrickProfiler 

pub struct BrickProfiler { /* private fields */ }

Per-brick profiler using pure Rust timing.

Design (PAR-073, PAR-200)

• Uses std::time::Instant for timing (no CUDA event FFI)
• PAR-200: O(1) hot path with BrickId enum + array storage
• GPU operations require explicit sync before timing point
• Supports deferred sync mode for low-overhead production profiling
• Aggregates statistics per brick name

Usage

use trueno::brick::{BrickProfiler, BrickId, SyncMode};

let mut profiler = BrickProfiler::new();
profiler.enable();

// Fast path: use BrickId for known bricks (PAR-200)
let timer = profiler.start_brick(BrickId::RmsNorm);
// ... do work ...
// For GPU: cuda_stream.synchronize() HERE
profiler.stop_brick(timer, 1);

// Legacy path: string-based (slower, for unknown bricks)
let timer = profiler.start("CustomBrick");
profiler.stop(timer, 1);

// Deferred sync mode (production)
profiler.set_sync_mode(SyncMode::Deferred);
profiler.record_deferred(BrickId::RmsNorm, start_ns, 1);
// ... more operations ...
cuda_stream.synchronize();
profiler.finalize(end_ns);

// Get statistics
let stats = profiler.brick_stats(BrickId::RmsNorm);
println!("RmsNorm avg: {:.2}µs", stats.avg_us());

// Get category breakdown
let cats = profiler.category_stats();
println!("Attention: {:.1}%", cats[BrickCategory::Attention as usize].percentage(profiler.total_ns()));

Implementations

impl BrickProfiler

pub fn record_checksum( &mut self, name: &str, layer_idx: usize, position: u32, output: &[f32], )

Record a kernel trace with output checksum for divergence detection.

This enables automated CPU/GPU divergence detection by capturing output checksums alongside timing data. When GPU produces wrong output, this identifies WHICH kernel diverged without hours of manual debugging.

Five-Whys Root Cause: Hours of manual “let me check X in Y” debugging -> No automated tool identified which kernel diverged -> BrickProfiler only captured timing, not checksums -> Missing feature: per-kernel checksum capture

Arguments

• name: Brick/kernel name
• layer_idx: Layer index (0-N for transformer layers)
• position: Position in sequence
• output: Output tensor data (first 64 floats checksummed)

Example

// After RoPE kernel
profiler.record_checksum("RopeNeox", layer_idx, position, &q_rotated);

pub fn get_checksums(&self) -> &[KernelChecksum]

Get all kernel checksums for divergence comparison.

pub fn find_divergence( &self, reference: &BrickProfiler, ) -> Option<DivergenceInfo>

Compare checksums with a reference profiler (e.g., CPU baseline).

Returns None if all checksums match, or the first divergent kernel.

pub fn reset_checksums(&mut self)

Reset checksum tracking (call before new forward pass).

impl BrickProfiler

pub fn enable_graph(&mut self)

Enable execution graph tracking.

When enabled, the profiler records the execution hierarchy:

• Layer → Brick → Kernel relationships
• PTX hashes for kernel identity
• Timing data per node

pub fn disable_graph(&mut self)

Disable execution graph tracking.

pub fn is_graph_enabled(&self) -> bool

Check if execution graph tracking is enabled.

pub fn execution_graph(&self) -> &ExecutionGraph

Get the execution graph (immutable).

pub fn execution_graph_mut(&mut self) -> &mut ExecutionGraph

Get the execution graph (mutable).

pub fn graph_push_scope( &mut self, node: ExecutionNode, ) -> Option<ExecutionNodeId>

Push a scope for hierarchical graph recording.

Example

profiler.enable_graph();
profiler.graph_push_scope(ExecutionNode::Layer { index: 0 });
// ... record bricks and kernels ...
profiler.graph_pop_scope();

pub fn graph_pop_scope(&mut self) -> Option<ExecutionNodeId>

Pop the current scope.

pub fn graph_record_brick( &mut self, brick_id: BrickId, timing_ns: u64, elements: u64, ) -> Option<ExecutionNodeId>

Record a brick in the execution graph.

This should be called after stop_brick() with the timing data.

pub fn graph_record_kernel( &mut self, name: &str, ptx_hash: u64, grid: (u32, u32, u32), block: (u32, u32, u32), shared_mem: u32, ) -> Option<ExecutionNodeId>

Record a kernel launch in the execution graph.

Arguments

• name: Kernel name (e.g., “batched_q4k_gemv”)
• ptx_hash: FNV-1a hash of PTX source for identity
• grid: Grid dimensions (blocks)
• block: Block dimensions (threads)
• shared_mem: Shared memory bytes

pub fn graph_to_dot(&self) -> String

Export execution graph to DOT format for visualization.

Use with Graphviz: dot -Tsvg output.dot -o graph.svg

pub fn graph_to_csr(&self) -> CsrGraph

Export execution graph to trueno-graph CsrGraph.

pub fn graph_clear(&mut self)

Clear the execution graph.

pub fn graph_is_scope_balanced(&self) -> bool

Check if the execution graph scope stack is balanced.

impl BrickProfiler

pub fn start(&self, name: &str) -> BrickTimer

Start timing a brick. Returns timer handle.

IMPORTANT: For GPU operations, call sync AFTER the operation completes but BEFORE calling stop().

pub fn stop(&mut self, timer: BrickTimer, elements: u64)

Stop timing and record the sample.

Arguments

• timer: Timer handle from start()
• elements: Number of elements (tokens) processed

pub fn record_elapsed(&mut self, name: &str, elapsed: Duration, elements: u64)

Record a pre-measured duration for a brick.

PAR-073: This method allows timing with raw Instant calls, avoiding borrow conflicts when profiling CUDA operations that also need &mut self.

Arguments

• name: Brick name
• elapsed: Duration of the operation (from Instant::elapsed())
• elements: Number of elements (tokens) processed

Example

let start = std::time::Instant::now();
cuda_stream.synchronize()?;
self.some_cuda_operation()?;
cuda_stream.synchronize()?;
let elapsed = start.elapsed();
self.profiler.record_elapsed("SomeBrick", elapsed, 1);

pub fn record_elapsed_with_bytes( &mut self, name: &str, elapsed: Duration, elements: u64, input_bytes: u64, output_bytes: u64, )

PMAT-451: Record elapsed time with byte metrics for compression workloads.

Arguments

• name: Brick name
• elapsed: Duration of the operation
• elements: Number of elements (pages) processed
• input_bytes: Original uncompressed size
• output_bytes: Compressed output size

Example

let start = std::time::Instant::now();
let compressed = zstd_compress(&page_data);
let elapsed = start.elapsed();
profiler.record_elapsed_with_bytes(
    "ZstdCompress",
    elapsed,
    1,
    page_data.len() as u64,
    compressed.len() as u64,
);

pub fn set_brick_bottleneck(&mut self, name: &str, bottleneck: BrickBottleneck)

PMAT-451: Set bottleneck classification for a brick.

pub fn stats(&self, name: &str) -> Option<&BrickStats>

Get statistics for a specific brick by name.

First checks known BrickId types (O(1)), then falls back to dynamic stats.

pub fn all_stats(&self) -> &HashMap<String, BrickStats>

👎Deprecated since 0.12.0:

Use all_brick_stats() for complete statistics

Get all brick statistics (legacy API, returns dynamic stats only).

For full statistics including known bricks, use all_brick_stats() instead.

pub fn all_brick_stats(&self) -> impl Iterator<Item = &BrickStats>

Get all brick statistics including both known and dynamic bricks.

pub fn brick_names(&self) -> Vec<String>

Get all brick names.

pub fn reset(&mut self)

Reset all statistics.

impl BrickProfiler

pub fn print_category_stats(&self)

Print category breakdown to console.

pub fn summary(&self) -> String

Generate a summary report.

pub fn to_json(&self) -> String

Export profiling data as JSON for pmat metrics integration.

Format compatible with .pmat-metrics/trends/ structure:

{
  "total_tokens": 1000,
  "total_ns": 5000000,
  "total_throughput": 200000.0,
  "bricks": [
    {
      "name": "RmsNorm",
      "count": 10,
      "total_ns": 1000000,
      "avg_us": 100.0,
      "min_us": 90.0,
      "max_us": 120.0,
      "throughput": 10000.0,
      "pct": 20.0
    }
  ]
}

pub fn write_json(&self, path: &Path) -> Result<()>

Write profiling data to a JSON file for pmat tracking.

Errors

Returns error if file cannot be written.

pub fn tile_summary(&self) -> String

Generate tile profiling summary report.

Example Output

=== Tile Profiling Summary (TILING-SPEC-001) ===
Level       Samples   Avg µs    GFLOP/s   AI      Elements
Macro           128    1234.5     12.34  0.50    1048576
Midi           2048      78.2     45.67  2.00      65536
Micro         32768       4.9     89.12  4.00       4096

pub fn tile_stats_to_json(&self) -> String

Export tile statistics as JSON.

Compatible with pmat metrics integration.

impl BrickProfiler

pub fn enable_tile_profiling(&mut self)

Enable tile-level profiling.

When enabled, start_tile()/stop_tile() record per-tile statistics for Macro/Midi/Micro tile hierarchy.

pub fn disable_tile_profiling(&mut self)

Disable tile-level profiling.

pub fn is_tile_profiling_enabled(&self) -> bool

Check if tile profiling is enabled.

pub fn start_tile(&self, level: TileLevel, row: u32, col: u32) -> TileTimer

Start timing a tile execution.

Returns a TileTimer that should be passed to stop_tile() after the tile computation completes.

Arguments

• level: Tile hierarchy level (Macro/Midi/Micro)
• row: Row index within parent tile
• col: Column index within parent tile

Example

let timer = profiler.start_tile(TileLevel::Macro, 0, 0);
// ... execute tile computation ...
profiler.stop_tile(timer, 256 * 256, 2 * 256 * 256 * 256);

pub fn stop_tile(&mut self, timer: TileTimer, elements: u64, flops: u64)

Stop timing and record tile statistics.

Arguments

• timer: Timer handle from start_tile()
• elements: Number of elements processed by this tile
• flops: Number of floating-point operations performed

pub fn tile_stats(&self, level: TileLevel) -> &TileStats

Get tile statistics for a given level.

pub fn tile_stats_mut(&mut self, level: TileLevel) -> &mut TileStats

Get mutable tile statistics for a given level.

pub fn all_tile_stats(&self) -> &[TileStats; 3]

Get all tile statistics as a slice.

pub fn reset_tile_stats(&mut self)

Reset tile statistics for all levels.

impl BrickProfiler

pub fn new() -> Self

Create a new profiler (disabled by default for zero overhead).

pub fn enabled() -> Self

Create an enabled profiler.

pub fn set_sync_mode(&mut self, mode: SyncMode)

Set the synchronization mode for GPU profiling.

Modes

• Immediate: Sync after each kernel (accurate but slow)
• PerLayer: Sync once per transformer layer
• Deferred: Sync once per forward pass (default, fast)
• None: No synchronization

pub fn sync_mode(&self) -> SyncMode

Get the current synchronization mode.

pub fn reset_epoch(&mut self)

Reset the epoch for deferred timing. Call this at the start of a forward pass.

pub fn elapsed_ns(&self) -> u64

Get nanoseconds elapsed since epoch.

pub fn start_brick(&self, brick_id: BrickId) -> BrickIdTimer

Start timing a brick using BrickId (O(1) hot path).

This is the preferred API for known brick types. For GPU operations, call stream.synchronize() before stop_brick().

pub fn stop_brick(&mut self, timer: BrickIdTimer, elements: u64)

Stop timing and record the sample (O(1) hot path).

pub fn brick_stats(&self, brick_id: BrickId) -> &BrickStats

Get statistics for a known brick type (O(1)).

pub fn brick_stats_mut(&mut self, brick_id: BrickId) -> &mut BrickStats

Get mutable statistics for a known brick type (O(1)).

pub fn record_deferred( &mut self, brick_id: BrickId, start_ns: u64, elements: u64, )

Record a measurement without GPU sync (deferred mode).

Call finalize() after GPU sync to apply all pending measurements.

Arguments

• brick_id: The brick type
• start_ns: Start time (from elapsed_ns() at operation start)
• elements: Number of elements processed

pub fn record_deferred_dynamic( &mut self, name: &str, start_ns: u64, elements: u64, )

Record a measurement for a dynamic brick (deferred mode).

pub fn finalize(&mut self, end_ns: u64)

Finalize all pending measurements after GPU sync.

Must be called after stream.synchronize() to get accurate timing.

Arguments

• end_ns: End time (from elapsed_ns() after sync)

pub fn has_pending(&self) -> bool

Check if there are pending measurements.

pub fn pending_count(&self) -> usize

Get number of pending measurements.

pub fn category_stats(&self) -> [CategoryStats; 7]

Get aggregated statistics by category.

Returns an array indexed by BrickCategory as usize.

pub fn set_l2_cache_hit_rate(&mut self, rate: f32)

Set L2 cache hit rate (v1.1.0 OBSERVE phase)

pub fn l2_cache_hit_rate(&self) -> Option<f32>

Get L2 cache hit rate

pub fn set_zero_copy(&mut self, enabled: bool)

Set zero-copy mode (v1.1.0 OBSERVE phase)

pub fn is_zero_copy(&self) -> bool

Check if zero-copy is enabled

pub fn enable(&mut self)

Enable profiling.

pub fn disable(&mut self)

Disable profiling.

pub fn is_enabled(&self) -> bool

Check if profiling is enabled.

pub fn total_throughput(&self) -> f64

Get total throughput across all bricks.

pub fn total_tokens(&self) -> u64

Get total tokens processed.

pub fn total_ns(&self) -> u64

Get total time in nanoseconds.

impl BrickProfiler

pub fn get_tuner_recommendations( &self, config: &RunConfig, ) -> Option<TunerRecommendation>

Get ML-based tuning recommendations.

Extracts features from current profile and returns recommendations.

pub fn print_tuner_recommendations(&self, config: &RunConfig)

Print tuner recommendations to console.

pub fn tokens_per_sec(&self) -> Option<f32>

Get tokens per second from profiler.

Trait Implementations

impl Debug for BrickProfiler

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for BrickProfiler

fn default() -> Self

Returns the “default value” for a type.