perf_event_open/sample/rb/
mod.rs

1use std::borrow::Cow;
2use std::cmp::Ordering as Ord;
3use std::mem::transmute;
4use std::ptr::copy_nonoverlapping;
5use std::slice;
6use std::sync::atomic::{AtomicU64, Ordering as MemOrd};
7
8pub use cow::CowChunk;
9
10mod cow;
11
12pub(super) struct Rb<'a> {
13    alloc: &'a [u8],
14    tail: &'a AtomicU64,
15    head: &'a AtomicU64,
16}
17
18impl<'a> Rb<'a> {
19    pub fn new(alloc: &'a [u8], tail: &'a AtomicU64, head: &'a AtomicU64) -> Self {
20        Self { alloc, tail, head }
21    }
22
23    pub fn lending_pop(&self) -> Option<CowChunk<'a>> {
24        let rb_ptr = self.alloc.as_ptr();
25        let size = self.alloc.len();
26
27        // Thread safe since no more threads set the tail
28        let tail = unsafe { *self.tail.as_ptr() };
29        // About acquire:
30        // https://github.com/torvalds/linux/blob/v6.13/include/uapi/linux/perf_event.h#L720
31        // https://github.com/torvalds/linux/blob/v6.13/kernel/events/ring_buffer.c#L99
32        let head = self.head.load(MemOrd::Acquire) % size as u64;
33
34        if tail == head {
35            return None;
36        }
37
38        // https://github.com/torvalds/linux/blob/v6.13/include/uapi/linux/perf_event.h#L824
39        // struct perf_event_header {
40        //     u32 type; # 4 bytes
41        //     u16 misc; # 2 bytes
42        //     u16 size; # 2 bytes
43        // };
44        let chunk_len = {
45            let d = size as u64 - tail;
46            match d.cmp(&7) {
47                Ord::Greater => unsafe {
48                    let ptr = rb_ptr.add((tail + 6) as _);
49                    *(ptr as *const u16)
50                },
51                Ord::Less => unsafe {
52                    let ptr = rb_ptr.add((6 - d) as _);
53                    *(ptr as *const u16)
54                },
55                Ord::Equal => unsafe {
56                    let hi_part_ptr = rb_ptr.add((tail + 6) as _);
57                    let lo_part_ptr = rb_ptr;
58                    let buf = [*hi_part_ptr, *lo_part_ptr];
59                    transmute::<[u8; 2], u16>(buf)
60                },
61            }
62        };
63
64        let new_tail = (tail + chunk_len as u64) % size as u64;
65
66        let chunk = match size as i64 - (tail + chunk_len as u64) as i64 {
67            d if d >= 0 => {
68                let buf = unsafe {
69                    let ptr = rb_ptr.add(tail as _);
70                    slice::from_raw_parts(ptr, chunk_len as _)
71                };
72                Cow::Borrowed(buf)
73            }
74            d => {
75                let mut buf = Vec::with_capacity(chunk_len as _);
76                let buf_ptr = buf.as_mut_ptr();
77
78                unsafe {
79                    let hi_part_ptr = rb_ptr.add(tail as _);
80                    let hi_part_len = (chunk_len as i64 + d) as _;
81                    copy_nonoverlapping(hi_part_ptr, buf_ptr, hi_part_len);
82
83                    let lo_part_ptr = rb_ptr;
84                    let lo_part_len = -d as _;
85                    copy_nonoverlapping(lo_part_ptr, buf_ptr.add(hi_part_len), lo_part_len);
86                    buf.set_len(chunk_len as _);
87                }
88
89                // https://github.com/torvalds/linux/blob/v6.13/include/uapi/linux/perf_event.h#L723
90                self.tail.store(new_tail, MemOrd::Release);
91
92                Cow::Owned(buf)
93            }
94        };
95
96        Some(CowChunk {
97            tail: self.tail,
98            new_tail,
99            chunk,
100        })
101    }
102}
perf_event_open/sample/rb/mod.rs

perf_event_open/sample/rb/
mod.rs
