1use std::alloc::{GlobalAlloc, Layout, System};
18use std::sync::atomic::{AtomicI64, AtomicU64, AtomicU8, Ordering::Relaxed};
19
20pub struct Counting;
21
22static STATE: AtomicU8 = AtomicU8::new(0);
23static LIVE_BYTES: AtomicI64 = AtomicI64::new(0);
24static LIVE_BLOCKS: AtomicI64 = AtomicI64::new(0);
25static PEAK_BYTES: AtomicI64 = AtomicI64::new(0);
26static ALLOCS: AtomicU64 = AtomicU64::new(0);
27
28const CLASSES: usize = 34;
29#[allow(clippy::declare_interior_mutable_const)]
30const ZERO: AtomicI64 = AtomicI64::new(0);
31static CLASS_BYTES: [AtomicI64; CLASSES] = [ZERO; CLASSES];
32static CLASS_BLOCKS: [AtomicI64; CLASSES] = [ZERO; CLASSES];
33
34const DEFAULT_ON: bool = !cfg!(feature = "prod");
35
36#[cfg(windows)]
37#[link(name = "kernel32")]
38extern "system" {
39 fn GetEnvironmentVariableW(name: *const u16, buffer: *mut u16, size: u32) -> u32;
40}
41
42#[cfg(windows)]
43fn memstats_env() -> Option<bool> {
44 const NAME: [u16; 12] = [
45 b'V' as u16,
46 b'N' as u16,
47 b'_' as u16,
48 b'M' as u16,
49 b'E' as u16,
50 b'M' as u16,
51 b'S' as u16,
52 b'T' as u16,
53 b'A' as u16,
54 b'T' as u16,
55 b'S' as u16,
56 0,
57 ];
58 let mut buf = [0u16; 8];
59 let n = unsafe { GetEnvironmentVariableW(NAME.as_ptr(), buf.as_mut_ptr(), buf.len() as u32) };
60 (n != 0).then(|| !(n == 1 && buf[0] == b'0' as u16))
61}
62
63#[cfg(not(windows))]
64fn memstats_env() -> Option<bool> {
65 extern "C" {
66 fn getenv(name: *const core::ffi::c_char) -> *const core::ffi::c_char;
67 }
68 let value = unsafe { getenv(c"SK_MEMSTATS".as_ptr()) };
70 if value.is_null() {
71 return None;
72 }
73 let first = unsafe { *value } as u8;
74 if first == 0 {
75 return None;
76 }
77 let second = unsafe { *value.add(1) } as u8;
78 Some(!(first == b'0' && second == 0))
79}
80
81#[cold]
82fn probe() -> bool {
83 let on = memstats_env().unwrap_or(DEFAULT_ON);
84 STATE.store(if on { 2 } else { 1 }, Relaxed);
85 on
86}
87
88#[inline]
89fn counting() -> bool {
90 match STATE.load(Relaxed) {
91 2 => true,
92 1 => false,
93 _ => probe(),
94 }
95}
96
97#[inline]
98fn class(size: usize) -> usize {
99 ((usize::BITS - size.leading_zeros()) as usize).min(CLASSES - 1)
100}
101
102#[inline]
103fn add(size: usize, sign: i64) {
104 let bytes = sign * size as i64;
105 let now = LIVE_BYTES.fetch_add(bytes, Relaxed) + bytes;
106 LIVE_BLOCKS.fetch_add(sign, Relaxed);
107 if sign > 0 {
108 PEAK_BYTES.fetch_max(now, Relaxed);
109 }
110 let k = class(size);
111 CLASS_BYTES[k].fetch_add(bytes, Relaxed);
112 CLASS_BLOCKS[k].fetch_add(sign, Relaxed);
113}
114
115unsafe impl GlobalAlloc for Counting {
116 #[inline]
117 unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
118 let p = unsafe { System.alloc(layout) };
119 if !p.is_null() {
120 recording::allocated(p as usize, layout.size());
121 }
122 if !p.is_null() && counting() {
123 add(layout.size(), 1);
124 ALLOCS.fetch_add(1, Relaxed);
125 }
126 p
127 }
128
129 #[inline]
130 unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
131 let p = unsafe { System.alloc_zeroed(layout) };
132 if !p.is_null() {
133 recording::allocated(p as usize, layout.size());
134 }
135 if !p.is_null() && counting() {
136 add(layout.size(), 1);
137 ALLOCS.fetch_add(1, Relaxed);
138 }
139 p
140 }
141
142 #[inline]
143 unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
144 recording::freed(ptr as usize, layout.size());
145 unsafe { System.dealloc(ptr, layout) };
146 if counting() {
147 add(layout.size(), -1);
148 }
149 }
150
151 #[inline]
152 unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
153 let ticket = recording::begin_realloc();
154 let p = unsafe { System.realloc(ptr, layout, new_size) };
155 recording::finish_realloc(ticket, ptr as usize, layout.size(), p as usize, new_size);
156 if !p.is_null() && counting() {
157 add(layout.size(), -1);
158 add(new_size, 1);
159 }
160 p
161 }
162}
163
164pub fn snapshot() -> Option<(i64, i64, i64, u64)> {
165 on().then(|| {
166 (
167 LIVE_BYTES.load(Relaxed),
168 LIVE_BLOCKS.load(Relaxed),
169 PEAK_BYTES.load(Relaxed),
170 ALLOCS.load(Relaxed),
171 )
172 })
173}
174
175pub fn classes() -> Vec<(u64, i64, i64)> {
176 (0..CLASSES)
177 .filter_map(|k| {
178 let blocks = CLASS_BLOCKS[k].load(Relaxed);
179 let bytes = CLASS_BYTES[k].load(Relaxed);
180 let upper = if k == CLASSES - 1 {
181 u64::MAX
182 } else {
183 1u64 << k
184 };
185 (blocks != 0).then_some((upper, bytes, blocks))
186 })
187 .collect()
188}
189
190pub fn on() -> bool {
194 counting() && ALLOCS.load(Relaxed) > 0
195}
196
197pub mod recording {
201 use std::{
202 cell::{Cell, UnsafeCell},
203 marker::PhantomData,
204 rc::Rc,
205 sync::{
206 atomic::{AtomicBool, AtomicPtr, AtomicU64, AtomicUsize, Ordering},
207 Mutex, OnceLock,
208 },
209 time::{Instant, SystemTime, UNIX_EPOCH},
210 };
211
212 pub const MAX_EVENTS: usize = 65_536;
213 pub const MAX_SITES: usize = 512;
214 static EPOCH: AtomicU64 = AtomicU64::new(0);
215 static WRITERS: AtomicUsize = AtomicUsize::new(0);
216 static RECORDER: AtomicPtr<Recorder> = AtomicPtr::new(std::ptr::null_mut());
217 static NEXT_SESSION: AtomicU64 = AtomicU64::new(1);
218 static NEXT_THREAD: AtomicUsize = AtomicUsize::new(1);
219 thread_local! {
220 static SUPPRESSED: Cell<u32> = const { Cell::new(0) };
221 static SITE: Cell<u32> = const { Cell::new(0) };
222 static THREAD: Cell<u32> = const { Cell::new(0) };
223 }
224
225 #[derive(Clone, Copy, Default)]
226 pub struct Event {
227 pub kind: u8,
228 pub address: usize,
229 pub size: usize,
230 pub new_address: usize,
231 pub new_size: usize,
232 pub site: u32,
233 pub thread: u32,
234 }
235
236 struct Slot {
237 ready: AtomicBool,
238 event: UnsafeCell<Event>,
239 }
240 unsafe impl Sync for Slot {}
243 struct Recorder {
244 slots: Box<[Slot]>,
245 next: AtomicUsize,
246 dropped: AtomicU64,
247 capacity: AtomicUsize,
248 sample_every: AtomicUsize,
249 }
250 struct Session {
251 id: u64,
252 started: Instant,
253 started_ms: f64,
254 stopped: Option<Instant>,
255 }
256 static SESSION: Mutex<Option<Session>> = Mutex::new(None);
257 #[derive(Clone)]
258 pub struct Site {
259 pub category: String,
260 pub name: String,
261 }
262 static SITES: Mutex<Vec<Site>> = Mutex::new(Vec::new());
263
264 pub struct Suppress {
265 _thread: PhantomData<Rc<()>>,
266 }
267 impl Drop for Suppress {
268 fn drop(&mut self) {
269 let _ = SUPPRESSED.try_with(|value| value.set(value.get().saturating_sub(1)));
270 }
271 }
272 pub fn suppress() -> Suppress {
273 let _ = SUPPRESSED.try_with(|value| value.set(value.get().saturating_add(1)));
274 Suppress {
275 _thread: PhantomData,
276 }
277 }
278
279 pub struct Scope {
280 previous: Option<u32>,
281 _thread: PhantomData<Rc<()>>,
282 }
283 impl Drop for Scope {
284 fn drop(&mut self) {
285 if let Some(previous) = self.previous {
286 let _ = SITE.try_with(|site| site.set(previous));
287 }
288 }
289 }
290 #[track_caller]
291 pub fn scope(category: &str, name: &str) -> Scope {
292 if EPOCH.load(Ordering::Relaxed) == 0 {
293 return Scope {
294 previous: None,
295 _thread: PhantomData,
296 };
297 }
298 let _suppressed = suppress();
299 let location = std::panic::Location::caller();
300 let label: String = format!("{name} ({}:{})", location.file(), location.line())
301 .chars()
302 .take(240)
303 .collect();
304 let category: String = category.chars().take(80).collect();
305 let id = {
306 let mut sites = SITES.lock().unwrap_or_else(|error| error.into_inner());
307 match sites
308 .iter()
309 .position(|site| site.category == category && site.name == label)
310 {
311 Some(index) => index as u32 + 1,
312 None if sites.len() < MAX_SITES => {
313 sites.push(Site {
314 category,
315 name: label,
316 });
317 sites.len() as u32
318 }
319 None => 0,
320 }
321 };
322 Scope {
323 previous: SITE.try_with(|site| site.replace(id)).ok(),
324 _thread: PhantomData,
325 }
326 }
327
328 #[derive(Clone)]
329 pub struct Status {
330 pub session: u64,
331 pub recording: bool,
332 pub started_at_ms: f64,
333 pub elapsed_ms: f64,
334 pub sample_every: usize,
335 pub capacity: usize,
336 pub reserved: usize,
337 pub dropped: u64,
338 }
339 fn recorder() -> &'static Recorder {
340 static STORAGE: OnceLock<Recorder> = OnceLock::new();
341 STORAGE.get_or_init(|| Recorder {
342 slots: (0..MAX_EVENTS)
343 .map(|_| Slot {
344 ready: AtomicBool::new(false),
345 event: UnsafeCell::new(Event::default()),
346 })
347 .collect::<Vec<_>>()
348 .into_boxed_slice(),
349 next: AtomicUsize::new(0),
350 dropped: AtomicU64::new(0),
351 capacity: AtomicUsize::new(MAX_EVENTS),
352 sample_every: AtomicUsize::new(1),
353 })
354 }
355 fn stop_writers() {
356 EPOCH.store(0, Ordering::SeqCst);
357 while WRITERS.load(Ordering::SeqCst) != 0 {
358 std::thread::yield_now();
359 }
360 }
361 pub fn start(capacity: usize, sample_every: usize) -> Status {
362 let _suppressed = suppress();
363 let mut session = SESSION.lock().unwrap_or_else(|error| error.into_inner());
364 stop_writers();
365 let recorder = recorder();
366 for slot in recorder.slots.iter() {
367 slot.ready.store(false, Ordering::Relaxed);
368 }
369 recorder.next.store(0, Ordering::Relaxed);
370 recorder.dropped.store(0, Ordering::Relaxed);
371 recorder
372 .capacity
373 .store(capacity.clamp(1024, MAX_EVENTS), Ordering::Relaxed);
374 recorder
375 .sample_every
376 .store(sample_every.clamp(1, 4096), Ordering::Relaxed);
377 let id = NEXT_SESSION.fetch_add(1, Ordering::Relaxed);
378 *session = Some(Session {
379 id,
380 started: Instant::now(),
381 started_ms: SystemTime::now()
382 .duration_since(UNIX_EPOCH)
383 .unwrap_or_default()
384 .as_secs_f64()
385 * 1000.0,
386 stopped: None,
387 });
388 RECORDER.store(recorder as *const _ as *mut _, Ordering::Release);
389 EPOCH.store(id, Ordering::SeqCst);
390 drop(session);
391 status()
392 }
393 pub fn stop() -> Status {
394 let _suppressed = suppress();
395 let mut session = SESSION.lock().unwrap_or_else(|error| error.into_inner());
396 stop_writers();
397 if let Some(session) = session.as_mut() {
398 session.stopped.get_or_insert_with(Instant::now);
399 }
400 drop(session);
401 status()
402 }
403 pub fn clear() {
404 let _suppressed = suppress();
405 let mut session = SESSION.lock().unwrap_or_else(|error| error.into_inner());
406 stop_writers();
407 let pointer = RECORDER.load(Ordering::Acquire);
408 if !pointer.is_null() {
409 let recorder = unsafe { &*pointer };
410 recorder.next.store(0, Ordering::Relaxed);
411 recorder.dropped.store(0, Ordering::Relaxed);
412 }
413 *session = None;
414 }
415 pub fn reserved_bytes() -> usize {
416 if RECORDER.load(Ordering::Acquire).is_null() {
417 0
418 } else {
419 MAX_EVENTS * std::mem::size_of::<Slot>()
420 }
421 }
422 pub fn status() -> Status {
423 let _suppressed = suppress();
424 let session = SESSION.lock().unwrap_or_else(|error| error.into_inner());
425 let pointer = RECORDER.load(Ordering::Acquire);
426 let (sample_every, capacity, reserved, dropped) = if pointer.is_null() {
427 (1, 0, 0, 0)
428 } else {
429 let recorder = unsafe { &*pointer };
430 (
431 recorder.sample_every.load(Ordering::Relaxed),
432 recorder.capacity.load(Ordering::Relaxed),
433 recorder.next.load(Ordering::Acquire),
434 recorder.dropped.load(Ordering::Relaxed),
435 )
436 };
437 Status {
438 session: session.as_ref().map_or(0, |session| session.id),
439 recording: EPOCH.load(Ordering::Acquire) != 0,
440 started_at_ms: session.as_ref().map_or(0.0, |session| session.started_ms),
441 elapsed_ms: session.as_ref().map_or(0.0, |session| {
442 session
443 .stopped
444 .unwrap_or_else(Instant::now)
445 .duration_since(session.started)
446 .as_secs_f64()
447 * 1000.0
448 }),
449 sample_every,
450 capacity,
451 reserved,
452 dropped,
453 }
454 }
455 pub fn sites() -> Vec<Site> {
456 let _suppressed = suppress();
457 SITES
458 .lock()
459 .unwrap_or_else(|error| error.into_inner())
460 .clone()
461 }
462 pub fn drain(cursor: &mut usize, limit: usize) -> Vec<Event> {
463 let _suppressed = suppress();
464 let _session = SESSION.lock().unwrap_or_else(|error| error.into_inner());
465 let pointer = RECORDER.load(Ordering::Acquire);
466 if pointer.is_null() {
467 return Vec::new();
468 }
469 let recorder = unsafe { &*pointer };
470 let end = recorder
471 .next
472 .load(Ordering::Acquire)
473 .min(recorder.capacity.load(Ordering::Relaxed));
474 let mut events = Vec::with_capacity(end.saturating_sub(*cursor).min(limit));
475 while *cursor < end && events.len() < limit {
476 let slot = &recorder.slots[*cursor];
477 if !slot.ready.load(Ordering::Acquire) {
478 break;
479 }
480 events.push(unsafe { *slot.event.get() });
481 *cursor += 1;
482 }
483 events
484 }
485 pub fn selected(address: usize, sample_every: usize) -> bool {
486 let mut hash = address as u64;
487 hash ^= hash >> 33;
488 hash = hash.wrapping_mul(0xff51afd7ed558ccd);
489 hash ^= hash >> 33;
490 hash % sample_every.max(1) as u64 == 0
491 }
492 pub struct Ticket {
493 recorder: &'static Recorder,
494 index: usize,
495 site: u32,
496 thread: u32,
497 }
498 #[inline]
499 fn reserve(address: Option<usize>) -> Option<Ticket> {
500 if EPOCH.load(Ordering::Relaxed) == 0 {
501 return None;
502 }
503 if SUPPRESSED
504 .try_with(|value| value.get() != 0)
505 .unwrap_or(true)
506 {
507 return None;
508 }
509 WRITERS.fetch_add(1, Ordering::SeqCst);
510 if EPOCH.load(Ordering::SeqCst) == 0 {
511 WRITERS.fetch_sub(1, Ordering::SeqCst);
512 return None;
513 }
514 let pointer = RECORDER.load(Ordering::Acquire);
515 let recorder = unsafe { &*pointer };
516 if address.is_some_and(|address| {
517 !selected(address, recorder.sample_every.load(Ordering::Relaxed))
518 }) {
519 WRITERS.fetch_sub(1, Ordering::SeqCst);
520 return None;
521 }
522 let index = recorder.next.fetch_add(1, Ordering::AcqRel);
523 if index >= recorder.capacity.load(Ordering::Relaxed) {
524 recorder.dropped.fetch_add(1, Ordering::Relaxed);
525 WRITERS.fetch_sub(1, Ordering::SeqCst);
526 return None;
527 }
528 let site = SITE.try_with(Cell::get).unwrap_or(0);
529 let thread = THREAD
530 .try_with(|thread| {
531 let mut id = thread.get();
532 if id == 0 {
533 id = NEXT_THREAD.fetch_add(1, Ordering::Relaxed) as u32;
534 thread.set(id);
535 }
536 id
537 })
538 .unwrap_or(0);
539 Some(Ticket {
540 recorder,
541 index,
542 site,
543 thread,
544 })
545 }
546 #[inline]
547 fn publish(ticket: Ticket, mut event: Event) {
548 event.site = ticket.site;
549 event.thread = ticket.thread;
550 let slot = &ticket.recorder.slots[ticket.index];
551 unsafe {
552 *slot.event.get() = event;
553 }
554 slot.ready.store(true, Ordering::Release);
555 WRITERS.fetch_sub(1, Ordering::SeqCst);
556 }
557 #[inline]
558 pub(super) fn allocated(address: usize, size: usize) {
559 if let Some(ticket) = reserve(Some(address)) {
560 publish(
561 ticket,
562 Event {
563 kind: 1,
564 address,
565 size,
566 ..Event::default()
567 },
568 );
569 }
570 }
571 #[inline]
572 pub(super) fn freed(address: usize, size: usize) {
573 if let Some(ticket) = reserve(Some(address)) {
574 publish(
575 ticket,
576 Event {
577 kind: 2,
578 address,
579 size,
580 ..Event::default()
581 },
582 );
583 }
584 }
585 #[inline]
586 pub(super) fn begin_realloc() -> Option<Ticket> {
587 reserve(None)
588 }
589 #[inline]
590 pub(super) fn finish_realloc(
591 ticket: Option<Ticket>,
592 address: usize,
593 size: usize,
594 new_address: usize,
595 new_size: usize,
596 ) {
597 if let Some(ticket) = ticket {
598 publish(
599 ticket,
600 Event {
601 kind: if new_address == 0 { 0 } else { 3 },
602 address,
603 size,
604 new_address,
605 new_size,
606 ..Event::default()
607 },
608 );
609 }
610 }
611
612 #[cfg(test)]
613 mod tests {
614 use super::*;
615 use std::alloc::{GlobalAlloc, Layout};
616 static TEST_SESSION: Mutex<()> = Mutex::new(());
617 #[test]
618 fn capture_guards_cannot_move_or_share_between_threads() {
619 trait NotSend<Marker> {
623 fn check() {}
624 }
625 impl<T: ?Sized> NotSend<()> for T {}
626 impl<T: ?Sized + Send> NotSend<u8> for T {}
627 trait NotSync<Marker> {
628 fn check() {}
629 }
630 impl<T: ?Sized> NotSync<()> for T {}
631 impl<T: ?Sized + Sync> NotSync<u8> for T {}
632 let _ = <Scope as NotSend<_>>::check;
633 let _ = <Scope as NotSync<_>>::check;
634 let _ = <Suppress as NotSend<_>>::check;
635 let _ = <Suppress as NotSync<_>>::check;
636 }
637 #[test]
638 fn allocator_records_paired_reallocation_without_recording_profiler_work() {
639 let _session = TEST_SESSION.lock().unwrap();
640 start(4096, 1);
641 let _scope = scope("fixture", "paired realloc");
642 let id = SITE.with(Cell::get);
643 let layout = Layout::from_size_align(48, 8).unwrap();
644 unsafe {
645 let original = crate::heapcount::Counting.alloc(layout);
646 assert!(!original.is_null());
647 let resized = crate::heapcount::Counting.realloc(original, layout, 96);
648 assert!(!resized.is_null());
649 crate::heapcount::Counting
650 .dealloc(resized, Layout::from_size_align(96, 8).unwrap());
651 }
652 stop();
653 let mut cursor = 0;
654 let events: Vec<_> = drain(&mut cursor, MAX_EVENTS)
655 .into_iter()
656 .filter(|event| event.site == id)
657 .collect();
658 assert_eq!(
659 events.iter().map(|event| event.kind).collect::<Vec<_>>(),
660 vec![1, 3, 2]
661 );
662 assert_eq!(events[0].size, 48);
663 assert_eq!(events[1].new_size, 96);
664 assert_eq!(events[1].new_address, events[2].address);
665 let before = status().reserved;
666 unsafe {
667 let pointer = crate::heapcount::Counting.alloc(layout);
668 crate::heapcount::Counting.dealloc(pointer, layout);
669 }
670 assert_eq!(
671 status().reserved,
672 before,
673 "disabled recorder retained events"
674 );
675 clear();
676 }
677 #[test]
678 fn capture_overflow_is_bounded_and_reported() {
679 let _session = TEST_SESSION.lock().unwrap();
680 start(1024, 1);
681 let layout = Layout::from_size_align(24, 8).unwrap();
682 for _ in 0..800 {
683 unsafe {
684 let pointer = crate::heapcount::Counting.alloc(layout);
685 assert!(!pointer.is_null());
686 crate::heapcount::Counting.dealloc(pointer, layout);
687 }
688 }
689 let status = stop();
690 assert!(status.dropped >= 576);
691 let mut cursor = 0;
692 assert_eq!(drain(&mut cursor, MAX_EVENTS).len(), 1024);
693 clear();
694 }
695 }
696}