Skip to main content

sk_app/
heapcount.rs

1//! Live Rust heap bytes: the devtools `mem` command's "Rust heap" row.
2//!
3//! The process's private bytes span several allocators (the Rust heap, which the static CRT's
4//! `malloc` shares; JSC's mimalloc and executable memory; D3D's driver allocations; libwebrtc). Only
5//! the Rust heap can be counted from Rust, and the retained tree, cascade, gpui's arena and every
6//! decoded image live in it, so it is counted here and the rest is attributed by subtraction in
7//! `devtools::mem`.
8//!
9//! The wrapper is the global allocator of every build that has the devtools (`cfg(sk_devtools)`). It
10//! counts by default in a dev build and not in a `--features prod` one; `SK_MEMSTATS` overrides
11//! either way (`0` off, anything else on).
12//!
13//! The switch is read on the very first allocation, before `main`, through `GetEnvironmentVariableW`
14//! into a stack buffer (`std::env` would allocate inside the allocator), so the count starts at the
15//! first byte and a free can never be of something that was not counted.
16
17use std::alloc::{GlobalAlloc, Layout, System};
18use std::sync::atomic::{AtomicI64, AtomicU64, AtomicU8, Ordering::Relaxed};
19
20pub struct Counting;
21
22static STATE: AtomicU8 = AtomicU8::new(0);
23static LIVE_BYTES: AtomicI64 = AtomicI64::new(0);
24static LIVE_BLOCKS: AtomicI64 = AtomicI64::new(0);
25static PEAK_BYTES: AtomicI64 = AtomicI64::new(0);
26static ALLOCS: AtomicU64 = AtomicU64::new(0);
27
28const CLASSES: usize = 34;
29#[allow(clippy::declare_interior_mutable_const)]
30const ZERO: AtomicI64 = AtomicI64::new(0);
31static CLASS_BYTES: [AtomicI64; CLASSES] = [ZERO; CLASSES];
32static CLASS_BLOCKS: [AtomicI64; CLASSES] = [ZERO; CLASSES];
33
34const DEFAULT_ON: bool = !cfg!(feature = "prod");
35
36#[cfg(windows)]
37#[link(name = "kernel32")]
38extern "system" {
39    fn GetEnvironmentVariableW(name: *const u16, buffer: *mut u16, size: u32) -> u32;
40}
41
42#[cfg(windows)]
43fn memstats_env() -> Option<bool> {
44    const NAME: [u16; 12] = [
45        b'V' as u16,
46        b'N' as u16,
47        b'_' as u16,
48        b'M' as u16,
49        b'E' as u16,
50        b'M' as u16,
51        b'S' as u16,
52        b'T' as u16,
53        b'A' as u16,
54        b'T' as u16,
55        b'S' as u16,
56        0,
57    ];
58    let mut buf = [0u16; 8];
59    let n = unsafe { GetEnvironmentVariableW(NAME.as_ptr(), buf.as_mut_ptr(), buf.len() as u32) };
60    (n != 0).then(|| !(n == 1 && buf[0] == b'0' as u16))
61}
62
63#[cfg(not(windows))]
64fn memstats_env() -> Option<bool> {
65    extern "C" {
66        fn getenv(name: *const core::ffi::c_char) -> *const core::ffi::c_char;
67    }
68    // `getenv` returns the environment's own storage: no copy, no allocation.
69    let value = unsafe { getenv(c"SK_MEMSTATS".as_ptr()) };
70    if value.is_null() {
71        return None;
72    }
73    let first = unsafe { *value } as u8;
74    if first == 0 {
75        return None;
76    }
77    let second = unsafe { *value.add(1) } as u8;
78    Some(!(first == b'0' && second == 0))
79}
80
81#[cold]
82fn probe() -> bool {
83    let on = memstats_env().unwrap_or(DEFAULT_ON);
84    STATE.store(if on { 2 } else { 1 }, Relaxed);
85    on
86}
87
88#[inline]
89fn counting() -> bool {
90    match STATE.load(Relaxed) {
91        2 => true,
92        1 => false,
93        _ => probe(),
94    }
95}
96
97#[inline]
98fn class(size: usize) -> usize {
99    ((usize::BITS - size.leading_zeros()) as usize).min(CLASSES - 1)
100}
101
102#[inline]
103fn add(size: usize, sign: i64) {
104    let bytes = sign * size as i64;
105    let now = LIVE_BYTES.fetch_add(bytes, Relaxed) + bytes;
106    LIVE_BLOCKS.fetch_add(sign, Relaxed);
107    if sign > 0 {
108        PEAK_BYTES.fetch_max(now, Relaxed);
109    }
110    let k = class(size);
111    CLASS_BYTES[k].fetch_add(bytes, Relaxed);
112    CLASS_BLOCKS[k].fetch_add(sign, Relaxed);
113}
114
115unsafe impl GlobalAlloc for Counting {
116    #[inline]
117    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
118        let p = unsafe { System.alloc(layout) };
119        if !p.is_null() {
120            recording::allocated(p as usize, layout.size());
121        }
122        if !p.is_null() && counting() {
123            add(layout.size(), 1);
124            ALLOCS.fetch_add(1, Relaxed);
125        }
126        p
127    }
128
129    #[inline]
130    unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
131        let p = unsafe { System.alloc_zeroed(layout) };
132        if !p.is_null() {
133            recording::allocated(p as usize, layout.size());
134        }
135        if !p.is_null() && counting() {
136            add(layout.size(), 1);
137            ALLOCS.fetch_add(1, Relaxed);
138        }
139        p
140    }
141
142    #[inline]
143    unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
144        recording::freed(ptr as usize, layout.size());
145        unsafe { System.dealloc(ptr, layout) };
146        if counting() {
147            add(layout.size(), -1);
148        }
149    }
150
151    #[inline]
152    unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
153        let ticket = recording::begin_realloc();
154        let p = unsafe { System.realloc(ptr, layout, new_size) };
155        recording::finish_realloc(ticket, ptr as usize, layout.size(), p as usize, new_size);
156        if !p.is_null() && counting() {
157            add(layout.size(), -1);
158            add(new_size, 1);
159        }
160        p
161    }
162}
163
164pub fn snapshot() -> Option<(i64, i64, i64, u64)> {
165    on().then(|| {
166        (
167            LIVE_BYTES.load(Relaxed),
168            LIVE_BLOCKS.load(Relaxed),
169            PEAK_BYTES.load(Relaxed),
170            ALLOCS.load(Relaxed),
171        )
172    })
173}
174
175pub fn classes() -> Vec<(u64, i64, i64)> {
176    (0..CLASSES)
177        .filter_map(|k| {
178            let blocks = CLASS_BLOCKS[k].load(Relaxed);
179            let bytes = CLASS_BYTES[k].load(Relaxed);
180            let upper = if k == CLASSES - 1 {
181                u64::MAX
182            } else {
183                1u64 << k
184            };
185            (blocks != 0).then_some((upper, bytes, blocks))
186        })
187        .collect()
188}
189
190/// Is the Rust heap being counted? Only when [`Counting`] IS the global
191/// allocator — the exe's is (src/main.rs), an embedder's need not be — and
192/// then it has counted the first allocation long before anyone asks.
193pub fn on() -> bool {
194    counting() && ALLOCS.load(Relaxed) > 0
195}
196
197/// Bounded allocation events. The hook only loads TLS, updates atomics and
198/// writes its uniquely reserved slot; it never allocates, locks or unwinds a
199/// stack. Session control and site registration happen outside the allocator.
200pub mod recording {
201    use std::{
202        cell::{Cell, UnsafeCell},
203        marker::PhantomData,
204        rc::Rc,
205        sync::{
206            atomic::{AtomicBool, AtomicPtr, AtomicU64, AtomicUsize, Ordering},
207            Mutex, OnceLock,
208        },
209        time::{Instant, SystemTime, UNIX_EPOCH},
210    };
211
212    pub const MAX_EVENTS: usize = 65_536;
213    pub const MAX_SITES: usize = 512;
214    static EPOCH: AtomicU64 = AtomicU64::new(0);
215    static WRITERS: AtomicUsize = AtomicUsize::new(0);
216    static RECORDER: AtomicPtr<Recorder> = AtomicPtr::new(std::ptr::null_mut());
217    static NEXT_SESSION: AtomicU64 = AtomicU64::new(1);
218    static NEXT_THREAD: AtomicUsize = AtomicUsize::new(1);
219    thread_local! {
220        static SUPPRESSED: Cell<u32> = const { Cell::new(0) };
221        static SITE: Cell<u32> = const { Cell::new(0) };
222        static THREAD: Cell<u32> = const { Cell::new(0) };
223    }
224
225    #[derive(Clone, Copy, Default)]
226    pub struct Event {
227        pub kind: u8,
228        pub address: usize,
229        pub size: usize,
230        pub new_address: usize,
231        pub new_size: usize,
232        pub site: u32,
233        pub thread: u32,
234    }
235
236    struct Slot {
237        ready: AtomicBool,
238        event: UnsafeCell<Event>,
239    }
240    // Each slot has one reserved writer; the reader only touches published
241    // slots. Reset happens with capture disabled and every writer drained.
242    unsafe impl Sync for Slot {}
243    struct Recorder {
244        slots: Box<[Slot]>,
245        next: AtomicUsize,
246        dropped: AtomicU64,
247        capacity: AtomicUsize,
248        sample_every: AtomicUsize,
249    }
250    struct Session {
251        id: u64,
252        started: Instant,
253        started_ms: f64,
254        stopped: Option<Instant>,
255    }
256    static SESSION: Mutex<Option<Session>> = Mutex::new(None);
257    #[derive(Clone)]
258    pub struct Site {
259        pub category: String,
260        pub name: String,
261    }
262    static SITES: Mutex<Vec<Site>> = Mutex::new(Vec::new());
263
264    pub struct Suppress {
265        _thread: PhantomData<Rc<()>>,
266    }
267    impl Drop for Suppress {
268        fn drop(&mut self) {
269            let _ = SUPPRESSED.try_with(|value| value.set(value.get().saturating_sub(1)));
270        }
271    }
272    pub fn suppress() -> Suppress {
273        let _ = SUPPRESSED.try_with(|value| value.set(value.get().saturating_add(1)));
274        Suppress {
275            _thread: PhantomData,
276        }
277    }
278
279    pub struct Scope {
280        previous: Option<u32>,
281        _thread: PhantomData<Rc<()>>,
282    }
283    impl Drop for Scope {
284        fn drop(&mut self) {
285            if let Some(previous) = self.previous {
286                let _ = SITE.try_with(|site| site.set(previous));
287            }
288        }
289    }
290    #[track_caller]
291    pub fn scope(category: &str, name: &str) -> Scope {
292        if EPOCH.load(Ordering::Relaxed) == 0 {
293            return Scope {
294                previous: None,
295                _thread: PhantomData,
296            };
297        }
298        let _suppressed = suppress();
299        let location = std::panic::Location::caller();
300        let label: String = format!("{name} ({}:{})", location.file(), location.line())
301            .chars()
302            .take(240)
303            .collect();
304        let category: String = category.chars().take(80).collect();
305        let id = {
306            let mut sites = SITES.lock().unwrap_or_else(|error| error.into_inner());
307            match sites
308                .iter()
309                .position(|site| site.category == category && site.name == label)
310            {
311                Some(index) => index as u32 + 1,
312                None if sites.len() < MAX_SITES => {
313                    sites.push(Site {
314                        category,
315                        name: label,
316                    });
317                    sites.len() as u32
318                }
319                None => 0,
320            }
321        };
322        Scope {
323            previous: SITE.try_with(|site| site.replace(id)).ok(),
324            _thread: PhantomData,
325        }
326    }
327
328    #[derive(Clone)]
329    pub struct Status {
330        pub session: u64,
331        pub recording: bool,
332        pub started_at_ms: f64,
333        pub elapsed_ms: f64,
334        pub sample_every: usize,
335        pub capacity: usize,
336        pub reserved: usize,
337        pub dropped: u64,
338    }
339    fn recorder() -> &'static Recorder {
340        static STORAGE: OnceLock<Recorder> = OnceLock::new();
341        STORAGE.get_or_init(|| Recorder {
342            slots: (0..MAX_EVENTS)
343                .map(|_| Slot {
344                    ready: AtomicBool::new(false),
345                    event: UnsafeCell::new(Event::default()),
346                })
347                .collect::<Vec<_>>()
348                .into_boxed_slice(),
349            next: AtomicUsize::new(0),
350            dropped: AtomicU64::new(0),
351            capacity: AtomicUsize::new(MAX_EVENTS),
352            sample_every: AtomicUsize::new(1),
353        })
354    }
355    fn stop_writers() {
356        EPOCH.store(0, Ordering::SeqCst);
357        while WRITERS.load(Ordering::SeqCst) != 0 {
358            std::thread::yield_now();
359        }
360    }
361    pub fn start(capacity: usize, sample_every: usize) -> Status {
362        let _suppressed = suppress();
363        let mut session = SESSION.lock().unwrap_or_else(|error| error.into_inner());
364        stop_writers();
365        let recorder = recorder();
366        for slot in recorder.slots.iter() {
367            slot.ready.store(false, Ordering::Relaxed);
368        }
369        recorder.next.store(0, Ordering::Relaxed);
370        recorder.dropped.store(0, Ordering::Relaxed);
371        recorder
372            .capacity
373            .store(capacity.clamp(1024, MAX_EVENTS), Ordering::Relaxed);
374        recorder
375            .sample_every
376            .store(sample_every.clamp(1, 4096), Ordering::Relaxed);
377        let id = NEXT_SESSION.fetch_add(1, Ordering::Relaxed);
378        *session = Some(Session {
379            id,
380            started: Instant::now(),
381            started_ms: SystemTime::now()
382                .duration_since(UNIX_EPOCH)
383                .unwrap_or_default()
384                .as_secs_f64()
385                * 1000.0,
386            stopped: None,
387        });
388        RECORDER.store(recorder as *const _ as *mut _, Ordering::Release);
389        EPOCH.store(id, Ordering::SeqCst);
390        drop(session);
391        status()
392    }
393    pub fn stop() -> Status {
394        let _suppressed = suppress();
395        let mut session = SESSION.lock().unwrap_or_else(|error| error.into_inner());
396        stop_writers();
397        if let Some(session) = session.as_mut() {
398            session.stopped.get_or_insert_with(Instant::now);
399        }
400        drop(session);
401        status()
402    }
403    pub fn clear() {
404        let _suppressed = suppress();
405        let mut session = SESSION.lock().unwrap_or_else(|error| error.into_inner());
406        stop_writers();
407        let pointer = RECORDER.load(Ordering::Acquire);
408        if !pointer.is_null() {
409            let recorder = unsafe { &*pointer };
410            recorder.next.store(0, Ordering::Relaxed);
411            recorder.dropped.store(0, Ordering::Relaxed);
412        }
413        *session = None;
414    }
415    pub fn reserved_bytes() -> usize {
416        if RECORDER.load(Ordering::Acquire).is_null() {
417            0
418        } else {
419            MAX_EVENTS * std::mem::size_of::<Slot>()
420        }
421    }
422    pub fn status() -> Status {
423        let _suppressed = suppress();
424        let session = SESSION.lock().unwrap_or_else(|error| error.into_inner());
425        let pointer = RECORDER.load(Ordering::Acquire);
426        let (sample_every, capacity, reserved, dropped) = if pointer.is_null() {
427            (1, 0, 0, 0)
428        } else {
429            let recorder = unsafe { &*pointer };
430            (
431                recorder.sample_every.load(Ordering::Relaxed),
432                recorder.capacity.load(Ordering::Relaxed),
433                recorder.next.load(Ordering::Acquire),
434                recorder.dropped.load(Ordering::Relaxed),
435            )
436        };
437        Status {
438            session: session.as_ref().map_or(0, |session| session.id),
439            recording: EPOCH.load(Ordering::Acquire) != 0,
440            started_at_ms: session.as_ref().map_or(0.0, |session| session.started_ms),
441            elapsed_ms: session.as_ref().map_or(0.0, |session| {
442                session
443                    .stopped
444                    .unwrap_or_else(Instant::now)
445                    .duration_since(session.started)
446                    .as_secs_f64()
447                    * 1000.0
448            }),
449            sample_every,
450            capacity,
451            reserved,
452            dropped,
453        }
454    }
455    pub fn sites() -> Vec<Site> {
456        let _suppressed = suppress();
457        SITES
458            .lock()
459            .unwrap_or_else(|error| error.into_inner())
460            .clone()
461    }
462    pub fn drain(cursor: &mut usize, limit: usize) -> Vec<Event> {
463        let _suppressed = suppress();
464        let _session = SESSION.lock().unwrap_or_else(|error| error.into_inner());
465        let pointer = RECORDER.load(Ordering::Acquire);
466        if pointer.is_null() {
467            return Vec::new();
468        }
469        let recorder = unsafe { &*pointer };
470        let end = recorder
471            .next
472            .load(Ordering::Acquire)
473            .min(recorder.capacity.load(Ordering::Relaxed));
474        let mut events = Vec::with_capacity(end.saturating_sub(*cursor).min(limit));
475        while *cursor < end && events.len() < limit {
476            let slot = &recorder.slots[*cursor];
477            if !slot.ready.load(Ordering::Acquire) {
478                break;
479            }
480            events.push(unsafe { *slot.event.get() });
481            *cursor += 1;
482        }
483        events
484    }
485    pub fn selected(address: usize, sample_every: usize) -> bool {
486        let mut hash = address as u64;
487        hash ^= hash >> 33;
488        hash = hash.wrapping_mul(0xff51afd7ed558ccd);
489        hash ^= hash >> 33;
490        hash % sample_every.max(1) as u64 == 0
491    }
492    pub struct Ticket {
493        recorder: &'static Recorder,
494        index: usize,
495        site: u32,
496        thread: u32,
497    }
498    #[inline]
499    fn reserve(address: Option<usize>) -> Option<Ticket> {
500        if EPOCH.load(Ordering::Relaxed) == 0 {
501            return None;
502        }
503        if SUPPRESSED
504            .try_with(|value| value.get() != 0)
505            .unwrap_or(true)
506        {
507            return None;
508        }
509        WRITERS.fetch_add(1, Ordering::SeqCst);
510        if EPOCH.load(Ordering::SeqCst) == 0 {
511            WRITERS.fetch_sub(1, Ordering::SeqCst);
512            return None;
513        }
514        let pointer = RECORDER.load(Ordering::Acquire);
515        let recorder = unsafe { &*pointer };
516        if address.is_some_and(|address| {
517            !selected(address, recorder.sample_every.load(Ordering::Relaxed))
518        }) {
519            WRITERS.fetch_sub(1, Ordering::SeqCst);
520            return None;
521        }
522        let index = recorder.next.fetch_add(1, Ordering::AcqRel);
523        if index >= recorder.capacity.load(Ordering::Relaxed) {
524            recorder.dropped.fetch_add(1, Ordering::Relaxed);
525            WRITERS.fetch_sub(1, Ordering::SeqCst);
526            return None;
527        }
528        let site = SITE.try_with(Cell::get).unwrap_or(0);
529        let thread = THREAD
530            .try_with(|thread| {
531                let mut id = thread.get();
532                if id == 0 {
533                    id = NEXT_THREAD.fetch_add(1, Ordering::Relaxed) as u32;
534                    thread.set(id);
535                }
536                id
537            })
538            .unwrap_or(0);
539        Some(Ticket {
540            recorder,
541            index,
542            site,
543            thread,
544        })
545    }
546    #[inline]
547    fn publish(ticket: Ticket, mut event: Event) {
548        event.site = ticket.site;
549        event.thread = ticket.thread;
550        let slot = &ticket.recorder.slots[ticket.index];
551        unsafe {
552            *slot.event.get() = event;
553        }
554        slot.ready.store(true, Ordering::Release);
555        WRITERS.fetch_sub(1, Ordering::SeqCst);
556    }
557    #[inline]
558    pub(super) fn allocated(address: usize, size: usize) {
559        if let Some(ticket) = reserve(Some(address)) {
560            publish(
561                ticket,
562                Event {
563                    kind: 1,
564                    address,
565                    size,
566                    ..Event::default()
567                },
568            );
569        }
570    }
571    #[inline]
572    pub(super) fn freed(address: usize, size: usize) {
573        if let Some(ticket) = reserve(Some(address)) {
574            publish(
575                ticket,
576                Event {
577                    kind: 2,
578                    address,
579                    size,
580                    ..Event::default()
581                },
582            );
583        }
584    }
585    #[inline]
586    pub(super) fn begin_realloc() -> Option<Ticket> {
587        reserve(None)
588    }
589    #[inline]
590    pub(super) fn finish_realloc(
591        ticket: Option<Ticket>,
592        address: usize,
593        size: usize,
594        new_address: usize,
595        new_size: usize,
596    ) {
597        if let Some(ticket) = ticket {
598            publish(
599                ticket,
600                Event {
601                    kind: if new_address == 0 { 0 } else { 3 },
602                    address,
603                    size,
604                    new_address,
605                    new_size,
606                    ..Event::default()
607                },
608            );
609        }
610    }
611
612    #[cfg(test)]
613    mod tests {
614        use super::*;
615        use std::alloc::{GlobalAlloc, Layout};
616        static TEST_SESSION: Mutex<()> = Mutex::new(());
617        #[test]
618        fn capture_guards_cannot_move_or_share_between_threads() {
619            // The second impl makes inference ambiguous if a guard ever
620            // acquires Send/Sync. This checks the negative bound without a
621            // dependency or a compile-fail test that could fail for other reasons.
622            trait NotSend<Marker> {
623                fn check() {}
624            }
625            impl<T: ?Sized> NotSend<()> for T {}
626            impl<T: ?Sized + Send> NotSend<u8> for T {}
627            trait NotSync<Marker> {
628                fn check() {}
629            }
630            impl<T: ?Sized> NotSync<()> for T {}
631            impl<T: ?Sized + Sync> NotSync<u8> for T {}
632            let _ = <Scope as NotSend<_>>::check;
633            let _ = <Scope as NotSync<_>>::check;
634            let _ = <Suppress as NotSend<_>>::check;
635            let _ = <Suppress as NotSync<_>>::check;
636        }
637        #[test]
638        fn allocator_records_paired_reallocation_without_recording_profiler_work() {
639            let _session = TEST_SESSION.lock().unwrap();
640            start(4096, 1);
641            let _scope = scope("fixture", "paired realloc");
642            let id = SITE.with(Cell::get);
643            let layout = Layout::from_size_align(48, 8).unwrap();
644            unsafe {
645                let original = crate::heapcount::Counting.alloc(layout);
646                assert!(!original.is_null());
647                let resized = crate::heapcount::Counting.realloc(original, layout, 96);
648                assert!(!resized.is_null());
649                crate::heapcount::Counting
650                    .dealloc(resized, Layout::from_size_align(96, 8).unwrap());
651            }
652            stop();
653            let mut cursor = 0;
654            let events: Vec<_> = drain(&mut cursor, MAX_EVENTS)
655                .into_iter()
656                .filter(|event| event.site == id)
657                .collect();
658            assert_eq!(
659                events.iter().map(|event| event.kind).collect::<Vec<_>>(),
660                vec![1, 3, 2]
661            );
662            assert_eq!(events[0].size, 48);
663            assert_eq!(events[1].new_size, 96);
664            assert_eq!(events[1].new_address, events[2].address);
665            let before = status().reserved;
666            unsafe {
667                let pointer = crate::heapcount::Counting.alloc(layout);
668                crate::heapcount::Counting.dealloc(pointer, layout);
669            }
670            assert_eq!(
671                status().reserved,
672                before,
673                "disabled recorder retained events"
674            );
675            clear();
676        }
677        #[test]
678        fn capture_overflow_is_bounded_and_reported() {
679            let _session = TEST_SESSION.lock().unwrap();
680            start(1024, 1);
681            let layout = Layout::from_size_align(24, 8).unwrap();
682            for _ in 0..800 {
683                unsafe {
684                    let pointer = crate::heapcount::Counting.alloc(layout);
685                    assert!(!pointer.is_null());
686                    crate::heapcount::Counting.dealloc(pointer, layout);
687                }
688            }
689            let status = stop();
690            assert!(status.dropped >= 576);
691            let mut cursor = 0;
692            assert_eq!(drain(&mut cursor, MAX_EVENTS).len(), 1024);
693            clear();
694        }
695    }
696}