#![allow(
clippy::branches_sharing_code,
clippy::if_same_then_else,
clippy::uninlined_format_args,
unused_variables
)]
#![deny(unsafe_op_in_unsafe_fn)]
use crate::{
debug_log, file_stem_from_path, find_profile, flush_debug_log, fn_name,
get_global_profile_type, get_root_module, internal_doc, is_detailed_memory,
mem_attribution::{DetailedAddressRegistry, ProfileReg},
profiling::{
build_stack, clean_function_name, extract_detailed_alloc_callstack,
get_memory_detail_dealloc_path, get_memory_detail_path, get_memory_path,
is_profiling_state_enabled, MemoryDetailDeallocFile, MemoryDetailFile, MemoryProfileFile,
},
safe_alloc, warn_once, Profile, ProfileRef, ProfileType,
};
use backtrace::{resolve_frame, trace};
use parking_lot::Mutex;
use regex::Regex;
use std::{
alloc::{GlobalAlloc, Layout, System},
collections::{HashMap, HashSet},
env, fmt,
io::{self, Write},
sync::{
atomic::{AtomicUsize, Ordering},
LazyLock,
},
time::Instant,
};
use thag_common::{lazy_static_var, re};
use std::{cell::Cell, thread_local};
pub static ALLOC_START_PATTERN: LazyLock<&'static Regex> =
LazyLock::new(|| re!("thag_profiler::mem_tracking.+Dispatcher"));
thread_local! {
static USING_SYSTEM_ALLOCATOR: Cell<bool> = const { Cell::new(false) };
}
#[internal_doc]
#[inline]
#[must_use]
pub fn get_using_system() -> bool {
USING_SYSTEM_ALLOCATOR.with(Cell::get)
}
#[internal_doc]
#[inline]
pub fn set_using_system(value: bool) {
USING_SYSTEM_ALLOCATOR.with(|cell| cell.set(value));
}
#[internal_doc]
#[inline]
pub fn compare_exchange_using_system(current: bool, new: bool) -> Result<bool, bool> {
USING_SYSTEM_ALLOCATOR.with(|cell| {
let actual = cell.get();
if actual == current {
cell.set(new);
Ok(actual)
} else {
Err(actual)
}
})
}
#[internal_doc]
pub fn reset_allocator_state() {
USING_SYSTEM_ALLOCATOR.with(|flag| flag.set(false));
}
const MAX_SAFE_ALLOCATION: usize = 1024 * 1024 * 1024;
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum Allocator {
Tracking,
System,
}
impl fmt::Display for Allocator {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Tracking => write!(f, "Tracking"),
Self::System => write!(f, "System"),
}
}
}
#[internal_doc]
#[must_use]
pub fn current_allocator() -> Allocator {
let using_system = USING_SYSTEM_ALLOCATOR.with(Cell::get) || !crate::is_profiling_enabled();
if using_system {
Allocator::System
} else {
Allocator::Tracking
}
}
pub struct Dispatcher {
pub tracking: TrackingAllocator,
pub system: std::alloc::System,
}
impl Dispatcher {
#[must_use]
pub const fn new() -> Self {
Self {
tracking: TrackingAllocator,
system: std::alloc::System,
}
}
}
impl Default for Dispatcher {
fn default() -> Self {
Self::new()
}
}
unsafe impl GlobalAlloc for Dispatcher {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
let current = current_allocator();
match current {
Allocator::System => unsafe { self.system.alloc(layout) },
Allocator::Tracking => {
unsafe { self.tracking.alloc(layout) }
}
}
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
if ptr.is_null() {
return;
}
if layout.size() > MAX_SAFE_ALLOCATION {
safe_alloc! {
eprintln!(
"WARNING: Extremely large deallocation request of {} bytes",
layout.size()
)
}
}
match current_allocator() {
Allocator::System => unsafe { self.system.dealloc(ptr, layout) },
Allocator::Tracking => {
unsafe { self.tracking.dealloc(ptr, layout) }
}
}
}
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
if ptr.is_null() {
return unsafe {
self.alloc(Layout::from_size_align_unchecked(new_size, layout.align()))
};
}
match current_allocator() {
Allocator::System => unsafe { self.system.realloc(ptr, layout, new_size) },
Allocator::Tracking => {
unsafe { self.tracking.realloc(ptr, layout, new_size) }
}
}
}
}
pub struct TrackingAllocator;
static TRACKING_ALLOCATOR: TrackingAllocator = TrackingAllocator;
#[must_use]
pub fn get_allocator() -> &'static TrackingAllocator {
&TRACKING_ALLOCATOR
}
#[allow(clippy::unused_self)]
impl TrackingAllocator {
#[internal_doc]
pub fn create_task_context(&'static self) -> TaskMemoryContext {
let task_id = TASK_STATE.next_task_id.fetch_add(1, Ordering::SeqCst);
activate_task(task_id);
TaskMemoryContext { task_id }
}
}
unsafe impl GlobalAlloc for TrackingAllocator {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
let ptr = unsafe { System.alloc(layout) };
safe_alloc! {
if !ptr.is_null() && is_profiling_state_enabled() {
let size = layout.size();
if size > *SIZE_TRACKING_THRESHOLD {
let address = ptr as usize;
record_alloc(address, size);
}
}
};
ptr
}
#[allow(clippy::too_many_lines)]
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
safe_alloc! {
if !ptr.is_null()
&& is_profiling_state_enabled()
&& lazy_static_var!(bool, deref, is_detailed_memory())
{
let size = layout.size();
if size > *SIZE_TRACKING_THRESHOLD {
let address = ptr as usize;
record_dealloc(address, size);
}
}
};
unsafe { System.dealloc(ptr, layout) };
}
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
safe_alloc! {
if !ptr.is_null()
&& is_profiling_state_enabled()
&& lazy_static_var!(bool, deref, is_detailed_memory())
{
let dealloc_size = layout.size();
if dealloc_size > *SIZE_TRACKING_THRESHOLD {
let address = ptr as usize;
record_dealloc(address, dealloc_size);
}
}
if new_size > *SIZE_TRACKING_THRESHOLD {
let address = ptr as usize;
record_alloc(address, new_size);
}
};
unsafe { System.realloc(ptr, layout, new_size) }
}
}
#[allow(
clippy::cognitive_complexity,
clippy::too_many_lines,
unreachable_code,
unused_variables
)]
fn record_alloc(address: usize, size: usize) {
static mut IN_TRACKING: bool = false;
struct Guard;
impl Drop for Guard {
fn drop(&mut self) {
unsafe {
IN_TRACKING = false;
}
}
}
safe_alloc! {
if size == 0 {
debug_log!("Zero-sized allocation found");
return;
}
let profile_type = lazy_static_var!(ProfileType, deref, get_global_profile_type());
if profile_type != ProfileType::Memory && profile_type != ProfileType::Both {
return;
}
let in_tracking = unsafe { IN_TRACKING };
if in_tracking {
debug_log!("*** Caution: already tracking: proceeding for allocation of {size} B");
}
unsafe {
IN_TRACKING = true;
}
let _guard = Guard;
let start_ident = Instant::now();
let file_names = {
safe_alloc! {
ProfileReg::get()
.get_file_names()
}
};
debug_log!("file_names={file_names:#?}");
let Some(frames) =
extract_callstack_with_recursion_check(&file_names)
else {
debug_log!("Recursion detected");
return;
};
safe_alloc! {
if frames.is_empty() {
debug_log!("No eligible profile found");
return;
}
let in_profile_code = frames.iter().any(|(_, _, frame, _, _)| {
frame.contains("Profile::new")
});
if in_profile_code {
debug_log!("Ignoring allocation request of size {size} for profiler code");
return;
}
let (filename, lineno, frame, fn_name, profile_ref) = &frames[0];
let detailed_memory = lazy_static_var!(bool, deref, is_detailed_memory());
debug_log!("Found filename (file_name)={filename}, lineno={lineno}, fn_name: {fn_name:?}, frame: {frame:?} for size {size}");
if detailed_memory {
record_detailed_alloc(
address,
size,
&ALLOC_START_PATTERN,
true,
);
}
if !filename.is_empty()
&& *lineno > 0
&& record_allocation(filename, fn_name, *lineno, size)
{
debug_log!("Recorded allocation of {size} bytes in {filename}::{fn_name}:{lineno} to a profile");
debug_log!(
"size={size}, time to assign = {}ms",
start_ident.elapsed().as_millis()
);
}
};
};
}
type FrameSummary = (String, u32, String, String, ProfileRef);
#[fn_name]
pub fn extract_callstack_with_recursion_check(file_names: &[String]) -> Option<Vec<FrameSummary>> {
safe_alloc! {
let capacity = 100;
let mut frames: Vec<(String, u32, String, String, ProfileRef)> = Vec::with_capacity(capacity); let mut found_recursion = false;
let mut fin = false;
let mut i = 0;
trace(|frame| {
let mut suppress = false;
resolve_frame(frame, |symbol| {
'process_symbol: {
let Some(name) = symbol.name() else {
suppress = true;
break 'process_symbol;
};
let name = name.to_string();
if name.contains("__rust_begin_short_backtrace") {
fin = true;
suppress = true;
}
if name.starts_with("backtrace::backtrace::") || name.starts_with('<') {
suppress = true;
}
if suppress { break 'process_symbol; }
if i > 0 && name.contains(fn_name) {
found_recursion = true;
break 'process_symbol;
}
let maybe_filename = symbol.filename();
let maybe_lineno = symbol.lineno();
if maybe_filename.is_none()
|| maybe_lineno.is_none()
{
suppress = true;
break 'process_symbol;
}
let filename = safe_alloc! { file_stem_from_path(maybe_filename.unwrap()) };
let lineno = safe_alloc! { maybe_lineno.unwrap() };
if !file_names.contains(&filename) {
suppress = true;
break 'process_symbol;
}
let fn_name = clean_function_name(&mut name.clone());
let maybe_profile_ref = find_profile(&filename, &fn_name, lineno);
if let Some(profile_ref) = maybe_profile_ref {
frames.push((filename, lineno, name, fn_name, profile_ref));
i += 1;
if i >= capacity {
safe_alloc! {
println!("frames={frames:#?}");
};
panic!("Max limit of {capacity} frames exceeded");
}
} else {
debug_log!("No profile found for {filename}, {fn_name}, {lineno}");
}
}
});
!found_recursion && !fin
});
if found_recursion {
None } else {
Some(frames)
}
}
}
#[internal_doc]
#[must_use]
pub fn record_allocation(file_name: &str, fn_name: &str, line: u32, size: usize) -> bool {
safe_alloc! {
debug_log!(
"Looking for profile to record allocation: module={file_name}, fn={fn_name}, line={line}, size={size}"
);
flush_debug_log();
{
let modules = ProfileReg::get()
.get_file_names();
debug_log!("Available modules in registry: {modules:?}");
flush_debug_log();
}
let result;
{
debug_log!("About to call record_allocation on registry");
result = ProfileReg::get().record_allocation(
file_name,
fn_name,
line,
size,
);
debug_log!("record_allocation on registry returned {result}");
}
if result {
debug_log!(
"Successfully recorded allocation of {size} bytes in module {file_name}::{fn_name} at line {line}"
);
} else {
debug_log!("No matching profile found to record allocation of {size} bytes in module {file_name}::{fn_name} at line {line}");
}
result
}
}
pub fn register_detailed_allocation(address: usize, size: usize, stack: Vec<String>) {
safe_alloc! {
if is_detailed_memory() {
DetailedAddressRegistry::get().insert(address, (stack, size));
}
}
}
pub fn record_detailed_alloc(
address: usize,
size: usize,
start_pattern: &Regex,
write_to_detail_file: bool,
) {
let detailed_stack = extract_detailed_alloc_callstack(start_pattern);
write_detailed_stack_alloc(size, write_to_detail_file, &detailed_stack);
register_detailed_allocation(address, size, detailed_stack);
}
#[allow(
clippy::ptr_arg,
clippy::missing_panics_doc,
reason = "debug_assertions"
)]
pub fn write_detailed_stack_alloc(
size: usize,
write_to_detail_file: bool,
detailed_stack: &Vec<String>,
) {
safe_alloc! {
let root_module = lazy_static_var!(
String,
get_root_module()
.as_ref()
.map_or("root module", |v| v)
.to_string()
);
let entry = if detailed_stack.is_empty() {
format!("[Out of `{root_module}` scope] {size}")
} else {
let descr_stack = build_stack(detailed_stack, None, ";");
debug_log!("descr_stack={descr_stack}");
format!("{descr_stack} {size}")
};
let (memory_path, file) = if write_to_detail_file {
(get_memory_detail_path().unwrap(), MemoryDetailFile::get())
} else {
(get_memory_path().unwrap(), MemoryProfileFile::get())
};
let _ = Profile::write_profile_event(memory_path, file, &entry);
}
}
#[allow(
clippy::too_many_lines,
clippy::missing_panics_doc,
reason = "debug_assertions"
)]
pub fn record_dealloc(address: usize, size: usize) {
static mut IN_TRACKING: bool = false;
struct Guard;
impl Drop for Guard {
fn drop(&mut self) {
unsafe {
IN_TRACKING = false;
}
}
}
let root_module = lazy_static_var!(
String,
get_root_module()
.as_ref()
.map_or("root module", |v| v)
.to_string()
);
let profile_type = lazy_static_var!(ProfileType, deref, get_global_profile_type());
let is_mem_prof = lazy_static_var!(bool, {
profile_type == ProfileType::Memory || profile_type == ProfileType::Both
});
warn_once!(
!is_mem_prof,
|| {
debug_log!("Skipping deallocation recording because profile_type={profile_type:?}");
},
return
);
let in_tracking = unsafe { IN_TRACKING };
if in_tracking {
debug_log!("*** Caution: already tracking: proceeding for deallocation of {size} B");
}
unsafe {
IN_TRACKING = true;
}
let _guard = Guard;
let start_pattern: &Regex = re!("thag_profiler::mem_tracking.+Dispatcher");
let detailed_memory = lazy_static_var!(bool, deref, is_detailed_memory());
if size > 0 && detailed_memory {
let detailed_stack = extract_detailed_alloc_callstack(start_pattern);
let in_profile_code = detailed_stack
.iter()
.any(|frame| frame.contains("::profiling::Profile"));
if in_profile_code {
debug_log!(
"Detailed memory tracking ignoring detailed deallocation request of size {size} for profiler code: frame={:?}",
detailed_stack
.iter()
.find(|frame| frame.contains("::profiling::Profile"))
);
return;
}
let entry = if detailed_stack.is_empty() {
let stack_and_size = {
DetailedAddressRegistry::get()
.remove(&address)
.unwrap_or((0, (Vec::new(), size)))
};
let (stack, _) = stack_and_size.1;
let legend = if stack.is_empty() {
format!("[Dealloc out of `{root_module}` scope]")
} else {
stack.join(";")
};
format!("{legend} {size}")
} else {
format!("{} {size}", detailed_stack.join(";"))
};
let memory_detail_dealloc_path = get_memory_detail_dealloc_path().unwrap();
let _ = Profile::write_profile_event(
memory_detail_dealloc_path,
MemoryDetailDeallocFile::get(),
&entry,
);
}
}
#[global_allocator]
static ALLOCATOR: Dispatcher = Dispatcher::new();
pub static SIZE_TRACKING_THRESHOLD: LazyLock<usize> = LazyLock::new(|| {
let threshold = env::var("SIZE_TRACKING_THRESHOLD")
.or_else(|_| Ok::<String, &str>(String::from("0")))
.ok()
.and_then(|val| val.parse::<usize>().ok())
.expect("Value specified for SIZE_TRACKING_THRESHOLD must be a valid integer");
if threshold == 0 {
debug_log!("*** The SIZE_TRACKING_THRESHOLD environment variable is set or defaulted to 0, so all memory allocations and deallocations will be tracked.");
} else {
debug_log!("*** Only memory allocations and deallocations exceeding the specified threshold of {threshold} bytes will be tracked.");
}
threshold
});
pub fn activate_task(task_id: usize) {
safe_alloc! {
ProfileReg::get().activate_task(task_id);
};
}
#[allow(dead_code)]
pub fn deactivate_task(task_id: usize) {
safe_alloc! {
ProfileReg::get().deactivate_task(task_id);
};
}
#[internal_doc]
#[must_use]
pub fn get_last_active_task() -> Option<usize> {
safe_alloc! { ProfileReg::get().get_last_active_task() }
}
#[internal_doc]
#[derive(Debug, Clone)]
pub struct TaskMemoryContext {
pub task_id: usize,
}
impl TaskMemoryContext {
#[must_use]
pub const fn id(&self) -> usize {
self.task_id
}
}
#[cfg(not(feature = "full_profiling"))]
#[derive(Debug, Default, Clone, Copy)]
pub struct TaskMemoryContext;
#[internal_doc]
#[must_use]
pub fn create_memory_task() -> TaskMemoryContext {
let allocator = get_allocator();
allocator.create_task_context()
}
pub struct TaskState {
pub next_task_id: AtomicUsize,
}
pub static TASK_STATE: LazyLock<TaskState> = LazyLock::new(|| TaskState {
next_task_id: AtomicUsize::new(1),
});
#[internal_doc]
#[derive(Clone, Debug)]
pub struct TaskGuard {
task_id: usize,
}
impl TaskGuard {
#[must_use]
pub const fn new(task_id: usize) -> Self {
Self { task_id }
}
}
#[cfg(not(feature = "full_profiling"))]
#[derive(Debug, Default, Clone, Copy)]
pub struct TaskGuard;
impl Drop for TaskGuard {
fn drop(&mut self) {
safe_alloc! {
ProfileReg::get().deactivate_task(self.task_id);
debug_log!("Deactivated task {}", self.task_id);
if let Some(logger) = crate::DebugLogger::get() {
let _ = logger.lock().flush();
}
};
}
}
pub static TASK_PATH_REGISTRY: LazyLock<Mutex<HashMap<usize, Vec<String>>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[allow(clippy::missing_panics_doc)]
pub fn initialize_memory_profiling() {
reset_allocator_state();
safe_alloc! {
debug_log!("Memory profiling initialized");
flush_debug_log();
};
#[cfg(debug_assertions)]
assert_eq!(current_allocator(), Allocator::Tracking);
}
pub fn finalize_memory_profiling() {
write_final_memory_profile_data();
flush_debug_log();
}
fn write_final_memory_profile_data() {
use std::{collections::HashMap, fs::File, path::Path};
safe_alloc! {
let memory_path = get_memory_path().unwrap_or("memory.folded");
let file_exists = Path::new(memory_path).exists();
let file_result = if file_exists {
debug_log!("Opening existing file in append mode");
File::options().append(true).open(memory_path)
} else {
debug_log!("Creating new file");
match File::create(memory_path) {
Ok(file) => {
Ok(file)
}
Err(e) => {
debug_log!("Error creating file: {e}");
Err(e)
}
}
};
if let Ok(file) = file_result {
let mut writer = io::BufWriter::new(file);
let task_paths_map: HashMap<usize, Vec<String>> = {
let binding = TASK_PATH_REGISTRY.lock();
binding
.iter()
.map(|(task_id, path)| (*task_id, path.clone()))
.collect()
};
let mut already_written = HashSet::new();
for (task_id, path) in &task_paths_map {
let task_id = *task_id;
let path_str = build_stack(path, None, ";");
if already_written.contains(&path_str) {
continue;
}
debug_log!("Writing for task {task_id} from registry: '{path_str}' with 0 bytes");
write_alloc(task_id, 0, &mut writer, &mut already_written, &path_str);
}
if let Err(e) = writer.flush() {
debug_log!("Error flushing writer: {e}");
}
}
};
}
fn write_alloc(
task_id: usize,
allocation: usize,
writer: &mut io::BufWriter<std::fs::File>,
already_written: &mut HashSet<String>,
path_str: &str,
) {
match writeln!(writer, "{} {}", path_str, allocation) {
Ok(()) => {
already_written.insert(path_str.to_string());
}
Err(e) => {
debug_log!("Error writing line for task {task_id}: {e}");
}
}
}