#![allow(dead_code)]
use super::super::{ExecOutcome, Vm, VmError, VmResult};
#[cfg(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
))]
use super::JitTrace;
use super::{JitMetrics, JitTraceTerminal, native};
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
#[cfg(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
))]
use std::{cell::RefCell, thread_local};
#[cfg(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
))]
type NativeTraceEntry = unsafe extern "C" fn(*mut Vm) -> i32;
#[cfg(not(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
)))]
type NativeTraceEntry = fn(*mut Vm) -> i32;
pub(crate) struct NativeTrace {
_keepalive: Arc<Mutex<native::TraceKeepAlive>>,
entry: NativeTraceEntry,
pub(super) code: Arc<[u8]>,
root_ip: usize,
terminal: JitTraceTerminal,
has_call: bool,
has_yielding_call: bool,
lowering_kind: native::TraceLoweringKind,
interrupt_settings: Option<native::NativeInterruptSettings>,
compile_profile: native::NativeCompileProfile,
drop_contract_events_enabled: bool,
}
#[cfg(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
))]
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
struct NativeTraceCacheKey {
interrupt_settings: Option<native::NativeInterruptSettings>,
compile_profile: native::NativeCompileProfile,
drop_contract_events_enabled: bool,
root_ip: usize,
terminal: JitTraceTerminal,
ssa_text: String,
}
#[cfg(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
))]
#[derive(Clone)]
struct NativeTraceCacheEntry {
entry: NativeTraceEntry,
keepalive: Arc<Mutex<native::TraceKeepAlive>>,
code: Arc<[u8]>,
lowering_kind: native::TraceLoweringKind,
compile_profile: native::NativeCompileProfile,
}
#[cfg(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
))]
struct NativeTraceCache {
active_program_key: Option<u64>,
entries: HashMap<NativeTraceCacheKey, NativeTraceCacheEntry>,
}
#[cfg(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
))]
thread_local! {
static NATIVE_TRACE_CACHE: RefCell<NativeTraceCache> = RefCell::new(
NativeTraceCache {
active_program_key: None,
entries: HashMap::new(),
}
);
}
#[cfg(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
))]
fn with_native_trace_cache<R>(f: impl FnOnce(&mut NativeTraceCache) -> R) -> R {
NATIVE_TRACE_CACHE.with(|cell| {
let mut cache = cell.borrow_mut();
f(&mut cache)
})
}
#[cfg(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
))]
fn native_trace_cache_key(
trace: &JitTrace,
interrupt_settings: Option<native::NativeInterruptSettings>,
compile_profile: native::NativeCompileProfile,
drop_contract_events_enabled: bool,
) -> NativeTraceCacheKey {
NativeTraceCacheKey {
interrupt_settings,
compile_profile,
drop_contract_events_enabled,
root_ip: trace.root_ip,
terminal: trace.terminal.clone(),
ssa_text: trace.ssa_text(),
}
}
#[cfg(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
))]
fn compile_profile_satisfies(
compiled: native::NativeCompileProfile,
requested: native::NativeCompileProfile,
) -> bool {
compiled == requested
}
fn should_fallback_to_interpreter(err: &VmError) -> bool {
matches!(err, VmError::JitNative(detail)
if detail.contains("SSA native lowering does not support"))
}
#[cfg(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
))]
pub(crate) fn resume_linked_trace_entry_address() -> usize {
pd_vm_native_resume_linked_trace as *const () as usize
}
#[cfg(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
))]
pub(crate) extern "C" fn pd_vm_native_resume_linked_trace(vm: *mut Vm) -> i32 {
let Some(vm_ref) = (unsafe { vm.as_mut() }) else {
native::store_bridge_error(VmError::JitNative(
"native linked-trace helper received null vm pointer".to_string(),
));
return native::STATUS_ERROR;
};
if vm_ref.jit_native_link_dispatch_depth > 0 {
return native::STATUS_TRACE_EXIT;
}
vm_ref.jit_native_link_dispatch_depth = vm_ref.jit_native_link_dispatch_depth.saturating_add(1);
match vm_ref.continue_linked_native_trace_from_exit() {
Ok(status) => {
vm_ref.jit_native_link_dispatch_depth =
vm_ref.jit_native_link_dispatch_depth.saturating_sub(1);
status
}
Err(err) => {
vm_ref.jit_native_link_dispatch_depth =
vm_ref.jit_native_link_dispatch_depth.saturating_sub(1);
native::store_bridge_error(err);
native::STATUS_ERROR
}
}
}
impl Vm {
#[cfg(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
))]
fn continue_linked_native_trace_from_exit(&mut self) -> VmResult<i32> {
self.jit_trace_exit_count = self.jit_trace_exit_count.saturating_add(1);
let mut current_trace_id = {
let ip = self.ip;
let mut next_trace_id = self.jit.compiled_trace_for_ip(ip);
if next_trace_id.is_none() {
let program = &self.program;
next_trace_id = self.jit.observe_exit_ip(ip, program);
}
let Some(next_trace_id) = next_trace_id else {
return Ok(native::STATUS_LINKED_CONTINUE);
};
next_trace_id
};
self.record_jit_link_handoff();
if let Err(err) =
self.ensure_native_trace(current_trace_id, native::NativeCompileProfile::Jit)
{
if should_fallback_to_interpreter(&err) {
self.record_jit_helper_fallback();
self.jit.block_trace(current_trace_id);
return Ok(native::STATUS_LINKED_CONTINUE);
}
return Err(err);
}
let (mut entry, mut root_ip, mut terminal, mut has_call, mut has_yielding_call) =
self.native_trace_state(current_trace_id)?;
loop {
native::clear_bridge_error();
let status = unsafe { entry(self as *mut Vm) };
self.native_trace_exec_count = self.native_trace_exec_count.saturating_add(1);
self.jit.mark_trace_executed(current_trace_id);
match status {
native::STATUS_CONTINUE => {
if !has_yielding_call
&& let Some(next_trace_id) = self.jit.compiled_trace_for_ip(self.ip)
&& next_trace_id != current_trace_id
{
self.record_jit_link_handoff();
current_trace_id = next_trace_id;
if let Err(err) = self.ensure_native_trace(
current_trace_id,
native::NativeCompileProfile::Jit,
) {
if should_fallback_to_interpreter(&err) {
self.record_jit_helper_fallback();
self.jit.block_trace(current_trace_id);
return Ok(native::STATUS_LINKED_CONTINUE);
}
return Err(err);
}
(entry, root_ip, terminal, has_call, has_yielding_call) =
self.native_trace_state(current_trace_id)?;
continue;
}
return Ok(native::STATUS_LINKED_CONTINUE);
}
native::STATUS_TRACE_EXIT => {
self.jit_trace_exit_count = self.jit_trace_exit_count.saturating_add(1);
if has_call {
return Ok(native::STATUS_LINKED_CONTINUE);
}
if !has_yielding_call
&& terminal == JitTraceTerminal::LoopBack
&& self.ip == root_ip
{
self.jit_native_loop_back_count =
self.jit_native_loop_back_count.saturating_add(1);
continue;
}
if !has_yielding_call {
let ip = self.ip;
let mut next_trace_id = self.jit.compiled_trace_for_ip(ip);
if next_trace_id.is_none() {
let program = &self.program;
next_trace_id = self.jit.observe_exit_ip(ip, program);
}
if let Some(next_trace_id) = next_trace_id
&& next_trace_id != current_trace_id
{
self.record_jit_link_handoff();
current_trace_id = next_trace_id;
if let Err(err) = self.ensure_native_trace(
current_trace_id,
native::NativeCompileProfile::Jit,
) {
if should_fallback_to_interpreter(&err) {
self.record_jit_helper_fallback();
self.jit.block_trace(current_trace_id);
return Ok(native::STATUS_LINKED_CONTINUE);
}
return Err(err);
}
(entry, root_ip, terminal, has_call, has_yielding_call) =
self.native_trace_state(current_trace_id)?;
continue;
}
}
return Ok(native::STATUS_LINKED_CONTINUE);
}
native::STATUS_HALTED
| native::STATUS_YIELDED
| native::STATUS_WAITING
| native::STATUS_OUT_OF_FUEL
| native::STATUS_ERROR => return Ok(status),
other => {
return Err(VmError::JitNative(format!(
"unexpected linked native trace return status {}",
other
)));
}
}
}
}
fn active_native_interrupt_settings(&self) -> Option<native::NativeInterruptSettings> {
match self.interrupt_mode {
super::super::InterruptMode::None => None,
super::super::InterruptMode::Fuel => Some(native::NativeInterruptSettings::fuel(
self.fuel_check_interval,
)),
super::super::InterruptMode::Epoch => Some(native::NativeInterruptSettings::epoch(
self.fuel_check_interval,
)),
}
}
pub fn set_jit_config(&mut self, config: super::JitConfig) {
if config.enabled {
self.ensure_program_cache_key();
}
self.native_traces.clear();
self.native_trace_exec_count = 0;
self.jit_trace_exit_count = 0;
self.jit_native_loop_back_count = 0;
self.jit_native_link_handoff_count = 0;
self.jit_native_link_dispatch_depth = 0;
self.jit_helper_fallback_count = 0;
self.jit.set_config(config);
}
pub fn jit_config(&self) -> &super::JitConfig {
self.jit.config()
}
pub fn jit_snapshot(&self) -> super::JitSnapshot {
self.jit.snapshot(self.jit_runtime_metrics())
}
pub fn dump_jit_info(&self) -> String {
self.dump_jit_info_with_machine_code(true)
}
pub fn dump_jit_info_with_machine_code(&self, include_machine_code: bool) -> String {
let mut out = self
.jit
.dump_text(self.program.debug.as_ref(), self.jit_runtime_metrics());
out.push_str(&format!(
" native codegen backend: {}\n",
native::selected_codegen_backend()
));
out.push_str(&format!(
" native trace executions: {}\n",
self.native_trace_exec_count
));
out.push_str(&format!(
" native trace handoffs: {}\n",
self.jit_native_link_handoff_count
));
if self.jit_native_bridge_stats_enabled {
let mut bridge_entries: Vec<(&'static str, u64)> = self
.jit_native_bridge_counts
.iter()
.map(|(name, count)| (*name, *count))
.collect();
bridge_entries.sort_unstable_by_key(|(name, _)| *name);
let total_bridge_hits = bridge_entries
.iter()
.fold(0u64, |acc, (_, count)| acc.saturating_add(*count));
out.push_str(&format!(
" native bridge hits: {} (helpers={})\n",
total_bridge_hits,
bridge_entries.len()
));
for (name, count) in bridge_entries {
out.push_str(&format!(" bridge {}: {}\n", name, count));
}
}
if self.native_traces.is_empty() {
out.push_str(" native traces: 0\n");
return out;
}
out.push_str(&format!(" native traces: {}\n", self.native_traces.len()));
let mut ids: Vec<usize> = self.native_traces.keys().copied().collect();
ids.sort_unstable();
for id in ids {
if let Some(native) = self.native_traces.get(&id) {
out.push_str(&format!(
" native trace#{} entry=0x{:X} code_bytes={} lowering={}\n",
id,
native.entry as usize,
native.code.len(),
native.lowering_kind.as_str()
));
if include_machine_code {
out.push_str(" code:");
for byte in native.code.iter() {
out.push_str(&format!(" {:02X}", byte));
}
out.push('\n');
}
}
}
out
}
pub(in crate::vm) fn execute_jit_entry(&mut self, trace_id: usize) -> VmResult<ExecOutcome> {
#[cfg(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
))]
{
match self.execute_jit_native(trace_id) {
Ok(outcome) => Ok(outcome),
Err(err) if should_fallback_to_interpreter(&err) => {
self.record_jit_helper_fallback();
self.jit.block_trace(trace_id);
Ok(ExecOutcome::Continue)
}
Err(err) => Err(err),
}
}
#[cfg(not(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
)))]
{
let _ = trace_id;
Ok(ExecOutcome::Continue)
}
}
#[cfg(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
))]
fn execute_jit_native(&mut self, trace_id: usize) -> VmResult<ExecOutcome> {
let mut current_trace_id = trace_id;
if let Err(err) =
self.ensure_native_trace(current_trace_id, native::NativeCompileProfile::Jit)
{
if should_fallback_to_interpreter(&err) {
self.record_jit_helper_fallback();
self.jit.block_trace(current_trace_id);
return Ok(ExecOutcome::Continue);
}
return Err(err);
}
let (mut entry, mut root_ip, mut terminal, mut has_call, mut has_yielding_call) =
self.native_trace_state(current_trace_id)?;
loop {
native::clear_bridge_error();
unsafe { crate::vm::native::prepare_for_execution() };
let status = unsafe { entry(self as *mut Vm) };
self.native_trace_exec_count = self.native_trace_exec_count.saturating_add(1);
self.jit.mark_trace_executed(current_trace_id);
match status {
native::STATUS_CONTINUE => {
if !has_yielding_call
&& let Some(next_trace_id) = self.jit.compiled_trace_for_ip(self.ip)
&& next_trace_id != current_trace_id
{
current_trace_id = next_trace_id;
if let Err(err) = self.ensure_native_trace(
current_trace_id,
native::NativeCompileProfile::Jit,
) {
if should_fallback_to_interpreter(&err) {
self.record_jit_helper_fallback();
self.jit.block_trace(current_trace_id);
return Ok(ExecOutcome::Continue);
}
return Err(err);
}
(entry, root_ip, terminal, has_call, has_yielding_call) =
self.native_trace_state(current_trace_id)?;
continue;
}
return Ok(ExecOutcome::Continue);
}
native::STATUS_TRACE_EXIT => {
self.jit_trace_exit_count = self.jit_trace_exit_count.saturating_add(1);
if has_call {
return Ok(ExecOutcome::Continue);
}
if !has_yielding_call
&& terminal == JitTraceTerminal::LoopBack
&& self.ip == root_ip
{
self.jit_native_loop_back_count =
self.jit_native_loop_back_count.saturating_add(1);
continue;
}
if !has_yielding_call {
let ip = self.ip;
let mut next_trace_id = self.jit.compiled_trace_for_ip(ip);
if next_trace_id.is_none() {
next_trace_id = {
let program = &self.program;
self.jit.observe_exit_ip(ip, program)
};
}
if let Some(next_trace_id) = next_trace_id
&& next_trace_id != current_trace_id
{
current_trace_id = next_trace_id;
if let Err(err) = self.ensure_native_trace(
current_trace_id,
native::NativeCompileProfile::Jit,
) {
if should_fallback_to_interpreter(&err) {
self.record_jit_helper_fallback();
self.jit.block_trace(current_trace_id);
return Ok(ExecOutcome::Continue);
}
return Err(err);
}
(entry, root_ip, terminal, has_call, has_yielding_call) =
self.native_trace_state(current_trace_id)?;
continue;
}
}
return Ok(ExecOutcome::Continue);
}
native::STATUS_HALTED => return Ok(ExecOutcome::Halted),
native::STATUS_LINKED_CONTINUE => return Ok(ExecOutcome::Continue),
native::STATUS_YIELDED => {
self.last_yield_reason = Some(super::super::VmYieldReason::Host);
return Ok(ExecOutcome::Yielded);
}
native::STATUS_WAITING => {
let op_id = self.waiting_host_op.map(|op| op.op_id).ok_or_else(|| {
VmError::JitNative(
"native call bridge reported waiting without a pending op".to_string(),
)
})?;
return Ok(ExecOutcome::Waiting(op_id));
}
native::STATUS_OUT_OF_FUEL => {
return match self.interrupt_mode {
super::super::InterruptMode::Fuel => Err(VmError::OutOfFuel {
needed: u64::from(self.fuel_check_interval),
remaining: self.fuel_remaining,
}),
super::super::InterruptMode::Epoch => Err(VmError::EpochDeadlineReached {
current: self.current_epoch(),
deadline: self.epoch_deadline,
}),
super::super::InterruptMode::None => Err(VmError::JitNative(
"native interruption checkpoint fired while interruption was disabled"
.to_string(),
)),
};
}
native::STATUS_ERROR => {
let err = native::take_bridge_error().unwrap_or_else(|| {
let trace_meta = self.jit.trace_clone(current_trace_id).map(|trace| {
format!(
"trace_id={} root_ip={} terminal={:?} ops={}",
trace.id,
trace.root_ip,
trace.terminal,
trace.op_names.len()
)
});
VmError::JitNative(format!(
"jit bridge reported failure without VmError (ip={} stack_len={} {})",
self.ip,
self.stack.len(),
trace_meta.unwrap_or_else(|| "trace=<missing>".to_string())
))
});
return Err(err);
}
other => {
return Err(VmError::JitNative(format!(
"unexpected native trace return status {}",
other
)));
}
}
}
}
#[cfg(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
))]
fn native_trace_state(
&self,
trace_id: usize,
) -> VmResult<(NativeTraceEntry, usize, JitTraceTerminal, bool, bool)> {
let native = self.native_traces.get(&trace_id).ok_or_else(|| {
VmError::JitNative(format!("native trace entry for id {} missing", trace_id))
})?;
Ok((
native.entry,
native.root_ip,
native.terminal.clone(),
native.has_call,
native.has_yielding_call,
))
}
#[cfg(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
))]
fn ensure_native_trace(
&mut self,
trace_id: usize,
compile_profile: native::NativeCompileProfile,
) -> VmResult<()> {
let interrupt_settings = self.active_native_interrupt_settings();
self.ensure_native_trace_with_settings(trace_id, compile_profile, interrupt_settings)
}
#[cfg(any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
))]
pub(super) fn ensure_native_trace_with_settings(
&mut self,
trace_id: usize,
compile_profile: native::NativeCompileProfile,
interrupt_settings: Option<native::NativeInterruptSettings>,
) -> VmResult<()> {
if let Some(native) = self.native_traces.get(&trace_id)
&& native.interrupt_settings == interrupt_settings
&& compile_profile_satisfies(native.compile_profile, compile_profile)
&& native.drop_contract_events_enabled == self.drop_contract_events_enabled()
{
return Ok(());
}
self.native_traces.remove(&trace_id);
let program_cache_key = self.ensure_program_cache_key();
let trace = self.jit.trace_clone(trace_id).ok_or_else(|| {
VmError::JitNative(format!("trace {} missing for native compile", trace_id))
})?;
let drop_contract_events_enabled = self.drop_contract_events_enabled();
let key = native_trace_cache_key(
&trace,
interrupt_settings,
compile_profile,
drop_contract_events_enabled,
);
let cached = with_native_trace_cache(|cache| {
if cache.active_program_key != Some(program_cache_key) {
cache.entries.clear();
cache.active_program_key = Some(program_cache_key);
}
if let Some(cached) = cache.entries.get(&key).cloned() {
return Some(cached);
}
None
});
if let Some(cached) = cached {
self.native_traces.insert(
trace_id,
NativeTrace {
_keepalive: cached.keepalive,
entry: cached.entry,
code: cached.code,
root_ip: trace.root_ip,
terminal: trace.terminal,
has_call: trace.has_call,
has_yielding_call: trace.has_yielding_call,
lowering_kind: cached.lowering_kind,
interrupt_settings,
compile_profile: cached.compile_profile,
drop_contract_events_enabled,
},
);
return Ok(());
}
let compiled = native::compile_native_trace(
&trace,
interrupt_settings,
compile_profile,
drop_contract_events_enabled,
)?;
let entry = unsafe { std::mem::transmute::<*const u8, NativeTraceEntry>(compiled.entry) };
let code = Arc::<[u8]>::from(compiled.code.into_boxed_slice());
let keepalive = Arc::new(Mutex::new(compiled.keepalive));
let cached = NativeTraceCacheEntry {
entry,
keepalive: Arc::clone(&keepalive),
code: Arc::clone(&code),
lowering_kind: compiled.lowering_kind,
compile_profile,
};
with_native_trace_cache(|cache| {
if cache.active_program_key != Some(program_cache_key) {
cache.entries.clear();
cache.active_program_key = Some(program_cache_key);
}
cache.entries.insert(key, cached);
});
self.native_traces.insert(
trace_id,
NativeTrace {
_keepalive: keepalive,
entry,
code,
root_ip: trace.root_ip,
terminal: trace.terminal,
has_call: trace.has_call,
has_yielding_call: trace.has_yielding_call,
lowering_kind: compiled.lowering_kind,
interrupt_settings,
compile_profile,
drop_contract_events_enabled,
},
);
Ok(())
}
pub fn jit_native_trace_count(&self) -> usize {
self.native_traces.len()
}
pub fn jit_native_exec_count(&self) -> u64 {
self.native_trace_exec_count
}
pub fn jit_native_link_handoff_count(&self) -> u64 {
self.jit_native_link_handoff_count
}
fn jit_runtime_metrics(&self) -> JitMetrics {
JitMetrics {
boxed_load_site_count: 0,
boxed_store_site_count: 0,
trace_exit_count: self.jit_trace_exit_count,
native_loop_back_count: self.jit_native_loop_back_count,
helper_fallback_count: self.jit_helper_fallback_count,
native_trace_exec_count: self.native_trace_exec_count,
}
}
fn record_jit_helper_fallback(&mut self) {
self.jit_helper_fallback_count = self.jit_helper_fallback_count.saturating_add(1);
}
fn record_jit_link_handoff(&mut self) {
self.jit_native_link_handoff_count = self.jit_native_link_handoff_count.saturating_add(1);
}
}
#[cfg(all(
test,
any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
)
))]
pub(crate) fn clear_native_trace_cache_for_tests() {
with_native_trace_cache(|cache| {
cache.entries.clear();
cache.active_program_key = None;
});
}
#[cfg(all(
test,
any(
all(
target_arch = "x86_64",
any(target_os = "windows", all(unix, not(target_os = "macos")))
),
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
)
))]
pub(crate) fn native_trace_cache_snapshot_for_tests() -> (Option<u64>, usize) {
with_native_trace_cache(|cache| (cache.active_program_key, cache.entries.len()))
}