use super::Timer;
#[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "kperf"))]
use super::kperf::PmuTimer;
#[cfg(all(target_os = "linux", feature = "perf"))]
use super::perf::LinuxPerfTimer;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum TimerError {
UnknownOrUnavailable(String),
InitializationFailed(String),
}
impl std::fmt::Display for TimerError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
TimerError::UnknownOrUnavailable(name) => {
write!(
f,
"Timer '{}' is unknown or not available on this platform. \
Available timers: {}",
name,
TimerSpec::available_names().join(", ")
)
}
TimerError::InitializationFailed(msg) => {
write!(f, "Timer initialization failed: {}", msg)
}
}
}
}
impl std::error::Error for TimerError {}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum TimerFallbackReason {
#[default]
None,
Requested,
ConcurrentAccess,
NoPrivileges,
CycleCounterUnavailable,
}
impl TimerFallbackReason {
pub fn as_str(&self) -> Option<&'static str> {
match self {
TimerFallbackReason::None => None,
TimerFallbackReason::Requested => Some("user requested"),
TimerFallbackReason::ConcurrentAccess => Some("concurrent access"),
TimerFallbackReason::NoPrivileges => Some("no sudo"),
TimerFallbackReason::CycleCounterUnavailable => Some("unavailable"),
}
}
}
#[allow(clippy::large_enum_variant)] pub enum BoxedTimer {
Standard(Timer),
#[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "kperf"))]
Kperf(PmuTimer),
#[cfg(all(target_os = "linux", feature = "perf"))]
Perf(LinuxPerfTimer),
}
impl BoxedTimer {
#[inline]
pub fn measure_cycles<F, T>(&mut self, f: F) -> super::error::MeasurementResult
where
F: FnOnce() -> T,
{
match self {
BoxedTimer::Standard(t) => t.measure_cycles(f),
#[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "kperf"))]
BoxedTimer::Kperf(t) => t.measure_cycles(f),
#[cfg(all(target_os = "linux", feature = "perf"))]
BoxedTimer::Perf(t) => t.measure_cycles(f),
}
}
#[inline]
pub fn cycles_to_ns(&self, cycles: u64) -> f64 {
match self {
BoxedTimer::Standard(t) => t.cycles_to_ns(cycles),
#[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "kperf"))]
BoxedTimer::Kperf(t) => t.cycles_to_ns(cycles),
#[cfg(all(target_os = "linux", feature = "perf"))]
BoxedTimer::Perf(t) => t.cycles_to_ns(cycles),
}
}
pub fn resolution_ns(&self) -> f64 {
match self {
BoxedTimer::Standard(t) => t.resolution_ns(),
#[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "kperf"))]
BoxedTimer::Kperf(t) => t.resolution_ns(),
#[cfg(all(target_os = "linux", feature = "perf"))]
BoxedTimer::Perf(t) => t.resolution_ns(),
}
}
pub fn cycles_per_ns(&self) -> f64 {
match self {
BoxedTimer::Standard(t) => t.cycles_per_ns(),
#[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "kperf"))]
BoxedTimer::Kperf(t) => t.cycles_per_ns(),
#[cfg(all(target_os = "linux", feature = "perf"))]
BoxedTimer::Perf(t) => t.cycles_per_ns(),
}
}
pub fn name(&self) -> &'static str {
match self {
BoxedTimer::Standard(_) => {
#[cfg(target_arch = "x86_64")]
{
"rdtsc"
}
#[cfg(target_arch = "aarch64")]
{
"cntvct_el0"
}
#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
{
"Instant"
}
}
#[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "kperf"))]
BoxedTimer::Kperf(_) => "kperf",
#[cfg(all(target_os = "linux", feature = "perf"))]
BoxedTimer::Perf(_) => "perf_event",
}
}
}
impl std::fmt::Debug for BoxedTimer {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("BoxedTimer")
.field("name", &self.name())
.field("cycles_per_ns", &self.cycles_per_ns())
.field("resolution_ns", &self.resolution_ns())
.finish()
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum TimerSpec {
#[default]
Auto,
SystemTimer,
RequireHighPrecision,
RequireCycleAccurate,
#[cfg(target_arch = "x86_64")]
Rdtsc,
#[cfg(target_arch = "aarch64")]
VirtualTimer,
#[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "kperf"))]
Kperf,
#[cfg(all(target_os = "linux", feature = "perf"))]
PerfEvent,
StdInstant,
}
impl TimerSpec {
pub fn by_name(name: &str) -> Result<TimerSpec, TimerError> {
match name.to_lowercase().as_str() {
"auto" => Ok(TimerSpec::Auto),
"system" | "systemtimer" | "system_timer" => Ok(TimerSpec::SystemTimer),
"highprecision" | "high_precision" | "requirehighprecision" => {
Ok(TimerSpec::RequireHighPrecision)
}
"cycle" | "cycleaccurate" | "cycle_accurate" => Ok(TimerSpec::RequireCycleAccurate),
"instant" | "std" | "stdinstant" | "std_instant" => Ok(TimerSpec::StdInstant),
#[cfg(target_arch = "x86_64")]
"rdtsc" | "tsc" => Ok(TimerSpec::Rdtsc),
#[cfg(not(target_arch = "x86_64"))]
"rdtsc" | "tsc" => Err(TimerError::UnknownOrUnavailable(name.to_string())),
#[cfg(target_arch = "aarch64")]
"cntvct" | "cntvct_el0" | "virtualtimer" | "virtual_timer" => {
Ok(TimerSpec::VirtualTimer)
}
#[cfg(not(target_arch = "aarch64"))]
"cntvct" | "cntvct_el0" | "virtualtimer" | "virtual_timer" => {
Err(TimerError::UnknownOrUnavailable(name.to_string()))
}
#[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "kperf"))]
"kperf" | "pmu" | "pmccntr" | "pmccntr_el0" => Ok(TimerSpec::Kperf),
#[cfg(not(all(target_os = "macos", target_arch = "aarch64", feature = "kperf")))]
"kperf" | "pmu" | "pmccntr" | "pmccntr_el0" => {
Err(TimerError::UnknownOrUnavailable(name.to_string()))
}
#[cfg(all(target_os = "linux", feature = "perf"))]
"perf" | "perf_event" | "perfevent" => Ok(TimerSpec::PerfEvent),
#[cfg(not(all(target_os = "linux", feature = "perf")))]
"perf" | "perf_event" | "perfevent" => {
Err(TimerError::UnknownOrUnavailable(name.to_string()))
}
_ => Err(TimerError::UnknownOrUnavailable(name.to_string())),
}
}
pub fn available_names() -> &'static [&'static str] {
&[
"auto",
"system",
"highprecision",
"cycle",
"instant",
#[cfg(target_arch = "x86_64")]
"rdtsc",
#[cfg(target_arch = "aarch64")]
"cntvct_el0",
#[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "kperf"))]
"kperf",
#[cfg(all(target_os = "linux", feature = "perf"))]
"perf_event",
]
}
pub fn create_timer(&self) -> (BoxedTimer, TimerFallbackReason) {
match self {
TimerSpec::SystemTimer => (
BoxedTimer::Standard(Timer::new()),
TimerFallbackReason::Requested,
),
TimerSpec::StdInstant => {
(
BoxedTimer::Standard(Timer::new()),
TimerFallbackReason::Requested,
)
}
#[cfg(target_arch = "x86_64")]
TimerSpec::Rdtsc => {
(
BoxedTimer::Standard(Timer::new()),
TimerFallbackReason::None,
)
}
#[cfg(target_arch = "aarch64")]
TimerSpec::VirtualTimer => {
(
BoxedTimer::Standard(Timer::new()),
TimerFallbackReason::None,
)
}
#[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "kperf"))]
TimerSpec::Kperf => {
use super::kperf::PmuError;
match PmuTimer::new() {
Ok(pmu) => (BoxedTimer::Kperf(pmu), TimerFallbackReason::None),
Err(PmuError::ConcurrentAccess) => {
panic!(
"Kperf: cycle counter locked by another process. \
Run with --test-threads=1 for exclusive access, \
or use TimerSpec::Auto to fall back to system timer."
);
}
Err(e) => {
panic!("Kperf: initialization failed: {:?}", e);
}
}
}
#[cfg(all(target_os = "linux", feature = "perf"))]
TimerSpec::PerfEvent => match LinuxPerfTimer::new() {
Ok(perf) => (BoxedTimer::Perf(perf), TimerFallbackReason::None),
Err(e) => {
panic!("PerfEvent: initialization failed: {:?}", e);
}
},
#[allow(clippy::needless_return)] TimerSpec::Auto => {
#[cfg(target_arch = "x86_64")]
{
return (
BoxedTimer::Standard(Timer::new()),
TimerFallbackReason::None,
);
}
#[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "kperf"))]
{
use super::kperf::PmuError;
match PmuTimer::new() {
Ok(pmu) => (BoxedTimer::Kperf(pmu), TimerFallbackReason::None),
Err(PmuError::ConcurrentAccess) => {
tracing::warn!(
"Cycle counter (kperf) locked by another process. \
Falling back to system timer."
);
(
BoxedTimer::Standard(Timer::new()),
TimerFallbackReason::ConcurrentAccess,
)
}
Err(_) => {
(
BoxedTimer::Standard(Timer::new()),
TimerFallbackReason::NoPrivileges,
)
}
}
}
#[cfg(all(target_os = "linux", target_arch = "aarch64", feature = "perf"))]
{
match LinuxPerfTimer::new() {
Ok(perf) => return (BoxedTimer::Perf(perf), TimerFallbackReason::None),
Err(_) => {
(
BoxedTimer::Standard(Timer::new()),
TimerFallbackReason::NoPrivileges,
)
}
}
}
#[cfg(not(any(
target_arch = "x86_64",
all(target_os = "macos", target_arch = "aarch64", feature = "kperf"),
all(target_os = "linux", target_arch = "aarch64", feature = "perf")
)))]
{
(
BoxedTimer::Standard(Timer::new()),
TimerFallbackReason::None,
)
}
}
#[allow(clippy::needless_return)] TimerSpec::RequireHighPrecision => {
const HIGH_PRECISION_THRESHOLD_NS: f64 = 2.0;
let system_timer = Timer::new();
if system_timer.resolution_ns() <= HIGH_PRECISION_THRESHOLD_NS {
return (
BoxedTimer::Standard(system_timer),
TimerFallbackReason::None,
);
}
#[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "kperf"))]
{
use super::kperf::PmuError;
match PmuTimer::new() {
Ok(pmu) => return (BoxedTimer::Kperf(pmu), TimerFallbackReason::None),
Err(PmuError::ConcurrentAccess) => {
panic!(
"RequireHighPrecision: System timer resolution ({:.1}ns) exceeds \
{HIGH_PRECISION_THRESHOLD_NS}ns threshold, and kperf is locked by \
another process. Run with --test-threads=1 for exclusive kperf access.",
system_timer.resolution_ns()
);
}
Err(e) => {
panic!(
"RequireHighPrecision: System timer resolution ({:.1}ns) exceeds \
{HIGH_PRECISION_THRESHOLD_NS}ns threshold, and kperf initialization \
failed: {:?}. Run with sudo for kperf access.",
system_timer.resolution_ns(),
e
);
}
}
}
#[cfg(all(target_os = "linux", feature = "perf"))]
{
match LinuxPerfTimer::new() {
Ok(perf) => return (BoxedTimer::Perf(perf), TimerFallbackReason::None),
Err(e) => {
panic!(
"RequireHighPrecision: System timer resolution ({:.1}ns) exceeds \
{HIGH_PRECISION_THRESHOLD_NS}ns threshold, and perf_event \
initialization failed: {:?}. Run with sudo or CAP_PERFMON.",
system_timer.resolution_ns(),
e
);
}
}
}
#[cfg(not(any(
all(target_os = "macos", target_arch = "aarch64", feature = "kperf"),
all(target_os = "linux", feature = "perf")
)))]
{
panic!(
"RequireHighPrecision: System timer resolution ({:.1}ns) exceeds \
{HIGH_PRECISION_THRESHOLD_NS}ns threshold, and no PMU timer is available. \
On ARM64, build with --features kperf (macOS) or --features perf (Linux).",
system_timer.resolution_ns()
);
}
}
#[allow(clippy::needless_return)] TimerSpec::RequireCycleAccurate => {
#[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "kperf"))]
{
use super::kperf::PmuError;
match PmuTimer::new() {
Ok(pmu) => (BoxedTimer::Kperf(pmu), TimerFallbackReason::None),
Err(PmuError::ConcurrentAccess) => {
panic!(
"RequireCycleAccurate: kperf unavailable due to concurrent access. \
Run with --test-threads=1 for exclusive access, \
or use TimerSpec::Auto to fall back to system timer."
);
}
Err(e) => {
panic!("RequireCycleAccurate: kperf initialization failed: {:?}", e);
}
}
}
#[cfg(all(target_os = "linux", feature = "perf"))]
{
match LinuxPerfTimer::new() {
Ok(perf) => return (BoxedTimer::Perf(perf), TimerFallbackReason::None),
Err(e) => {
panic!(
"RequireCycleAccurate: perf_event initialization failed: {:?}",
e
);
}
}
}
#[cfg(all(
target_arch = "x86_64",
not(any(
all(target_os = "macos", target_arch = "aarch64", feature = "kperf"),
all(target_os = "linux", feature = "perf")
))
))]
{
(
BoxedTimer::Standard(Timer::new()),
TimerFallbackReason::None,
)
}
#[cfg(not(any(
target_arch = "x86_64",
all(target_os = "macos", target_arch = "aarch64", feature = "kperf"),
all(target_os = "linux", feature = "perf")
)))]
{
panic!(
"RequireCycleAccurate: Cycle-accurate timing not available on this platform. \
Use TimerSpec::Auto or TimerSpec::SystemTimer instead."
);
}
}
}
}
#[allow(clippy::needless_return)] pub fn cycle_accurate_available() -> bool {
#[cfg(target_arch = "x86_64")]
{
return true;
}
#[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "kperf"))]
{
return PmuTimer::new().is_ok();
}
#[cfg(all(target_os = "linux", target_arch = "aarch64", feature = "perf"))]
{
return LinuxPerfTimer::new().is_ok();
}
#[cfg(all(
target_arch = "aarch64",
not(all(target_os = "macos", feature = "kperf")),
not(all(target_os = "linux", feature = "perf"))
))]
{
return false;
}
#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
{
return false;
}
}
}
impl std::fmt::Display for TimerSpec {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
TimerSpec::Auto => write!(f, "Auto"),
TimerSpec::SystemTimer => write!(f, "SystemTimer"),
TimerSpec::RequireHighPrecision => write!(f, "RequireHighPrecision"),
TimerSpec::RequireCycleAccurate => write!(f, "RequireCycleAccurate"),
TimerSpec::StdInstant => write!(f, "StdInstant"),
#[cfg(target_arch = "x86_64")]
TimerSpec::Rdtsc => write!(f, "Rdtsc"),
#[cfg(target_arch = "aarch64")]
TimerSpec::VirtualTimer => write!(f, "VirtualTimer"),
#[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "kperf"))]
TimerSpec::Kperf => write!(f, "Kperf"),
#[cfg(all(target_os = "linux", feature = "perf"))]
TimerSpec::PerfEvent => write!(f, "PerfEvent"),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_by_name_user_friendly() {
assert_eq!(TimerSpec::by_name("auto").unwrap(), TimerSpec::Auto);
assert_eq!(
TimerSpec::by_name("system").unwrap(),
TimerSpec::SystemTimer
);
assert_eq!(
TimerSpec::by_name("systemtimer").unwrap(),
TimerSpec::SystemTimer
);
assert_eq!(
TimerSpec::by_name("cycle").unwrap(),
TimerSpec::RequireCycleAccurate
);
assert_eq!(
TimerSpec::by_name("cycleaccurate").unwrap(),
TimerSpec::RequireCycleAccurate
);
assert_eq!(
TimerSpec::by_name("highprecision").unwrap(),
TimerSpec::RequireHighPrecision
);
assert_eq!(
TimerSpec::by_name("high_precision").unwrap(),
TimerSpec::RequireHighPrecision
);
assert_eq!(
TimerSpec::by_name("instant").unwrap(),
TimerSpec::StdInstant
);
assert_eq!(TimerSpec::by_name("std").unwrap(), TimerSpec::StdInstant);
}
#[test]
fn test_by_name_case_insensitive() {
assert_eq!(TimerSpec::by_name("AUTO").unwrap(), TimerSpec::Auto);
assert_eq!(TimerSpec::by_name("Auto").unwrap(), TimerSpec::Auto);
assert_eq!(
TimerSpec::by_name("SYSTEM").unwrap(),
TimerSpec::SystemTimer
);
}
#[test]
fn test_by_name_unknown() {
assert!(matches!(
TimerSpec::by_name("unknown"),
Err(TimerError::UnknownOrUnavailable(_))
));
}
#[test]
#[cfg(target_arch = "x86_64")]
fn test_by_name_x86() {
assert_eq!(TimerSpec::by_name("rdtsc").unwrap(), TimerSpec::Rdtsc);
assert_eq!(TimerSpec::by_name("tsc").unwrap(), TimerSpec::Rdtsc);
}
#[test]
#[cfg(target_arch = "aarch64")]
fn test_by_name_arm64() {
assert_eq!(
TimerSpec::by_name("cntvct_el0").unwrap(),
TimerSpec::VirtualTimer
);
assert_eq!(
TimerSpec::by_name("virtualtimer").unwrap(),
TimerSpec::VirtualTimer
);
}
#[test]
fn test_available_names() {
let names = TimerSpec::available_names();
assert!(names.contains(&"auto"));
assert!(names.contains(&"system"));
assert!(names.contains(&"highprecision"));
assert!(names.contains(&"cycle"));
assert!(names.contains(&"instant"));
}
#[test]
fn test_fallback_reason_as_str() {
assert_eq!(TimerFallbackReason::None.as_str(), None);
assert_eq!(
TimerFallbackReason::Requested.as_str(),
Some("user requested")
);
assert_eq!(
TimerFallbackReason::ConcurrentAccess.as_str(),
Some("concurrent access")
);
assert_eq!(TimerFallbackReason::NoPrivileges.as_str(), Some("no sudo"));
assert_eq!(
TimerFallbackReason::CycleCounterUnavailable.as_str(),
Some("unavailable")
);
}
#[test]
fn test_timer_error_display() {
let err = TimerError::UnknownOrUnavailable("foo".to_string());
let msg = err.to_string();
assert!(msg.contains("foo"));
assert!(msg.contains("Available timers"));
}
}