use std::ffi::{CStr, CString, c_char, c_void};
use std::fmt;
pub const TDR_DEFAULT_LEVEL: u32 = 3;
pub const TDR_DEFAULT_DELAY_S: u32 = 2;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DriverModel {
Wddm,
Tcc,
Mcdm,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct WatchdogState {
pub delay_ns: Option<u64>,
pub basis: String,
}
impl WatchdogState {
pub fn applies(&self) -> bool {
self.delay_ns.is_some()
}
}
impl fmt::Display for WatchdogState {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.basis)
}
}
pub type TdrSettings = (u32, u32);
pub fn decide(
model: &Result<DriverModel, String>,
tdr: Option<&Result<TdrSettings, String>>,
) -> WatchdogState {
let (model_text, outside_tdr) = match model {
Ok(DriverModel::Tcc) => (format!("driver model {:?}", DriverModel::Tcc), true),
Ok(covered) => (format!("driver model {covered:?}"), false),
Err(err) => (format!("driver model unreadable ({err})"), false),
};
let Some(tdr) = tdr else {
return WatchdogState {
delay_ns: None,
basis: format!("{model_text}; no watchdog is known on this platform"),
};
};
if outside_tdr {
return WatchdogState {
delay_ns: None,
basis: format!("{model_text}; TCC devices are outside TDR"),
};
}
match tdr {
Ok((0, delay_s)) => WatchdogState {
delay_ns: None,
basis: format!("{model_text}; TdrLevel 0 disables detection (TdrDelay {delay_s} s)"),
},
Ok((level, delay_s)) => WatchdogState {
delay_ns: Some(u64::from(*delay_s) * 1_000_000_000),
basis: format!("{model_text}; TdrLevel {level}, TdrDelay {delay_s} s"),
},
Err(err) => WatchdogState {
delay_ns: Some(u64::from(TDR_DEFAULT_DELAY_S) * 1_000_000_000),
basis: format!(
"{model_text}; TDR settings unreadable ({err}), so the documented \
{TDR_DEFAULT_DELAY_S} s delay is taken"
),
},
}
}
pub fn detect(ordinal: usize) -> WatchdogState {
detect_with_model(ordinal).1
}
pub fn detect_with_model(ordinal: usize) -> (Result<DriverModel, String>, WatchdogState) {
const CACHED_ORDINALS: usize = 64;
type Reading = (Result<DriverModel, String>, WatchdogState);
static CACHE: [std::sync::OnceLock<Reading>; CACHED_ORDINALS] =
[const { std::sync::OnceLock::new() }; CACHED_ORDINALS];
match CACHE.get(ordinal) {
Some(slot) => slot.get_or_init(|| read_watchdog(ordinal)).clone(),
None => read_watchdog(ordinal),
}
}
fn read_watchdog(ordinal: usize) -> (Result<DriverModel, String>, WatchdogState) {
let model = pci_bus_id(ordinal).and_then(|bus_id| nvml_driver_model(&bus_id));
#[cfg(windows)]
let tdr = Some(tdr_settings());
#[cfg(not(windows))]
let tdr: Option<Result<TdrSettings, String>> = None;
let state = decide(&model, tdr.as_ref());
(model, state)
}
fn pci_bus_id(ordinal: usize) -> Result<String, String> {
use cudarc::driver::sys as cu;
if !crate::backend::detect::cuda_available() {
return Err("the CUDA driver is not loadable".to_string());
}
if !unsafe { cu::is_culib_present() } {
return Err("cudarc found no CUDA driver library".to_string());
}
let rc = unsafe { cu::cuInit(0) };
if rc != cu::CUresult::CUDA_SUCCESS {
return Err(format!("cuInit returned {rc:?}"));
}
let driver_ordinal = i32::try_from(ordinal)
.map_err(|err| format!("device ordinal {ordinal} does not fit the driver's type ({err})"))?;
let mut dev: cu::CUdevice = 0;
let rc = unsafe { cu::cuDeviceGet(&mut dev, driver_ordinal) };
if rc != cu::CUresult::CUDA_SUCCESS {
return Err(format!("cuDeviceGet({ordinal}) returned {rc:?}"));
}
let mut buf = [0 as c_char; 32];
let rc = unsafe { cu::cuDeviceGetPCIBusId(buf.as_mut_ptr(), buf.len() as i32, dev) };
if rc != cu::CUresult::CUDA_SUCCESS {
return Err(format!("cuDeviceGetPCIBusId({ordinal}) returned {rc:?}"));
}
let id = unsafe { CStr::from_ptr(buf.as_ptr()) };
id.to_str()
.map(str::to_string)
.map_err(|err| format!("the PCI bus id is not UTF-8 ({err})"))
}
#[cfg(windows)]
const NVML_LIBRARY: &str = "nvml.dll";
#[cfg(not(windows))]
const NVML_LIBRARY: &str = "libnvidia-ml.so.1";
const NVML_SUCCESS: i32 = 0;
const NVML_DRIVER_WDDM: i32 = 0;
const NVML_DRIVER_WDM: i32 = 1;
const NVML_DRIVER_MCDM: i32 = 2;
type NvmlDevice = *mut c_void;
type NvmlInit = unsafe extern "C" fn() -> i32;
type NvmlByBusId = unsafe extern "C" fn(*const c_char, *mut NvmlDevice) -> i32;
type NvmlDriverModel = unsafe extern "C" fn(NvmlDevice, *mut i32, *mut i32) -> i32;
fn nvml_driver_model(bus_id: &str) -> Result<DriverModel, String> {
let lib = unsafe { libloading::Library::new(NVML_LIBRARY) }
.map_err(|err| format!("{NVML_LIBRARY} did not load ({err})"))?;
let init: libloading::Symbol<NvmlInit> = unsafe { lib.get(b"nvmlInit_v2") }
.map_err(|err| format!("nvmlInit_v2 is missing ({err})"))?;
let shutdown: libloading::Symbol<NvmlInit> = unsafe { lib.get(b"nvmlShutdown") }
.map_err(|err| format!("nvmlShutdown is missing ({err})"))?;
let by_bus: libloading::Symbol<NvmlByBusId> =
unsafe { lib.get(b"nvmlDeviceGetHandleByPciBusId_v2") }
.map_err(|err| format!("nvmlDeviceGetHandleByPciBusId_v2 is missing ({err})"))?;
let model_fn: libloading::Symbol<NvmlDriverModel> =
match unsafe { lib.get(b"nvmlDeviceGetDriverModel_v2") } {
Ok(f) => f,
Err(v2_err) => unsafe { lib.get(b"nvmlDeviceGetDriverModel") }.map_err(|err| {
format!(
"neither nvmlDeviceGetDriverModel_v2 ({v2_err}) nor \
nvmlDeviceGetDriverModel ({err}) is exported"
)
})?,
};
let bus = CString::new(bus_id)
.map_err(|err| format!("the PCI bus id {bus_id:?} holds a NUL ({err})"))?;
let rc = unsafe { init() };
if rc != NVML_SUCCESS {
return Err(format!("nvmlInit_v2 returned {rc}"));
}
let mut device: NvmlDevice = std::ptr::null_mut();
let rc = unsafe { by_bus(bus.as_ptr(), &mut device) };
let result = if rc != NVML_SUCCESS {
Err(format!("nvmlDeviceGetHandleByPciBusId_v2({bus_id}) returned {rc}"))
} else {
let mut current = -1i32;
let mut pending = -1i32;
let rc = unsafe { model_fn(device, &mut current, &mut pending) };
match (rc, current) {
(NVML_SUCCESS, NVML_DRIVER_WDDM) => Ok(DriverModel::Wddm),
(NVML_SUCCESS, NVML_DRIVER_WDM) => Ok(DriverModel::Tcc),
(NVML_SUCCESS, NVML_DRIVER_MCDM) => Ok(DriverModel::Mcdm),
(NVML_SUCCESS, other) => Err(format!("NVML reported driver model {other}")),
(failed, word) => Err(format!(
"the driver model query returned {failed}, leaving the model word at {word}"
)),
}
};
let rc = unsafe { shutdown() };
if rc != NVML_SUCCESS {
notice!("flynnel gpu_peer watchdog: nvmlShutdown returned {rc}");
}
result
}
#[cfg(windows)]
fn tdr_settings() -> Result<TdrSettings, String> {
let level = match registry_dword("TdrLevel")? {
Some(value) => value,
None => TDR_DEFAULT_LEVEL,
};
let delay_s = match registry_dword("TdrDelay")? {
Some(value) => value,
None => TDR_DEFAULT_DELAY_S,
};
Ok((level, delay_s))
}
#[cfg(windows)]
fn registry_dword(name: &str) -> Result<Option<u32>, String> {
use std::os::windows::ffi::OsStrExt;
const HKEY_LOCAL_MACHINE: isize = 0x8000_0002_u32 as i32 as isize;
const RRF_RT_REG_DWORD: u32 = 0x0000_0010;
const ERROR_SUCCESS: i32 = 0;
const ERROR_FILE_NOT_FOUND: i32 = 2;
#[link(name = "advapi32")]
unsafe extern "system" {
fn RegGetValueW(
hkey: isize,
sub_key: *const u16,
value: *const u16,
flags: u32,
kind: *mut u32,
data: *mut c_void,
data_len: *mut u32,
) -> i32;
}
let wide = |s: &str| -> Vec<u16> { std::ffi::OsStr::new(s).encode_wide().chain(Some(0)).collect() };
let key = wide(r"SYSTEM\CurrentControlSet\Control\GraphicsDrivers");
let value_name = wide(name);
let mut data: u32 = 0;
let mut len: u32 = 4;
let status = unsafe {
RegGetValueW(
HKEY_LOCAL_MACHINE,
key.as_ptr(),
value_name.as_ptr(),
RRF_RT_REG_DWORD,
std::ptr::null_mut(),
(&mut data as *mut u32).cast(),
&mut len,
)
};
match status {
ERROR_SUCCESS => Ok(Some(data)),
ERROR_FILE_NOT_FOUND => Ok(None),
other => Err(format!("RegGetValueW(GraphicsDrivers\\{name}) returned {other}")),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_platform_without_tdr_has_no_watchdog() {
let state = decide(&Ok(DriverModel::Wddm), None);
assert!(!state.applies());
assert!(state.basis.contains("no watchdog is known"), "{}", state.basis);
}
#[test]
fn a_tcc_device_is_outside_tdr_whatever_the_settings() {
let state = decide(&Ok(DriverModel::Tcc), Some(&Ok((3, 2))));
assert!(!state.applies());
assert!(state.basis.contains("TCC"), "{}", state.basis);
}
#[test]
fn tdr_level_zero_disables_the_watchdog() {
let state = decide(&Ok(DriverModel::Wddm), Some(&Ok((0, 2))));
assert!(!state.applies());
assert!(state.basis.contains("TdrLevel 0"), "{}", state.basis);
}
#[test]
fn a_covered_device_takes_the_registry_delay() {
let wddm = decide(&Ok(DriverModel::Wddm), Some(&Ok((3, 8))));
assert_eq!(wddm.delay_ns, Some(8_000_000_000));
let mcdm = decide(&Ok(DriverModel::Mcdm), Some(&Ok((3, 2))));
assert_eq!(mcdm.delay_ns, Some(2_000_000_000));
}
#[test]
fn unreadable_settings_take_the_documented_delay_and_say_so() {
let state = decide(&Ok(DriverModel::Wddm), Some(&Err("denied".to_string())));
assert_eq!(state.delay_ns, Some(u64::from(TDR_DEFAULT_DELAY_S) * 1_000_000_000));
assert!(state.basis.contains("unreadable (denied)"), "{}", state.basis);
}
#[test]
fn an_unreadable_driver_model_is_treated_as_covered() {
let state = decide(&Err("no NVML".to_string()), Some(&Ok((3, 2))));
assert_eq!(state.delay_ns, Some(2_000_000_000));
assert!(state.basis.contains("driver model unreadable (no NVML)"), "{}", state.basis);
}
#[test]
fn the_model_returned_beside_a_state_is_the_one_that_state_was_decided_from() {
let (model, state) = detect_with_model(0);
let expected = match &model {
Ok(DriverModel::Tcc) => "driver model Tcc".to_string(),
Ok(covered) => format!("driver model {covered:?}"),
Err(err) => format!("driver model unreadable ({err})"),
};
assert!(
state.basis.starts_with(&expected),
"the state says {:?} and the model beside it is {:?}",
state.basis,
model
);
}
#[test]
fn detect_answers_the_state_half_of_the_pair() {
assert_eq!(detect(0), detect_with_model(0).1);
}
}