1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
//! The CUDA backend's error type. Never a panic path — every device or library absence is a `Result`.
use std::fmt;
use crate::engine::cuda_context::ContextError;
/// Everything that can go wrong talking to a CUDA device.
#[derive(Debug)]
pub enum GpuError {
/// No CUDA device is visible to this process.
NoDevice,
/// `libcuda` or `libnvrtc` could not be dynamically loaded.
LibraryMissing(&'static str),
/// A CUDA driver call failed.
Driver(cudarc::driver::DriverError),
/// NVRTC compilation failed.
Compile {
/// The width the compilation was for.
width: usize,
/// NVRTC's compile log.
log: String,
},
/// A device allocation ran out of memory.
OutOfMemory {
/// Which device ordinal, `None` when the failing call does not say.
device: Option<u32>,
/// The requested size in bytes, `None` when unknown.
bytes: Option<u64>,
},
/// A feature this backend does not implement.
Unsupported(&'static str),
/// A device placement did not resolve.
Topology(crate::engine::partitioned::TopologyError),
/// A partitioned split whose partition `rank` failed at layer `layer` of an earlier call; its partitions no longer hold one consistent sum, so every later call is refused until the caller scatters again.
Poisoned {
/// The partition that failed.
rank: usize,
/// The layer index within that call, in application order.
layer: usize,
},
/// An NCCL call reported a non-success result.
#[cfg(feature = "mpi")]
Nccl {
/// The `ncclResult_t` value, as an `i32`.
code: i32,
/// What was being attempted.
what: String,
},
/// A device wire of an exchange failed or was given up.
Wire(&'static str),
/// A bounded wait on a device wire exceeded its deadline; see `PAULISTRINGS_WIRE_TIMEOUT_S`.
Timeout {
/// What was being waited on.
what: &'static str,
},
}
impl fmt::Display for GpuError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
GpuError::NoDevice => write!(f, "no CUDA device is visible to this process"),
GpuError::LibraryMissing(lib) => write!(f, "{lib} could not be loaded"),
GpuError::Driver(e) => write!(f, "CUDA driver error: {e}"),
GpuError::Compile { width, log } => {
write!(f, "NVRTC compilation failed at W={width}: {log}")
},
GpuError::OutOfMemory { device, bytes } => {
match device {
Some(device) => write!(f, "device {device} out of memory")?,
None => write!(f, "CUDA device out of memory")?,
}
match bytes {
Some(bytes) => write!(f, " (requested {bytes} bytes)"),
None => Ok(()),
}
}
GpuError::Unsupported(what) => write!(f, "unsupported on the CUDA backend: {what}"),
GpuError::Topology(e) => write!(f, "device placement: {e}"),
GpuError::Poisoned { rank, layer } => write!(
f,
"device partition {rank} failed at layer {layer} of an earlier propagate; scatter again"
),
#[cfg(feature = "mpi")]
GpuError::Nccl { code, what } => write!(f, "NCCL error {code} during {what}"),
GpuError::Wire(what) => write!(f, "device wire failed: {what}"),
GpuError::Timeout { what } => write!(f, "timed out waiting on {what}"),
}
}
}
impl std::error::Error for GpuError {}
impl GpuError {
/// `e` with an out-of-memory result attributed to `device` and a request of `bytes`.
pub(crate) fn from_alloc(e: cudarc::driver::DriverError, device: u32, bytes: u64) -> Self {
match GpuError::from(e) {
GpuError::OutOfMemory { .. } => {
GpuError::OutOfMemory { device: Some(device), bytes: Some(bytes) }
}
other => other,
}
}
}
impl From<cudarc::driver::DriverError> for GpuError {
/// `CUDA_ERROR_OUT_OF_MEMORY` maps to [`GpuError::OutOfMemory`] with neither `device` nor `bytes`, since the driver error carries neither.
fn from(e: cudarc::driver::DriverError) -> Self {
if e.0 == cudarc::driver::sys::CUresult::CUDA_ERROR_OUT_OF_MEMORY {
GpuError::OutOfMemory { device: None, bytes: None }
} else {
GpuError::Driver(e)
}
}
}
impl From<ContextError> for GpuError {
fn from(e: ContextError) -> Self {
match e {
ContextError::LibraryMissing => GpuError::LibraryMissing("libcuda"),
ContextError::Driver(e) => GpuError::from(e),
}
}
}