1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
//! Runs that a plan requests, and the record of each finished run.
use std::fmt;
use std::str::FromStr;
use crate::explore::measure::SeriesBuffer;
/// One run of a plan.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PlannedRun {
/// Position of the run in the plan.
///
/// In a sweep it is `config_id * replicates + rep`. In a search it is `config_id * replicates` plus the run's
/// index within its candidate. A re-evaluation continues the `rep` numbering after the existing replicates of
/// the candidate it repeats.
pub run_id: u64,
/// Id of the config the run builds.
pub config_id: u64,
/// Replicate index within the config.
pub rep: u64,
/// Seed the model is built with.
pub seed: u64,
}
/// End state of a run.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RunStatus {
/// The run ended without a fault, and every sampled value was finite.
Ok,
/// The run ended without a fault, and some sampled value was not finite.
NonFinite,
/// The model panicked while building or stepping.
Panicked,
/// The device reported an error.
GpuError,
/// The host refused to build the model.
Refused,
/// A sample no longer fit the stat layout the plan fixed.
ShapeError,
/// The run passed its wall-clock timeout.
TimedOut,
}
impl RunStatus {
/// Returns the status's name in `runs.csv`, as in `timed_out`.
pub fn as_str(self) -> &'static str {
match self {
Self::Ok => "ok",
Self::NonFinite => "non_finite",
Self::Panicked => "panicked",
Self::GpuError => "gpu_error",
Self::Refused => "refused",
Self::ShapeError => "shape_error",
Self::TimedOut => "timed_out",
}
}
/// Returns whether the run ended on a fault or a timeout, so its values cover part of the run at most.
pub fn is_failure(self) -> bool {
match self {
Self::Ok | Self::NonFinite => false,
Self::Panicked | Self::GpuError | Self::Refused | Self::ShapeError | Self::TimedOut => true,
}
}
}
impl FromStr for RunStatus {
type Err = RunStatusError;
/// Reads a status as [`RunStatus::as_str`] writes it.
fn from_str(raw: &str) -> Result<Self, Self::Err> {
match raw {
"ok" => Ok(Self::Ok),
"non_finite" => Ok(Self::NonFinite),
"panicked" => Ok(Self::Panicked),
"gpu_error" => Ok(Self::GpuError),
"refused" => Ok(Self::Refused),
"shape_error" => Ok(Self::ShapeError),
"timed_out" => Ok(Self::TimedOut),
_ => Err(RunStatusError { raw: raw.to_owned() }),
}
}
}
/// Text that matches no [`RunStatus`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RunStatusError {
/// Text as given.
pub raw: String,
}
impl fmt::Display for RunStatusError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "unknown run status '{}'", self.raw)
}
}
impl std::error::Error for RunStatusError {}
/// Cause of the end of a run.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StopReason {
/// The run stepped to its total.
Steps,
/// The stop condition held at a sample.
Condition,
/// A fault ended the run early.
Fault,
/// The run passed its wall-clock timeout.
Timeout,
}
impl StopReason {
/// Returns the reason's name in `runs.csv`, as in `condition`.
pub fn as_str(self) -> &'static str {
match self {
Self::Steps => "steps",
Self::Condition => "condition",
Self::Fault => "fault",
Self::Timeout => "timeout",
}
}
}
impl FromStr for StopReason {
type Err = StopReasonError;
/// Reads a reason as [`StopReason::as_str`] writes it.
fn from_str(raw: &str) -> Result<Self, Self::Err> {
match raw {
"steps" => Ok(Self::Steps),
"condition" => Ok(Self::Condition),
"fault" => Ok(Self::Fault),
"timeout" => Ok(Self::Timeout),
_ => Err(StopReasonError { raw: raw.to_owned() }),
}
}
}
/// Text that matches no [`StopReason`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct StopReasonError {
/// Text as given.
pub raw: String,
}
impl fmt::Display for StopReasonError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "unknown stop reason '{}'", self.raw)
}
}
impl std::error::Error for StopReasonError {}
/// Record of one finished run.
#[derive(Debug, Clone, PartialEq)]
pub struct RunOutcome {
/// Run the record belongs to.
pub run: PlannedRun,
/// Hash that identifies the run's results, from [`crate::explore::fingerprint::run_key`].
pub run_key: u64,
/// State the run ended in.
pub status: RunStatus,
/// Cause of the run's end.
pub stop_reason: StopReason,
/// Tick the run ended on.
///
/// A GPU run that faulted or timed out reports the tick of its latest completed sample, or 0 before any sample
/// completes.
pub ticks: u64,
/// Population at the run's last sample.
pub population: u64,
/// Time in milliseconds spent building the model.
pub build_ms: f64,
/// Time in milliseconds spent stepping and sampling.
pub wall_ms: f64,
/// One value per reducer, `None` for a reducer that saw no finite value.
pub reducers: Vec<Option<f64>>,
/// Series rows that the run kept.
pub series: SeriesBuffer,
/// Actions the model rejected, then the fault message, the timeout or the first value that was not finite.
pub note: Option<String>,
}
impl RunOutcome {
/// Steps per second of wall time. Note that the value is not finite for a run with no wall time.
pub fn steps_per_s(&self) -> f64 {
self.ticks as f64 * 1000.0 / self.wall_ms
}
}
#[cfg(test)]
mod tests {
use super::{RunStatus, StopReason};
#[test]
fn statuses_are_written_in_snake_case() {
let statuses = [
RunStatus::Ok,
RunStatus::NonFinite,
RunStatus::Panicked,
RunStatus::GpuError,
RunStatus::Refused,
RunStatus::ShapeError,
RunStatus::TimedOut,
];
let names: Vec<&str> = statuses.iter().map(|status| status.as_str()).collect();
assert_eq!(
names,
[
"ok",
"non_finite",
"panicked",
"gpu_error",
"refused",
"shape_error",
"timed_out"
]
);
let failures: Vec<bool> = statuses.iter().map(|status| status.is_failure()).collect();
assert_eq!(failures, [false, false, true, true, true, true, true]);
for status in statuses {
assert_eq!(status.as_str().parse(), Ok(status));
}
assert!("timed out".parse::<RunStatus>().is_err());
let reasons = [
StopReason::Steps,
StopReason::Condition,
StopReason::Fault,
StopReason::Timeout,
];
let names: Vec<&str> = reasons.iter().map(|reason| reason.as_str()).collect();
assert_eq!(names, ["steps", "condition", "fault", "timeout"]);
for reason in reasons {
assert_eq!(reason.as_str().parse(), Ok(reason));
}
assert!("stopped".parse::<StopReason>().is_err());
}
}