1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
//! `GRD-005`: the backward pass reports failures instead of aborting.
//!
//! PROPOSALS.md sec. 3.9 states the obligation in two sentences: "Backward
//! closures must return structured errors. NaN checking is an execution policy
//! applied consistently across backends, not a panic-only backend helper."
//!
//! Neither held. `BackwardFn` was `Fn(&S) -> Vec<S>`, so a recipe that could
//! not produce a gradient had exactly one way to say so, and 115 sites across
//! the three backends took it. And the NaN check was a second entry point,
//! `Backend::backward_with_nan_check`, which panicked - so choosing it changed
//! both *what was checked* and *what happened on failure*, and wanting the
//! check without the abort had no spelling at all.
//!
//! What is under test is therefore mostly negative: that things which used to
//! end the process now return.
extern crate incin_core as incin;
use std::panic;
use incin_backends::cpu::{CpuBackendImpl, tape_depth};
use incin_core::exec::{
ExecutionPolicy, FallbackPolicy, GradMode, MathMode, NanPolicy, check_gradients,
};
use incin_core::exec::{TapeStorage, TensorId};
use incin_core::prelude::*;
use incin_macros::s;
type B = CpuBackendImpl;
/// A chain whose gradient is not finite, and the id of the operand whose
/// gradient goes bad first.
///
/// `d(a/b)/da = 1/b`, so a zero divisor puts an infinity in the *gradient*
/// while the forward value is whatever it is. That is the case worth catching:
/// a loss that still looks like a number, produced by an operation whose
/// derivative did not.
#[allow(clippy::type_complexity)]
fn non_finite_chain() -> (Tensor<s![2, 2], B, f32, Grad>, TensorId) {
let a = Tensor::<s![2, 2], B, f32, Grad>::from_slice(&[1.0, 2.0, 3.0, 4.0], ()).unwrap();
let zero = Tensor::<s![2, 2], B, f32, Grad>::zeros(()).unwrap();
let numerator = TapeStorage::id(a.inner());
(a.try_div(&zero).unwrap(), numerator)
}
#[allow(clippy::type_complexity)]
fn seeded_backward(
loss: &Tensor<s![2, 2], B, f32, Grad>,
) -> Result<incin_core::optim::Gradients<B>> {
let seed = Tensor::<s![2, 2], B, f32>::ones(()).unwrap();
loss.backward_with(&seed)
}
// ── The policy ───────────────────────────────────────────────────────────────
#[test]
fn checking_is_off_by_default() {
// The default has to be off, and not as a preference: the check reads
// every element of every gradient, which on a device backend is a full
// readback per contribution.
assert_eq!(NanPolicy::current(), NanPolicy::Permit);
assert_eq!(ExecutionPolicy::new().nan_policy, NanPolicy::Permit);
assert!(!NanPolicy::Permit.checks());
assert!(NanPolicy::Reject.checks());
}
#[test]
fn the_check_is_an_axis_beside_the_others_not_a_replacement_for_them() {
let policy = ExecutionPolicy::new()
.with_math_mode(MathMode::Fast)
.with_nan_policy(NanPolicy::Reject);
assert_eq!(policy.nan_policy, NanPolicy::Reject);
assert_eq!(policy.math_mode, MathMode::Fast);
assert_eq!(policy.fallback, FallbackPolicy::AllowComposition);
assert_eq!(policy.grad_mode, GradMode::Enabled);
}
#[test]
fn the_scope_ends_where_it_says_it_does() {
check_gradients(|| assert_eq!(NanPolicy::current(), NanPolicy::Reject));
assert_eq!(NanPolicy::current(), NanPolicy::Permit);
}
// ── The failure ──────────────────────────────────────────────────────────────
#[test]
fn a_non_finite_gradient_is_a_returned_error() {
let (loss, numerator) = non_finite_chain();
let Err(err) = check_gradients(|| seeded_backward(&loss)) else {
panic!("a non-finite gradient was not reported");
};
// Matched structurally rather than on the rendered string: the point of a
// typed error is that a caller can branch on it, and a test that only
// reads the message would pass against a `Msg` variant carrying prose.
let Error::Backward(BackwardError::NonFinite { tensor, operation }) = err else {
panic!("expected a NonFinite backward error, got: {err}");
};
assert_eq!(operation, NonFiniteSite::Contribution);
// Which tensor is the entire value of checking. A report naming some
// arbitrary id would satisfy a weaker assertion and still leave the caller
// bisecting the graph, so this names the operand whose gradient - 1/0 -
// is the one that actually went bad.
assert_eq!(tensor, numerator.get());
}
#[test]
fn the_error_message_names_the_tensor_and_where_it_was_found() {
let err: Error = BackwardError::NonFinite {
tensor: 42,
operation: NonFiniteSite::Accumulation,
}
.into();
let rendered = err.to_string();
assert!(rendered.contains("42"), "{rendered}");
assert!(
rendered.contains("accumulating two contributions"),
"{rendered}"
);
}
#[test]
fn a_contribution_and_an_accumulation_are_reported_differently() {
// Two finite contributions can sum to an infinity, which is a different
// fault from a recipe returning one. A report that cannot tell them apart
// sends the reader to the wrong operation.
assert_ne!(
NonFiniteSite::Contribution.to_string(),
NonFiniteSite::Accumulation.to_string()
);
}
#[test]
fn nothing_in_the_backward_path_panics_any_more() {
// The regression this row exists to prevent. Before it, this exact call
// aborted the process from inside a backward recipe, and a caller had no
// way to handle it.
let (loss, _) = non_finite_chain();
let outcome = panic::catch_unwind(|| check_gradients(|| seeded_backward(&loss)).is_err());
assert_eq!(
outcome.ok(),
Some(true),
"the backward pass panicked instead of returning an error"
);
}
// ── What must not change ─────────────────────────────────────────────────────
#[test]
fn the_default_pass_does_not_look_at_gradient_values() {
// Not merely "does not fail": the check must not run at all, or every
// training step pays for a debugging aid. Asserted through the only
// observable difference - an unchecked pass over a non-finite gradient
// succeeds, where a checked one over the same tape does not.
let (loss, _) = non_finite_chain();
assert!(seeded_backward(&loss).is_ok());
let (loss, _) = non_finite_chain();
assert!(check_gradients(|| seeded_backward(&loss)).is_err());
}
#[test]
fn a_checked_pass_over_finite_gradients_agrees_with_an_unchecked_one() {
// Values, not counts: a checked pass that quietly returned different
// numbers would satisfy a length comparison, and the whole promise is that
// turning the check on changes when a pass fails and nothing else.
let a = Tensor::<s![2, 2], B, f32, Grad>::from_slice(&[1.0, 2.0, 3.0, 4.0], ()).unwrap();
let b = Tensor::<s![2, 2], B, f32, Grad>::from_slice(&[5.0, 6.0, 7.0, 8.0], ()).unwrap();
let product = a.try_mul(&b).unwrap();
let plain = gradient_of(&a, &seeded_backward(&product).unwrap());
let product_checked = a.try_mul(&b).unwrap();
let checked = gradient_of(
&a,
&check_gradients(|| seeded_backward(&product_checked)).unwrap(),
);
assert_eq!(plain, vec![5.0, 6.0, 7.0, 8.0]);
assert_eq!(plain, checked);
}
/// `t`'s accumulated gradient, as `f32`s.
#[allow(clippy::type_complexity)]
fn gradient_of(
t: &Tensor<s![2, 2], B, f32, Grad>,
grads: &incin_core::optim::Gradients<B>,
) -> Vec<f32> {
grads.require(t).unwrap().to_vec1::<f32>().unwrap()
}
#[test]
fn a_failed_pass_still_drained_the_tape_it_walked() {
// The failure returns from inside the walk, and the nodes were taken
// before the first recipe ran (`D-06`). A pass that left them behind would
// make the next unrelated backward inherit a half-consumed graph.
let (loss, _) = non_finite_chain();
assert!(tape_depth() > 0);
assert!(check_gradients(|| seeded_backward(&loss)).is_err());
assert_eq!(tape_depth(), 0);
}
#[test]
fn a_recipe_failure_propagates_rather_than_aborting() {
// `unbroadcast` inside a backward recipe used to be
// `.expect("unbroadcast lhs (add)")`. It is a `?` now, so a recipe that
// cannot produce a gradient returns one error among the others rather
// than ending the process. Exercised through a real chain, since the
// recipe type is not something a core test can construct.
let a = Tensor::<s![2, 3], B, f32, Grad>::ones(()).unwrap();
let b = Tensor::<s![2, 3], B, f32, Grad>::ones(()).unwrap();
let loss = a.try_add(&b).unwrap().sum_all().unwrap();
assert!(loss.backward().is_ok());
}