1use std::{
6 any::Any,
7 panic::{AssertUnwindSafe, catch_unwind},
8};
9
10use logit_loom_core::{ControlFlow, CoreError, Digest};
11use serde::{Deserialize, Serialize};
12
13use crate::{Error, InterventionFailure, Result, StepContext, TensorSpec};
14
15const MAX_STEP_OBSERVERS: usize = 32;
16
17pub trait StepObserver {
19 fn implementation(&self) -> &Digest;
21
22 fn observe(
29 &mut self,
30 context: &StepContext,
31 state: &[f32],
32 ) -> std::result::Result<ControlFlow, String>;
33}
34
35#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
37#[serde(deny_unknown_fields)]
38pub struct ObserverReceipt {
39 pub implementation: Digest,
41 pub observed_steps: u32,
43 pub elements_seen: u64,
45 pub stop_requested: bool,
47 #[serde(default, skip_serializing_if = "Option::is_none")]
49 pub failure: Option<InterventionFailure>,
50}
51
52pub struct ObserverSet {
54 expected_plan: Digest,
55 tensor: TensorSpec,
56 max_steps: u32,
57 observers: Vec<Box<dyn StepObserver>>,
58 receipts: Vec<ObserverReceipt>,
59}
60
61impl std::fmt::Debug for ObserverSet {
62 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
63 formatter
64 .debug_struct("ObserverSet")
65 .field("expected_plan", &self.expected_plan)
66 .field("tensor", &self.tensor)
67 .field("max_steps", &self.max_steps)
68 .field("receipts", &self.receipts)
69 .finish_non_exhaustive()
70 }
71}
72
73impl ObserverSet {
74 pub fn new(
80 expected_plan: Digest,
81 tensor: TensorSpec,
82 max_steps: u32,
83 observers: Vec<Box<dyn StepObserver>>,
84 ) -> Result<Self> {
85 tensor.validate()?;
86 if max_steps == 0 {
87 return Err(
88 CoreError::invalid("diffusion observer step bound", "must be positive").into(),
89 );
90 }
91 if !(1..=MAX_STEP_OBSERVERS).contains(&observers.len()) {
92 return Err(CoreError::invalid(
93 "diffusion observers",
94 format!("requires 1..={MAX_STEP_OBSERVERS} entries"),
95 )
96 .into());
97 }
98 let receipts = observers
99 .iter()
100 .map(|observer| ObserverReceipt {
101 implementation: observer.implementation().clone(),
102 observed_steps: 0,
103 elements_seen: 0,
104 stop_requested: false,
105 failure: None,
106 })
107 .collect();
108 Ok(Self {
109 expected_plan,
110 tensor,
111 max_steps,
112 observers,
113 receipts,
114 })
115 }
116
117 pub fn observe(&mut self, context: &StepContext, state: &[f32]) -> Result<ControlFlow> {
126 self.validate_call(context, state)?;
127 let elements = u64::try_from(state.len())
128 .map_err(|_| Error::Incompatible("state length exceeds u64".to_owned()))?;
129 let mut control = ControlFlow::Continue;
130 for index in 0..self.observers.len() {
131 let outcome = catch_unwind(AssertUnwindSafe(|| {
132 self.observers[index].observe(context, state)
133 }));
134 self.receipts[index].observed_steps = self.receipts[index]
135 .observed_steps
136 .checked_add(1)
137 .ok_or_else(|| Error::Incompatible("observer step count overflowed".to_owned()))?;
138 self.receipts[index].elements_seen = self.receipts[index]
139 .elements_seen
140 .checked_add(elements)
141 .ok_or_else(|| {
142 Error::Incompatible("observer element accounting overflowed".to_owned())
143 })?;
144 match observer_outcome(outcome) {
145 Ok(ControlFlow::Continue) => {}
146 Ok(ControlFlow::Stop) => {
147 self.receipts[index].stop_requested = true;
148 control = ControlFlow::Stop;
149 }
150 Err(failure) => {
151 self.receipts[index].failure = Some(failure.clone());
152 return Err(Error::Observer {
153 observer: index,
154 message: failure.message,
155 });
156 }
157 }
158 }
159 Ok(control)
160 }
161
162 pub fn receipts(&self) -> &[ObserverReceipt] {
164 &self.receipts
165 }
166
167 fn validate_call(&self, context: &StepContext, state: &[f32]) -> Result<()> {
168 if context.plan != self.expected_plan || context.tensor != self.tensor {
169 return Err(Error::Incompatible(
170 "observer step plan, shape, dtype, layout, or device differs".to_owned(),
171 ));
172 }
173 let expected = usize::try_from(self.tensor.elements()?)
174 .map_err(|_| Error::Incompatible("tensor element count exceeds usize".to_owned()))?;
175 if state.len() != expected {
176 return Err(Error::Incompatible(format!(
177 "observer state contains {} elements; expected {expected}",
178 state.len()
179 )));
180 }
181 if state.iter().any(|value| !value.is_finite()) {
182 return Err(Error::Incompatible(
183 "observer state contains a non-finite value".to_owned(),
184 ));
185 }
186 if self
187 .receipts
188 .iter()
189 .any(|receipt| receipt.failure.is_some())
190 {
191 return Err(Error::Incompatible(
192 "observer set is failed and cannot continue".to_owned(),
193 ));
194 }
195 if self
196 .receipts
197 .iter()
198 .any(|receipt| receipt.observed_steps >= self.max_steps)
199 {
200 return Err(Error::Incompatible(
201 "observer step bound is exhausted".to_owned(),
202 ));
203 }
204 Ok(())
205 }
206}
207
208fn observer_outcome(
209 outcome: std::thread::Result<std::result::Result<ControlFlow, String>>,
210) -> std::result::Result<ControlFlow, InterventionFailure> {
211 match outcome {
212 Ok(Ok(control)) => Ok(control),
213 Ok(Err(message)) => Err(InterventionFailure::new(false, message)),
214 Err(payload) => Err(InterventionFailure::new(true, panic_message(&payload))),
215 }
216}
217
218fn panic_message(payload: &Box<dyn Any + Send>) -> String {
219 payload.downcast_ref::<&str>().map_or_else(
220 || {
221 payload
222 .downcast_ref::<String>()
223 .map_or_else(|| "callback panicked".to_owned(), Clone::clone)
224 },
225 |message| (*message).to_owned(),
226 )
227}
228
229#[cfg(test)]
230mod tests {
231 use std::{cell::RefCell, collections::BTreeMap, rc::Rc};
232
233 use crate::{DiffusionPlan, DiffusionSchedule, TensorDType, TensorLayout};
234
235 use super::*;
236
237 struct CallbackObserver {
238 implementation: Digest,
239 order: Rc<RefCell<Vec<&'static str>>>,
240 name: &'static str,
241 outcome: fn(&[f32]) -> std::result::Result<ControlFlow, String>,
242 }
243
244 impl StepObserver for CallbackObserver {
245 fn implementation(&self) -> &Digest {
246 &self.implementation
247 }
248
249 fn observe(
250 &mut self,
251 _context: &StepContext,
252 state: &[f32],
253 ) -> std::result::Result<ControlFlow, String> {
254 self.order.borrow_mut().push(self.name);
255 (self.outcome)(state)
256 }
257 }
258
259 fn fixture() -> (DiffusionPlan, StepContext) {
260 let tensor = TensorSpec::new(
261 vec![2, 2, 1],
262 TensorDType::F32,
263 TensorLayout::DimensionZeroFastest,
264 "test0",
265 )
266 .unwrap();
267 let schedule =
268 DiffusionSchedule::new(Digest::of_bytes("scheduler", b"v1"), vec![1.0, 0.0]).unwrap();
269 let mut components = BTreeMap::new();
270 components.insert("model".to_owned(), Digest::of_bytes("artifact", b"model"));
271 let plan = DiffusionPlan::new(
272 components,
273 Digest::of_bytes("conditioning", b"input"),
274 Digest::of_bytes("rng", b"v1"),
275 7,
276 tensor,
277 schedule,
278 )
279 .unwrap();
280 let context = StepContext::for_plan(&plan, 0).unwrap();
281 (plan, context)
282 }
283
284 fn observer(
285 order: Rc<RefCell<Vec<&'static str>>>,
286 name: &'static str,
287 outcome: fn(&[f32]) -> std::result::Result<ControlFlow, String>,
288 ) -> CallbackObserver {
289 CallbackObserver {
290 implementation: Digest::of_bytes("test-observer", name.as_bytes()),
291 order,
292 name,
293 outcome,
294 }
295 }
296
297 #[test]
298 fn stop_still_delivers_the_same_boundary_to_later_observers() {
299 let (plan, context) = fixture();
300 let order = Rc::new(RefCell::new(Vec::new()));
301 let stop = |_state: &[f32]| Ok(ControlFlow::Stop);
302 let continue_ = |_state: &[f32]| Ok(ControlFlow::Continue);
303 let mut observers = ObserverSet::new(
304 plan.digest().unwrap(),
305 plan.tensor.clone(),
306 1,
307 vec![
308 Box::new(observer(order.clone(), "stop", stop)),
309 Box::new(observer(order.clone(), "later", continue_)),
310 ],
311 )
312 .unwrap();
313 assert_eq!(
314 observers.observe(&context, &[0.0; 4]).unwrap(),
315 ControlFlow::Stop
316 );
317 assert_eq!(&*order.borrow(), &["stop", "later"]);
318 assert_eq!(observers.receipts()[1].observed_steps, 1);
319 }
320
321 #[test]
322 fn observer_error_and_panic_are_contained() {
323 let (plan, context) = fixture();
324 let order = Rc::new(RefCell::new(Vec::new()));
325 let fail = |_state: &[f32]| Err("failed".to_owned());
326 let mut observers = ObserverSet::new(
327 plan.digest().unwrap(),
328 plan.tensor.clone(),
329 1,
330 vec![Box::new(observer(order, "fail", fail))],
331 )
332 .unwrap();
333 assert!(observers.observe(&context, &[0.0; 4]).is_err());
334 assert!(
335 observers.receipts()[0]
336 .failure
337 .as_ref()
338 .is_some_and(|failure| !failure.panicked)
339 );
340
341 let order = Rc::new(RefCell::new(Vec::new()));
342 let panic_observer =
343 |_state: &[f32]| -> std::result::Result<ControlFlow, String> { panic!("contained") };
344 let mut observers = ObserverSet::new(
345 plan.digest().unwrap(),
346 plan.tensor.clone(),
347 1,
348 vec![Box::new(observer(order, "panic", panic_observer))],
349 )
350 .unwrap();
351 assert!(observers.observe(&context, &[0.0; 4]).is_err());
352 assert!(
353 observers.receipts()[0]
354 .failure
355 .as_ref()
356 .is_some_and(|failure| failure.panicked)
357 );
358 }
359
360 #[test]
361 fn observer_rejects_wrong_shape_device_and_nonfinite_state() {
362 let (plan, mut context) = fixture();
363 let order = Rc::new(RefCell::new(Vec::new()));
364 let continue_ = |_state: &[f32]| Ok(ControlFlow::Continue);
365 let mut observers = ObserverSet::new(
366 plan.digest().unwrap(),
367 plan.tensor.clone(),
368 1,
369 vec![Box::new(observer(order, "observe", continue_))],
370 )
371 .unwrap();
372 assert!(observers.observe(&context, &[0.0; 3]).is_err());
373 assert!(
374 observers
375 .observe(&context, &[0.0, 0.0, f32::INFINITY, 0.0])
376 .is_err()
377 );
378 context.tensor.device = "other0".to_owned();
379 assert!(observers.observe(&context, &[0.0; 4]).is_err());
380 }
381}