bacnet_server/
fault_detection.rs

1//! Fault detection / reliability evaluation.
2//!
3//! The [`FaultDetector`] periodically invokes each object's opt-in,
4//! object-owned reliability evaluation hook.
5
6use std::collections::HashMap;
7use std::sync::Mutex;
8
9use bacnet_objects::database::ObjectDatabase;
10use bacnet_objects::traits::ReliabilityEvaluation;
11use bacnet_types::enums::Reliability;
12use bacnet_types::primitives::ObjectIdentifier;
13
14/// A reliability change detected by the fault detector.
15#[derive(Debug, Clone, PartialEq)]
16pub struct ReliabilityChange {
17    /// The object whose reliability changed.
18    pub object_id: ObjectIdentifier,
19    /// Previous reliability value.
20    pub old_reliability: Reliability,
21    /// New reliability value.
22    pub new_reliability: Reliability,
23}
24
25/// Fault detection engine.
26///
27/// Call [`FaultDetector::evaluate`] periodically (the bundled server uses a
28/// 10-second interval) against the object database. Every object is visited;
29/// objects opt in by overriding
30/// [`BACnetObject::evaluate_reliability_internal`](bacnet_objects::traits::BACnetObject::evaluate_reliability_internal).
31/// The default object hook is a no-op.
32pub struct FaultDetector {
33    /// Timeout after which an object is considered to have a communication
34    /// failure. Set to `None` to disable communication-failure detection.
35    pub comm_timeout: Option<std::time::Duration>,
36    warned_internal_failures: Mutex<HashMap<ObjectIdentifier, String>>,
37}
38
39impl Default for FaultDetector {
40    fn default() -> Self {
41        Self {
42            comm_timeout: Some(std::time::Duration::from_secs(60)),
43            warned_internal_failures: Mutex::new(HashMap::new()),
44        }
45    }
46}
47
48impl FaultDetector {
49    /// Create a new fault detector with the given communication timeout.
50    pub fn new(comm_timeout: Option<std::time::Duration>) -> Self {
51        Self {
52            comm_timeout,
53            warned_internal_failures: Mutex::new(HashMap::new()),
54        }
55    }
56
57    fn should_warn_internal_failure(&self, oid: ObjectIdentifier, error: &str) -> bool {
58        let mut warned = self
59            .warned_internal_failures
60            .lock()
61            .unwrap_or_else(|poisoned| poisoned.into_inner());
62        if warned.get(&oid).is_some_and(|previous| previous == error) {
63            false
64        } else {
65            warned.insert(oid, error.to_owned());
66            true
67        }
68    }
69
70    fn clear_internal_failure(&self, oid: &ObjectIdentifier) {
71        self.warned_internal_failures
72            .lock()
73            .unwrap_or_else(|poisoned| poisoned.into_inner())
74            .remove(oid);
75    }
76
77    fn prune_internal_failures(&self, db: &ObjectDatabase) {
78        self.warned_internal_failures
79            .lock()
80            .unwrap_or_else(|poisoned| poisoned.into_inner())
81            .retain(|oid, _| db.get(oid).is_some());
82    }
83
84    /// Evaluate reliability for all objects in the database.
85    ///
86    /// Each object owns both the decision and any mutation. Engineering
87    /// `Min_Pres_Value` / `Max_Pres_Value` metadata is not a reliability fault
88    /// algorithm. Only a successful hook result of
89    /// [`ReliabilityEvaluation::Changed`] produces a [`ReliabilityChange`].
90    /// Hook errors are rate-limited per object and produce no change record.
91    ///
92    /// Returns a list of changes that object hooks successfully applied.
93    pub fn evaluate(&self, db: &mut ObjectDatabase) -> Vec<ReliabilityChange> {
94        let mut changes = Vec::new();
95        db.for_each_object_mut(|oid, object| {
96            if object.reliability_evaluation_inhibited_internal() {
97                self.clear_internal_failure(&oid);
98                return;
99            }
100            match object.evaluate_reliability_internal() {
101                Ok(ReliabilityEvaluation::Unchanged) => {
102                    self.clear_internal_failure(&oid);
103                }
104                Ok(ReliabilityEvaluation::Changed {
105                    old_reliability,
106                    new_reliability,
107                }) => {
108                    self.clear_internal_failure(&oid);
109                    changes.push(ReliabilityChange {
110                        object_id: oid,
111                        old_reliability,
112                        new_reliability,
113                    });
114                }
115                Err(error) => {
116                    let error_text = error.to_string();
117                    if self.should_warn_internal_failure(oid, &error_text) {
118                        tracing::warn!(
119                            object = %oid,
120                            error = %error,
121                            "Fault detection object-owned reliability evaluation failed"
122                        );
123                    }
124                }
125            }
126        });
127        self.prune_internal_failures(db);
128        changes
129    }
130}
131
132#[cfg(test)]
133mod tests {
134    use std::borrow::Cow;
135    use std::sync::Arc;
136    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
137
138    use bacnet_objects::analog::{AnalogInputObject, AnalogOutputObject, AnalogValueObject};
139    use bacnet_objects::traits::BACnetObject;
140    use bacnet_types::enums::{EventState, ObjectType, PropertyIdentifier};
141    use bacnet_types::error::Error;
142    use bacnet_types::primitives::PropertyValue;
143    use tracing::span::{Attributes, Id, Record};
144    use tracing::{Event, Level, Metadata, Subscriber};
145
146    use super::*;
147    use bacnet_types::enums::EngineeringUnits;
148
149    const HOOK_ERROR: &str = "test reliability hook failed";
150
151    struct WarningCounter {
152        warnings: Arc<AtomicUsize>,
153    }
154
155    impl Subscriber for WarningCounter {
156        fn enabled(&self, metadata: &Metadata<'_>) -> bool {
157            *metadata.level() == Level::WARN
158        }
159
160        fn new_span(&self, _span: &Attributes<'_>) -> Id {
161            Id::from_u64(1)
162        }
163
164        fn record(&self, _span: &Id, _values: &Record<'_>) {}
165
166        fn record_follows_from(&self, _span: &Id, _follows: &Id) {}
167
168        fn event(&self, event: &Event<'_>) {
169            if *event.metadata().level() == Level::WARN {
170                self.warnings.fetch_add(1, Ordering::SeqCst);
171            }
172        }
173
174        fn enter(&self, _span: &Id) {}
175
176        fn exit(&self, _span: &Id) {}
177    }
178
179    fn with_warning_counter(test: impl FnOnce(&AtomicUsize)) {
180        let warnings = Arc::new(AtomicUsize::new(0));
181        let subscriber = WarningCounter {
182            warnings: Arc::clone(&warnings),
183        };
184        tracing::subscriber::with_default(subscriber, || test(warnings.as_ref()));
185    }
186
187    fn read_reliability(db: &ObjectDatabase, oid: ObjectIdentifier) -> u32 {
188        match db
189            .get(&oid)
190            .unwrap()
191            .read_property(PropertyIdentifier::RELIABILITY, None)
192            .unwrap()
193        {
194            PropertyValue::Enumerated(value) => value,
195            other => panic!("expected Enumerated Reliability, got {other:?}"),
196        }
197    }
198
199    struct OptInReliabilityObject {
200        oid: ObjectIdentifier,
201        name: String,
202        reliability: Reliability,
203        target_reliability: Reliability,
204        fail: Arc<AtomicBool>,
205    }
206
207    impl OptInReliabilityObject {
208        fn new(
209            instance: u32,
210            name: &str,
211            target_reliability: Reliability,
212            fail: Arc<AtomicBool>,
213        ) -> Self {
214            Self {
215                oid: ObjectIdentifier::new(ObjectType::BINARY_INPUT, instance).unwrap(),
216                name: name.to_owned(),
217                reliability: Reliability::NO_FAULT_DETECTED,
218                target_reliability,
219                fail,
220            }
221        }
222    }
223
224    impl BACnetObject for OptInReliabilityObject {
225        fn object_identifier(&self) -> ObjectIdentifier {
226            self.oid
227        }
228
229        fn object_name(&self) -> &str {
230            &self.name
231        }
232
233        fn read_property(
234            &self,
235            property: PropertyIdentifier,
236            _array_index: Option<u32>,
237        ) -> Result<PropertyValue, Error> {
238            if property == PropertyIdentifier::RELIABILITY {
239                Ok(PropertyValue::Enumerated(self.reliability.to_raw()))
240            } else {
241                Err(Error::Encoding("test property is unsupported".into()))
242            }
243        }
244
245        fn write_property(
246            &mut self,
247            _property: PropertyIdentifier,
248            _array_index: Option<u32>,
249            _value: PropertyValue,
250            _priority: Option<u8>,
251        ) -> Result<(), Error> {
252            Err(Error::Encoding("test object is read-only".into()))
253        }
254
255        fn property_list(&self) -> Cow<'static, [PropertyIdentifier]> {
256            Cow::Borrowed(&[PropertyIdentifier::RELIABILITY])
257        }
258
259        fn evaluate_reliability_internal(&mut self) -> Result<ReliabilityEvaluation, Error> {
260            if self.fail.load(Ordering::SeqCst) {
261                return Err(Error::Encoding(HOOK_ERROR.into()));
262            }
263            if self.reliability == self.target_reliability {
264                return Ok(ReliabilityEvaluation::Unchanged);
265            }
266
267            let old_reliability = self.reliability;
268            self.reliability = self.target_reliability;
269            Ok(ReliabilityEvaluation::Changed {
270                old_reliability,
271                new_reliability: self.reliability,
272            })
273        }
274    }
275
276    struct InhibitedProbeObject {
277        oid: ObjectIdentifier,
278        calls: Arc<AtomicUsize>,
279    }
280
281    impl BACnetObject for InhibitedProbeObject {
282        fn object_identifier(&self) -> ObjectIdentifier {
283            self.oid
284        }
285
286        fn object_name(&self) -> &str {
287            "inhibited-probe"
288        }
289
290        fn read_property(
291            &self,
292            _property: PropertyIdentifier,
293            _array_index: Option<u32>,
294        ) -> Result<PropertyValue, Error> {
295            Err(Error::Encoding("test property is unsupported".into()))
296        }
297
298        fn write_property(
299            &mut self,
300            _property: PropertyIdentifier,
301            _array_index: Option<u32>,
302            _value: PropertyValue,
303            _priority: Option<u8>,
304        ) -> Result<(), Error> {
305            Err(Error::Encoding("test object is read-only".into()))
306        }
307
308        fn property_list(&self) -> Cow<'static, [PropertyIdentifier]> {
309            Cow::Borrowed(&[])
310        }
311
312        fn reliability_evaluation_inhibited_internal(&self) -> bool {
313            true
314        }
315
316        fn evaluate_reliability_internal(&mut self) -> Result<ReliabilityEvaluation, Error> {
317            self.calls.fetch_add(1, Ordering::SeqCst);
318            Err(Error::Encoding(HOOK_ERROR.into()))
319        }
320    }
321
322    #[test]
323    fn inhibited_object_skips_hook_and_clears_stale_failure_warning() {
324        let detector = FaultDetector::default();
325        let oid = ObjectIdentifier::new(ObjectType::ANALOG_INPUT, 77).unwrap();
326        let calls = Arc::new(AtomicUsize::new(0));
327        let mut db = ObjectDatabase::new();
328        db.add(Box::new(InhibitedProbeObject {
329            oid,
330            calls: Arc::clone(&calls),
331        }))
332        .unwrap();
333        assert!(detector.should_warn_internal_failure(oid, HOOK_ERROR));
334
335        assert!(detector.evaluate(&mut db).is_empty());
336        assert_eq!(calls.load(Ordering::SeqCst), 0, "inhibited hook was called");
337        assert!(
338            detector.should_warn_internal_failure(oid, HOOK_ERROR),
339            "inhibition must clear an old warning so recovery cannot leave stale state"
340        );
341    }
342
343    #[test]
344    fn identical_internal_failure_warning_is_suppressed_until_state_changes() {
345        let detector = FaultDetector::default();
346        let oid = ObjectIdentifier::new(ObjectType::ANALOG_INPUT, 1).unwrap();
347
348        assert!(detector.should_warn_internal_failure(oid, "first error"));
349        assert!(!detector.should_warn_internal_failure(oid, "first error"));
350        assert!(detector.should_warn_internal_failure(oid, "different error"));
351        assert!(!detector.should_warn_internal_failure(oid, "different error"));
352
353        detector.clear_internal_failure(&oid);
354        assert!(detector.should_warn_internal_failure(oid, "first error"));
355    }
356
357    #[test]
358    fn stock_analog_engineering_bounds_never_change_reliability() {
359        let mut db = ObjectDatabase::new();
360
361        let mut ai = AnalogInputObject::new(1, "AI-1", EngineeringUnits::DEGREES_CELSIUS).unwrap();
362        ai.set_min_pres_value(0.0);
363        ai.set_max_pres_value(100.0);
364        ai.set_present_value(200.0);
365        let ai_oid = ai.object_identifier();
366        db.add(Box::new(ai)).unwrap();
367
368        let mut ao = AnalogOutputObject::new(1, "AO-1", EngineeringUnits::DEGREES_CELSIUS).unwrap();
369        ao.set_min_pres_value(0.0);
370        ao.set_max_pres_value(100.0);
371        ao.write_property_from(
372            PropertyIdentifier::PRESENT_VALUE,
373            None,
374            PropertyValue::Real(150.0),
375            Some(8),
376            &crate::command_source::test_origin(),
377        )
378        .unwrap();
379        let ao_oid = ao.object_identifier();
380        db.add(Box::new(ao)).unwrap();
381
382        let mut av = AnalogValueObject::new(1, "AV-1", EngineeringUnits::DEGREES_CELSIUS).unwrap();
383        av.set_min_pres_value(0.0);
384        av.set_max_pres_value(100.0);
385        av.set_relinquish_default(-10.0).unwrap();
386        let av_oid = av.object_identifier();
387        db.add(Box::new(av)).unwrap();
388
389        let changes = FaultDetector::default().evaluate(&mut db);
390
391        assert!(changes.is_empty());
392        for oid in [ai_oid, ao_oid, av_oid] {
393            assert_eq!(
394                read_reliability(&db, oid),
395                Reliability::NO_FAULT_DETECTED.to_raw()
396            );
397        }
398        let ao = db.get(&ao_oid).unwrap();
399        assert_eq!(
400            ao.read_property(PropertyIdentifier::PRESENT_VALUE, None)
401                .unwrap(),
402            PropertyValue::Real(150.0)
403        );
404        assert_eq!(
405            ao.read_property(PropertyIdentifier::PRIORITY_ARRAY, Some(8))
406                .unwrap(),
407            PropertyValue::Real(150.0)
408        );
409    }
410
411    #[test]
412    fn configured_ai_and_av_changes_are_recorded_and_seen_by_intrinsic_fault_observation() {
413        let mut db = ObjectDatabase::new();
414
415        let mut ai = AnalogInputObject::new(1, "AI-1", EngineeringUnits::DEGREES_CELSIUS).unwrap();
416        ai.configure_fault_out_of_range(10.0, 20.0).unwrap();
417        ai.set_present_value(21.0);
418        let ai_oid = ai.object_identifier();
419        db.add(Box::new(ai)).unwrap();
420
421        let mut av = AnalogValueObject::new(1, "AV-1", EngineeringUnits::DEGREES_CELSIUS).unwrap();
422        av.configure_fault_out_of_range(10.0, 20.0).unwrap();
423        av.write_property_from(
424            PropertyIdentifier::PRESENT_VALUE,
425            None,
426            PropertyValue::Real(9.0),
427            Some(8),
428            &crate::command_source::test_origin(),
429        )
430        .unwrap();
431        let av_oid = av.object_identifier();
432        db.add(Box::new(av)).unwrap();
433
434        let detector = FaultDetector::default();
435        let changes = detector.evaluate(&mut db);
436        assert_eq!(changes.len(), 2);
437        assert!(changes.contains(&ReliabilityChange {
438            object_id: ai_oid,
439            old_reliability: Reliability::NO_FAULT_DETECTED,
440            new_reliability: Reliability::OVER_RANGE,
441        }));
442        assert!(changes.contains(&ReliabilityChange {
443            object_id: av_oid,
444            old_reliability: Reliability::NO_FAULT_DETECTED,
445            new_reliability: Reliability::UNDER_RANGE,
446        }));
447        assert!(detector.evaluate(&mut db).is_empty());
448
449        for oid in [ai_oid, av_oid] {
450            let proposal = db
451                .get_mut(&oid)
452                .unwrap()
453                .evaluate_intrinsic_reporting()
454                .expect("range Reliability must be observed as an intrinsic fault");
455            assert_eq!(proposal.change.from, EventState::NORMAL);
456            assert_eq!(proposal.change.to, EventState::FAULT);
457        }
458    }
459
460    // The public sensor setters intentionally retain their debug assertions.
461    // This release-only test exercises the production path where those setters
462    // can receive non-finite application data without changing their API.
463    #[cfg(not(debug_assertions))]
464    #[test]
465    fn non_finite_configured_input_create_no_change_and_keep_owned_faults() {
466        let mut ai = AnalogInputObject::new(1, "AI-1", EngineeringUnits::DEGREES_CELSIUS).unwrap();
467        ai.configure_fault_out_of_range(10.0, 20.0).unwrap();
468        ai.set_present_value(21.0);
469        ai.evaluate_reliability_internal().unwrap();
470        ai.set_present_value(f32::NAN);
471        let ai_oid = ai.object_identifier();
472
473        let mut db = ObjectDatabase::new();
474        db.add(Box::new(ai)).unwrap();
475        let detector = FaultDetector::default();
476
477        assert!(detector.evaluate(&mut db).is_empty());
478        assert!(detector.evaluate(&mut db).is_empty());
479        assert_eq!(
480            read_reliability(&db, ai_oid),
481            Reliability::OVER_RANGE.to_raw()
482        );
483
484        let ai = db.get_mut(&ai_oid).unwrap();
485        ai.write_property(
486            PropertyIdentifier::OUT_OF_SERVICE,
487            None,
488            PropertyValue::Boolean(true),
489            None,
490        )
491        .unwrap();
492        ai.write_property_from(
493            PropertyIdentifier::PRESENT_VALUE,
494            None,
495            PropertyValue::Real(9.0),
496            None,
497            &crate::command_source::test_origin(),
498        )
499        .unwrap();
500        ai.write_property(
501            PropertyIdentifier::OUT_OF_SERVICE,
502            None,
503            PropertyValue::Boolean(false),
504            None,
505        )
506        .unwrap();
507        let changes = detector.evaluate(&mut db);
508        assert_eq!(changes.len(), 1);
509        assert!(changes.contains(&ReliabilityChange {
510            object_id: ai_oid,
511            old_reliability: Reliability::OVER_RANGE,
512            new_reliability: Reliability::UNDER_RANGE,
513        }));
514    }
515
516    #[test]
517    fn existing_nonzero_in_service_reliability_is_preserved() {
518        let mut ai = AnalogInputObject::new(1, "AI-1", EngineeringUnits::DEGREES_CELSIUS).unwrap();
519        ai.set_min_pres_value(0.0);
520        ai.set_max_pres_value(100.0);
521        ai.set_present_value(50.0);
522        ai.set_reliability_internal(Reliability::NO_SENSOR).unwrap();
523        let oid = ai.object_identifier();
524        let mut db = ObjectDatabase::new();
525        db.add(Box::new(ai)).unwrap();
526
527        assert!(FaultDetector::default().evaluate(&mut db).is_empty());
528        assert_eq!(read_reliability(&db, oid), Reliability::NO_SENSOR.to_raw());
529    }
530
531    #[test]
532    fn out_of_service_client_write_and_restore_remain_object_owned() {
533        let mut ai = AnalogInputObject::new(1, "AI-1", EngineeringUnits::DEGREES_CELSIUS).unwrap();
534        ai.set_reliability_internal(Reliability::OVER_RANGE)
535            .unwrap();
536        let oid = ai.object_identifier();
537        let mut db = ObjectDatabase::new();
538        db.add(Box::new(ai)).unwrap();
539
540        let object = db.get_mut(&oid).unwrap();
541        object
542            .write_property(
543                PropertyIdentifier::OUT_OF_SERVICE,
544                None,
545                PropertyValue::Boolean(true),
546                None,
547            )
548            .unwrap();
549        object
550            .write_property(
551                PropertyIdentifier::RELIABILITY,
552                None,
553                PropertyValue::Enumerated(Reliability::NO_SENSOR.to_raw()),
554                None,
555            )
556            .unwrap();
557
558        assert!(FaultDetector::default().evaluate(&mut db).is_empty());
559        assert_eq!(read_reliability(&db, oid), Reliability::NO_SENSOR.to_raw());
560
561        db.get_mut(&oid)
562            .unwrap()
563            .write_property(
564                PropertyIdentifier::OUT_OF_SERVICE,
565                None,
566                PropertyValue::Boolean(false),
567                None,
568            )
569            .unwrap();
570        assert_eq!(read_reliability(&db, oid), Reliability::OVER_RANGE.to_raw());
571    }
572
573    #[test]
574    fn only_hook_returned_changed_creates_a_change_record() {
575        let fail = Arc::new(AtomicBool::new(false));
576        let object = OptInReliabilityObject::new(1, "custom-hook", Reliability::NO_SENSOR, fail);
577        let oid = object.object_identifier();
578        let mut db = ObjectDatabase::new();
579        db.add(Box::new(object)).unwrap();
580        let detector = FaultDetector::default();
581        assert!(detector.should_warn_internal_failure(oid, HOOK_ERROR));
582
583        assert_eq!(
584            detector.evaluate(&mut db),
585            vec![ReliabilityChange {
586                object_id: oid,
587                old_reliability: Reliability::NO_FAULT_DETECTED,
588                new_reliability: Reliability::NO_SENSOR,
589            }]
590        );
591        assert_eq!(read_reliability(&db, oid), Reliability::NO_SENSOR.to_raw());
592        assert!(detector.should_warn_internal_failure(oid, HOOK_ERROR));
593        assert!(detector.evaluate(&mut db).is_empty());
594        assert!(detector.should_warn_internal_failure(oid, HOOK_ERROR));
595    }
596
597    #[test]
598    fn failing_hook_warning_is_rate_limited_and_restored_after_recovery() {
599        let fail = Arc::new(AtomicBool::new(true));
600        let object = OptInReliabilityObject::new(
601            1,
602            "failing-hook",
603            Reliability::NO_SENSOR,
604            Arc::clone(&fail),
605        );
606        let oid = object.object_identifier();
607        let mut db = ObjectDatabase::new();
608        db.add(Box::new(object)).unwrap();
609        let detector = FaultDetector::default();
610
611        with_warning_counter(|warnings| {
612            assert!(detector.evaluate(&mut db).is_empty());
613            assert_eq!(warnings.load(Ordering::SeqCst), 1);
614            assert_eq!(
615                read_reliability(&db, oid),
616                Reliability::NO_FAULT_DETECTED.to_raw()
617            );
618
619            assert!(detector.evaluate(&mut db).is_empty());
620            assert_eq!(warnings.load(Ordering::SeqCst), 1);
621
622            fail.store(false, Ordering::SeqCst);
623            assert_eq!(
624                detector.evaluate(&mut db),
625                vec![ReliabilityChange {
626                    object_id: oid,
627                    old_reliability: Reliability::NO_FAULT_DETECTED,
628                    new_reliability: Reliability::NO_SENSOR,
629                }]
630            );
631            assert_eq!(warnings.load(Ordering::SeqCst), 1);
632
633            fail.store(true, Ordering::SeqCst);
634            assert!(detector.evaluate(&mut db).is_empty());
635            assert_eq!(warnings.load(Ordering::SeqCst), 2);
636
637            fail.store(false, Ordering::SeqCst);
638            assert!(detector.evaluate(&mut db).is_empty());
639            assert_eq!(warnings.load(Ordering::SeqCst), 2);
640
641            fail.store(true, Ordering::SeqCst);
642            assert!(detector.evaluate(&mut db).is_empty());
643            assert_eq!(warnings.load(Ordering::SeqCst), 3);
644        });
645    }
646
647    #[test]
648    fn deleted_failure_warning_is_pruned_and_oid_reuse_warns() {
649        let fail = Arc::new(AtomicBool::new(true));
650        let object = OptInReliabilityObject::new(
651            1,
652            "removed-failing-hook",
653            Reliability::NO_FAULT_DETECTED,
654            Arc::clone(&fail),
655        );
656        let oid = object.object_identifier();
657        let mut db = ObjectDatabase::new();
658        db.add(Box::new(object)).unwrap();
659        let detector = FaultDetector::default();
660
661        with_warning_counter(|warnings| {
662            assert!(detector.evaluate(&mut db).is_empty());
663            assert_eq!(warnings.load(Ordering::SeqCst), 1);
664
665            assert!(detector.evaluate(&mut db).is_empty());
666            assert_eq!(warnings.load(Ordering::SeqCst), 1);
667
668            assert!(db.remove(&oid).unwrap().is_some());
669            assert!(detector.evaluate(&mut db).is_empty());
670            assert_eq!(warnings.load(Ordering::SeqCst), 1);
671
672            db.add(Box::new(OptInReliabilityObject::new(
673                1,
674                "replacement-failing-hook",
675                Reliability::NO_FAULT_DETECTED,
676                fail,
677            )))
678            .unwrap();
679            assert!(detector.evaluate(&mut db).is_empty());
680            assert_eq!(warnings.load(Ordering::SeqCst), 2);
681        });
682    }
683}
bacnet_server/fault_detection.rs

bacnet_server/
fault_detection.rs
