Skip to main content

sightloom_analysis/
stat_anomaly.rs

1//! Statistical anomaly backend (z-score rules).
2//!
3//! Hosts must treat outputs as backend-neutral [`AnomalyEvent`] values.
4
5#![allow(
6    clippy::cast_possible_truncation,
7    clippy::cast_precision_loss,
8    clippy::cast_sign_loss
9)]
10
11extern crate alloc;
12
13use crate::anomaly::{AnomalyEvent, AnomalyReason, Severity};
14use crate::input::{AnalysisSeries, DurationSample, TimedSubjectEvent};
15use crate::stats::{
16    change_point_cusum, day_of_week_ns, hour_of_day_ns, mad, mean, median, robust_z_score, stddev,
17    z_score,
18};
19use alloc::{vec, vec::Vec};
20use sightloom_core::{AnomalyId, EventId, MediaTime, SubjectId};
21
22/// Configuration for the statistical anomaly detector.
23#[derive(Clone, Copy, Debug, PartialEq)]
24pub struct StatAnomalyConfig {
25    /// Absolute z-score threshold to flag an anomaly.
26    pub z_threshold: f32,
27    /// Minimum samples required to build a baseline.
28    pub min_samples: usize,
29    /// When true, also run robust MAD scoring and CUSUM change-points.
30    pub use_robust: bool,
31    /// CUSUM change-point score threshold (unitless; smoke default).
32    pub change_point_threshold: f32,
33    /// When true, flag appearances on weekdays that were rare in baseline.
34    pub use_day_of_week: bool,
35    /// Baseline day fraction below this is "rare" (e.g. `0.08` = 8%).
36    pub rare_day_fraction: f32,
37}
38
39impl Default for StatAnomalyConfig {
40    fn default() -> Self {
41        Self {
42            z_threshold: 2.5,
43            min_samples: 5,
44            use_robust: true,
45            change_point_threshold: 8.0,
46            use_day_of_week: true,
47            rare_day_fraction: 0.08,
48        }
49    }
50}
51
52/// Baseline statistics learned from historical series.
53#[derive(Clone, Debug, Default, PartialEq)]
54pub struct BaselineStats {
55    /// Mean dwell duration nanoseconds.
56    pub dwell_mean: Option<f32>,
57    /// Stddev dwell duration nanoseconds.
58    pub dwell_std: Option<f32>,
59    /// Mean inter-arrival nanoseconds.
60    pub gap_mean: Option<f32>,
61    /// Stddev inter-arrival nanoseconds.
62    pub gap_std: Option<f32>,
63    /// Mean hour-of-day (0..24) for appearances.
64    pub hour_mean: Option<f32>,
65    /// Stddev hour-of-day.
66    pub hour_std: Option<f32>,
67    /// Sample counts used.
68    pub dwell_n: usize,
69    /// Gap sample count.
70    pub gap_n: usize,
71    /// Timed sample count.
72    pub timed_n: usize,
73    /// Robust dwell median (when enough samples).
74    pub dwell_median: Option<f32>,
75    /// Robust dwell MAD.
76    pub dwell_mad: Option<f32>,
77    /// Fraction of timed events per weekday `0..7` (epoch-aligned).
78    pub day_fraction: [f32; 7],
79    /// Timed events used for day-of-week fractions.
80    pub day_n: usize,
81}
82
83/// Builds baseline stats from historical series.
84#[must_use]
85pub fn build_baseline(series: &AnalysisSeries, config: StatAnomalyConfig) -> BaselineStats {
86    let mut stats = BaselineStats::default();
87
88    let dwells: Vec<f32> = series
89        .durations
90        .iter()
91        .map(|d| d.duration_ns as f32)
92        .filter(|v| *v > 0.0 && v.is_finite())
93        .collect();
94    stats.dwell_n = dwells.len();
95    if dwells.len() >= config.min_samples {
96        stats.dwell_mean = mean(&dwells);
97        stats.dwell_std = stddev(&dwells);
98        let mut scratch = vec![0.0_f32; dwells.len()];
99        stats.dwell_median = median(&dwells, &mut scratch);
100        stats.dwell_mad = mad(&dwells, &mut scratch);
101    }
102
103    let mut gaps = Vec::new();
104    for (_, times) in group_subject_times(&series.timed) {
105        for window in times.windows(2) {
106            let gap = (window[1] - window[0]) as f32;
107            if gap > 0.0 {
108                gaps.push(gap);
109            }
110        }
111    }
112    stats.gap_n = gaps.len();
113    if gaps.len() >= config.min_samples {
114        stats.gap_mean = mean(&gaps);
115        stats.gap_std = stddev(&gaps);
116    }
117
118    let hours: Vec<f32> = series
119        .timed
120        .iter()
121        .map(|e| f32::from(hour_of_day_ns(e.at_ns)))
122        .collect();
123    stats.timed_n = hours.len();
124    if hours.len() >= config.min_samples {
125        stats.hour_mean = mean(&hours);
126        stats.hour_std = stddev(&hours);
127    }
128
129    // Day-of-week seasonality histogram.
130    let mut day_counts = [0_u32; 7];
131    for e in &series.timed {
132        let d = day_of_week_ns(e.at_ns) as usize % 7;
133        day_counts[d] = day_counts[d].saturating_add(1);
134    }
135    let day_n = day_counts.iter().sum::<u32>() as usize;
136    stats.day_n = day_n;
137    if day_n > 0 {
138        let inv = 1.0 / day_n as f32;
139        for (frac, count) in stats.day_fraction.iter_mut().zip(day_counts.iter()) {
140            *frac = *count as f32 * inv;
141        }
142    }
143
144    stats
145}
146
147/// Detects statistical anomalies in `series` against `baseline`.
148#[must_use]
149pub fn detect_statistical(
150    series: &AnalysisSeries,
151    baseline: &BaselineStats,
152    config: StatAnomalyConfig,
153    next_id: &mut u64,
154) -> Vec<AnomalyEvent> {
155    let mut out = Vec::new();
156    out.extend(detect_dwell_anomalies(
157        &series.durations,
158        baseline,
159        config,
160        next_id,
161    ));
162    out.extend(detect_frequency_anomalies(
163        &series.timed,
164        baseline,
165        config,
166        next_id,
167    ));
168    out.extend(detect_time_anomalies(
169        &series.timed,
170        baseline,
171        config,
172        next_id,
173    ));
174    out.extend(detect_missing_expected(
175        &series.timed,
176        baseline,
177        config,
178        next_id,
179    ));
180    out.extend(detect_sudden_change(
181        &series.durations,
182        baseline,
183        config,
184        next_id,
185    ));
186    if config.use_robust {
187        out.extend(detect_robust_dwell(
188            &series.durations,
189            baseline,
190            config,
191            next_id,
192        ));
193        out.extend(detect_sequence_change_points(
194            &series.durations,
195            config,
196            next_id,
197        ));
198        out.extend(detect_subject_specific_gaps(&series.timed, config, next_id));
199    }
200    if config.use_day_of_week {
201        out.extend(detect_day_of_week_anomalies(
202            &series.timed,
203            baseline,
204            config,
205            next_id,
206        ));
207    }
208    out
209}
210
211/// Flags timed events on weekdays that were rare in the baseline seasonality.
212fn detect_day_of_week_anomalies(
213    events: &[TimedSubjectEvent],
214    baseline: &BaselineStats,
215    config: StatAnomalyConfig,
216    next_id: &mut u64,
217) -> Vec<AnomalyEvent> {
218    if baseline.day_n < config.min_samples.saturating_mul(2) {
219        return Vec::new();
220    }
221    let rare = config.rare_day_fraction.clamp(0.0, 0.5);
222    let mut out = Vec::new();
223    for event in events {
224        let d = day_of_week_ns(event.at_ns) as usize % 7;
225        let frac = baseline.day_fraction[d];
226        if frac > rare {
227            continue;
228        }
229        // Score: inverse rarity (higher when day was almost never seen).
230        let score = 1.0 / (frac + 1e-3);
231        // Map into z-like scale for severity helpers.
232        let z = (score * 0.5).min(10.0);
233        if z < config.z_threshold * 0.5 {
234            // Only flag clearly rare days (fraction well below rare threshold).
235            continue;
236        }
237        out.push(make_event(
238            next_id,
239            z,
240            AnomalyReason::UnusualAppearanceTime,
241            event.subject_id,
242            event.event_id,
243            event.at_ns,
244        ));
245    }
246    out
247}
248
249fn detect_robust_dwell(
250    samples: &[DurationSample],
251    baseline: &BaselineStats,
252    config: StatAnomalyConfig,
253    next_id: &mut u64,
254) -> Vec<AnomalyEvent> {
255    let (Some(med), Some(mad_v)) = (baseline.dwell_median, baseline.dwell_mad) else {
256        return Vec::new();
257    };
258    let mut out = Vec::new();
259    for sample in samples {
260        let value = sample.duration_ns as f32;
261        let Some(z) = robust_z_score(value, med, mad_v) else {
262            continue;
263        };
264        if z < config.z_threshold {
265            continue;
266        }
267        out.push(make_event(
268            next_id,
269            z,
270            AnomalyReason::UnusualDwell,
271            sample.subject_id,
272            sample.event_id,
273            sample.at_ns,
274        ));
275    }
276    out
277}
278
279fn detect_sequence_change_points(
280    samples: &[DurationSample],
281    config: StatAnomalyConfig,
282    next_id: &mut u64,
283) -> Vec<AnomalyEvent> {
284    if samples.len() < config.min_samples.saturating_mul(2) {
285        return Vec::new();
286    }
287    let values: Vec<f32> = samples.iter().map(|s| s.duration_ns as f32).collect();
288    let Some((idx, score)) = change_point_cusum(&values) else {
289        return Vec::new();
290    };
291    if score < config.change_point_threshold {
292        return Vec::new();
293    }
294    let sample = samples[idx.min(samples.len().saturating_sub(1))];
295    vec![make_event(
296        next_id,
297        score,
298        AnomalyReason::SuddenBehaviourChange,
299        sample.subject_id,
300        sample.event_id,
301        sample.at_ns,
302    )]
303}
304
305/// Per-subject gap baselines (subject-specific frequency anomalies).
306fn detect_subject_specific_gaps(
307    events: &[TimedSubjectEvent],
308    config: StatAnomalyConfig,
309    next_id: &mut u64,
310) -> Vec<AnomalyEvent> {
311    let mut out = Vec::new();
312    for (subject, times) in group_subject_times(events) {
313        if times.len() < config.min_samples.saturating_add(1) {
314            continue;
315        }
316        let mut gaps: Vec<f32> = Vec::new();
317        for w in times.windows(2) {
318            let g = (w[1] - w[0]) as f32;
319            if g > 0.0 {
320                gaps.push(g);
321            }
322        }
323        if gaps.len() < config.min_samples {
324            continue;
325        }
326        let mut scratch = vec![0.0_f32; gaps.len()];
327        let Some(med) = median(&gaps, &mut scratch) else {
328            continue;
329        };
330        let Some(mad_v) = mad(&gaps, &mut scratch) else {
331            continue;
332        };
333        // Flag the latest gap if robust-outlier vs this subject's own history.
334        if let Some(last) = gaps.last().copied() {
335            let Some(z) = robust_z_score(last, med, mad_v) else {
336                continue;
337            };
338            if z >= config.z_threshold {
339                let at = *times.last().unwrap_or(&0);
340                out.push(make_event(
341                    next_id,
342                    z,
343                    AnomalyReason::UnusualFrequency,
344                    subject,
345                    None,
346                    at,
347                ));
348            }
349        }
350    }
351    out
352}
353
354fn detect_dwell_anomalies(
355    samples: &[DurationSample],
356    baseline: &BaselineStats,
357    config: StatAnomalyConfig,
358    next_id: &mut u64,
359) -> Vec<AnomalyEvent> {
360    let (Some(mu), Some(sd)) = (baseline.dwell_mean, baseline.dwell_std) else {
361        return Vec::new();
362    };
363    let mut out = Vec::new();
364    for sample in samples {
365        let value = sample.duration_ns as f32;
366        let Some(z) = z_score(value, mu, sd) else {
367            continue;
368        };
369        if z < config.z_threshold {
370            continue;
371        }
372        out.push(make_event(
373            next_id,
374            z,
375            AnomalyReason::UnusualDwell,
376            sample.subject_id,
377            sample.event_id,
378            sample.at_ns,
379        ));
380    }
381    out
382}
383
384fn detect_frequency_anomalies(
385    events: &[TimedSubjectEvent],
386    baseline: &BaselineStats,
387    config: StatAnomalyConfig,
388    next_id: &mut u64,
389) -> Vec<AnomalyEvent> {
390    let (Some(mu), Some(sd)) = (baseline.gap_mean, baseline.gap_std) else {
391        return Vec::new();
392    };
393    let mut out = Vec::new();
394    for (subject, times) in group_subject_times(events) {
395        for window in times.windows(2) {
396            let gap = (window[1] - window[0]) as f32;
397            let Some(z) = z_score(gap, mu, sd) else {
398                continue;
399            };
400            if z < config.z_threshold {
401                continue;
402            }
403            out.push(make_event(
404                next_id,
405                z,
406                AnomalyReason::UnusualFrequency,
407                subject,
408                None,
409                window[1],
410            ));
411        }
412    }
413    out
414}
415
416fn detect_time_anomalies(
417    events: &[TimedSubjectEvent],
418    baseline: &BaselineStats,
419    config: StatAnomalyConfig,
420    next_id: &mut u64,
421) -> Vec<AnomalyEvent> {
422    let (Some(mu), Some(sd)) = (baseline.hour_mean, baseline.hour_std) else {
423        return Vec::new();
424    };
425    let mut out = Vec::new();
426    for event in events {
427        let hour = f32::from(hour_of_day_ns(event.at_ns));
428        // Circular hour distance approx via min wrap.
429        let delta = hour_delta(hour, mu);
430        let Some(z) = z_score(mu + delta, mu, sd.max(0.5)) else {
431            continue;
432        };
433        if z < config.z_threshold {
434            continue;
435        }
436        out.push(make_event(
437            next_id,
438            z,
439            AnomalyReason::UnusualAppearanceTime,
440            event.subject_id,
441            event.event_id,
442            event.at_ns,
443        ));
444    }
445    out
446}
447
448fn detect_missing_expected(
449    events: &[TimedSubjectEvent],
450    baseline: &BaselineStats,
451    config: StatAnomalyConfig,
452    next_id: &mut u64,
453) -> Vec<AnomalyEvent> {
454    let (Some(mu), Some(sd)) = (baseline.gap_mean, baseline.gap_std) else {
455        return Vec::new();
456    };
457    let mut out = Vec::new();
458    for (subject, times) in group_subject_times(events) {
459        if times.len() < 2 {
460            continue;
461        }
462        let last = *times.last().unwrap();
463        let prev = times[times.len() - 2];
464        let gap = (last - prev) as f32;
465        // Missing expected: gap much larger than baseline mean.
466        let Some(z) = z_score(gap, mu, sd) else {
467            continue;
468        };
469        if gap > mu && z >= config.z_threshold {
470            out.push(make_event(
471                next_id,
472                z,
473                AnomalyReason::MissingExpectedAppearance,
474                subject,
475                None,
476                last,
477            ));
478        }
479    }
480    out
481}
482
483fn detect_sudden_change(
484    samples: &[DurationSample],
485    baseline: &BaselineStats,
486    config: StatAnomalyConfig,
487    next_id: &mut u64,
488) -> Vec<AnomalyEvent> {
489    let (Some(mu), Some(sd)) = (baseline.dwell_mean, baseline.dwell_std) else {
490        return Vec::new();
491    };
492    if samples.len() < config.min_samples {
493        return Vec::new();
494    }
495    // Compare recent window mean vs baseline mean.
496    let recent_n = config.min_samples.min(samples.len());
497    let recent: Vec<f32> = samples[samples.len() - recent_n..]
498        .iter()
499        .map(|s| s.duration_ns as f32)
500        .collect();
501    let Some(recent_mu) = mean(&recent) else {
502        return Vec::new();
503    };
504    let Some(z) = z_score(recent_mu, mu, sd) else {
505        return Vec::new();
506    };
507    if z < config.z_threshold {
508        return Vec::new();
509    }
510    let last = samples.last();
511    let at_ns = last.map_or(0, |s| s.at_ns);
512    let subject = last.and_then(|s| s.subject_id);
513    let event_id = last.and_then(|s| s.event_id);
514    vec![make_event(
515        next_id,
516        z,
517        AnomalyReason::SuddenBehaviourChange,
518        subject,
519        event_id,
520        at_ns,
521    )]
522}
523
524fn make_event(
525    next_id: &mut u64,
526    z: f32,
527    reason: AnomalyReason,
528    subject_id: Option<SubjectId>,
529    event_id: Option<EventId>,
530    at_ns: i64,
531) -> AnomalyEvent {
532    let id = AnomalyId(*next_id);
533    *next_id = next_id.saturating_add(1);
534    let mut evidence = Vec::new();
535    if let Some(event_id) = event_id {
536        evidence.push(event_id);
537    }
538    AnomalyEvent {
539        anomaly_id: id,
540        score: if z.is_finite() { z } else { 100.0 },
541        severity: severity_from_z(z),
542        reasons: alloc::vec![reason],
543        evidence,
544        subject_id,
545        source_id: None,
546        at: MediaTime::new(at_ns, 1_000_000_000).unwrap_or_default(),
547    }
548}
549
550fn severity_from_z(z: f32) -> Severity {
551    if !z.is_finite() {
552        return Severity::Critical;
553    }
554    if z >= 6.0 {
555        Severity::Critical
556    } else if z >= 4.5 {
557        Severity::High
558    } else if z >= 3.5 {
559        Severity::Medium
560    } else {
561        Severity::Low
562    }
563}
564
565fn hour_delta(a: f32, b: f32) -> f32 {
566    let d = (a - b).abs();
567    d.min(24.0 - d)
568}
569
570fn group_subject_times(events: &[TimedSubjectEvent]) -> Vec<(Option<SubjectId>, Vec<i64>)> {
571    let mut groups: Vec<(Option<SubjectId>, Vec<i64>)> = Vec::new();
572    for event in events {
573        if let Some(slot) = groups.iter_mut().find(|(s, _)| *s == event.subject_id) {
574            slot.1.push(event.at_ns);
575        } else {
576            groups.push((event.subject_id, alloc::vec![event.at_ns]));
577        }
578    }
579    for (_, times) in &mut groups {
580        times.sort_unstable();
581    }
582    groups
583}
584
585#[cfg(test)]
586mod tests {
587    use super::*;
588    use crate::input::AnalysisSeries;
589
590    const DAY_NS: i64 = 86_400_i64 * 1_000_000_000;
591
592    fn timed(day: i64, subject: u64) -> TimedSubjectEvent {
593        TimedSubjectEvent {
594            subject_id: Some(SubjectId(subject)),
595            source_id: None,
596            at_ns: day * DAY_NS + 10 * 3_600_000_000_000, // 10:00
597            event_id: Some(EventId(day as u64 + 1)),
598            kind_tag: 0,
599        }
600    }
601
602    #[test]
603    fn baseline_day_fraction_concentrates_on_seen_weekdays() {
604        let mut series = AnalysisSeries::default();
605        // All events on epoch weekday 1 (days 1, 8, 15, …).
606        for k in 0..12 {
607            series.timed.push(timed(1 + k * 7, 1));
608        }
609        let cfg = StatAnomalyConfig {
610            min_samples: 5,
611            use_day_of_week: true,
612            ..StatAnomalyConfig::default()
613        };
614        let baseline = build_baseline(&series, cfg);
615        assert_eq!(baseline.day_n, 12);
616        assert!((baseline.day_fraction[1] - 1.0).abs() < 1e-5);
617        for d in [0, 2, 3, 4, 5, 6] {
618            assert!(baseline.day_fraction[d].abs() < 1e-6);
619        }
620    }
621
622    #[test]
623    fn rare_weekday_flags_unusual_appearance_time() {
624        let mut history = AnalysisSeries::default();
625        for k in 0..14 {
626            history.timed.push(timed(1 + k * 7, 1)); // weekday 1 only
627        }
628        let cfg = StatAnomalyConfig {
629            min_samples: 5,
630            use_day_of_week: true,
631            rare_day_fraction: 0.08,
632            use_robust: false,
633            z_threshold: 2.5,
634            change_point_threshold: 100.0,
635        };
636        let baseline = build_baseline(&history, cfg);
637
638        let mut live = AnalysisSeries::default();
639        // Weekday 3 never seen in baseline → rare.
640        live.timed.push(timed(3, 1));
641
642        let mut next_id = 1;
643        let anomalies = detect_statistical(&live, &baseline, cfg, &mut next_id);
644        assert!(
645            anomalies
646                .iter()
647                .any(|a| a.reasons.contains(&AnomalyReason::UnusualAppearanceTime)),
648            "expected day-of-week anomaly, got {anomalies:?}"
649        );
650    }
651
652    #[test]
653    fn day_of_week_can_be_disabled() {
654        let mut history = AnalysisSeries::default();
655        for k in 0..14 {
656            history.timed.push(timed(1 + k * 7, 1));
657        }
658        let cfg = StatAnomalyConfig {
659            use_day_of_week: false,
660            use_robust: false,
661            ..StatAnomalyConfig::default()
662        };
663        let baseline = build_baseline(&history, cfg);
664        let mut live = AnalysisSeries::default();
665        live.timed.push(timed(3, 1));
666        let mut next_id = 1;
667        let with = detect_statistical(
668            &live,
669            &baseline,
670            StatAnomalyConfig {
671                use_day_of_week: true,
672                use_robust: false,
673                ..StatAnomalyConfig::default()
674            },
675            &mut next_id,
676        );
677        let mut next_id = 1;
678        let without = detect_statistical(&live, &baseline, cfg, &mut next_id);
679        assert!(
680            with.iter()
681                .any(|a| a.reasons.contains(&AnomalyReason::UnusualAppearanceTime))
682        );
683        assert!(
684            without
685                .iter()
686                .all(|a| !a.reasons.contains(&AnomalyReason::UnusualAppearanceTime)),
687            "use_day_of_week=false must not emit weekday seasonality hits"
688        );
689    }
690}