beatwise 0.2.1

A lightweight heartbeat-based task liveness monitor with fixed and learned timing
Documentation
fn status_for(entry: &TaskEntry, now: u64) -> TaskStatus {
    let heartbeat_count = entry.heartbeat_count.load(Ordering::Acquire);
    let (health, timing) = if entry.lifecycle.load(Ordering::Acquire) == STOPPED {
        let stopped_tick = entry.stopped_tick.load(Ordering::Acquire);
        let reason = match entry.stop_reason.load(Ordering::Acquire) {
            STOP_EXPLICIT => StopReason::Explicit,
            _ => StopReason::LastHandleDropped,
        };

        (
            Health::Stopped {
                stopped_for: tick_duration(now, stopped_tick),
                reason,
            },
            timing_status(&entry.timing),
        )
    } else {
        running_status(entry, now)
    };

    TaskStatus {
        id: entry.id,
        name: Arc::clone(&entry.name),
        health,
        timing,
        heartbeat_count,
    }
}

fn running_status(entry: &TaskEntry, now: u64) -> (Health, TimingStatus) {
    let last_tick = entry.last_tick.load(Ordering::Acquire);

    if last_tick == NO_TICK {
        let elapsed = tick_duration(now, entry.created_tick);
        let health = if elapsed <= entry.startup_grace {
            Health::Starting {
                elapsed,
                startup_grace: entry.startup_grace,
            }
        } else {
            Health::Late {
                silent_for: elapsed,
                deadline: entry.startup_grace,
                overdue_by: elapsed.saturating_sub(entry.startup_grace),
                missed_intervals: 0,
            }
        };

        return (health, timing_status(&entry.timing));
    }

    let silent_for = tick_duration(now, last_tick);

    match &entry.timing {
        TimingState::Fixed(fixed) => {
            let deadline = fixed.deadline();
            let health = classify_health(silent_for, deadline, fixed.interval);
            (
                health,
                TimingStatus::Fixed {
                    interval: fixed.interval,
                    grace: fixed.grace,
                },
            )
        }
        TimingState::Learned { config, state } => {
            let state = lock_learning(state);
            let status = learned_timing_status(*config, &state);

            if state.is_trained() {
                let interval = state
                    .estimated_interval()
                    .expect("trained learned timing must have an interval");
                let deadline = state
                    .deadline(*config)
                    .expect("trained learned timing must have a deadline");
                return (classify_health(silent_for, deadline, interval), status);
            }

            let provisional_deadline = state.deadline(*config).unwrap_or(entry.startup_grace);
            if silent_for > provisional_deadline {
                let interval = state.estimated_interval().unwrap_or(provisional_deadline);
                return (
                    classify_health(silent_for, provisional_deadline, interval),
                    status,
                );
            }

            (
                Health::Learning {
                    samples: state.samples(),
                    required: config.minimum_samples,
                    silent_for,
                    estimated_interval: state.estimated_interval(),
                },
                status,
            )
        }
    }
}

fn classify_health(silent_for: Duration, deadline: Duration, interval: Duration) -> Health {
    if silent_for <= deadline {
        Health::Healthy {
            silent_for,
            deadline,
        }
    } else {
        Health::Late {
            silent_for,
            deadline,
            overdue_by: silent_for.saturating_sub(deadline),
            missed_intervals: interval_count(silent_for, interval),
        }
    }
}

fn timing_status(timing: &TimingState) -> TimingStatus {
    match timing {
        TimingState::Fixed(fixed) => TimingStatus::Fixed {
            interval: fixed.interval,
            grace: fixed.grace,
        },
        TimingState::Learned { config, state } => {
            let state = lock_learning(state);
            learned_timing_status(*config, &state)
        }
    }
}

fn learned_timing_status(config: LearnedTiming, state: &LearningState) -> TimingStatus {
    let confidence = state.confidence(config);

    if let Some(pattern_config) = state.pattern_config() {
        if state.is_trained() {
            return TimingStatus::PatternLearned {
                config: pattern_config,
                samples: state.samples(),
                rejected_samples: state.rejected_samples(),
                period: state
                    .pattern_period()
                    .expect("trained pattern timing must have a period"),
                next_phase: state
                    .pattern_next_phase()
                    .expect("trained pattern timing must have a next phase"),
                intervals: Arc::from(
                    state
                        .pattern_intervals()
                        .expect("trained pattern timing must have phase intervals"),
                ),
                deviations: Arc::from(
                    state
                        .pattern_deviations()
                        .expect("trained pattern timing must have phase deviations"),
                ),
                expected_interval: state
                    .estimated_interval()
                    .expect("trained pattern timing must have an expected interval"),
                deadline: state
                    .deadline(config)
                    .expect("trained pattern timing must have a deadline"),
                adaptation: config.adaptation,
                confidence,
            };
        }

        return TimingStatus::PatternLearning {
            config: pattern_config,
            samples: state.samples(),
            rejected_samples: state.rejected_samples(),
            minimum_samples: config.minimum_samples,
            candidate_period: state.pattern_candidate_period(),
            confidence,
        };
    }

    if state.is_trained() {
        TimingStatus::Learned {
            model: state.model(),
            samples: state.samples(),
            rejected_samples: state.rejected_samples(),
            interval: state
                .estimated_interval()
                .expect("trained learned timing must have an interval"),
            deviation: state
                .estimated_deviation()
                .expect("trained learned timing must have a deviation"),
            deadline: state
                .deadline(config)
                .expect("trained learned timing must have a deadline"),
            adaptation: config.adaptation,
            confidence,
        }
    } else {
        TimingStatus::Learning {
            model: state.model(),
            samples: state.samples(),
            rejected_samples: state.rejected_samples(),
            required: config.minimum_samples,
            estimated_interval: state.estimated_interval(),
            estimated_deviation: state.estimated_deviation(),
            confidence,
        }
    }
}

fn tick_duration(later: u64, earlier: u64) -> Duration {
    Duration::from_micros(later.saturating_sub(earlier))
}

fn interval_count(elapsed: Duration, interval: Duration) -> u64 {
    if interval.is_zero() {
        return 0;
    }

    (elapsed.as_nanos() / interval.as_nanos()).min(u128::from(u64::MAX)) as u64
}