modem-mgmt 0.2.0

The management plane of zenoh-modem as a library: serve every device on a host over Zenoh, for any driver that links the contract
Documentation
//! The operational status of every device, as `state` and as `events` (#121).
//!
//! RFC 2863 in two documents: `state/modem/device/{device}/oper` is where
//! things stand, refreshed like every other state document and rewritten the
//! moment they change; `events/modem/device/{device}/oper_change/{event_id}`
//! is the transition, which a reader arriving an hour later still finds. The
//! status comes from the device — [`Device::oper`](modem_device::Device::oper),
//! through the lane driver where there is one — and is **polled**, at the
//! telemetry cadence: it is sync and cached by contract, so a poll costs an
//! atomic load, and a device that pushed changes would need a channel every
//! implementor had to remember to feed.
//!
//! **Damped.** RFC 2863 §3.1.15 requires that a flapping interface be
//! rate-limited without fixing the rate, and `device_fault`'s sixty-second
//! floor is reused: the first transition publishes at once, the ones that
//! follow inside the floor fold, and when the floor expires with anything
//! folded one trailing event carries the status as it stands and how many
//! transitions it stands for. The *document* is never damped — it always says
//! what is true now — so damping loses nothing a reader could act on; it only
//! stops a carrier that flaps ten times a second from costing ten events a
//! second on a bus that may be a 2400 bit/s channel.

use std::sync::Arc;
use std::sync::atomic::AtomicU64;
use std::time::Duration;

use modem_contract::registry::modem::{Subject, key};
use modem_device::OperStatus;
use tokio::time::Instant;
use zenkey::origin::LocalOrigin;
use zenoh::Session;

use crate::clock::{clock_synced, now_ms, ulid};
use crate::host::{Host, Node, Plane};
use crate::plane::SduPlane;

/// The floor under `oper_change`: `device_fault`'s, for the same reason.
pub(crate) const FLOOR: Duration = Duration::from_secs(60);

/// Where a device's status is read from.
pub(crate) enum Source {
    /// Through the lane driver, which overlays what it knows.
    Sdu(Arc<dyn SduPlane>),
    /// The device itself: a netdev has no driver to know better.
    Device(Arc<dyn modem_device::Device>),
}

impl Source {
    pub(crate) fn of(node: &Node) -> Self {
        match node.plane() {
            Plane::Sdu(plane) => Self::Sdu(plane.clone()),
            Plane::Netdev(dev) => Self::Device(dev.clone()),
        }
    }

    pub(super) fn oper(&self) -> OperStatus {
        match self {
            Self::Sdu(plane) => plane.oper(),
            Self::Device(dev) => dev.oper(),
        }
    }

    pub(super) fn report(&self) -> modem_device::Report {
        match self {
            Self::Sdu(plane) => plane.report(),
            Self::Device(dev) => dev.report(),
        }
    }

    /// The device's configuration schema, if it declares one (#125).
    pub(super) fn schema(&self) -> Option<modem_device::config::Schema> {
        match self {
            Self::Sdu(plane) => plane.configuration().map(|c| c.schema()),
            Self::Device(dev) => dev.configuration().map(|c| c.schema()),
        }
    }

    /// What the device runs with, if it declares a configuration plane.
    pub(super) fn effective(&self) -> Option<modem_device::config::Effective> {
        match self {
            Self::Sdu(plane) => plane.configuration().map(|c| c.effective()),
            Self::Device(dev) => dev.configuration().map(|c| c.effective()),
        }
    }

    /// The device's operator actions, if it offers any (#129).
    pub(super) fn action_specs(&self) -> Option<Vec<modem_device::action::ActionSpec>> {
        match self {
            Self::Sdu(plane) => plane.actions().map(|a| a.specs()),
            Self::Device(dev) => dev.actions().map(|a| a.specs()),
        }
    }

    /// Run one of them.
    pub(super) async fn invoke(
        &self,
        name: &str,
        arguments: &std::collections::BTreeMap<String, modem_device::config::ConfigValue>,
    ) -> Result<modem_device::action::ActionOutcome, modem_device::action::ActionError> {
        let a = match self {
            Self::Sdu(plane) => plane.actions(),
            Self::Device(dev) => dev.actions(),
        };
        match a {
            Some(a) => a.invoke(name, arguments).await,
            None => Err(modem_device::action::ActionError::Unsupported(
                name.to_string(),
            )),
        }
    }

    pub(super) fn identity(&self) -> modem_device::Identity {
        match self {
            Self::Sdu(plane) => plane.identity(),
            Self::Device(dev) => dev.identity(),
        }
    }

    /// A wake-up the poll need not wait for: the driver's own health flag,
    /// which flips the moment a lane operation fails.
    fn health(&self) -> Option<tokio::sync::watch::Receiver<bool>> {
        match self {
            Self::Sdu(plane) => Some(plane.health()),
            Self::Device(_) => None,
        }
    }
}

/// The rate floor, as a value a test can drive with a clock of its own.
///
/// `observe` is called on every transition and answers whether to publish it
/// now; `flush` is called whenever the loop wakes and answers whether the
/// floor has expired over folded transitions. `deadline` is when that will
/// next be true, so the loop can sleep exactly until then.
pub(crate) struct Damper {
    floor: Duration,
    last: Option<Instant>,
    folded: u64,
}

impl Damper {
    pub(crate) fn new(floor: Duration) -> Self {
        Self {
            floor,
            last: None,
            folded: 0,
        }
    }

    /// A transition happened now. `Some(1)`: publish it. `None`: folded.
    pub(crate) fn observe(&mut self, now: Instant) -> Option<u64> {
        match self.last {
            Some(t) if now.duration_since(t) < self.floor => {
                self.folded += 1;
                None
            }
            _ => {
                self.last = Some(now);
                Some(1)
            }
        }
    }

    /// The floor has expired over folded transitions: publish one event
    /// standing for that many, and start a new floor from it.
    pub(crate) fn flush(&mut self, now: Instant) -> Option<u64> {
        if self.folded == 0 {
            return None;
        }
        let expired = self
            .last
            .is_none_or(|t| now.duration_since(t) >= self.floor);
        if !expired {
            return None;
        }
        let n = self.folded;
        self.folded = 0;
        self.last = Some(now);
        Some(n)
    }

    /// When `flush` will next answer, if anything is folded.
    pub(crate) fn deadline(&self) -> Option<Instant> {
        if self.folded == 0 {
            return None;
        }
        Some(self.last.map_or_else(Instant::now, |t| t + self.floor))
    }
}

#[allow(clippy::too_many_lines)]
pub(super) async fn publish_loop(
    session: Session,
    origin: LocalOrigin,
    host: Arc<Host>,
    device: String,
    source: Source,
    ttl_s: u64,
    poll: Duration,
) {
    let subject = Subject::device_oper(&device);
    let doc_key = key(&origin, &subject);
    let qos = subject.qos();
    let refresh = Duration::from_secs((ttl_s / 2).max(1));
    let counter = AtomicU64::new(0);
    let mut damper = Damper::new(FLOOR);
    let mut health = source.health();

    let mut current = source.oper();
    let document = |status: &OperStatus| {
        super::Oper::of(
            &device,
            status,
            host.uptime_s(),
            host.boot_id(),
            clock_synced(),
        )
    };
    crate::state::put(&session, &doc_key, &document(&current), qos).await;
    let mut refreshed = Instant::now();

    loop {
        let wake = [
            Some(Instant::now() + poll),
            Some(refreshed + refresh),
            damper.deadline(),
        ]
        .into_iter()
        .flatten()
        .min()
        .expect("the poll is always a deadline");
        tokio::select! {
            biased;
            changed = health_changed(health.as_mut()) => {
                if changed.is_err() {
                    return;
                }
            }
            () = tokio::time::sleep_until(wake) => {}
        }

        let now = Instant::now();
        let observed = source.oper();
        if observed != current {
            current = observed;
            crate::state::put(&session, &doc_key, &document(&current), qos).await;
            refreshed = now;
            if let Some(n) = damper.observe(now) {
                let event_id = ulid(&counter);
                let subject = Subject::device_oper_change(&device, &event_id);
                let doc = super::OperChange::of(&device, event_id, now_ms(), &current, n);
                crate::events::publish(&session, &origin, &subject, &doc, "oper_change").await;
            }
        } else if now >= refreshed + refresh {
            crate::state::put(&session, &doc_key, &document(&current), qos).await;
            refreshed = now;
        }
        if let Some(n) = damper.flush(now) {
            let event_id = ulid(&counter);
            let subject = Subject::device_oper_change(&device, &event_id);
            let doc = super::OperChange::of(&device, event_id, now_ms(), &current, n);
            crate::events::publish(&session, &origin, &subject, &doc, "oper_change").await;
        }
    }
}

/// Resolves when the health flag changes; never, for a source without one.
/// `Err` means the driver is gone, and so is this loop.
async fn health_changed(
    health: Option<&mut tokio::sync::watch::Receiver<bool>>,
) -> Result<(), tokio::sync::watch::error::RecvError> {
    match health {
        Some(rx) => rx.changed().await,
        None => std::future::pending().await,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn at(base: Instant, s: u64) -> Instant {
        base + Duration::from_secs(s)
    }

    #[test]
    fn the_first_transition_publishes_at_once() {
        let base = Instant::now();
        let mut d = Damper::new(FLOOR);
        assert_eq!(d.observe(base), Some(1));
        assert_eq!(d.flush(base), None);
        assert_eq!(d.deadline(), None);
    }

    #[test]
    fn transitions_inside_the_floor_fold_into_one_trailing_event() {
        let base = Instant::now();
        let mut d = Damper::new(FLOOR);
        assert_eq!(d.observe(base), Some(1));
        // A carrier flapping every second for a while.
        for s in 1..=5 {
            assert_eq!(d.observe(at(base, s)), None, "folded at {s}s");
            assert_eq!(
                d.flush(at(base, s)),
                None,
                "the floor has not expired at {s}s"
            );
        }
        assert_eq!(d.deadline(), Some(at(base, 60)));
        assert_eq!(d.flush(at(base, 59)), None);
        // The floor expires: one event standing for the five folded flaps.
        assert_eq!(d.flush(at(base, 60)), Some(5));
        assert_eq!(d.deadline(), None, "nothing is folded after a flush");
        // And a new floor starts from that event, so a flap right after it
        // folds again rather than publishing.
        assert_eq!(d.observe(at(base, 61)), None);
        assert_eq!(
            d.flush(at(base, 119)),
            None,
            "not yet: the floor restarted at 60"
        );
        assert_eq!(d.flush(at(base, 120)), Some(1));
    }

    #[test]
    fn a_transition_after_a_quiet_floor_publishes_at_once() {
        let base = Instant::now();
        let mut d = Damper::new(FLOOR);
        assert_eq!(d.observe(base), Some(1));
        assert_eq!(
            d.observe(at(base, 61)),
            Some(1),
            "the floor expired with nothing folded"
        );
    }
}