Skip to main content

rws_gateway_api/gateway_api/
lease.rs

1//! Leader election over a `coordination.k8s.io/v1` Lease: every replica
2//! serves traffic, only the holder writes status and Services. The holder
3//! renews every [`RENEW`]; another replica takes over once a renewal is older
4//! than the lease's duration. Updates carry the Lease's `resourceVersion`,
5//! so two replicas can't both win. Since 18.35.0 the work is done by
6//! `rws_k8s`'s `LeaderElector`, shared with operators.
7
8use std::sync::atomic::{AtomicBool, Ordering};
9use std::sync::Arc;
10use std::time::Duration;
11
12use k8s_openapi::api::coordination::v1::Lease;
13use kube::Client;
14use tokio::sync::Notify;
15
16/// How long a holder's renewal counts.
17pub const DURATION: Duration = Duration::from_secs(15);
18/// How often the holder renews (and others check).
19pub const RENEW: Duration = Duration::from_secs(5);
20
21pub struct LeaseSettings {
22    pub namespace: String,
23    pub name: String,
24    /// This replica (the pod's name).
25    pub identity: String,
26}
27
28/// What a replica should do with the Lease as it is now.
29#[derive(Debug, PartialEq)]
30pub enum Decision {
31    /// Create it, holding it.
32    Create,
33    /// Write it back holding it (renewing, or taking over an expired one).
34    Take,
35    /// Someone else holds it.
36    Follow,
37}
38
39pub fn decide(lease: Option<&Lease>, identity: &str, now: k8s_openapi::jiff::Timestamp) -> Decision {
40    match rws_k8s::k8s::leader::decide(lease, identity, now, DURATION) {
41        rws_k8s::k8s::leader::Decision::Create => Decision::Create,
42        rws_k8s::k8s::leader::Decision::Take => Decision::Take,
43        rws_k8s::k8s::leader::Decision::Follow => Decision::Follow,
44    }
45}
46
47/// Hold or follow the Lease forever (through `rws_k8s`'s `LeaderElector`,
48/// 18.35.0); `leader` says which, `gained` fires when this replica becomes
49/// the leader (so it can reconcile at once).
50pub async fn run(client: Client, settings: LeaseSettings, leader: Arc<AtomicBool>, gained: Arc<Notify>) {
51    let elector = rws_k8s::k8s::leader::LeaderElector::new(client, &settings.namespace, &settings.name).identity(&settings.identity).timing(DURATION, RENEW);
52    loop {
53        let holding = elector.step().await;
54        let was = leader.swap(holding, Ordering::SeqCst);
55        if holding && !was {
56            gained.notify_one();
57        }
58        tokio::time::sleep(RENEW).await;
59    }
60}
61
62/// The Lease held by this replica, based on `current` (keeping its
63/// `resourceVersion` and counting a change of holder).
64#[cfg(test)]
65fn held(settings: &LeaseSettings, current: Option<&Lease>, now: k8s_openapi::jiff::Timestamp) -> Lease {
66    rws_k8s::k8s::leader::held(&settings.namespace, &settings.name, &settings.identity, DURATION, current, now)
67}
68
69#[cfg(test)]
70mod tests {
71    use super::*;
72    use k8s_openapi::api::coordination::v1::LeaseSpec;
73    use k8s_openapi::apimachinery::pkg::apis::meta::v1::MicroTime;
74
75    fn lease(holder: &str, renewed: i64) -> Lease {
76        Lease {
77            metadata: Default::default(),
78            spec: Some(LeaseSpec {
79                holder_identity: Some(holder.to_string()),
80                lease_duration_seconds: Some(15),
81                renew_time: Some(MicroTime(k8s_openapi::jiff::Timestamp::from_second(renewed).unwrap())),
82                ..Default::default()
83            }),
84        }
85    }
86
87    #[test]
88    fn replicas_create_renew_follow_and_take_over() {
89        let now = k8s_openapi::jiff::Timestamp::from_second(1_000).unwrap();
90        assert_eq!(Decision::Create, decide(None, "a", now));
91        assert_eq!(Decision::Take, decide(Some(&lease("a", 999)), "a", now), "the holder renews");
92        assert_eq!(Decision::Follow, decide(Some(&lease("b", 990)), "a", now), "a live holder keeps it");
93        assert_eq!(Decision::Take, decide(Some(&lease("b", 980)), "a", now), "an expired one is taken over");
94        assert_eq!(Decision::Take, decide(Some(&lease("", 999)), "a", now), "nobody holds it");
95    }
96
97    #[test]
98    fn taking_over_counts_a_transition_and_keeps_the_resource_version() {
99        let settings = LeaseSettings { namespace: "ns".to_string(), name: "l".to_string(), identity: "a".to_string() };
100        let mut current = lease("b", 0);
101        current.metadata.resource_version = Some("42".to_string());
102        let now = k8s_openapi::jiff::Timestamp::from_second(100).unwrap();
103        let taken = held(&settings, Some(&current), now);
104        assert_eq!(Some("42".to_string()), taken.metadata.resource_version);
105        let spec = taken.spec.unwrap();
106        assert_eq!((Some("a".to_string()), Some(1)), (spec.holder_identity.clone(), spec.lease_transitions));
107        let renewed = held(&settings, Some(&Lease { spec: Some(spec.clone()), ..current }), now);
108        assert_eq!(Some(1), renewed.spec.unwrap().lease_transitions, "renewing isn't a transition");
109    }
110}