coactor 0.1.0

An embedded distributed Actor framework for Rust applications
Documentation
use std::{
    sync::{
        Arc,
        atomic::{AtomicBool, Ordering},
    },
    time::Duration,
};

use parking_lot::Mutex;
use tokio::sync::{oneshot, watch};

use super::confirm_node_lease;
use crate::{
    __macro::ServerInner,
    MutationOutcome, NodeLeaseStore, NodeRecord, Revision,
    transport::{Endpoint, ServerTransport},
};

pub struct NodeAuthority {
    valid: AtomicBool,
    deadline: Mutex<tokio::time::Instant>,
    ttl: Duration,
    fenced: watch::Sender<bool>,
}

impl NodeAuthority {
    pub fn new(
        operation_started: tokio::time::Instant,
        ttl: Duration,
        fenced: watch::Sender<bool>,
    ) -> Self {
        Self {
            valid: AtomicBool::new(true),
            deadline: Mutex::new(operation_started + ttl),
            ttl,
            fenced,
        }
    }
    pub fn is_valid(&self) -> bool {
        self.valid.load(Ordering::Acquire) && tokio::time::Instant::now() < *self.deadline.lock()
    }
    fn renew(&self, operation_started: tokio::time::Instant) {
        *self.deadline.lock() = operation_started + self.ttl;
    }
    fn remaining(&self) -> Option<Duration> {
        self.deadline
            .lock()
            .checked_duration_since(tokio::time::Instant::now())
    }
    fn deadline(&self) -> tokio::time::Instant {
        *self.deadline.lock()
    }
    fn fence(&self) {
        if self.valid.swap(false, Ordering::AcqRel) {
            let _ = self.fenced.send(true);
        }
    }
}

pub struct TransportTask {
    pub shutdown: watch::Sender<bool>,
    pub force: oneshot::Sender<()>,
    pub task: tokio::task::JoinHandle<()>,
    pub stopped: watch::Receiver<bool>,
}

pub struct RenewalTask {
    shutdown: oneshot::Sender<()>,
    task: tokio::task::JoinHandle<RenewalExit>,
    pub stopped: watch::Receiver<bool>,
}
struct RenewalExit {
    leases: Arc<dyn NodeLeaseStore>,
    node_id: String,
    revision: Revision,
    release: bool,
}

pub struct ClusterTasks {
    pub transport: TransportTask,
    pub renewal: RenewalTask,
    pub fenced: watch::Receiver<bool>,
}
impl ClusterTasks {
    pub async fn shutdown(self) {
        let _ = self.renewal.shutdown.send(());
        let _ = self.transport.shutdown.send(true);
        if let Ok(exit) = self.renewal.task.await {
            if exit.release {
                let _ = exit
                    .leases
                    .release_node(&exit.node_id, &exit.revision)
                    .await;
            }
        }
        let _ = self.transport.force.send(());
        let _ = self.transport.task.await;
    }
}

pub fn spawn_transport<S>(
    runtime: Arc<ServerInner<S>>,
    transport: Arc<dyn ServerTransport>,
    advertised: &Endpoint,
    listener: Option<tokio::net::TcpListener>,
) -> TransportTask
where
    S: Send + Sync + 'static,
{
    let (shutdown, shutdown_receiver) = watch::channel(false);
    let (force, force_receiver) = oneshot::channel();
    let (stopped_sender, stopped) = watch::channel(false);
    let mut listener = transport
        .listen(advertised, listener)
        .expect("transport listener bind");
    let task = tokio::spawn(async move {
        let mut shutdown_receiver = shutdown_receiver;
        let mut force_receiver = force_receiver;
        loop {
            tokio::select! {
                changed = shutdown_receiver.changed() => if changed.is_ok() && *shutdown_receiver.borrow() { listener.shutdown(); },
                _ = &mut force_receiver => break,
                stream = listener.accept() => {
                    let Some(stream) = stream else { break };
                    let task_runtime = runtime.clone();
                    let handle = tokio::spawn(async move {
                        let mut stream = stream;
                        let sender = stream.sender();
                        while let Some(envelope) = stream.recv().await { task_runtime.dispatch_inbound(envelope, Some(sender.clone())).await; }
                        task_runtime.close_sessions_for_sender(&sender).await;
                        task_runtime.retain_inbound_tasks();
                    });
                    runtime.register_inbound_task(handle.abort_handle());
                }
            }
        }
        let _ = stopped_sender.send(true);
    });
    TransportTask {
        shutdown,
        force,
        task,
        stopped,
    }
}

pub(crate) struct RenewalTiming {
    pub ttl: Duration,
    pub operation_timeout: Duration,
    pub interval: Duration,
}

pub fn spawn_lease_renewal<S>(
    runtime: Arc<ServerInner<S>>,
    authority: Arc<NodeAuthority>,
    leases: Arc<dyn NodeLeaseStore>,
    mut node: NodeRecord,
    mut revision: Revision,
    timing: RenewalTiming,
) -> RenewalTask
where
    S: Send + Sync + 'static,
{
    let (shutdown, mut shutdown_receiver) = oneshot::channel();
    let (stopped_sender, stopped) = watch::channel(false);
    let task = tokio::spawn(async move {
        let _stopped_sender = stopped_sender;
        loop {
            let jitter = 0.8 + (rand::random::<u16>() % 401) as f64 / 1000.0;
            let renewal_due = tokio::time::Instant::now() + timing.interval.mul_f64(jitter);
            let wake_at = renewal_due.min(authority.deadline());
            tokio::select! {
                _ = tokio::time::sleep_until(wake_at) => {}
                _ = &mut shutdown_receiver => return RenewalExit { leases, node_id: node.node_id, revision, release: true },
            }
            if !authority.is_valid() {
                authority.fence();
                runtime.fence().await;
                return RenewalExit {
                    leases,
                    node_id: node.node_id,
                    revision,
                    release: false,
                };
            }
            let operation_started = tokio::time::Instant::now();
            node.lease_generation = node.lease_generation.saturating_add(1);
            runtime.update_capacity_sample(&mut node);
            let Some(remaining) = authority.remaining() else {
                authority.fence();
                runtime.fence().await;
                return RenewalExit {
                    leases,
                    node_id: node.node_id,
                    revision,
                    release: false,
                };
            };
            let outcome = tokio::time::timeout(
                timing.operation_timeout.min(remaining),
                leases.renew_node(node.clone(), timing.ttl, &revision),
            )
            .await;
            match outcome {
                Ok(Ok(MutationOutcome::Applied(next))) => {
                    revision = next;
                    authority.renew(operation_started);
                }
                Ok(Ok(MutationOutcome::Indeterminate(_))) => {
                    let Some(next) = confirm_node_lease(
                        leases.as_ref(),
                        &node,
                        authority.deadline(),
                        timing.operation_timeout,
                    )
                    .await
                    else {
                        authority.fence();
                        runtime.fence().await;
                        return RenewalExit {
                            leases,
                            node_id: node.node_id,
                            revision,
                            release: false,
                        };
                    };
                    revision = next;
                    authority.renew(operation_started);
                }
                Ok(Ok(MutationOutcome::Conflict)) => {
                    authority.fence();
                    runtime.fence().await;
                    return RenewalExit {
                        leases,
                        node_id: node.node_id,
                        revision,
                        release: false,
                    };
                }
                _ if !authority.is_valid() => {
                    authority.fence();
                    runtime.fence().await;
                    return RenewalExit {
                        leases,
                        node_id: node.node_id,
                        revision,
                        release: false,
                    };
                }
                _ => {}
            }
        }
    });
    RenewalTask {
        shutdown,
        task,
        stopped,
    }
}