use super::liveness;
use crate::serve::ConnectionSlot;
use indicatrix_net::messages::{Backend, PayloadEncoding, RenderCapability};
use std::{
collections::BTreeMap,
io::{Read, Write},
net::{SocketAddr, TcpStream},
sync::{
Arc, Mutex, MutexGuard, PoisonError,
atomic::{AtomicU32, AtomicU64, Ordering},
mpsc,
},
time::{Duration, Instant},
};
pub trait WorkerConn: Read + Write + Send {
fn set_timeouts(
&mut self,
read: Option<Duration>,
write: Option<Duration>,
) -> std::io::Result<()>;
}
impl WorkerConn for TcpStream {
fn set_timeouts(
&mut self,
read: Option<Duration>,
write: Option<Duration>,
) -> std::io::Result<()> {
self.set_read_timeout(read)?;
self.set_write_timeout(write)
}
}
impl WorkerConn for crate::serve::TlsStream {
fn set_timeouts(
&mut self,
read: Option<Duration>,
write: Option<Duration>,
) -> std::io::Result<()> {
self.sock.set_read_timeout(read)?;
self.sock.set_write_timeout(write)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct LivenessConfig {
pub ping_interval: Duration,
pub dead_after: Duration,
pub tick: Duration,
}
impl Default for LivenessConfig {
fn default() -> Self {
Self {
ping_interval: Duration::from_secs(10),
dead_after: Duration::from_secs(30),
tick: Duration::from_secs(1),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct WorkerInfo {
pub worker_id: u32,
pub capability: RenderCapability,
pub peer: Option<SocketAddr>,
pub label: Option<String>,
pub payload_encoding: PayloadEncoding,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct Capacity {
pub workers: u32,
pub threads: u32,
pub gpus: u32,
pub max_pixels: u32,
pub hdr_workers: u32,
}
const MAX_REPORTED_THREADS: u32 = 1024;
impl Capacity {
fn add(&mut self, capability: &RenderCapability) {
self.workers = self.workers.saturating_add(1);
self.hdr_workers = self.hdr_workers.saturating_add(u32::from(capability.hdr));
match &capability.backend {
Backend::Cpu { threads } => {
self.threads = self
.threads
.saturating_add((*threads).min(MAX_REPORTED_THREADS));
}
Backend::Gpu { .. } => self.gpus = self.gpus.saturating_add(1),
Backend::Coordinator { threads, gpus, .. } => {
self.threads = self.threads.saturating_add(*threads);
self.gpus = self.gpus.saturating_add(*gpus);
}
}
self.max_pixels = self.max_pixels.max(capability.max_pixels);
}
}
enum Slot {
Idle(Box<dyn WorkerConn>),
CheckedOut,
Pinging,
}
struct Entry {
info: WorkerInfo,
slot: Slot,
last_traffic: Instant,
_connection_slot: Option<ConnectionSlot>,
}
pub struct Registry {
entries: Mutex<BTreeMap<u32, Entry>>,
subscribers: Mutex<Vec<mpsc::Sender<Capacity>>>,
next_id: AtomicU32,
next_nonce: AtomicU64,
liveness: LivenessConfig,
assets_forwarded: AtomicU64,
}
impl Registry {
#[must_use]
pub fn new(liveness: LivenessConfig) -> Arc<Self> {
Arc::new(Self {
entries: Mutex::new(BTreeMap::new()),
subscribers: Mutex::new(Vec::new()),
next_id: AtomicU32::new(1),
next_nonce: AtomicU64::new(1),
liveness,
assets_forwarded: AtomicU64::new(0),
})
}
#[must_use]
pub fn assets_forwarded(&self) -> u64 {
self.assets_forwarded.load(Ordering::Relaxed)
}
pub(super) fn note_assets_forwarded(&self, count: u32) {
self.assets_forwarded
.fetch_add(u64::from(count), Ordering::Relaxed);
}
pub fn spawn_liveness(this: &Arc<Self>) {
liveness::spawn(Arc::downgrade(this));
}
#[must_use]
pub const fn liveness(&self) -> LivenessConfig {
self.liveness
}
pub fn allocate_id(&self) -> u32 {
self.next_id.fetch_add(1, Ordering::Relaxed)
}
pub(super) fn next_nonce(&self) -> u64 {
self.next_nonce.fetch_add(1, Ordering::Relaxed)
}
pub fn insert(
&self,
info: WorkerInfo,
conn: Box<dyn WorkerConn>,
connection_slot: Option<ConnectionSlot>,
) {
let id = info.worker_id;
self.lock_entries().insert(
id,
Entry {
info,
slot: Slot::Idle(conn),
last_traffic: Instant::now(),
_connection_slot: connection_slot,
},
);
self.notify();
}
#[must_use]
pub fn capacity(&self) -> Capacity {
let mut capacity = Capacity::default();
for entry in self.lock_entries().values() {
capacity.add(&entry.info.capability);
}
capacity
}
#[must_use]
pub fn workers(&self) -> Vec<(WorkerInfo, bool)> {
self.lock_entries()
.values()
.map(|e| (e.info.clone(), matches!(e.slot, Slot::Idle(_))))
.collect()
}
#[must_use]
pub fn subscribe(&self) -> mpsc::Receiver<Capacity> {
let (tx, rx) = mpsc::channel();
self.subscribers
.lock()
.unwrap_or_else(PoisonError::into_inner)
.push(tx);
rx
}
pub fn checkout(
this: &Arc<Self>,
filter: impl Fn(&WorkerInfo) -> bool,
) -> Option<WorkerHandle> {
let (info, conn) = this.lock_entries().values_mut().find_map(|entry| {
if !filter(&entry.info) {
return None;
}
match std::mem::replace(&mut entry.slot, Slot::CheckedOut) {
Slot::Idle(conn) => Some((entry.info.clone(), conn)),
busy => {
entry.slot = busy;
None
}
}
})?;
Some(WorkerHandle {
registry: Arc::clone(this),
info,
conn: Some(conn),
})
}
pub(super) fn check_in(&self, worker_id: u32, conn: Box<dyn WorkerConn>) {
if let Some(entry) = self.lock_entries().get_mut(&worker_id) {
entry.slot = Slot::Idle(conn);
entry.last_traffic = Instant::now();
}
}
pub fn remove(&self, worker_id: u32, why: &str) {
let removed = self.lock_entries().remove(&worker_id);
if let Some(entry) = removed {
tracing::warn!(
"coordinator: worker #{worker_id} ({:?}) dropped: {why}",
entry.info.peer
);
drop(entry);
self.notify();
}
}
pub(super) fn take_due_pings(&self, now: Instant) -> Vec<(u32, Box<dyn WorkerConn>, Instant)> {
let mut due = Vec::new();
for (id, entry) in self.lock_entries().iter_mut() {
if !matches!(entry.slot, Slot::Idle(_))
|| now.duration_since(entry.last_traffic) < self.liveness.ping_interval
{
continue;
}
if let Slot::Idle(conn) = std::mem::replace(&mut entry.slot, Slot::Pinging) {
due.push((*id, conn, entry.last_traffic + self.liveness.dead_after));
}
}
due
}
fn lock_entries(&self) -> MutexGuard<'_, BTreeMap<u32, Entry>> {
self.entries.lock().unwrap_or_else(PoisonError::into_inner)
}
fn notify(&self) {
let capacity = self.capacity();
self.subscribers
.lock()
.unwrap_or_else(PoisonError::into_inner)
.retain(|tx| tx.send(capacity).is_ok());
}
}
pub struct WorkerHandle {
registry: Arc<Registry>,
info: WorkerInfo,
conn: Option<Box<dyn WorkerConn>>,
}
impl WorkerHandle {
#[must_use]
pub const fn info(&self) -> &WorkerInfo {
&self.info
}
pub fn stream(&mut self) -> &mut dyn WorkerConn {
self.conn
.as_deref_mut()
.expect("a WorkerHandle holds its connection until dropped or discarded")
}
pub fn discard(mut self, why: &str) {
drop(self.conn.take());
self.registry.remove(self.info.worker_id, why);
}
}
impl Drop for WorkerHandle {
fn drop(&mut self) {
if let Some(conn) = self.conn.take() {
self.registry.check_in(self.info.worker_id, conn);
}
}
}