mod chunk;
pub use chunk::LaneTimeouts;
pub(in crate::coordinator) use chunk::{POLL, PatientReader, WRITE_TIMEOUT};
use super::registry::{Registry, WorkerHandle, WorkerInfo};
use crate::{assets::HeldAsset, cli::ComputeMode, validate::MAX_SAMPLES_PER_REQUEST};
use glam::Vec3;
use indicatrix::renderer::gpu_backend::GpuBackend;
use indicatrix_dispatch::{CancelToken, ChunkResult, SampleRange, WorkerLane};
use indicatrix_net::{
SceneState,
messages::{ClientMessage, StreamEvent, error_codes},
};
use std::{
sync::{
Arc, Mutex, PoisonError,
atomic::{AtomicBool, AtomicU32, Ordering},
},
thread,
time::{Duration, Instant},
};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct LaneNeed {
pub pixels: u32,
pub hdr: bool,
}
impl LaneNeed {
#[must_use]
pub const fn pixels(pixels: u32) -> Self {
Self { pixels, hdr: false }
}
#[must_use]
pub const fn accepts(self, info: &WorkerInfo) -> bool {
info.capability.max_pixels >= self.pixels && (!self.hdr || info.capability.hdr)
}
}
pub struct JobLanes {
need: LaneNeed,
asset: Option<Arc<HeldAsset>>,
timeouts: LaneTimeouts,
excluded: Mutex<Vec<u32>>,
}
impl JobLanes {
#[must_use]
pub const fn new(
need: LaneNeed,
asset: Option<Arc<HeldAsset>>,
timeouts: LaneTimeouts,
) -> Self {
Self {
need,
asset,
timeouts,
excluded: Mutex::new(Vec::new()),
}
}
#[must_use]
pub(in crate::coordinator) const fn need(&self) -> LaneNeed {
self.need
}
pub(in crate::coordinator) fn accepts(&self, info: &WorkerInfo) -> bool {
self.need.accepts(info) && !self.lock_excluded().contains(&info.worker_id)
}
fn exclude(&self, worker_id: u32) {
let mut excluded = self.lock_excluded();
if !excluded.contains(&worker_id) {
excluded.push(worker_id);
}
}
fn lock_excluded(&self) -> std::sync::MutexGuard<'_, Vec<u32>> {
self.excluded.lock().unwrap_or_else(PoisonError::into_inner)
}
}
static NEXT_WORKER_REQUEST_ID: AtomicU32 = AtomicU32::new(1);
pub(in crate::coordinator) fn next_worker_request_id() -> u32 {
NEXT_WORKER_REQUEST_ID.fetch_add(1, Ordering::Relaxed)
}
struct Held {
handle: Mutex<Option<WorkerHandle>>,
last_traffic: Mutex<Instant>,
gone: AtomicBool,
}
const HELD_PING_INTERVAL: Duration = Duration::from_secs(15);
const HELD_PING_POLL: Duration = Duration::from_millis(200);
const HELD_PING_DEADLINE: Duration = Duration::from_secs(5);
pub struct JoinedWorkerLane {
name: String,
registry: Arc<Registry>,
job: Arc<JobLanes>,
held: Arc<Held>,
heartbeat_stop: Arc<AtomicBool>,
heartbeat: Option<thread::JoinHandle<()>>,
}
impl JoinedWorkerLane {
#[must_use]
pub fn new(handle: WorkerHandle, registry: Arc<Registry>, job: Arc<JobLanes>) -> Self {
let name = format!("joined worker #{}", handle.info().worker_id);
let held = Arc::new(Held {
handle: Mutex::new(Some(handle)),
last_traffic: Mutex::new(Instant::now()),
gone: AtomicBool::new(false),
});
let heartbeat_stop = Arc::new(AtomicBool::new(false));
let heartbeat = {
let held = Arc::clone(&held);
let stop = Arc::clone(&heartbeat_stop);
thread::Builder::new()
.name(format!("{name}-heartbeat"))
.spawn(move || held_ping_loop(&held, &stop))
.ok()
};
Self {
name,
registry,
job,
held,
heartbeat_stop,
heartbeat,
}
}
fn run_on(
&self,
handle: &mut WorkerHandle,
scene: &SceneState,
range: SampleRange,
cancel: &CancelToken,
) -> (ChunkResult, Option<String>) {
let pixels = scene.width as usize * scene.height as usize;
let worker_id = handle.info().worker_id;
let mut total = ChunkResult::complete(vec![Vec3::ZERO; pixels], 0, None);
let mut rest = range;
let mut parts = 0;
while !rest.is_empty() {
let take = rest.samples.min(MAX_SAMPLES_PER_REQUEST);
let request =
chunk::worker_request(next_worker_request_id(), scene, rest.first_sample, take);
let exchange = chunk::Exchange {
worker_id,
cancel,
timeouts: self.job.timeouts,
asset: self.job.asset.as_deref(),
};
let reply = chunk::run_request(handle.stream(), &request, exchange);
parts += 1;
if reply.assets_sent > 0 {
self.registry.note_assets_forwarded(reply.assets_sent);
}
if self.job.need.hdr
&& matches!(
reply.worker_code,
Some(error_codes::ASSET_FAILED | error_codes::UNSUPPORTED_REQUEST)
)
{
tracing::info!(
"coordinator job: worker #{worker_id} refused the job's HDR map; it takes no further \
chunk of this job"
);
self.job.exclude(worker_id);
}
let usable = if reply.done == take || reply.prefix {
reply.done
} else {
0
};
if usable > 0 {
for (acc, v) in total.sum.iter_mut().zip(&reply.sum) {
*acc += *v;
}
total.done += usable;
}
total.rate = if parts == 1 { reply.rate } else { None };
if usable < take || reply.broken.is_some() {
total.rate = None;
total.error = reply
.error
.or_else(|| cancel.is_cancelled().then(|| "cancelled".to_string()))
.or_else(|| Some(format!("worker #{worker_id} ended the chunk early")));
return (total, reply.broken);
}
rest = rest.after_prefix(take);
}
(total, None)
}
}
impl WorkerLane for JoinedWorkerLane {
fn name(&self) -> &str {
&self.name
}
fn render_chunk(
&self,
scene: &SceneState,
range: SampleRange,
cancel: &CancelToken,
) -> ChunkResult {
let mut slot = self
.held
.handle
.lock()
.unwrap_or_else(PoisonError::into_inner);
let handle = slot
.take()
.or_else(|| Registry::checkout(&self.registry, |w| self.job.accepts(w)));
let Some(mut handle) = handle else {
self.held.gone.store(true, Ordering::Relaxed);
return ChunkResult::failed("no idle joined worker to take the chunk".to_string());
};
self.held.gone.store(false, Ordering::Relaxed);
let (result, broken) = self.run_on(&mut handle, scene, range, cancel);
*self
.held
.last_traffic
.lock()
.unwrap_or_else(PoisonError::into_inner) = Instant::now();
match broken {
Some(why) => {
self.held.gone.store(true, Ordering::Relaxed);
handle.discard(&why);
}
None if result.error.is_none() => *slot = Some(handle),
None => drop(handle),
}
drop(slot);
result
}
fn lost(&self) -> bool {
self.held.gone.load(Ordering::Relaxed)
}
}
impl Drop for JoinedWorkerLane {
fn drop(&mut self) {
self.heartbeat_stop.store(true, Ordering::Relaxed);
if let Some(thread) = self.heartbeat.take() {
let _ = thread.join();
}
}
}
fn held_ping_loop(held: &Held, stop: &AtomicBool) {
while !stop.load(Ordering::Relaxed) {
thread::sleep(HELD_PING_POLL);
if stop.load(Ordering::Relaxed) {
return;
}
let due = held
.last_traffic
.lock()
.unwrap_or_else(PoisonError::into_inner)
.elapsed()
>= HELD_PING_INTERVAL;
if due {
try_heartbeat_once(held);
}
}
}
fn try_heartbeat_once(held: &Held) {
let mut slot = match held.handle.try_lock() {
Ok(guard) => guard,
Err(std::sync::TryLockError::Poisoned(poisoned)) => poisoned.into_inner(),
Err(std::sync::TryLockError::WouldBlock) => {
return;
}
};
let Some(mut handle) = slot.take() else {
return;
};
drop(slot);
match ping_held(&mut handle) {
Ok(()) => {
*held
.last_traffic
.lock()
.unwrap_or_else(PoisonError::into_inner) = Instant::now();
*held.handle.lock().unwrap_or_else(PoisonError::into_inner) = Some(handle);
}
Err(why) => {
tracing::info!(
"coordinator job: heartbeat ping to worker #{} failed while it idled between \
chunks: {why}",
handle.info().worker_id
);
handle.discard(&why);
held.gone.store(true, Ordering::Relaxed);
}
}
}
fn ping_held(handle: &mut WorkerHandle) -> Result<(), String> {
const NONCE: u64 = 0x4845_4152_5442_4954; let mut conn = handle.stream();
conn.set_timeouts(Some(HELD_PING_DEADLINE), Some(HELD_PING_DEADLINE))
.map_err(|e| format!("could not arm the heartbeat ping deadline: {e}"))?;
indicatrix_net::messages::write_message(&mut conn, &ClientMessage::Ping { nonce: NONCE })
.map_err(|e| format!("heartbeat PING failed: {e}"))?;
loop {
let (event, _payload) = indicatrix_net::messages::read_stream_event(&mut conn)
.map_err(|e| format!("heartbeat: no PONG ({e})"))?;
match event {
StreamEvent::Pong { nonce: NONCE } => {
let _ = conn.set_timeouts(None, None);
return Ok(());
}
other => tracing::debug!("coordinator: ignoring {other:?} from a held idle worker"),
}
}
}
pub struct OwnLane {
gpu: Arc<GpuBackend>,
threads: usize,
compute_mode: ComputeMode,
}
impl OwnLane {
#[must_use]
pub const fn new(gpu: Arc<GpuBackend>, threads: usize, compute_mode: ComputeMode) -> Self {
Self {
gpu,
threads,
compute_mode,
}
}
}
impl WorkerLane for OwnLane {
fn name(&self) -> &'static str {
"coordinator's own lane"
}
fn render_chunk(
&self,
scene: &SceneState,
range: SampleRange,
cancel: &CancelToken,
) -> ChunkResult {
let traced = crate::stream_emit::trace_range(
scene,
(range.first_sample, range.samples),
self.threads,
self.compute_mode,
&self.gpu,
cancel.as_flag(),
);
if traced.panicked {
return ChunkResult::partial(
traced.sum,
traced.done,
"the own lane's tracer panicked on this scene".to_string(),
);
}
if traced.done < range.samples {
return ChunkResult::partial(traced.sum, traced.done, "cancelled".to_string());
}
ChunkResult::complete(traced.sum, traced.done, None)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::coordinator::LivenessConfig;
use indicatrix_net::messages::{Backend, PayloadEncoding, RenderCapability};
use std::{
net::{TcpListener, TcpStream},
thread,
};
fn checked_out_handle() -> (WorkerHandle, TcpStream) {
let registry = Registry::new(LivenessConfig::default());
let listener = TcpListener::bind("127.0.0.1:0").expect("bind a loopback listener");
let addr = listener
.local_addr()
.expect("a bound listener has a local address");
let near = TcpStream::connect(addr).expect("connect to our own listener");
let far = listener
.accept()
.expect("accept the connection just made")
.0;
let worker_id = registry.allocate_id();
let info = WorkerInfo {
worker_id,
capability: RenderCapability {
backend: Backend::Cpu { threads: 1 },
max_pixels: 1_000,
min_cadence_ms: 100,
hdr: false,
},
peer: None,
label: None,
payload_encoding: PayloadEncoding::Raw,
};
registry.insert(info, Box::new(near), None);
let handle =
Registry::checkout(®istry, |_| true).expect("the just-inserted worker is idle");
(handle, far)
}
fn answer_one_ping(mut far: TcpStream) {
thread::spawn(move || {
let message: ClientMessage = indicatrix_net::messages::read_message(&mut far)
.expect("the heartbeat sends exactly one PING");
let ClientMessage::Ping { nonce } = message else {
panic!("expected ClientMessage::Ping, got {message:?}");
};
indicatrix_net::messages::write_stream_event(
&mut far,
&StreamEvent::Pong { nonce },
None,
)
.expect("writing PONG must not fail on a live loopback socket");
});
}
#[test]
fn ping_held_succeeds_against_a_peer_that_answers_pong() {
let (mut handle, far) = checked_out_handle();
answer_one_ping(far);
ping_held(&mut handle).expect("a live peer answering PONG must succeed");
}
#[test]
fn ping_held_fails_against_a_silent_peer() {
let (mut handle, _far) = checked_out_handle();
assert!(ping_held(&mut handle).is_err());
}
}