use crate::{assets::HeldAsset, stream_emit::is_stream_timeout};
use glam::Vec3;
use indicatrix_dispatch::{CancelToken, marginal_rate};
use indicatrix_net::{
SceneState,
messages::{
Cancel, ClientMessage, ContentHash, FrameHeader, RenderRequest, RequestIntent,
StreamConfig, StreamEvent, TransferMode, hash_hex, write_asset_message,
},
radiance::PayloadDecoder,
};
use std::{
io::{Read, Write},
time::{Duration, Instant},
};
use super::super::WorkerConn;
pub(in crate::coordinator) const POLL: Duration = Duration::from_millis(50);
pub(in crate::coordinator) const WRITE_TIMEOUT: Duration = Duration::from_secs(10);
const WORKER_CADENCE_MS: u32 = 1000;
const PROGRESS_STALL: Duration = Duration::from_secs(60);
const ASSET_WRITE_PIECE: usize = 1024 * 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct LaneTimeouts {
pub first_event: Duration,
pub liveness: Duration,
pub cancel_wait: Duration,
}
impl Default for LaneTimeouts {
fn default() -> Self {
Self {
first_event: Duration::from_secs(30),
liveness: Duration::from_secs(8),
cancel_wait: Duration::from_secs(10),
}
}
}
pub(in crate::coordinator) struct PatientReader<'a, F: FnMut() -> bool> {
conn: &'a mut dyn WorkerConn,
request_id: u32,
timeouts: LaneTimeouts,
should_cancel: F,
last_traffic: Instant,
seen_traffic: bool,
cancel_sent_at: Option<Instant>,
last_progress: Option<u32>,
last_progress_advance: Instant,
}
impl<'a, F: FnMut() -> bool> PatientReader<'a, F> {
pub(in crate::coordinator) fn new(
conn: &'a mut dyn WorkerConn,
request_id: u32,
timeouts: LaneTimeouts,
should_cancel: F,
) -> Self {
Self {
conn,
request_id,
timeouts,
should_cancel,
last_traffic: Instant::now(),
seen_traffic: false,
cancel_sent_at: None,
last_progress: None,
last_progress_advance: Instant::now(),
}
}
fn note_progress(&mut self, samples_done: u32) {
if self.last_progress.is_none_or(|prev| samples_done > prev) {
self.last_progress = Some(samples_done);
self.last_progress_advance = Instant::now();
}
}
fn on_idle(&mut self) -> std::io::Result<()> {
if self.cancel_sent_at.is_none() && (self.should_cancel)() {
let cancel = ClientMessage::Cancel(Cancel {
request_id: self.request_id,
});
indicatrix_net::messages::write_message(&mut self.conn, &cancel)
.map_err(|e| std::io::Error::other(format!("CANCEL failed: {e}")))?;
self.cancel_sent_at = Some(Instant::now());
}
if self.cancel_sent_at.is_none()
&& self.last_progress.is_some()
&& self.last_progress_advance.elapsed() > PROGRESS_STALL
{
return Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
format!(
"worker reported no sample progress for {PROGRESS_STALL:?} \
(heartbeats only, no advance)"
),
));
}
let (since, limit, what) = match self.cancel_sent_at {
Some(sent) => (sent, self.timeouts.cancel_wait, "no DONE after CANCEL"),
None if self.seen_traffic => {
(self.last_traffic, self.timeouts.liveness, "worker silent")
}
None => (self.last_traffic, self.timeouts.first_event, "no reply"),
};
if since.elapsed() > limit {
return Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
format!("{what} for {limit:?}"),
));
}
Ok(())
}
fn cancel_now(&mut self) -> std::io::Result<()> {
if self.cancel_sent_at.is_some() {
return Ok(());
}
let cancel = ClientMessage::Cancel(Cancel {
request_id: self.request_id,
});
indicatrix_net::messages::write_message(&mut self.conn, &cancel)
.map_err(|e| std::io::Error::other(format!("CANCEL failed: {e}")))?;
self.cancel_sent_at = Some(Instant::now());
Ok(())
}
pub(in crate::coordinator) fn send_asset(&mut self, bytes: &[u8]) -> std::io::Result<()> {
let mut paced = PacedWriter { reader: self };
write_asset_message(&mut paced, bytes).map_err(|e| std::io::Error::other(e.to_string()))?;
paced.flush()?;
self.last_traffic = Instant::now();
Ok(())
}
}
struct PacedWriter<'r, 'a, F: FnMut() -> bool> {
reader: &'r mut PatientReader<'a, F>,
}
impl<F: FnMut() -> bool> Write for PacedWriter<'_, '_, F> {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
if (self.reader.should_cancel)() {
return Err(std::io::Error::other(
"the job was cancelled while forwarding its HDR map",
));
}
let piece = &buf[..buf.len().min(ASSET_WRITE_PIECE)];
let written = self.reader.conn.write(piece)?;
self.reader.last_traffic = Instant::now();
Ok(written)
}
fn flush(&mut self) -> std::io::Result<()> {
self.reader.conn.flush()
}
}
impl<F: FnMut() -> bool> Read for PatientReader<'_, F> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
loop {
match self.conn.read(buf) {
Ok(n) => {
if n > 0 {
self.last_traffic = Instant::now();
self.seen_traffic = true;
}
return Ok(n);
}
Err(e) if is_stream_timeout(&e) => self.on_idle()?,
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
Err(e) => return Err(e),
}
}
}
}
pub(super) struct Reply {
pub(super) sum: Vec<Vec3>,
pub(super) done: u32,
pub(super) prefix: bool,
pub(super) rate: Option<f64>,
pub(super) error: Option<String>,
pub(super) worker_code: Option<u32>,
pub(super) assets_sent: u32,
pub(super) broken: Option<String>,
spans: Vec<(u32, u32)>,
dropped_pixels: u64,
}
impl Reply {
fn new(pixels: usize) -> Self {
Self {
sum: vec![Vec3::ZERO; pixels],
done: 0,
prefix: true,
rate: None,
error: None,
worker_code: None,
assets_sent: 0,
broken: None,
spans: Vec::new(),
dropped_pixels: 0,
}
}
fn broke(mut self, why: String) -> Self {
self.error.get_or_insert_with(|| why.clone());
self.broken = Some(why);
self
}
}
pub(super) fn worker_request(
request_id: u32,
scene: &SceneState,
first_sample: u32,
samples: u32,
) -> RenderRequest {
RenderRequest {
request_id,
scene: scene.clone(),
first_sample,
samples,
stream: StreamConfig {
transfer_mode: TransferMode::FinalOnly,
cadence_ms: WORKER_CADENCE_MS,
preview: None,
},
intent: RequestIntent::Batch,
}
}
#[derive(Clone, Copy)]
pub(super) struct Exchange<'a> {
pub(super) worker_id: u32,
pub(super) cancel: &'a CancelToken,
pub(super) timeouts: LaneTimeouts,
pub(super) asset: Option<&'a HeldAsset>,
}
pub(super) fn run_request(
mut conn: &mut dyn WorkerConn,
request: &RenderRequest,
exchange: Exchange<'_>,
) -> Reply {
let (width, height) = (request.scene.width, request.scene.height);
let reply = Reply::new(width as usize * height as usize);
if let Err(e) = conn.set_timeouts(Some(POLL), Some(WRITE_TIMEOUT)) {
return reply.broke(format!("could not arm the socket deadlines: {e}"));
}
let message = ClientMessage::RenderRequest(Box::new(request.clone()));
let reply = if let Err(e) = indicatrix_net::messages::write_message(&mut conn, &message) {
reply.broke(format!("sending the chunk failed: {e}"))
} else {
let cancel = exchange.cancel;
let mut reader = PatientReader::new(conn, request.request_id, exchange.timeouts, || {
cancel.is_cancelled()
});
read_reply(&mut reader, exchange, request, reply)
};
if reply.broken.is_none() {
let _ = conn.set_timeouts(None, None);
}
reply
}
enum ForwardAssetError {
Protocol(String),
Missing(String),
Transport(String),
}
fn forward_asset<F: FnMut() -> bool>(
reader: &mut PatientReader<'_, F>,
exchange: Exchange<'_>,
hash: &ContentHash,
) -> Result<(), ForwardAssetError> {
let worker_id = exchange.worker_id;
let Some(asset) = exchange.asset.filter(|a| a.content_hash() == *hash) else {
return Err(ForwardAssetError::Protocol(format!(
"worker #{worker_id} asked for asset {}, which is not this job's HDR map (protocol violation)",
hash_hex(hash)
)));
};
let Some(bytes) = asset.bytes() else {
return Err(ForwardAssetError::Missing(format!(
"the coordinator no longer holds HDR map {} to forward to worker #{worker_id}; \
failing this chunk, not the connection",
hash_hex(hash)
)));
};
tracing::info!(
"coordinator: forwarding HDR map {} ({} bytes) to worker #{worker_id}",
hash_hex(hash),
bytes.len()
);
reader.send_asset(&bytes).map_err(|e| {
ForwardAssetError::Transport(format!(
"forwarding HDR map {} to worker #{worker_id}: {e}",
hash_hex(hash)
))
})
}
fn read_reply<F: FnMut() -> bool>(
reader: &mut PatientReader<'_, F>,
exchange: Exchange<'_>,
request: &RenderRequest,
mut reply: Reply,
) -> Reply {
let worker_id = exchange.worker_id;
let mut decoder = PayloadDecoder::new();
let mut first_progress: Option<(Instant, u32)> = None;
let mut last_progress: Option<(Instant, u32)> = None;
loop {
let (event, payload) = match indicatrix_net::messages::read_stream_event(reader) {
Ok(read) => read,
Err(e) => return reply.broke(format!("worker #{worker_id}: {e}")),
};
if event
.request_id()
.is_some_and(|id| id != request.request_id)
{
continue; }
match event {
StreamEvent::Frame(header) => {
if let Err(why) = fold_frame(&mut reply, &mut decoder, request, &header, payload) {
return reply.broke(format!("worker #{worker_id} {why}"));
}
}
StreamEvent::Progress(p) => {
reader.note_progress(p.samples_done);
let now = Instant::now();
if first_progress.is_none() && p.samples_done > 0 {
first_progress = Some((now, p.samples_done));
}
if p.samples_done > 0 && last_progress.is_none_or(|(_, done)| p.samples_done > done)
{
last_progress = Some((now, p.samples_done));
}
}
StreamEvent::Done(done) => {
if reply.dropped_pixels > 0 {
tracing::warn!(
"coordinator: worker #{worker_id} sent {} pixels with non-finite or negative \
radiance; they were skipped",
reply.dropped_pixels
);
}
return finish_done(
reply,
request,
worker_id,
done.cancelled,
first_progress,
last_progress,
);
}
StreamEvent::Error(e) => {
reply.error = Some(format!(
"worker #{worker_id} refused or failed the chunk (worker error code {})",
e.code
));
reply.worker_code = Some(e.code);
return reply;
}
StreamEvent::NeedAsset { content_hash } => {
match forward_asset(reader, exchange, &content_hash) {
Ok(()) => reply.assets_sent += 1,
Err(ForwardAssetError::Missing(why)) => {
reply.error.get_or_insert(why);
if let Err(e) = reader.cancel_now() {
return reply.broke(format!("worker #{worker_id}: {e}"));
}
}
Err(ForwardAssetError::Protocol(why) | ForwardAssetError::Transport(why)) => {
return reply.broke(why);
}
}
}
_ => {}
}
}
}
fn fold_frame(
reply: &mut Reply,
decoder: &mut PayloadDecoder,
request: &RenderRequest,
header: &FrameHeader,
payload: Option<Vec<u8>>,
) -> Result<(), String> {
let end = u64::from(request.first_sample) + u64::from(request.samples);
let contained = header.samples > 0
&& header.first_sample >= request.first_sample
&& u64::from(header.first_sample) + u64::from(header.samples) <= end
&& reply
.done
.checked_add(header.samples)
.is_some_and(|total| total <= request.samples);
if !contained {
return Err(format!(
"sent a FRAME [{}, +{}) outside its chunk",
header.first_sample, header.samples
));
}
let bytes = payload.unwrap_or_default();
let dropped = decoder
.decode_and_add(
header.encoding,
header.raw_len,
&bytes,
request.scene.width,
request.scene.height,
&mut reply.sum,
)
.map_err(|e| format!("sent an undecodable FRAME: {e}"))?;
reply.dropped_pixels = reply.dropped_pixels.saturating_add(u64::from(dropped));
reply.prefix &= header.first_sample == request.first_sample + reply.done;
reply.done += header.samples;
reply.spans.push((header.first_sample, header.samples));
Ok(())
}
fn frames_tile(spans: &mut [(u32, u32)], first: u32, samples: u32) -> bool {
spans.sort_unstable();
let spans: &[(u32, u32)] = spans;
let mut next = u64::from(first);
for &(start, len) in spans {
if u64::from(start) != next {
return false;
}
next += u64::from(len);
}
next == u64::from(first) + u64::from(samples)
}
fn rate_from_progress(
first: Option<(Instant, u32)>,
last: Option<(Instant, u32)>,
samples: u32,
) -> Option<f64> {
let (first_at, first_done) = first?;
last.filter(|&(_, done)| done > first_done)
.and_then(|(last_at, last_done)| {
marginal_rate(
last_done - first_done,
last_at.saturating_duration_since(first_at),
)
})
.or_else(|| marginal_rate(samples.saturating_sub(first_done), first_at.elapsed()))
}
fn finish_done(
mut reply: Reply,
request: &RenderRequest,
worker_id: u32,
cancelled: bool,
first_progress: Option<(Instant, u32)>,
last_progress: Option<(Instant, u32)>,
) -> Reply {
let samples = request.samples;
if reply.done == samples && !frames_tile(&mut reply.spans, request.first_sample, samples) {
reply.done = 0;
reply.prefix = false;
reply.error = Some(format!(
"worker #{worker_id} sent {samples} samples whose FRAMEs do not tile the chunk exactly; \
result discarded"
));
return reply;
}
if reply.done == samples {
reply.rate = rate_from_progress(first_progress, last_progress, samples);
reply.error = None;
} else if !cancelled {
reply.error = Some(format!(
"worker #{worker_id} reported DONE after {} of {samples} samples",
reply.done
));
}
reply
}
#[cfg(test)]
mod tests {
use super::*;
use std::net::{TcpListener, TcpStream};
fn loopback_pair() -> (TcpStream, TcpStream) {
let listener = TcpListener::bind("127.0.0.1:0").expect("bind a loopback listener");
let addr = listener
.local_addr()
.expect("a bound listener has a local address");
let client = TcpStream::connect(addr).expect("connect to our own listener");
let (server, _) = listener.accept().expect("accept the connection just made");
(client, server)
}
#[test]
fn note_progress_only_advances_the_clock_on_a_genuine_increase() {
let (mut conn, _peer) = loopback_pair();
let mut reader = PatientReader::new(&mut conn, 1, LaneTimeouts::default(), || false);
reader.note_progress(10);
let after_first = reader.last_progress_advance;
reader.note_progress(10);
assert_eq!(
reader.last_progress_advance, after_first,
"a repeated samples_done (heartbeat only) must not look like an advance"
);
reader.note_progress(11);
assert_eq!(reader.last_progress, Some(11));
}
#[test]
fn on_idle_fails_a_chunk_whose_progress_has_stalled_past_the_budget() {
let (mut conn, _peer) = loopback_pair();
let mut reader = PatientReader::new(&mut conn, 1, LaneTimeouts::default(), || false);
reader.note_progress(5);
reader.last_progress_advance = Instant::now()
.checked_sub(PROGRESS_STALL + Duration::from_secs(1))
.expect("the process has been running for at least PROGRESS_STALL + 1s");
let err = reader
.on_idle()
.expect_err("a stalled chunk must fail, not idle forever");
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
assert!(err.to_string().contains("no sample progress"));
}
#[test]
fn on_idle_ignores_a_stale_clock_before_any_progress_was_seen() {
let (mut conn, _peer) = loopback_pair();
let timeouts = LaneTimeouts {
first_event: Duration::from_secs(3600),
..LaneTimeouts::default()
};
let mut reader = PatientReader::new(&mut conn, 1, timeouts, || false);
reader.last_traffic = Instant::now()
.checked_sub(PROGRESS_STALL + Duration::from_secs(1))
.expect("the process has been running for at least PROGRESS_STALL + 1s");
assert!(reader.on_idle().is_ok());
}
#[test]
fn on_idle_defers_to_cancel_wait_once_cancel_has_been_sent() {
let (mut conn, _peer) = loopback_pair();
let timeouts = LaneTimeouts {
cancel_wait: Duration::from_secs(3600),
..LaneTimeouts::default()
};
let mut reader = PatientReader::new(&mut conn, 1, timeouts, || true);
reader.note_progress(5);
reader.last_progress_advance = Instant::now()
.checked_sub(PROGRESS_STALL + Duration::from_secs(1))
.expect("the process has been running for at least PROGRESS_STALL + 1s");
assert!(reader.on_idle().is_ok());
assert!(reader.cancel_sent_at.is_some());
assert!(reader.on_idle().is_ok());
}
#[test]
fn rate_from_progress_ends_at_the_last_advance_not_at_done() {
let t0 = Instant::now();
let first = Some((t0, 10));
let last = Some((t0 + Duration::from_secs(2), 50));
let rate = rate_from_progress(first, last, 50).expect("two advances are measurable");
assert!((rate - 20.0).abs() < 1e-9, "{rate}");
let through_done = marginal_rate(40, Duration::from_secs(10));
assert_eq!(through_done, Some(4.0), "the rate this replaces");
}
#[test]
fn rate_from_progress_falls_back_to_the_whole_window_without_two_advances() {
let started = Instant::now()
.checked_sub(Duration::from_secs(4))
.expect("the process has been running for at least 4 s");
let first = Some((started, 10));
for last in [None, first, Some((started + Duration::from_secs(1), 10))] {
let rate = rate_from_progress(first, last, 50).expect("40 samples in 4 s or more");
assert!(rate > 0.0 && rate <= 10.0, "{rate}");
assert!(rate > 5.0, "the test's own clock ran far over: {rate}");
}
}
#[test]
fn rate_from_progress_is_none_without_any_progress() {
let last = Some((Instant::now(), 50));
assert_eq!(rate_from_progress(None, None, 50), None);
assert_eq!(rate_from_progress(None, last, 50), None);
}
}