use super::{
Coordinator, LaneKey, RateBook,
late::{LaneTable, LateLanes},
limits::{
Admission, JobParkedBudget, Reservation, contribution_bytes, hdr_bytes, job_bytes,
job_estimate_bytes, lane_bytes,
},
over_budget,
plan::{InteractivePin, JobPlan, WorkerPick, rank_fastest},
};
use crate::{
assets::HeldAsset,
coordinator::{JobLanes, JoinedWorkerLane, LaneNeed, OwnLane, Registry, WorkerInfo},
stream_emit::{Awaited, ContributionSlot, ProducerOutcome, ProducerSink},
};
use glam::Vec3;
use indicatrix_dispatch::{
CancelToken, ChunkResult, LaneFeed, LanePool, Merger, PoolEvent, PoolStatus, RateModel,
SampleRange, WorkerLane,
};
use indicatrix_net::{
SceneState,
messages::{ErrorMsg, error_codes},
};
use std::{
collections::BTreeSet,
sync::{
Arc, Mutex, PoisonError,
atomic::{AtomicBool, AtomicU32, Ordering},
},
thread,
time::{Duration, Instant},
};
const LANE_RECHECK: Duration = Duration::from_millis(50);
pub(super) const INITIAL_RATE_GUESS: f64 = 10.0;
pub struct Job {
pub coordinator: Arc<Coordinator>,
pub viewer: Arc<str>,
pub rates: Arc<Mutex<RateBook>>,
pub scene: SceneState,
pub range: SampleRange,
pub plan: JobPlan,
pub asset: Option<Arc<HeldAsset>>,
pub contribution: Option<Arc<ContributionSlot>>,
pub lanes_run: Arc<AtomicU32>,
}
pub(super) type PendingChunk = (u32, u32, Vec<Vec3>);
pub(super) struct SinkLane {
pub(super) inner: Arc<dyn WorkerLane>,
pub(super) pixels: usize,
pub(super) pending: Arc<Mutex<Option<PendingChunk>>>,
}
impl WorkerLane for SinkLane {
fn name(&self) -> &str {
self.inner.name()
}
fn lost(&self) -> bool {
self.inner.lost()
}
fn render_chunk(
&self,
scene: &SceneState,
range: SampleRange,
cancel: &CancelToken,
) -> ChunkResult {
let result = self.inner.render_chunk(scene, range, cancel);
if result.done > 0 && result.done <= range.samples && result.sum.len() == self.pixels {
*self.pending.lock().unwrap_or_else(PoisonError::into_inner) =
Some((range.first_sample, result.done, result.sum.clone()));
}
result
}
}
pub fn run(job: &Job, sink: ProducerSink) {
let Some(admission) = admit(job, &sink) else {
return sink.finish(ProducerOutcome::Cancelled);
};
let bytes = job_bytes(job.scene.width, job.scene.height).saturating_add(extra_bytes(job));
let Ok(_reservation) = job.coordinator.budget().try_reserve(bytes) else {
let error = over_budget(&job.coordinator, bytes);
drop(admission);
return sink.finish(ProducerOutcome::Failed(error));
};
let shared = job_lanes(job);
let lanes = acquire_lanes(job, &sink, &shared);
if lanes.is_empty() {
drop(admission);
let outcome = if sink.is_cancelled() {
ProducerOutcome::Cancelled
} else {
ProducerOutcome::Failed(lost("no joined worker became idle to take the request"))
};
return sink.finish(outcome);
}
let (lanes, lane_frames) = match reserve_lane_frames(job, lanes) {
Ok(reserved) => reserved,
Err(error) => {
drop(admission);
return sink.finish(ProducerOutcome::Failed(error));
}
};
job.lanes_run.store(
u32::try_from(lanes.len()).unwrap_or(u32::MAX),
Ordering::Relaxed,
);
let sink = Arc::new(sink);
let (status, merger) = run_pool(job, lanes, &shared, &sink, job.range);
drop(lane_frames);
let Ok(sink) = Arc::try_unwrap(sink) else {
drop(admission);
return;
};
match status {
PoolStatus::Complete => finish_complete(job, sink, merger, admission),
PoolStatus::Cancelled { .. } => {
drop(admission);
sink.finish(ProducerOutcome::Cancelled);
}
PoolStatus::LanesExhausted { missing } => {
drop(admission);
sink.finish(ProducerOutcome::Failed(lost(&format!(
"every render lane failed with {missing} sample(s) of the request left"
))));
}
}
}
fn admit<'a>(job: &'a Job, sink: &ProducerSink) -> Option<Admission<'a>> {
let queues = &job.coordinator.queues;
if job.plan.whole_image {
let cap = job.coordinator.job_config().jobs_per_viewer;
return queues
.wait_slot(&job.viewer, cap, || sink.is_cancelled())
.map(Admission::slot);
}
if job.plan.fifo {
return queues
.wait_turn(&job.viewer, || sink.is_cancelled())
.map(Admission::turn);
}
Some(Admission::default())
}
fn extra_bytes(job: &Job) -> u64 {
let contribution = if job.contribution.is_some() {
contribution_bytes(job.scene.width, job.scene.height)
} else {
0
};
let hdr = job.asset.as_ref().map_or(0, |asset| {
let encoded = asset
.bytes()
.map_or(0, |bytes| u64::try_from(bytes.len()).unwrap_or(u64::MAX));
let texels = asset
.map()
.map(|map| map.width() as u64 * map.height() as u64);
hdr_bytes(encoded, texels)
});
contribution.saturating_add(hdr)
}
type LaneSet = Vec<(LaneKey, Arc<dyn WorkerLane>)>;
fn reserve_lane_frames(
job: &Job,
mut lanes: LaneSet,
) -> Result<(LaneSet, Reservation<'_>), ErrorMsg> {
let (width, height) = (job.scene.width, job.scene.height);
let wanted = lanes.len();
for count in (1..=wanted).rev() {
let n = u32::try_from(count).unwrap_or(u32::MAX);
if let Ok(reservation) = job
.coordinator
.budget()
.try_reserve(lane_bytes(width, height, n))
{
if count < wanted {
tracing::info!(
"coordinator job: running on {count} of {wanted} lane(s) -- the frame buffers \
of more would exceed the memory budget (--max-job-memory-mib)"
);
lanes.truncate(count);
}
return Ok((lanes, reservation));
}
}
Err(over_budget(
&job.coordinator,
job_estimate_bytes(width, height, 1, extra_bytes(job)),
))
}
fn finish_complete(job: &Job, sink: ProducerSink, merger: Merger, admission: Admission<'_>) {
let Some(slot) = &job.contribution else {
drop(admission);
warn_dropped_pixels(merger.dropped_pixels());
let (sum, _count) = merger.into_parts();
return sink.finish(ProducerOutcome::Complete {
final_total: Some(sum),
reclaimed_samples: 0,
});
};
let reserved = slot.reserved();
let wait = job.coordinator.job_config().contribution_wait;
match slot.await_until(wait, || sink.is_cancelled()) {
Awaited::Arrived(sum) => {
drop(admission);
sink.add_chunk(reserved.samples, &sum);
let _ = merger.add(reserved.first_sample, reserved.samples, sum);
warn_dropped_pixels(merger.dropped_pixels());
let (total, _count) = merger.into_parts();
sink.finish(ProducerOutcome::Complete {
final_total: Some(total),
reclaimed_samples: 0,
});
}
Awaited::Cancelled => {
drop(admission);
sink.finish(ProducerOutcome::Cancelled);
}
Awaited::Reclaim { reason } => {
tracing::info!("coordinator job: reclaiming viewer range {reserved:?}: {reason}");
reclaim(job, sink, merger, admission, reserved);
}
}
}
fn warn_dropped_pixels(dropped: u64) {
if dropped > 0 {
tracing::warn!(
"coordinator job: {dropped} pixel(s) of merged chunks were zeroed for a non-finite or negative \
component (a lane returned invalid radiance)"
);
}
}
fn reclaim(
job: &Job,
sink: ProducerSink,
merger: Merger,
admission: Admission<'_>,
reserved: SampleRange,
) {
let shared = job_lanes(job);
let lanes = acquire_lanes(job, &sink, &shared);
if lanes.is_empty() {
drop(admission);
let outcome = if sink.is_cancelled() {
ProducerOutcome::Cancelled
} else {
ProducerOutcome::Failed(lost(
"no joined worker became idle to reclaim the viewer's share",
))
};
return sink.finish(outcome);
}
let (lanes, lane_frames) = match reserve_lane_frames(job, lanes) {
Ok(reserved) => reserved,
Err(error) => {
drop(admission);
return sink.finish(ProducerOutcome::Failed(error));
}
};
let sink = Arc::new(sink);
let (status, extra) = run_pool(job, lanes, &shared, &sink, reserved);
drop(lane_frames);
let Ok(sink) = Arc::try_unwrap(sink) else {
drop(admission);
return;
};
drop(admission);
match status {
PoolStatus::Complete => {
let extra_dropped = extra.dropped_pixels();
let (sum, count) = extra.into_parts();
let _ = merger.add(reserved.first_sample, count, sum);
warn_dropped_pixels(merger.dropped_pixels().saturating_add(extra_dropped));
let (total, _count) = merger.into_parts();
sink.finish(ProducerOutcome::Complete {
final_total: Some(total),
reclaimed_samples: reserved.samples,
});
}
PoolStatus::Cancelled { .. } => sink.finish(ProducerOutcome::Cancelled),
PoolStatus::LanesExhausted { missing } => {
sink.finish(ProducerOutcome::Failed(lost(&format!(
"every render lane failed reclaiming the viewer's share, {missing} sample(s) left"
))));
}
}
}
fn lost(why: &str) -> ErrorMsg {
ErrorMsg {
code: error_codes::ALL_WORKERS_LOST,
message: format!("the coordinator lost every lane that could finish this request: {why}"),
request_id: None,
}
}
fn run_pool(
job: &Job,
lanes: LaneSet,
shared: &Arc<JobLanes>,
sink: &Arc<ProducerSink>,
range: SampleRange,
) -> (PoolStatus, Merger) {
let pixels = job.scene.width as usize * job.scene.height as usize;
let image_pixels = job.scene.width * job.scene.height;
let table = LaneTable::default();
let (pool, joined) = seed_pool(job, lanes, &table);
let feed = LateLanes::for_job(job, shared, &table, joined);
let merger = Merger::new(pixels, range.first_sample)
.with_parked_budget(Arc::new(JobParkedBudget(Arc::clone(&job.coordinator))));
let token = CancelToken::new();
let running = AtomicBool::new(true);
let events = |event: PoolEvent| {
if let PoolEvent::ChunkMerged {
lane, range, done, ..
} = &event
&& let Some(slot) = table.pending(*lane)
&& let Some((first_sample, pending_done, sum)) =
slot.lock().unwrap_or_else(PoisonError::into_inner).take()
&& first_sample == range.first_sample
&& pending_done == *done
{
sink.add_chunk(pending_done, &sum);
}
log_event(event);
};
let status = thread::scope(|scope| {
scope.spawn(|| {
while running.load(Ordering::Relaxed) {
if sink.is_cancelled() {
token.cancel();
break;
}
thread::sleep(Duration::from_millis(20));
}
});
let late = feed.as_ref().map(|feed| feed as &dyn LaneFeed);
let status = pool.run_into_fed(&job.scene, range, &merger, &token, &events, late);
running.store(false, Ordering::Relaxed);
status
});
drop(feed);
{
let mut book = job.rates.lock().unwrap_or_else(PoisonError::into_inner);
for (index, key) in table.keys() {
if let Some(rate) = pool.rate(index).and_then(|r| r.estimate()) {
book.set(key, image_pixels, rate);
}
}
}
let status = status.unwrap_or(PoolStatus::LanesExhausted {
missing: range.samples,
});
(status, merger)
}
fn seed_pool(job: &Job, lanes: LaneSet, table: &LaneTable) -> (LanePool, Vec<Arc<dyn WorkerLane>>) {
let pixels = job.scene.width as usize * job.scene.height as usize;
let image_pixels = job.scene.width * job.scene.height;
let mut pool = LanePool::new(job.plan.pool);
let mut joined = Vec::new();
let book = job.rates.lock().unwrap_or_else(PoisonError::into_inner);
let mut seeded = BTreeSet::new();
for (key, inner) in lanes {
let rate = lane_rate(&book, &key, image_pixels, &mut seeded);
let pending = Arc::new(Mutex::new(None));
if matches!(key, LaneKey::Worker(_)) {
joined.push(Arc::clone(&inner));
}
let lane = SinkLane {
inner,
pixels,
pending: Arc::clone(&pending),
};
let index = pool.add_lane(Arc::new(lane), rate);
table.register(index, key, pending);
}
drop(book);
(pool, joined)
}
pub(super) fn lane_rate(
book: &RateBook,
key: &LaneKey,
pixels: u32,
seeded: &mut BTreeSet<LaneKey>,
) -> RateModel {
let repeated = !seeded.insert(key.clone());
book.get(key, pixels).map_or_else(
|| RateModel::new(INITIAL_RATE_GUESS),
|rate| {
if repeated {
RateModel::staggered(rate)
} else {
RateModel::calibrated(rate)
}
},
)
}
fn log_event(event: PoolEvent) {
match event {
PoolEvent::LaneFailed {
lane,
error,
returned,
pause,
..
} => tracing::info!(
"coordinator job: lane {lane} failed ({error}); {} sample(s) returned to the pool, pause {pause:?}",
returned.samples
),
PoolEvent::LaneRetired {
lane,
consecutive_failures,
} => tracing::warn!(
"coordinator job: lane {lane} retired after {consecutive_failures} failed chunk(s) in a row"
),
PoolEvent::ChunkDeferred {
lane,
range,
frontier,
total_done,
deferrals,
} => tracing::warn!(
"coordinator job: lane {lane} chunk [{}, +{}) deferred {deferrals} time(s) by the parked-bytes budget; frontier {frontier}, {total_done} sample(s) merged or parked",
range.first_sample,
range.samples
),
other => tracing::trace!("coordinator job: {other:?}"),
}
}
fn own_lane(coordinator: &Coordinator) -> Option<(LaneKey, Arc<dyn WorkerLane>)> {
let own = coordinator.own.as_ref()?;
let lane = OwnLane::new(Arc::clone(&own.gpu), own.threads, own.compute_mode);
Some((LaneKey::Own, Arc::new(lane)))
}
pub(super) fn whole_image_lane(
coordinator: &Coordinator,
rates: &Mutex<RateBook>,
shared: &Arc<JobLanes>,
need: LaneNeed,
) -> Option<(LaneKey, Arc<dyn WorkerLane>)> {
if let Some(registry) = coordinator.registry.as_ref()
&& let Some(handle) = checkout_fastest(registry, rates, need, 1, None)
.into_iter()
.next()
{
let key = LaneKey::for_worker(handle.info());
let lane = JoinedWorkerLane::new(handle, Arc::clone(registry), Arc::clone(shared));
return Some((key, Arc::new(lane)));
}
let own = own_lane(coordinator)?;
tracing::debug!(
"coordinator job: no joined worker is idle; a whole-image job runs on the own lane"
);
Some(own)
}
fn job_lanes(job: &Job) -> Arc<JobLanes> {
let need = LaneNeed {
pixels: job.scene.width * job.scene.height,
hdr: job.scene.hdr().is_some(),
};
Arc::new(JobLanes::new(
need,
job.asset.clone(),
job.coordinator.job_config().lane,
))
}
fn acquire_lanes(job: &Job, sink: &ProducerSink, shared: &Arc<JobLanes>) -> LaneSet {
let config = job.coordinator.job_config();
let deadline = Instant::now() + config.lane_wait;
let need = shared.need();
let mut lanes: LaneSet = Vec::new();
if job.plan.whole_image {
if let Some(lane) = whole_image_lane(&job.coordinator, &job.rates, shared, need) {
return vec![lane];
}
} else if job.plan.own {
lanes.extend(own_lane(&job.coordinator));
}
let Some(registry) = job.coordinator.registry.as_ref() else {
return lanes;
};
let pin = if job.plan.whole_image {
None
} else {
job.coordinator.interactive_pin.as_ref()
};
loop {
let handles = match job.plan.workers {
WorkerPick::None => Vec::new(),
WorkerPick::All => {
std::iter::from_fn(|| Registry::checkout(registry, |w| need.accepts(w))).collect()
}
WorkerPick::Fastest(n) => checkout_fastest(registry, &job.rates, need, n, pin),
};
for handle in handles {
let key = LaneKey::for_worker(handle.info());
let lane = JoinedWorkerLane::new(handle, Arc::clone(registry), Arc::clone(shared));
lanes.push((key, Arc::new(lane)));
}
if !lanes.is_empty() || sink.is_cancelled() || Instant::now() >= deadline {
return lanes;
}
thread::sleep(LANE_RECHECK);
}
}
pub(super) fn checkout_fastest(
registry: &Arc<Registry>,
rates: &Mutex<RateBook>,
need: LaneNeed,
n: u32,
pin: Option<&InteractivePin>,
) -> Vec<crate::coordinator::WorkerHandle> {
checkout_ranked(registry, rates, need.pixels, |w| need.accepts(w), n, pin)
}
pub(super) fn checkout_ranked(
registry: &Arc<Registry>,
rates: &Mutex<RateBook>,
pixels: u32,
accepts: impl Fn(&WorkerInfo) -> bool,
n: u32,
pin: Option<&InteractivePin>,
) -> Vec<crate::coordinator::WorkerHandle> {
let mut idle: Vec<WorkerInfo> = registry
.workers()
.into_iter()
.filter(|(w, idle)| *idle && accepts(w))
.map(|(w, _)| w)
.collect();
if idle.is_empty() {
return Vec::new();
}
rank_fastest(
&mut idle,
&rates.lock().unwrap_or_else(PoisonError::into_inner),
pixels,
);
if let Some(pin) = pin {
pin.apply(&mut idle);
}
idle.iter()
.filter_map(|w| Registry::checkout(registry, |c| c.worker_id == w.worker_id))
.take(n as usize)
.collect()
}