use super::{
LaneKey,
limits::{Reservation, lane_bytes},
plan::WorkerPick,
producer::{INITIAL_RATE_GUESS, Job, PendingChunk, SinkLane, checkout_ranked},
};
use crate::coordinator::{
Capacity, JobLanes, JoinedWorkerLane, Registry, WorkerHandle, WorkerInfo,
};
use indicatrix_dispatch::{LaneFeed, RateModel, WorkerLane};
use std::{
collections::BTreeMap,
sync::{
Arc, Mutex, MutexGuard, PoisonError,
mpsc::{self, RecvTimeoutError},
},
thread,
time::Duration,
};
pub(super) type PendingSlot = Arc<Mutex<Option<PendingChunk>>>;
#[derive(Default)]
pub(super) struct LaneTable {
lanes: Mutex<BTreeMap<usize, (LaneKey, PendingSlot)>>,
}
impl LaneTable {
fn lock(&self) -> MutexGuard<'_, BTreeMap<usize, (LaneKey, PendingSlot)>> {
self.lanes.lock().unwrap_or_else(PoisonError::into_inner)
}
pub(super) fn register(&self, index: usize, key: LaneKey, pending: PendingSlot) {
self.lock().insert(index, (key, pending));
}
pub(super) fn pending(&self, index: usize) -> Option<PendingSlot> {
self.lock()
.get(&index)
.map(|(_, pending)| Arc::clone(pending))
}
pub(super) fn keys(&self) -> Vec<(usize, LaneKey)> {
self.lock()
.iter()
.map(|(index, (key, _))| (*index, key.clone()))
.collect()
}
}
struct HeldLane<'a> {
lane: Arc<dyn WorkerLane>,
_frames: Option<Reservation<'a>>,
}
pub(super) struct LateLanes<'a> {
registry: &'a Arc<Registry>,
shared: &'a Arc<JobLanes>,
table: &'a LaneTable,
budget: &'a super::limits::MemoryBudget,
rates: &'a Mutex<super::RateBook>,
pin: Option<&'a super::InteractivePin>,
workers: WorkerPick,
size: (u32, u32),
changes: Mutex<mpsc::Receiver<Capacity>>,
held: Mutex<Vec<HeldLane<'a>>>,
}
impl<'a> LateLanes<'a> {
pub(super) fn for_job(
job: &'a Job,
shared: &'a Arc<JobLanes>,
table: &'a LaneTable,
joined: Vec<Arc<dyn WorkerLane>>,
) -> Option<Self> {
if job.plan.whole_image || job.plan.workers == WorkerPick::None {
return None;
}
let registry = job.coordinator.registry.as_ref()?;
Some(Self {
registry,
shared,
table,
budget: job.coordinator.budget(),
rates: &job.rates,
pin: job.coordinator.interactive_pin(),
workers: job.plan.workers,
size: (job.scene.width, job.scene.height),
changes: Mutex::new(registry.subscribe()),
held: Mutex::new(
joined
.into_iter()
.map(|lane| HeldLane {
lane,
_frames: None,
})
.collect(),
),
})
}
fn wait_for_change(&self, wait: Duration) {
let changes = self.changes.lock().unwrap_or_else(PoisonError::into_inner);
match changes.recv_timeout(wait) {
Ok(_) => while changes.try_recv().is_ok() {},
Err(RecvTimeoutError::Timeout) => {}
Err(RecvTimeoutError::Disconnected) => thread::sleep(wait),
}
}
fn take_one(&self) -> Option<(WorkerHandle, Reservation<'a>)> {
let (width, height) = self.size;
let frames = self.budget.try_reserve(lane_bytes(width, height, 1)).ok()?;
let accepts = |w: &WorkerInfo| self.shared.accepts(w);
let handle = match self.workers {
WorkerPick::Fastest(_) => checkout_ranked(
self.registry,
self.rates,
width * height,
accepts,
1,
self.pin,
)
.into_iter()
.next(),
_ => Registry::checkout(self.registry, accepts),
}?;
Some((handle, frames))
}
fn start_rate(&self, key: &LaneKey) -> RateModel {
let (width, height) = self.size;
self.rates
.lock()
.unwrap_or_else(PoisonError::into_inner)
.get(key, width * height)
.map_or_else(|| RateModel::new(INITIAL_RATE_GUESS), RateModel::staggered)
}
}
impl LaneFeed for LateLanes<'_> {
fn next(&self, first_index: usize, wait: Duration) -> Vec<(Arc<dyn WorkerLane>, RateModel)> {
self.wait_for_change(wait);
let mut held = self.held.lock().unwrap_or_else(PoisonError::into_inner);
held.retain(|h| !h.lane.lost());
let room = match self.workers {
WorkerPick::Fastest(n) => (n as usize).saturating_sub(held.len()),
_ => usize::MAX,
};
let pixels = self.size.0 as usize * self.size.1 as usize;
let mut late: Vec<(Arc<dyn WorkerLane>, RateModel)> = Vec::new();
while late.len() < room {
let Some((handle, frames)) = self.take_one() else {
break;
};
let key = LaneKey::for_worker(handle.info());
tracing::info!(
"coordinator job: worker #{} became available; it joins the running job",
handle.info().worker_id
);
let inner: Arc<dyn WorkerLane> = Arc::new(JoinedWorkerLane::new(
handle,
Arc::clone(self.registry),
Arc::clone(self.shared),
));
let pending: PendingSlot = Arc::new(Mutex::new(None));
let rate = self.start_rate(&key);
self.table
.register(first_index + late.len(), key, Arc::clone(&pending));
let sink = SinkLane {
inner: Arc::clone(&inner),
pixels,
pending,
};
held.push(HeldLane {
lane: inner,
_frames: Some(frames),
});
late.push((Arc::new(sink), rate));
}
drop(held);
late
}
}