use super::{Coordinator, JobConfig, RateBook};
use crate::coordinator::{LaneNeed, WorkerInfo};
use indicatrix_dispatch::PoolConfig;
use indicatrix_net::messages::{Backend, ErrorMsg, RequestIntent, TransferMode, error_codes};
use std::sync::{
PoisonError,
atomic::{AtomicBool, Ordering},
};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WorkerPick {
None,
All,
Fastest(u32),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct JobPlan {
pub own: bool,
pub workers: WorkerPick,
pub fifo: bool,
pub whole_image: bool,
pub pool: PoolConfig,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Route {
Direct,
Job(JobPlan),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Ask {
pub intent: RequestIntent,
pub transfer_mode: TransferMode,
pub pixels: u32,
pub samples: u32,
pub hdr: bool,
}
impl Ask {
#[must_use]
pub const fn need(self) -> LaneNeed {
LaneNeed {
pixels: self.pixels,
hdr: self.hdr,
}
}
const fn is_throughput(self) -> bool {
matches!(self.intent, RequestIntent::Batch)
&& matches!(self.transfer_mode, TransferMode::FinalOnly)
}
}
pub fn plan(coordinator: &Coordinator, ask: Ask) -> Result<Route, ErrorMsg> {
let own = coordinator.own.is_some();
let workers = coordinator
.registry
.as_ref()
.map(|r| r.workers())
.unwrap_or_default();
let need = ask.need();
let eligible: Vec<&WorkerInfo> = workers
.iter()
.map(|(w, _)| w)
.filter(|w| need.accepts(w))
.collect();
if !own && eligible.is_empty() {
return Err(refusal(&workers, need));
}
let config = coordinator.job_config();
if ask.intent == RequestIntent::Batch && is_whole_image(coordinator, &config, ask, &eligible) {
return Ok(Route::Job(JobPlan {
own: false,
workers: WorkerPick::Fastest(1),
fifo: false,
whole_image: true,
pool: config.batch,
}));
}
let wanted = if ask.intent == RequestIntent::Interactive {
match coordinator.interactive_workers {
0 if own => WorkerPick::None,
0 => WorkerPick::Fastest(1),
n => WorkerPick::Fastest(n),
}
} else {
WorkerPick::All
};
let workers = if eligible.is_empty() {
WorkerPick::None
} else {
wanted
};
if own && workers == WorkerPick::None {
return Ok(Route::Direct);
}
Ok(Route::Job(JobPlan {
own,
workers,
fifo: ask.is_throughput(),
whole_image: false,
pool: if ask.is_throughput() {
config.batch
} else {
config.interactive
},
}))
}
fn is_whole_image(
coordinator: &Coordinator,
config: &JobConfig,
ask: Ask,
eligible: &[&WorkerInfo],
) -> bool {
if eligible.is_empty() {
return false;
}
let fastest = {
let book = coordinator
.rates()
.lock()
.unwrap_or_else(PoisonError::into_inner);
eligible
.iter()
.filter_map(|w| book.worker(w, ask.pixels))
.filter(|rate| rate.is_finite() && *rate > 0.0)
.max_by(f64::total_cmp)
};
fastest.map_or_else(
|| u64::from(ask.pixels) * u64::from(ask.samples) <= config.whole_image_pixel_samples,
|rate| f64::from(ask.samples) / rate < config.whole_image_secs,
)
}
fn refusal(workers: &[(WorkerInfo, bool)], need: LaneNeed) -> ErrorMsg {
let pixels = need.pixels;
if need.hdr
&& workers
.iter()
.any(|(w, _)| LaneNeed::pixels(pixels).accepts(w))
{
return ErrorMsg {
code: error_codes::UNSUPPORTED_REQUEST,
request_id: None,
message: "the scene is lit by an HDR map, but no joined worker renders HDR environments \
(their asset caches are disabled) and this coordinator has no own render lane \
(--render)"
.to_string(),
};
}
if workers.is_empty() {
return ErrorMsg {
code: error_codes::NO_RENDER_CAPACITY,
request_id: None,
message:
"this coordinator has no render lane -- no joined workers and no --render; its \
WELCOME advertised no render capability"
.to_string(),
};
}
let largest = workers
.iter()
.map(|(w, _)| w.capability.max_pixels)
.max()
.unwrap_or(0);
ErrorMsg {
code: error_codes::UNSUPPORTED_REQUEST,
request_id: None,
message: format!(
"the image has {pixels} px, more than any joined worker accepts (largest max_pixels \
{largest}) and this coordinator has no own render lane (--render)"
),
}
}
pub fn rank_fastest(candidates: &mut [WorkerInfo], rates: &RateBook, pixels: u32) {
candidates.sort_by(|a, b| {
let key = |w: &WorkerInfo| {
let measured = rates.worker(w, pixels);
let guess = match w.capability.backend {
Backend::Gpu { .. } => f64::from(u32::MAX),
Backend::Cpu { threads } => f64::from(threads),
Backend::Coordinator { threads, gpus, .. } => {
f64::from(gpus).mul_add(f64::from(u32::MAX), f64::from(threads))
}
};
(measured.is_some(), measured.unwrap_or(guess))
};
let (a_measured, a_rate) = key(a);
let (b_measured, b_rate) = key(b);
a_measured
.cmp(&b_measured)
.then_with(|| b_rate.total_cmp(&a_rate))
.then_with(|| a.worker_id.cmp(&b.worker_id))
});
}
#[derive(Debug)]
pub struct InteractivePin {
label: String,
falling_back: AtomicBool,
}
impl InteractivePin {
#[must_use]
pub const fn new(label: String) -> Self {
Self {
label,
falling_back: AtomicBool::new(false),
}
}
#[must_use]
pub fn label(&self) -> &str {
&self.label
}
pub fn apply(&self, ranked: &mut [WorkerInfo]) -> bool {
let found = ranked
.iter()
.position(|w| w.label.as_deref() == Some(self.label.as_str()));
let Some(index) = found else {
if !self.falling_back.swap(true, Ordering::Relaxed) {
tracing::info!(
"coordinator: pinned interactive worker {:?} is not connected, idle and able to \
take the image; live-view requests fall back to the fastest idle worker(s)",
self.label
);
}
return false;
};
ranked[..=index].rotate_right(1);
if self.falling_back.swap(false, Ordering::Relaxed) {
tracing::info!(
"coordinator: pinned interactive worker {:?} is available again; live-view \
requests use it first",
self.label
);
}
true
}
}
#[cfg(test)]
mod tests {
use super::{super::producer::checkout_fastest, *};
use crate::coordinator::{LivenessConfig, Registry, WorkerHandle};
use indicatrix_net::messages::{PayloadEncoding, RenderCapability};
use std::{
net::{TcpListener, TcpStream},
sync::{Arc, Mutex},
};
fn info(worker_id: u32, backend: Backend, label: Option<&str>) -> WorkerInfo {
WorkerInfo {
worker_id,
capability: RenderCapability {
backend,
max_pixels: 1_000,
min_cadence_ms: 100,
hdr: false,
},
peer: None,
label: label.map(str::to_string),
payload_encoding: PayloadEncoding::Raw,
}
}
fn registry() -> (Arc<Registry>, Vec<TcpStream>) {
let registry = Registry::new(LivenessConfig::default());
let mut far_ends = Vec::new();
let gpu = Backend::Gpu {
adapter: "test".to_string(),
};
for (backend, label) in [
(gpu, "gpu-box"),
(Backend::Cpu { threads: 2 }, "slow"),
(Backend::Cpu { threads: 8 }, "mid"),
] {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let near = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
far_ends.push(listener.accept().unwrap().0);
let worker_id = registry.allocate_id();
registry.insert(info(worker_id, backend, Some(label)), Box::new(near), None);
}
(registry, far_ends)
}
fn labels(handles: &[WorkerHandle]) -> Vec<&str> {
handles
.iter()
.map(|h| h.info().label.as_deref().unwrap_or("-"))
.collect()
}
#[test]
fn an_available_pinned_worker_is_used_first() {
let (registry, _far) = registry();
let rates = Mutex::new(RateBook::default());
let pin = InteractivePin::new("slow".to_string());
let one = checkout_fastest(®istry, &rates, LaneNeed::pixels(100), 1, Some(&pin));
assert_eq!(labels(&one), ["slow"]);
drop(one);
let two = checkout_fastest(®istry, &rates, LaneNeed::pixels(100), 2, Some(&pin));
assert_eq!(labels(&two), ["slow", "gpu-box"]);
assert!(!pin.falling_back.load(Ordering::Relaxed));
}
#[test]
fn an_unavailable_pinned_worker_falls_back_to_the_fastest() {
let (registry, _far) = registry();
let rates = Mutex::new(RateBook::default());
let absent = InteractivePin::new("not-joined".to_string());
let picked = checkout_fastest(®istry, &rates, LaneNeed::pixels(100), 1, Some(&absent));
assert_eq!(labels(&picked), ["gpu-box"]);
assert!(absent.falling_back.load(Ordering::Relaxed));
let busy = InteractivePin::new("mid".to_string());
let held = checkout_fastest(®istry, &rates, LaneNeed::pixels(100), 1, Some(&busy));
assert_eq!(labels(&held), ["mid"]);
let next = checkout_fastest(®istry, &rates, LaneNeed::pixels(100), 1, Some(&busy));
assert_eq!(
labels(&next),
["slow"],
"gpu-box and mid are both checked out"
);
assert!(busy.falling_back.load(Ordering::Relaxed));
}
#[test]
fn apply_moves_only_the_pinned_worker() {
let mut ranked = vec![
info(1, Backend::Cpu { threads: 8 }, Some("a")),
info(2, Backend::Cpu { threads: 4 }, None),
info(3, Backend::Cpu { threads: 2 }, Some("c")),
];
assert!(InteractivePin::new("c".to_string()).apply(&mut ranked));
let ids: Vec<u32> = ranked.iter().map(|w| w.worker_id).collect();
assert_eq!(ids, [3, 1, 2]);
assert!(!InteractivePin::new("zzz".to_string()).apply(&mut ranked));
let ids: Vec<u32> = ranked.iter().map(|w| w.worker_id).collect();
assert_eq!(ids, [3, 1, 2]);
}
}