mod session;
pub use session::{
IdleDeadlineStream, JOIN_IDLE_TIMEOUT, JoinError, JoinTarget, Session, WorkerSetup,
bundle_role, join_once,
};
use crate::cli::{ComputeMode, JoinArgs};
use indicatrix::renderer::gpu_backend::GpuBackend;
use indicatrix_net::messages::PeerRole;
use std::{
path::Path,
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
thread,
time::Duration,
};
pub const BACKOFF_MIN: Duration = Duration::from_secs(1);
pub const BACKOFF_MAX: Duration = Duration::from_secs(60);
pub fn run(args: &JoinArgs) -> Result<(), String> {
if let Some(token) = &args.token {
let enroll_addr = match &args.enroll_addr {
Some(addr) => addr.clone(),
None => default_enroll_addr(&args.coordinator)?,
};
let bundle = indicatrix_net::enroll::claim(token, &enroll_addr)
.map_err(|e| format!("claiming the worker token at {enroll_addr}: {e}"))?;
crate::enroll_client::write_bundle(&args.cert_dir, &bundle)?;
tracing::info!(
"indicatrix-worker join: claimed a worker certificate into {}",
args.cert_dir.display()
);
}
check_worker_bundle(&args.cert_dir)?;
let target = Arc::new(JoinTarget::from_bundle(&args.coordinator, &args.cert_dir)?);
let gpu = Arc::new(if args.compute_mode == ComputeMode::OnlyCpu {
GpuBackend::disabled()
} else {
GpuBackend::acquire()
});
if args.compute_mode == ComputeMode::Hybrid && gpu.adapter_label().is_some() {
crate::render_core::hybrid::calibrate_now(
&gpu,
&probe_scene(),
args.threads,
args.compute_mode,
);
}
let assets = crate::assets::open_configured(&args.cert_dir);
let capability = indicatrix_net::messages::RenderCapability {
hdr: assets.is_some(),
..crate::serve::local_render_capability(&gpu, args.threads)
};
tracing::info!(
"indicatrix-worker join: joining {} with {} slot(s) as {:?} (HDR environments: {})",
args.coordinator,
args.slots,
capability.backend,
if capability.hdr { "yes" } else { "no" }
);
let setup = Arc::new(WorkerSetup {
gpu,
threads: args.threads,
compute_mode: args.compute_mode,
capability,
assets,
payload: args.payload_encoding,
});
let stop = Arc::new(AtomicBool::new(false));
let slots: Vec<_> = (0..args.slots)
.map(|slot| {
let (target, setup, stop) =
(Arc::clone(&target), Arc::clone(&setup), Arc::clone(&stop));
thread::spawn(move || run_slot(slot, &target, &setup, &stop))
})
.collect();
for slot in slots {
if slot.join().is_err() {
tracing::error!("indicatrix-worker join: a slot thread panicked");
}
}
Ok(())
}
fn probe_scene() -> indicatrix_net::SceneState {
use indicatrix::{
geometry::cuts::StandardGemCuts,
optics::{materials::GemMaterial, raytracer::LightingPreset},
};
indicatrix_net::SceneState {
width: 512,
height: 512,
yaw: 0.4,
pitch: 0.3,
distance: 3.0,
light_yaw: 0.85,
light_pitch: 0.95,
exposure: 1.0,
max_bounces: 4,
lighting_preset: LightingPreset::Daylight,
material: GemMaterial::diamond(),
planes: StandardGemCuts::standard_round_brilliant(),
girdle_frosted: false,
backdrop: 0.0,
environment: indicatrix_net::scene::SceneEnvironment::Studio,
surface_glare: 1.0,
tools: Vec::new(),
fluorescence: indicatrix::optics::fluorescence::Fluorescence::default(),
head_shadow_deg: 16.0,
}
}
fn check_worker_bundle(cert_dir: &Path) -> Result<(), String> {
match bundle_role(cert_dir).map_err(|e| format!("--cert-dir {}: {e}", cert_dir.display()))? {
PeerRole::Worker => Ok(()),
PeerRole::Viewer => Err(format!(
"--cert-dir {}: this is a VIEWER certificate -- `join` needs a worker certificate (`cert issue-client \
--role worker` on the coordinator, or `join --token` with a `cert issue-token --role worker` token)",
cert_dir.display()
)),
}
}
pub fn default_enroll_addr(coordinator: &str) -> Result<String, String> {
let (host, port) = coordinator
.rsplit_once(':')
.ok_or_else(|| format!("coordinator address {coordinator:?} must be host:port"))?;
let port: u16 = port
.parse()
.map_err(|_| format!("coordinator address {coordinator:?}: bad port"))?;
let enroll_port = port
.checked_add(1)
.ok_or_else(|| "pass --enroll-addr explicitly (the worker port is 65535)".to_string())?;
Ok(format!("{host}:{enroll_port}"))
}
pub fn run_slot(slot: usize, target: &JoinTarget, setup: &WorkerSetup, stop: &AtomicBool) {
let mut backoff = Backoff::new();
while !stop.load(Ordering::Relaxed) {
match join_once(target, setup) {
Ok(session) => {
backoff.reset();
match session.ended {
Ok(()) => tracing::info!(
"indicatrix-worker join: slot {slot}: the coordinator closed worker #{}'s connection",
session.worker_id
),
Err(e) => tracing::warn!(
"indicatrix-worker join: slot {slot}: worker #{}'s connection ended: {e}",
session.worker_id
),
}
}
Err(e) => tracing::warn!(
"indicatrix-worker join: slot {slot}: {} -- {e}",
target.addr
),
}
let delay = backoff.next_delay();
tracing::info!("indicatrix-worker join: slot {slot}: reconnecting in {delay:.1?}");
sleep_unless_stopped(delay, stop);
}
}
fn sleep_unless_stopped(delay: Duration, stop: &AtomicBool) {
let step = Duration::from_millis(200);
let mut left = delay;
while !left.is_zero() && !stop.load(Ordering::Relaxed) {
let now = left.min(step);
thread::sleep(now);
left -= now;
}
}
#[derive(Debug)]
pub struct Backoff {
next: Duration,
}
impl Backoff {
#[must_use]
pub const fn new() -> Self {
Self { next: BACKOFF_MIN }
}
pub const fn reset(&mut self) {
self.next = BACKOFF_MIN;
}
pub fn next_delay(&mut self) -> Duration {
let base = self.next;
self.next = (self.next * 2).min(BACKOFF_MAX);
base.mul_f64(random_unit().mul_add(0.5, 0.5))
}
}
impl Default for Backoff {
fn default() -> Self {
Self::new()
}
}
fn random_unit() -> f64 {
use ring::rand::SecureRandom;
let mut bytes = [0_u8; 4];
if ring::rand::SystemRandom::new().fill(&mut bytes).is_err() {
return 0.5;
}
f64::from(u32::from_le_bytes(bytes)) / (f64::from(u32::MAX) + 1.0)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn backoff_doubles_to_the_cap_with_jitter_and_resets() {
let mut backoff = Backoff::new();
let mut bases = Vec::new();
for _ in 0..10 {
let base = backoff.next;
let delay = backoff.next_delay();
assert!(
delay >= base / 2 && delay <= base,
"{delay:?} vs base {base:?}"
);
bases.push(base.as_secs());
}
assert_eq!(bases, [1, 2, 4, 8, 16, 32, 60, 60, 60, 60]);
backoff.reset();
assert_eq!(backoff.next, BACKOFF_MIN);
}
#[test]
fn the_enroll_address_defaults_to_one_port_above_the_worker_port() {
assert_eq!(
default_enroll_addr("coord.lan:7880").unwrap(),
"coord.lan:7881"
);
assert_eq!(default_enroll_addr("[::1]:7880").unwrap(), "[::1]:7881");
assert!(default_enroll_addr("coord.lan").is_err());
assert!(default_enroll_addr("coord.lan:65535").is_err());
}
}