use crate::{
assets::AssetCache,
cli::ComputeMode,
pki,
serve::{self, RequestContext},
stream_emit::{TimeoutRead, TimeoutWrite},
};
use indicatrix::renderer::gpu_backend::GpuBackend;
use indicatrix_net::{
client::{ClientError, handshake_with_hello},
handshake,
messages::{
ErrorMsg, Hello, NetError, PayloadEncoding, RenderCapability, adaptive::PayloadChoice,
error_codes,
},
};
use rustls::pki_types::ServerName;
use std::{
fmt,
io::{Read, Write},
net::{SocketAddr, TcpStream, ToSocketAddrs},
path::Path,
sync::Arc,
time::Duration,
};
pub const JOIN_IDLE_TIMEOUT: Duration = Duration::from_secs(45);
const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
pub struct JoinTarget {
pub addr: String,
server_name: ServerName<'static>,
client_config: Arc<rustls::ClientConfig>,
}
impl JoinTarget {
pub fn from_bundle(addr: &str, cert_dir: &Path) -> Result<Self, String> {
let host = indicatrix_net::tls::host_for_server_name(addr)
.ok_or_else(|| format!("coordinator address {addr:?} must be host:port"))?;
let server_name = ServerName::try_from(host.to_string())
.map_err(|e| format!("coordinator address {addr:?}: invalid host for TLS: {e}"))?;
let ca_path = cert_dir.join(pki::CA_CERT_FILE);
let cert_path = cert_dir.join(pki::CLIENT_CERT_FILE);
let key_path = cert_dir.join(pki::CLIENT_KEY_FILE);
let ca = indicatrix_net::tls::load_ca(&ca_path).map_err(|e| e.to_string())?;
let certs = indicatrix_net::tls::load_certs(&cert_path).map_err(|e| e.to_string())?;
let key = indicatrix_net::tls::load_private_key(&key_path).map_err(|e| e.to_string())?;
let client_config = indicatrix_net::tls::client_config(ca, certs, key)
.map_err(|e| format!("failed to build the TLS client config: {e}"))?;
Ok(Self {
addr: addr.to_string(),
server_name,
client_config,
})
}
}
pub fn bundle_role(cert_dir: &Path) -> Result<indicatrix_net::messages::PeerRole, String> {
let certs = indicatrix_net::tls::load_certs(&cert_dir.join(pki::CLIENT_CERT_FILE))
.map_err(|e| e.to_string())?;
pki::role_of_certificate(&certs[0])
}
pub struct WorkerSetup {
pub gpu: Arc<GpuBackend>,
pub threads: usize,
pub compute_mode: ComputeMode,
pub capability: RenderCapability,
pub assets: Option<Arc<AssetCache>>,
pub payload: PayloadChoice,
}
impl WorkerSetup {
#[must_use]
pub fn current_capability(&self) -> RenderCapability {
if !self.gpu.try_recover() {
tracing::debug!(
"indicatrix-worker join: the GPU is lost and not yet recoverable; advertising \
the CPU backend"
);
}
RenderCapability {
backend: serve::local_render_capability(&self.gpu, self.threads).backend,
..self.capability.clone()
}
}
}
#[derive(Debug)]
pub enum JoinError {
Connect(String),
Tls(String),
Refused(ErrorMsg),
Handshake(String),
NotACoordinator,
}
impl fmt::Display for JoinError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Connect(e) => write!(f, "could not connect: {e}"),
Self::Tls(e) => write!(f, "TLS handshake failed: {e}"),
Self::Refused(e) if e.code == error_codes::ROLE_REFUSED => write!(
f,
"refused by the coordinator (ROLE_REFUSED): {} -- `join` needs a WORKER certificate on the \
coordinator's WORKER port",
e.message
),
Self::Refused(e) => write!(
f,
"refused by the coordinator (code {}): {}",
e.code, e.message
),
Self::Handshake(e) => write!(f, "handshake failed: {e}"),
Self::NotACoordinator => write!(
f,
"the peer answered WELCOME without a worker registration -- that is a viewer port or a plain \
worker, not a coordinator's worker port (default 7880)"
),
}
}
}
impl std::error::Error for JoinError {}
#[derive(Debug)]
pub struct Session {
pub worker_id: u32,
pub ended: Result<(), NetError>,
}
pub fn join_once(target: &JoinTarget, setup: &WorkerSetup) -> Result<Session, JoinError> {
let mut tls = dial(target)?;
let hello = handshake::local_worker_hello(setup.current_capability());
let welcome = handshake_with_hello(&mut tls, &hello).map_err(|e| match e {
ClientError::Refused(refusal) => JoinError::Refused(refusal),
other => JoinError::Handshake(other.to_string()),
})?;
let registration = welcome.registration.ok_or(JoinError::NotACoordinator)?;
let remote = Hello::viewer(
welcome.protocol_version,
welcome.build_hash,
welcome.source_hash,
);
handshake::verify_compatible(&hello, &remote)
.map_err(|e| JoinError::Handshake(format!("refusing to serve this coordinator: {e}")))?;
tracing::info!(
"indicatrix-worker join: joined coordinator {} as worker #{} (payload encoding {:?})",
target.addr,
registration.worker_id,
welcome.payload_encoding
);
let _ = tls.set_read_timeout(None);
let _ = tls.set_write_timeout(None);
let link = serve::LinkSettings {
choice: setup.payload,
peer: target.addr.clone(),
loopback: serve::is_loopback_peer(tls.sock.tcp.peer_addr().ok()),
}
.open(&PayloadEncoding::default_accept_list());
let ended = serve::serve_requests(
&mut tls,
&RequestContext {
threads: setup.threads,
gpu: &setup.gpu,
db: None,
compute_mode: setup.compute_mode,
payload_encoding: welcome.payload_encoding,
link: Some(&link),
own_lane: true,
session: None,
assets: setup.assets.as_deref(),
},
);
Ok(Session {
worker_id: registration.worker_id,
ended,
})
}
type JoinStream = rustls::StreamOwned<rustls::ClientConnection, IdleDeadlineStream>;
fn dial(target: &JoinTarget) -> Result<JoinStream, JoinError> {
let addrs: Vec<SocketAddr> = target
.addr
.to_socket_addrs()
.map_err(|e| JoinError::Connect(format!("{}: {e}", target.addr)))?
.collect();
let mut last_error = format!("{}: no address", target.addr);
let tcp = addrs
.iter()
.find_map(|addr| {
TcpStream::connect_timeout(addr, CONNECT_TIMEOUT)
.map_err(|e| last_error = format!("{addr}: {e}"))
.ok()
})
.ok_or_else(|| JoinError::Connect(last_error.clone()))?;
let peer = tcp.peer_addr().ok();
serve::tune_accepted_socket(&tcp, peer);
serve::apply_handshake_timeout(&tcp, peer);
let conn = rustls::ClientConnection::new(
Arc::clone(&target.client_config),
target.server_name.clone(),
)
.map_err(|e| JoinError::Tls(e.to_string()))?;
let mut tls = rustls::StreamOwned::new(
conn,
IdleDeadlineStream {
tcp,
idle: JOIN_IDLE_TIMEOUT,
},
);
tls.conn
.complete_io(&mut tls.sock)
.map_err(|e| JoinError::Tls(e.to_string()))?;
Ok(tls)
}
pub struct IdleDeadlineStream {
tcp: TcpStream,
idle: Duration,
}
impl Read for IdleDeadlineStream {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
self.tcp.read(buf)
}
}
impl Write for IdleDeadlineStream {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.tcp.write(buf)
}
fn flush(&mut self) -> std::io::Result<()> {
self.tcp.flush()
}
}
impl TimeoutRead for IdleDeadlineStream {
fn set_read_timeout(&mut self, duration: Option<Duration>) -> std::io::Result<()> {
self.tcp
.set_read_timeout(Some(duration.unwrap_or(self.idle)))
}
}
impl TimeoutWrite for IdleDeadlineStream {
fn set_write_timeout(&mut self, duration: Option<Duration>) -> std::io::Result<()> {
self.tcp.set_write_timeout(duration)
}
}