use alloc::string::{String, ToString};
use alloc::vec::Vec;
use core::marker::PhantomData;
use std::net::SocketAddr;
use std::process::Command;
use std::time::Duration;
use typenum::{U0, U1, U2};
use crate::dist::mesh::ValidMesh;
use crate::shapes::Dyn;
use super::bootstrap::{parse_env, read_env, rendezvous, validate_dyn_launch, validate_timeout};
use super::error::ContextError;
use super::identity::RunId;
use super::lifecycle::DistributedContext;
use super::rank::StaticTwoRank;
use super::{
LOCAL_CUDA_DEVICE_ENV, RANK_ENV, RENDEZVOUS_ADDR_ENV, RENDEZVOUS_TIMEOUT_MS_ENV, RUN_ID_ENV,
TWO_RANK_WORLD, WORLD_SIZE_ENV,
};
impl<M, R> DistributedContext<M, R>
where
M: ValidMesh<World = U2>,
R: StaticTwoRank,
{
pub fn rendezvous_static(config: StaticRendezvousConfig<R>) -> Result<Self, ContextError> {
rendezvous(
config.run_id,
config.endpoint,
R::RANK,
TWO_RANK_WORLD,
config.local_cuda_device,
config.timeout,
)
}
}
impl DistributedContext<Dyn, Dyn> {
pub fn rendezvous_dyn(config: DynRendezvousConfig) -> Result<Self, ContextError> {
validate_dyn_launch(config.endpoint, config.rank, config.world, config.timeout)?;
rendezvous(
config.run_id,
config.endpoint,
config.rank,
config.world,
config.local_cuda_device,
config.timeout,
)
}
pub fn from_env() -> Result<Self, ContextError> {
Self::rendezvous_dyn(DynRendezvousConfig::from_env()?)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum RendezvousEndpoint {
Root {
bind: SocketAddr,
},
Peer {
root: SocketAddr,
},
}
impl RendezvousEndpoint {
#[must_use]
pub const fn rank(self) -> usize {
match self {
Self::Root { .. } => 0,
Self::Peer { .. } => 1,
}
}
#[must_use]
pub const fn address(self) -> SocketAddr {
match self {
Self::Root { bind } => bind,
Self::Peer { root } => root,
}
}
}
#[derive(Debug, Clone)]
pub struct StaticRendezvousConfig<R> {
run_id: RunId,
endpoint: RendezvousEndpoint,
local_cuda_device: usize,
timeout: Duration,
marker: PhantomData<R>,
}
impl StaticRendezvousConfig<U0> {
#[must_use]
pub const fn root(
run_id: RunId,
bind: SocketAddr,
local_cuda_device: usize,
timeout: Duration,
) -> Self {
Self {
run_id,
endpoint: RendezvousEndpoint::Root { bind },
local_cuda_device,
timeout,
marker: PhantomData,
}
}
}
impl StaticRendezvousConfig<U1> {
#[must_use]
pub const fn peer(
run_id: RunId,
root: SocketAddr,
local_cuda_device: usize,
timeout: Duration,
) -> Self {
Self {
run_id,
endpoint: RendezvousEndpoint::Peer { root },
local_cuda_device,
timeout,
marker: PhantomData,
}
}
}
#[derive(Debug, Clone)]
pub struct DynRendezvousConfig {
run_id: RunId,
endpoint: RendezvousEndpoint,
rank: usize,
world: usize,
local_cuda_device: usize,
timeout: Duration,
}
impl DynRendezvousConfig {
#[must_use]
pub const fn new(
run_id: RunId,
endpoint: RendezvousEndpoint,
rank: usize,
world: usize,
local_cuda_device: usize,
timeout: Duration,
) -> Self {
Self {
run_id,
endpoint,
rank,
world,
local_cuda_device,
timeout,
}
}
pub fn from_env() -> Result<Self, ContextError> {
let run_id = RunId::new(read_env(RUN_ID_ENV)?)?;
let rank = parse_env::<usize>(RANK_ENV)?;
let world = parse_env::<usize>(WORLD_SIZE_ENV)?;
let address = parse_env::<SocketAddr>(RENDEZVOUS_ADDR_ENV)?;
let local_cuda_device = parse_env::<usize>(LOCAL_CUDA_DEVICE_ENV)?;
let timeout_ms = parse_env::<u64>(RENDEZVOUS_TIMEOUT_MS_ENV)?;
let endpoint = match rank {
0 => RendezvousEndpoint::Root { bind: address },
_ => RendezvousEndpoint::Peer { root: address },
};
Ok(Self::new(
run_id,
endpoint,
rank,
world,
local_cuda_device,
Duration::from_millis(timeout_ms),
))
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RankLaunch {
run_id: RunId,
root: SocketAddr,
rank: usize,
world: usize,
local_cuda_device: usize,
timeout_ms: u64,
}
impl RankLaunch {
#[must_use]
pub const fn rank(&self) -> usize {
self.rank
}
#[must_use]
pub const fn local_cuda_device(&self) -> usize {
self.local_cuda_device
}
#[must_use]
pub fn environment(&self) -> Vec<(&'static str, String)> {
vec![
(RUN_ID_ENV, self.run_id.as_str().to_string()),
(RANK_ENV, self.rank.to_string()),
(WORLD_SIZE_ENV, self.world.to_string()),
(RENDEZVOUS_ADDR_ENV, self.root.to_string()),
(LOCAL_CUDA_DEVICE_ENV, self.local_cuda_device.to_string()),
(RENDEZVOUS_TIMEOUT_MS_ENV, self.timeout_ms.to_string()),
]
}
pub fn apply(&self, command: &mut Command) {
command.envs(self.environment());
}
}
#[derive(Debug, Clone)]
pub struct TwoRankLaunchPlan<M = Dyn> {
run_id: RunId,
root: SocketAddr,
local_cuda_devices: [usize; TWO_RANK_WORLD],
timeout: Duration,
marker: PhantomData<M>,
}
impl<M> TwoRankLaunchPlan<M>
where
M: ValidMesh<World = U2>,
{
pub fn new_static(
run_id: RunId,
root: SocketAddr,
local_cuda_devices: [usize; TWO_RANK_WORLD],
timeout: Duration,
) -> Result<Self, ContextError> {
validate_timeout(timeout)?;
Ok(Self {
run_id,
root,
local_cuda_devices,
timeout,
marker: PhantomData,
})
}
#[must_use]
pub fn rank_static<R: StaticTwoRank>(&self) -> RankLaunch {
self.rank_unchecked(R::RANK)
}
}
impl TwoRankLaunchPlan<Dyn> {
pub fn new_dyn(
run_id: RunId,
root: SocketAddr,
world: usize,
local_cuda_devices: Vec<usize>,
timeout: Duration,
) -> Result<Self, ContextError> {
if world != TWO_RANK_WORLD {
return Err(ContextError::WorldSize {
expected: TWO_RANK_WORLD,
found: world,
});
}
if local_cuda_devices.len() != TWO_RANK_WORLD {
return Err(ContextError::DeviceCount {
expected: TWO_RANK_WORLD,
found: local_cuda_devices.len(),
});
}
validate_timeout(timeout)?;
Ok(Self {
run_id,
root,
local_cuda_devices: [local_cuda_devices[0], local_cuda_devices[1]],
timeout,
marker: PhantomData,
})
}
pub fn rank_dyn(&self, rank: usize) -> Result<RankLaunch, ContextError> {
if rank >= TWO_RANK_WORLD {
return Err(ContextError::RankOutOfRange {
rank,
world: TWO_RANK_WORLD,
});
}
Ok(self.rank_unchecked(rank))
}
}
impl<M> TwoRankLaunchPlan<M> {
fn rank_unchecked(&self, rank: usize) -> RankLaunch {
RankLaunch {
run_id: self.run_id.clone(),
root: self.root,
rank,
world: TWO_RANK_WORLD,
local_cuda_device: self.local_cuda_devices[rank],
timeout_ms: self.timeout.as_millis().min(u128::from(u64::MAX)) as u64,
}
}
}