burn_tensor/tensor/
distributed.rs

1//! Distributed execution utilities.
2//!
3//! The core component of this module is [`DistributedContext`], which manages
4//! the lifecycle of distributed synchronization clients.
5
6use alloc::vec::Vec;
7use burn_backend::TensorMetadata;
8use burn_backend::{DeviceOps, distributed::DistributedOps};
9use burn_dispatch::{Dispatch, DispatchTensor};
10pub use burn_std::distributed::*;
11
12use crate::{Device, Tensor, ops::BridgeTensor};
13
14/// This structure acts as a resource handle for multi-device synchronization.
15///
16/// Spawning this context automatically initializes the underlying distributed communication
17/// servers, while dropping it guarantees a clean and safe teardown of all network resources.
18#[derive(Debug)]
19pub struct DistributedContext {
20    devices: Vec<Device>,
21}
22
23impl DistributedContext {
24    /// Starts a distributed communication server for the provided devices.
25    ///
26    /// # Arguments
27    ///
28    /// * `devices` - The collection of compute devices participating in the distributed operations.
29    /// * `config` - Parameter aggregation settings, such as global reduction strategies (`Mean`, `Sum`, etc.).
30    pub fn init(devices: Vec<Device>, config: DistributedConfig) -> Self {
31        let dispatch_devices = devices
32            .iter()
33            .map(|d| d.as_dispatch().clone())
34            .collect::<Vec<_>>();
35        Dispatch::start_communication_server(&dispatch_devices, config);
36
37        Self { devices }
38    }
39}
40
41impl Drop for DistributedContext {
42    fn drop(&mut self) {
43        if !self.devices.is_empty() {
44            Dispatch::close_communication_server(self.devices[0].as_dispatch());
45        }
46    }
47}
48
49/// A tensor handle used for a collective operation, that is not yet valid for use.
50/// We must ensure collective operations are completed before accessing the underlying data.
51#[derive(new, Clone)]
52pub struct CollectiveTensor<const D: usize> {
53    handle: DispatchTensor,
54}
55
56impl<const D: usize> CollectiveTensor<D> {
57    /// Synchronizes the collective operation and returns a valid tensor handle.
58    pub fn resolve(self) -> Tensor<D> {
59        Dispatch::sync_collective(&self.handle.device());
60        Tensor::new(BridgeTensor::float(self.handle))
61    }
62
63    /// Returns the tensor handle without synchronizing.
64    ///
65    /// # Safety
66    ///
67    /// The caller must ensure that `sync_collective()` is called before
68    /// the returned handle is used in any computation.
69    pub unsafe fn assume_resolved(self) -> Tensor<D> {
70        Tensor::new(BridgeTensor::float(self.handle))
71    }
72}
73
74/// Performs an all_reduce operation on the input tensor.
75///
76/// # Arguments
77/// - `input`: The input tensor.
78/// - `op`: The aggregation operation.
79/// - `device_ids`: The list of all devices with which to `all_reduce`
80///
81/// # Returns
82/// A [CollectiveTensor] containing the handle of the result.
83pub fn all_reduce<const D: usize>(
84    input: Tensor<D>,
85    op: ReduceOperation,
86    device_ids: Vec<Device>,
87) -> CollectiveTensor<D> {
88    let device_ids = device_ids.iter().map(|d| d.as_dispatch().id()).collect();
89    let collective = Dispatch::all_reduce(input.primitive.into_float(), op, device_ids);
90    // Safety: we call `assume_resolved` only to wrap it in `burn_tensor`'s [CollectiveTensor].
91    CollectiveTensor::new(unsafe { collective.assume_resolved() })
92}
burn_tensor/tensor/distributed.rs

burn_tensor/tensor/
distributed.rs
