ruCCL 0.21.1

Ruda collective communication algorithms and orchestration.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

use crate::rank::ElementType;
use ruda_tensor::{Element, bf16, f16};

mod sealed {
    pub trait Wire: Sized {
        fn append_le(self, bytes: &mut Vec<u8>);
        fn read_le(bytes: &[u8]) -> Self;
    }
}

/// Storage types supported by the shared collective device adapter.
///
/// The sealed implementations preserve F32/F16/BF16/I32 storage bits on the
/// little-endian rank transport; they do not convert half values through F32.
pub trait TensorElement: Element + sealed::Wire {
    const ELEMENT_TYPE: ElementType;
}

macro_rules! native_element {
    ($ty:ty, $kind:ident) => {
        impl TensorElement for $ty {
            const ELEMENT_TYPE: ElementType = ElementType::$kind;
        }
        impl sealed::Wire for $ty {
            fn append_le(self, bytes: &mut Vec<u8>) {
                bytes.extend_from_slice(&self.to_le_bytes());
            }
            fn read_le(bytes: &[u8]) -> Self {
                Self::from_le_bytes(bytes.try_into().expect("complete wire element"))
            }
        }
    };
}

macro_rules! half_element {
    ($ty:ty, $kind:ident) => {
        impl TensorElement for $ty {
            const ELEMENT_TYPE: ElementType = ElementType::$kind;
        }
        impl sealed::Wire for $ty {
            fn append_le(self, bytes: &mut Vec<u8>) {
                bytes.extend_from_slice(&self.to_bits().to_le_bytes());
            }
            fn read_le(bytes: &[u8]) -> Self {
                Self::from_bits(u16::from_le_bytes(bytes.try_into().expect("complete wire element")))
            }
        }
    };
}

native_element!(f32, F32);
native_element!(i32, I32);
half_element!(f16, F16);
half_element!(bf16, BF16);

pub(super) fn encode<T: TensorElement>(values: &[T]) -> Vec<u8> {
    let mut bytes = Vec::with_capacity(std::mem::size_of_val(values));
    for value in values {
        value.append_le(&mut bytes);
    }
    bytes
}

pub(super) fn decode<T: TensorElement>(bytes: &[u8]) -> Result<Vec<T>, super::TensorDeviceError> {
    let width = std::mem::size_of::<T>();
    if !bytes.len().is_multiple_of(width) {
        return Err(super::TensorDeviceError::InvalidBuffer("incomplete wire element"));
    }
    Ok(bytes.chunks_exact(width).map(T::read_le).collect())
}