libmir-cuda 0.3.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

mod execution;
mod routed;
mod scratch;
#[cfg(all(test, target_os = "linux"))]
mod tests;
mod weights;

pub use execution::CudaAffineSharedExpertMoeExecution;
pub use weights::AffineSharedExpertMoeWeights;

use crate::{CudaBackend, CudaTensorSet, Error, GatedActivation, Result};

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct AffineSharedExpertMoeConfig {
    pub hidden_size: usize,
    pub routed_intermediate_size: usize,
    pub shared_intermediate_size: usize,
    pub expert_count: usize,
    pub top_k: usize,
    pub group_size: usize,
    pub expert_bits: usize,
    pub router_bits: usize,
    pub activation: GatedActivation,
}

impl AffineSharedExpertMoeConfig {
    fn validate(self) -> Result<()> {
        if self.hidden_size == 0
            || self.routed_intermediate_size == 0
            || self.shared_intermediate_size == 0
            || self.expert_count == 0
            || self.expert_count > 256
            || self.top_k == 0
            || self.top_k > self.expert_count
            || !valid_storage(self.group_size, self.expert_bits, self.router_bits)
            || (self.group_size > 0
                && (!self.hidden_size.is_multiple_of(self.group_size)
                    || !self.routed_intermediate_size.is_multiple_of(self.group_size)
                    || !self.shared_intermediate_size.is_multiple_of(self.group_size)))
        {
            return Err(Error::InvalidDecoderKernel("invalid affine shared-expert MoE config"));
        }
        Ok(())
    }
}

const fn valid_storage(group_size: usize, expert_bits: usize, router_bits: usize) -> bool {
    (group_size == 0 && expert_bits == 0 && router_bits == 0)
        || (group_size > 0
            && matches!(expert_bits, 2 | 3 | 4 | 5 | 6 | 8)
            && matches!(router_bits, 2 | 3 | 4 | 5 | 6 | 8))
}

#[derive(Clone, Debug)]
pub struct CudaAffineSharedExpertMoe {
    backend: CudaBackend,
    config: AffineSharedExpertMoeConfig,
    weights: AffineSharedExpertMoeWeights,
}

impl CudaAffineSharedExpertMoe {
    pub fn from_tensors(
        backend: &CudaBackend,
        tensors: &CudaTensorSet,
        prefix: &str,
        config: AffineSharedExpertMoeConfig,
    ) -> Result<Self> {
        Self::new(backend, config, AffineSharedExpertMoeWeights::load(tensors, prefix)?)
    }

    pub fn new(
        backend: &CudaBackend,
        config: AffineSharedExpertMoeConfig,
        weights: AffineSharedExpertMoeWeights,
    ) -> Result<Self> {
        config.validate()?;
        weights.validate(config)?;
        Ok(Self {
            backend: backend.clone(),
            config,
            weights,
        })
    }

    pub fn prepare(&self, tokens: usize) -> Result<CudaAffineSharedExpertMoeExecution> {
        CudaAffineSharedExpertMoeExecution::new(&self.backend, self.config, &self.weights, tokens)
    }
}

pub(super) fn checked(left: usize, right: usize) -> Result<usize> {
    left.checked_mul(right)
        .ok_or(Error::InvalidDecoderKernel("affine shared-expert MoE shape overflow"))
}