libmir-cuda 0.2.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92

use mircuda::{DeviceBuffer, bf16};
use runtime::backend::{DecodeOutput, LogitsTrace, SamplingLogits, TokenEvent};

use crate::{
    CudaBackend, CudaClampedRoutedModelSession, CudaMoeModelSession, CudaSharedRoutedModelSession,
    Result,
};

pub(in crate::engine) struct Output {
    pub(in crate::engine) token: Option<u32>,
    pub(in crate::engine) logits: Option<LogitsTrace>,
}

pub(in crate::engine) trait DeviceLogitsSession {
    fn sample(&mut self, policy: SamplingLogits) -> Result<&DeviceBuffer<u32>>;
    fn logits(&self) -> &DeviceBuffer<bf16>;
}

impl DeviceLogitsSession for CudaMoeModelSession {
    fn sample(&mut self, policy: SamplingLogits) -> Result<&DeviceBuffer<u32>> {
        Self::sample(self, policy)
    }

    fn logits(&self) -> &DeviceBuffer<bf16> {
        Self::logits(self)
    }
}

impl DeviceLogitsSession for CudaSharedRoutedModelSession {
    fn sample(&mut self, policy: SamplingLogits) -> Result<&DeviceBuffer<u32>> {
        Self::sample(self, policy)
    }

    fn logits(&self) -> &DeviceBuffer<bf16> {
        Self::logits(self)
    }
}

impl DeviceLogitsSession for CudaClampedRoutedModelSession {
    fn sample(&mut self, policy: SamplingLogits) -> Result<&DeviceBuffer<u32>> {
        Self::sample(self, policy)
    }

    fn logits(&self) -> &DeviceBuffer<bf16> {
        Self::logits(self)
    }
}

pub(in crate::engine) const fn device_sampling(policy: SamplingLogits) -> bool {
    matches!(
        policy,
        SamplingLogits::None | SamplingLogits::SampleTopK { .. } | SamplingLogits::Sample { .. }
    )
}

pub(in crate::engine) fn generation_output(
    backend: &CudaBackend,
    session: &mut impl DeviceLogitsSession,
    policy: SamplingLogits,
) -> Result<Output> {
    if device_sampling(policy) {
        let selected = session.sample(policy)?;
        return Ok(Output {
            token: Some(backend.read_token(selected)?),
            logits: None,
        });
    }
    output_logits(backend, session.logits())
}

pub(super) fn output_logits(backend: &CudaBackend, logits: &DeviceBuffer<bf16>) -> Result<Output> {
    let values = backend.read_logits(logits)?;
    Ok(Output {
        token: None,
        logits: Some(LogitsTrace {
            shape: vec![1, 1, i32::try_from(values.len())?],
            values,
        }),
    })
}

pub(super) fn decode_output(output: Output) -> DecodeOutput {
    DecodeOutput {
        event: TokenEvent {
            token_id: output.token,
            text: "cuda.decode=device-token-pipeline".into(),
            finished: false,
        },
        logits: output.logits,
        candidates: None,
    }
}