libmir-cuda 0.3.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157

use mircuda::{DeviceBuffer, LaunchConfig, Stream, bf16};

use super::ClampedRoutedAttention;
use crate::{PagedPrefillBatch, Result};

impl ClampedRoutedAttention {
    #[allow(clippy::too_many_arguments)]
    pub(crate) fn execute_prefill_batch(
        &self,
        stream: &Stream,
        query: &DeviceBuffer<bf16>,
        current_keys: &DeviceBuffer<bf16>,
        current_values: &DeviceBuffer<bf16>,
        key_pages: &DeviceBuffer<u8>,
        value_pages: &DeviceBuffer<u8>,
        batch: &PagedPrefillBatch,
        tables: &DeviceBuffer<u32>,
        sinks: &DeviceBuffer<bf16>,
        softmax_lse: &mut DeviceBuffer<bf16>,
        output: &mut DeviceBuffer<bf16>,
        window: Option<usize>,
        scale: f32,
    ) -> Result<()> {
        if self.execute_fmha(
            stream, query, key_pages, value_pages, batch, tables, sinks, softmax_lse, output, scale,
        )? {
            return Ok(());
        }
        Ok(self.batch_prefill.launch(
            stream,
            Self::launch(batch.tokens() * self.query_heads, self.head_dim)?,
            (
                query,
                key_pages,
                current_keys,
                current_values,
                value_pages,
                tables,
                batch.request_indices(),
                batch.positions(),
                batch.query_starts(),
                batch.block_counts(),
                sinks,
                output,
                narrow(batch.tokens())?,
                narrow(batch.max_blocks())?,
                narrow(self.block_size)?,
                narrow(self.query_heads)?,
                narrow(self.kv_heads)?,
                narrow(self.head_dim)?,
                narrow(window.unwrap_or(0))?,
                scale,
            ),
        )?)
    }

    #[allow(clippy::too_many_arguments)]
    pub(crate) fn execute(
        &self,
        stream: &Stream,
        query: &DeviceBuffer<bf16>,
        current_keys: &DeviceBuffer<bf16>,
        current_values: &DeviceBuffer<bf16>,
        key_pages: &DeviceBuffer<u8>,
        value_pages: &DeviceBuffer<u8>,
        table: &DeviceBuffer<u32>,
        sinks: &DeviceBuffer<bf16>,
        output: &mut DeviceBuffer<bf16>,
        tokens: usize,
        blocks: usize,
        window: Option<usize>,
        scale: f32,
    ) -> Result<()> {
        Ok(self.decode.launch(
            stream,
            Self::launch(self.query_heads, self.head_dim)?,
            (
                query,
                current_keys,
                current_values,
                key_pages,
                value_pages,
                table,
                sinks,
                output,
                narrow(tokens)?,
                narrow(blocks)?,
                narrow(self.block_size)?,
                narrow(self.query_heads)?,
                narrow(self.kv_heads)?,
                narrow(self.head_dim)?,
                narrow(window.unwrap_or(0))?,
                scale,
            ),
        )?)
    }

    #[allow(clippy::too_many_arguments)]
    pub(crate) fn execute_prefill(
        &self,
        stream: &Stream,
        query: &DeviceBuffer<bf16>,
        current_keys: &DeviceBuffer<bf16>,
        current_values: &DeviceBuffer<bf16>,
        key_pages: &DeviceBuffer<u8>,
        value_pages: &DeviceBuffer<u8>,
        table: &DeviceBuffer<u32>,
        sinks: &DeviceBuffer<bf16>,
        output: &mut DeviceBuffer<bf16>,
        query_tokens: usize,
        start: usize,
        blocks: usize,
        window: Option<usize>,
        scale: f32,
    ) -> Result<()> {
        Ok(self.prefill.launch(
            stream,
            Self::launch(query_tokens * self.query_heads, self.head_dim)?,
            (
                query,
                current_keys,
                current_values,
                key_pages,
                value_pages,
                table,
                sinks,
                output,
                narrow(query_tokens)?,
                narrow(start)?,
                narrow(blocks)?,
                narrow(self.block_size)?,
                narrow(self.query_heads)?,
                narrow(self.kv_heads)?,
                narrow(self.head_dim)?,
                narrow(window.unwrap_or(0))?,
                scale,
            ),
        )?)
    }

    pub(super) fn launch(blocks: usize, head_dim: usize) -> Result<LaunchConfig> {
        let threads = if head_dim == 64 {
            32
        } else {
            head_dim.next_multiple_of(32).min(256)
        };
        Ok(LaunchConfig {
            grid: (narrow(blocks)?, 1, 1),
            block: (narrow(threads)?, 1, 1),
            shared_memory_bytes: 0,
        })
    }
}

pub(super) fn narrow(value: usize) -> Result<u32> {
    Ok(u32::try_from(value)?)
}