libmir-cuda 0.1.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

use mircuda::{DeviceBuffer, PinnedBuffer, Stream};

use super::CudaModelSessionConfig;
use crate::{CudaBackend, Error, Result};

pub(super) struct PrefillTokenBuffer {
    device: DeviceBuffer<u32>,
    staging: PinnedBuffer<u32>,
    capacity: usize,
}

impl PrefillTokenBuffer {
    pub(super) fn new(backend: &CudaBackend, config: CudaModelSessionConfig) -> Result<Self> {
        let capacity = config.validate()?.prefill_chunk_tokens;
        Ok(Self {
            device: backend.inner.pool.allocate::<u32>(&backend.inner.stream, capacity)?,
            staging: backend.inner.context.allocate_pinned(capacity)?,
            capacity,
        })
    }

    pub(super) fn upload(&mut self, stream: &Stream, tokens: &[u32]) -> Result<()> {
        if tokens.is_empty() || tokens.len() > self.capacity {
            return Err(Error::InvalidDecoderKernel("invalid CUDA prefill token chunk"));
        }
        self.staging.write_prefix(tokens)?;
        stream.copy_to_device(&mut self.staging, &mut self.device)?;
        Ok(())
    }

    pub(super) const fn capacity(&self) -> usize {
        self.capacity
    }

    pub(super) const fn device(&self) -> &DeviceBuffer<u32> {
        &self.device
    }
}