libmir-cuda 0.1.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

use crate::{Error, Result};

/// Explicit allocation policy for one CUDA model session.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct CudaModelSessionConfig {
    /// Maximum prompt tokens uploaded in one pinned-host transfer.
    pub prefill_chunk_tokens: usize,
}

impl Default for CudaModelSessionConfig {
    fn default() -> Self {
        Self { prefill_chunk_tokens: 256 }
    }
}

impl CudaModelSessionConfig {
    pub(super) fn validate(self) -> Result<Self> {
        if self.prefill_chunk_tokens == 0 {
            Err(Error::InvalidDecoderKernel("CUDA prefill chunk cannot be empty"))
        } else {
            Ok(self)
        }
    }
}