libmir-cuda 0.1.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

/// Explicit CUDA runtime and planning policy.
use std::path::PathBuf;

use crate::backend::{CudaModelSessionConfig, CudaPlanningPolicy};

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct CudaConfig {
    /// CUDA device ordinal selected during backend construction.
    pub device_ordinal: usize,
    /// Bytes retained by CUDA's default memory pool after use.
    pub memory_pool_release_threshold: u64,
    /// Header roots passed explicitly to NVRTC for toolkit-backed kernels.
    pub nvrtc_include_paths: Vec<PathBuf>,
    /// Optional directory for persistent, architecture-specific PTX artifacts.
    pub nvrtc_cache_directory: Option<PathBuf>,
    /// Explicit model-level execution planning policy.
    pub planning: CudaPlanningPolicy,
    /// Session-local prompt allocation policy.
    pub model_session: CudaModelSessionConfig,
}

impl Default for CudaConfig {
    fn default() -> Self {
        Self {
            device_ordinal: 0,
            memory_pool_release_threshold: 512 * 1_024 * 1_024,
            nvrtc_include_paths: vec![
                PathBuf::from("/usr/local/cuda/include"),
                PathBuf::from("/usr/local/cuda/include/cccl"),
            ],
            nvrtc_cache_directory: None,
            planning: CudaPlanningPolicy::default(),
            model_session: CudaModelSessionConfig::default(),
        }
    }
}