libmir-cuda 0.2.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

use models::{
    layout::VisionConfig,
    weights::{TensorCatalog, TensorReadiness},
};

use crate::{
    CudaBackend, Result,
    backend::{CudaPooledVisionTower, CudaSpatialMergeVisionTower},
    checkpoint::load_vision_tensors,
};

pub(in crate::engine) enum LoadedVisionModel {
    Pooled(CudaPooledVisionTower),
    SpatialMerge(CudaSpatialMergeVisionTower),
}

pub(in crate::engine) fn load_vision_model(
    backend: &CudaBackend,
    config: Option<&VisionConfig>,
    readiness: Option<&TensorReadiness>,
    catalog: &TensorCatalog,
) -> Result<Option<LoadedVisionModel>> {
    if !readiness.is_some_and(TensorReadiness::is_ready) {
        return Ok(None);
    }
    match config {
        Some(VisionConfig::PooledEncoder(config)) => {
            let tensors = load_vision_tensors(
                backend,
                &VisionConfig::PooledEncoder(config.clone()),
                catalog,
            )?;
            Ok(Some(LoadedVisionModel::Pooled(CudaPooledVisionTower::new(
                backend,
                config.clone(),
                tensors,
            )?)))
        },
        Some(VisionConfig::SpatialMergeEncoder(config)) => {
            let tensors = load_vision_tensors(
                backend,
                &VisionConfig::SpatialMergeEncoder(config.clone()),
                catalog,
            )?;
            Ok(Some(LoadedVisionModel::SpatialMerge(CudaSpatialMergeVisionTower::new(
                backend,
                config.clone(),
                tensors,
            )?)))
        },
        None => Ok(None),
    }
}