libmir-cuda 0.1.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

mod batch;
mod config;
mod decode;
pub(in crate::backend) mod graph;
mod prefill;
mod projection;
mod scratch;
mod template;

pub(super) use batch::BatchedDecodeDenseLayer;
pub use config::{DenseDownWeight, DenseGateUpWeights, DenseSwiGluConfig, DenseSwiGluWeights};
pub use decode::DecodeDenseSwiGlu;
use decode::validate;
pub use prefill::PrefillDenseSwiGlu;
use projection::{DownProjection, GateUpBuffers, GateUpProjection};
use scratch::DenseScratch;
pub use template::{
    DenseDownSource, DenseGateUpSource, DenseOutputSource, DenseQkvSource,
    DenseSwiGluLayerTemplate, DenseWeightSource,
};

use super::{Bf16LinearPairWeights, CudaBackend, DecodeAttentionConfig, DecodeAttentionWeights};
use crate::CudaTensor;