libmir-cuda 0.3.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

mod attention;
mod batch;
mod prefill_batch;
#[cfg(test)]
mod profile;
mod storage;
#[cfg(test)]
mod tests;

pub use attention::{
    BatchedPagedAttentionBf16, BatchedPrefillPagedAttentionBf16, PagedAttentionBf16,
    autotune::{
        SplitMeasurement as AttentionSplitMeasurement, candidate_partitions,
        execution_average as attention_execution_average, sample_contexts,
        select_execution as select_attention_execution,
    },
};
pub(in crate::backend) use attention::{
    CapturedPagedAttentionKernels, CapturedPagedAttentionNodes,
};
pub use batch::PagedDecodeBatch;
pub use prefill_batch::PagedPrefillBatch;
pub use storage::PagedKvCache;