libmir-cuda 0.1.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

mod attention;
mod batch;
#[cfg(test)]
mod batch_tests;
#[cfg(test)]
mod profile;
#[cfg(test)]
mod split_tests;
mod storage;
#[cfg(test)]
mod tests;

pub use attention::{BatchedPagedAttentionBf16, PagedAttentionBf16};
pub(in crate::backend) use attention::{
    CapturedPagedAttentionKernels, CapturedPagedAttentionNodes,
};
pub use batch::PagedDecodeBatch;
pub use storage::PagedKvCache;