libmir-cuda 0.2.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14

mod attention;
mod batch;
#[cfg(test)]
mod profile;
mod storage;
#[cfg(test)]
mod tests;

pub use attention::{BatchedPagedAttentionBf16, PagedAttentionBf16};
pub(in crate::backend) use attention::{
    CapturedPagedAttentionKernels, CapturedPagedAttentionNodes,
};
pub use batch::PagedDecodeBatch;
pub use storage::PagedKvCache;