libmir-cuda 0.3.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

use runtime::kv::BlockTable;

#[derive(Debug)]
pub struct PrefillBatchRow {
    table: BlockTable,
    start: usize,
    tokens: usize,
}

impl PrefillBatchRow {
    pub(super) const fn new(table: BlockTable, start: usize, tokens: usize) -> Self {
        Self { table, start, tokens }
    }

    pub const fn table(&self) -> &BlockTable {
        &self.table
    }

    pub const fn start(&self) -> usize {
        self.start
    }

    pub const fn tokens(&self) -> usize {
        self.tokens
    }
}