libmir-metal 0.2.0

Metal inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

use crate::engine::{Array, DecoderCache};

#[derive(Debug)]
pub(super) struct SessionState {
    pub(super) cache: DecoderCache,
    pub(super) position: usize,
    pub(super) rope_position_delta: i32,
    pub(super) pending: Option<PendingDecode>,
}

impl SessionState {
    pub(super) const fn new(cache: DecoderCache) -> Self {
        Self {
            cache,
            position: 0,
            rope_position_delta: 0,
            pending: None,
        }
    }

    pub(super) const fn from_prefix(cache: DecoderCache, position: usize) -> Self {
        Self {
            cache,
            position,
            rope_position_delta: 0,
            pending: None,
        }
    }

    pub(super) fn model_position(&self) -> crate::native::error::Result<usize> {
        let position = i64::try_from(self.position)? + i64::from(self.rope_position_delta);
        usize::try_from(position).map_err(Into::into)
    }
}

#[derive(Debug)]
pub(super) struct PendingDecode {
    pub(super) token_id: u32,
    pub(super) logits: Array,
}