slm_inference 0.1.1

Backend-agnostic trait layer for running Small Language Model (SLM) inference in Rust.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

use std::marker::PhantomData;
use std::sync::Arc;

pub trait Free<T> {
    unsafe fn free(ptr: *mut T);
}

pub struct UniquePtr<T, F: Free<T>> {
    ptr: *mut T,
    _marker: PhantomData<F>,
}

unsafe impl<T, F: Free<T>> Send for UniquePtr<T, F> {}

impl<T, F: Free<T>> Drop for UniquePtr<T, F> {
    fn drop(&mut self) {
        if !self.ptr.is_null() {
            unsafe {
                F::free(self.ptr);
            }
        }
    }
}

impl<T, F: Free<T>> UniquePtr<T, F> {
    pub fn new(ptr: *mut T) -> Self {
        Self {
            ptr,
            _marker: PhantomData,
        }
    }

    pub fn get_const_ptr(&self) -> *const T {
        self.ptr
    }
    pub fn get_ptr(&self) -> *mut T {
        self.ptr
    }
    pub fn is_null(&self) -> bool {
        self.ptr.is_null()
    }
}

#[derive(Clone)]
pub struct SharedPtr<T, F: Free<T>> {
    inner: Arc<UniquePtr<T, F>>,
}

pub trait ThreadSafe {}
unsafe impl<T, F: Free<T> + ThreadSafe> Send for SharedPtr<T, F> {}
unsafe impl<T, F: Free<T> + ThreadSafe> Sync for SharedPtr<T, F> {}

impl<T, F: Free<T>> SharedPtr<T, F> {
    pub fn new(ptr: *mut T) -> Self {
        Self {
            inner: Arc::new(UniquePtr::new(ptr)),
        }
    }

    pub fn get_const_ptr(&self) -> *const T {
        self.inner.ptr
    }
    pub fn get_ptr(&self) -> *mut T {
        self.inner.ptr
    }
    pub fn is_null(&self) -> bool {
        self.inner.ptr.is_null()
    }
}