use std::ffi::CString;
use std::os::raw::{c_char, c_void};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use libloading::Library;
use super::{NpuBackend, NpuSession};
type InitFn = unsafe extern "C" fn(*const c_char, u32, *mut u32, *mut *mut c_void) -> i32;
type ForwardFn = unsafe extern "C" fn(*mut c_void, *const u32, u32, u32, *mut f32) -> i32;
type ResetFn = unsafe extern "C" fn(*mut c_void) -> i32;
type FreeFn = unsafe extern "C" fn(*mut c_void);
type MediaPrefillFn = unsafe extern "C" fn(
*mut c_void,
*const c_char,
*const *const u8,
*const u64,
u32,
*mut u32,
*mut f32,
) -> i32;
pub struct VendorLibrary {
lib: Library,
path: PathBuf,
}
impl VendorLibrary {
pub fn open(path: &Path) -> std::result::Result<Self, String> {
let lib = unsafe { Library::new(path) }
.map_err(|e| format!("failed to load NPU plugin {path:?}: {e}"))?;
let this = Self {
lib,
path: path.to_path_buf(),
};
for symbol in [
"joshua_npu_init",
"joshua_npu_forward",
"joshua_npu_reset",
"joshua_npu_free",
] {
this.get_raw(symbol)?;
}
Ok(this)
}
pub fn path(&self) -> &Path {
&self.path
}
fn get_raw(&self, name: &str) -> std::result::Result<*mut c_void, String> {
let cname = format!("{name}\0");
unsafe {
self.lib
.get::<*mut c_void>(cname.as_bytes())
.map(|s| *s)
.map_err(|e| format!("NPU plugin {:?} is missing symbol {name}: {e}", self.path))
}
}
fn sym<T: Copy>(&self, name: &str) -> std::result::Result<T, String> {
let cname = format!("{name}\0");
unsafe {
self.lib
.get::<T>(cname.as_bytes())
.map(|s| *s)
.map_err(|e| format!("NPU plugin {:?} is missing symbol {name}: {e}", self.path))
}
}
pub fn init(
self: &Arc<Self>,
model_path: &Path,
n_ctx: u32,
) -> std::result::Result<VendorSession, String> {
let init: InitFn = self.sym("joshua_npu_init")?;
let cpath = CString::new(model_path.to_string_lossy().as_bytes())
.map_err(|_| "model path contains a NUL byte".to_string())?;
let mut vocab: u32 = 0;
let mut handle: *mut c_void = std::ptr::null_mut();
let rc = unsafe { init(cpath.as_ptr(), n_ctx, &mut vocab, &mut handle) };
if rc != 0 || handle.is_null() {
return Err(format!(
"NPU plugin {:?} init failed for {model_path:?} (code {rc})",
self.path
));
}
if vocab == 0 {
if let Ok(free) = self.sym::<FreeFn>("joshua_npu_free") {
unsafe { free(handle) };
}
return Err(format!("NPU plugin {:?} reported a zero vocab", self.path));
}
Ok(VendorSession {
has_media: self.has_media(),
lib: Arc::clone(self),
handle,
vocab: vocab as usize,
})
}
pub fn has_media(&self) -> bool {
self.get_raw("joshua_npu_media_prefill").is_ok()
}
}
pub struct VendorSession {
lib: Arc<VendorLibrary>,
handle: *mut c_void,
vocab: usize,
has_media: bool,
}
unsafe impl Send for VendorSession {}
impl VendorSession {
pub fn forward_into(
&mut self,
tokens: &[u32],
pos: usize,
out: &mut [f32],
) -> std::result::Result<(), String> {
if tokens.is_empty() {
return Err("empty token batch".to_string());
}
if out.len() != self.vocab {
return Err(format!(
"logit buffer has {} slots, vocab is {}",
out.len(),
self.vocab
));
}
let forward: ForwardFn = self.lib.sym("joshua_npu_forward")?;
let rc = unsafe {
forward(
self.handle,
tokens.as_ptr(),
tokens.len() as u32,
pos as u32,
out.as_mut_ptr(),
)
};
if rc != 0 {
return Err(format!("NPU plugin forward failed (code {rc})"));
}
Ok(())
}
pub fn vocab(&self) -> usize {
self.vocab
}
pub fn has_media(&self) -> bool {
self.has_media
}
pub fn media_prefill_into(
&mut self,
prompt: &str,
images: &[Vec<u8>],
out: &mut [f32],
) -> std::result::Result<usize, String> {
if !self.has_media {
return Err("plugin does not export joshua_npu_media_prefill".to_string());
}
if out.len() != self.vocab {
return Err(format!(
"logit buffer has {} slots, vocab is {}",
out.len(),
self.vocab
));
}
let media_prefill: MediaPrefillFn = self.lib.sym("joshua_npu_media_prefill")?;
let cprompt = CString::new(prompt.as_bytes())
.map_err(|_| "prompt contains a NUL byte".to_string())?;
let ptrs: Vec<*const u8> = images.iter().map(|i| i.as_ptr()).collect();
let lens: Vec<u64> = images.iter().map(|i| i.len() as u64).collect();
let mut n_past: u32 = 0;
let rc = unsafe {
media_prefill(
self.handle,
cprompt.as_ptr(),
ptrs.as_ptr(),
lens.as_ptr(),
images.len() as u32,
&mut n_past,
out.as_mut_ptr(),
)
};
if rc != 0 {
return Err(format!("NPU plugin media_prefill failed (code {rc})"));
}
Ok(n_past as usize)
}
pub fn reset(&mut self) -> std::result::Result<(), String> {
let reset: ResetFn = self.lib.sym("joshua_npu_reset")?;
let rc = unsafe { reset(self.handle) };
if rc != 0 {
return Err(format!("NPU plugin reset failed (code {rc})"));
}
Ok(())
}
}
impl Drop for VendorSession {
fn drop(&mut self) {
if let Ok(free) = self.lib.sym::<FreeFn>("joshua_npu_free") {
unsafe { free(self.handle) };
}
}
}
pub struct InProcessBackend {
lib: Arc<VendorLibrary>,
}
impl InProcessBackend {
pub fn load(library: &Path) -> std::result::Result<Self, String> {
Ok(Self {
lib: Arc::new(VendorLibrary::open(library)?),
})
}
}
impl NpuBackend for InProcessBackend {
fn name(&self) -> String {
format!("npu-inprocess({})", self.lib.path().display())
}
fn create_session(
&self,
model_path: &Path,
n_ctx: u32,
) -> std::result::Result<Box<dyn NpuSession>, String> {
Ok(Box::new(self.lib.init(model_path, n_ctx)?))
}
}
impl NpuSession for VendorSession {
fn vocab_size(&self) -> usize {
self.vocab
}
fn forward(&mut self, tokens: &[u32], pos: usize) -> std::result::Result<Vec<f32>, String> {
let mut out = vec![0f32; self.vocab];
self.forward_into(tokens, pos, &mut out)?;
Ok(out)
}
fn reset(&mut self) -> bool {
VendorSession::reset(self).is_ok()
}
fn supports_media(&self) -> bool {
self.has_media
}
fn media_prefill(
&mut self,
prompt: &str,
images: &[Vec<u8>],
) -> std::result::Result<(usize, Vec<f32>), String> {
let mut out = vec![0f32; self.vocab];
let n_past = self.media_prefill_into(prompt, images, &mut out)?;
Ok((n_past, out))
}
}