use std::cell::Cell;
use std::cell::RefCell;
use std::ffi::{c_int, CString};
use std::marker::PhantomData;
use std::ptr;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use anyhow::{anyhow, Result};
use super::ffi::{self, Api, GgmlBackend, GgmlBackendBuft, GgmlBackendSched, GgmlContext, Tensor};
pub struct Backend {
pub api: &'static Api,
pub gpu: GgmlBackend,
pub cpu: GgmlBackend,
weight_buft: GgmlBackendBuft,
sched: GgmlBackendSched,
max_nodes: usize,
probe: Option<(Ctx, *mut ffi::GgmlCgraph)>,
gpu_failed: Cell<bool>,
abort: Option<Arc<AtomicBool>>,
}
unsafe impl Send for Backend {}
impl Backend {
pub fn new(
api: &'static Api,
use_gpu: bool,
n_threads: i32,
max_nodes: usize,
) -> Result<Backend> {
unsafe {
let cpu =
(api.ggml_backend_init_by_type)(ffi::GGML_BACKEND_DEVICE_TYPE_CPU, ptr::null());
if cpu.is_null() {
return Err(anyhow!("failed to initialize the CPU backend"));
}
let gpu = if use_gpu {
init_gpu(api)
} else {
ptr::null_mut()
};
let mut probe = None;
let weight_buft = if !gpu.is_null() {
let name = std::ffi::CStr::from_ptr((api.ggml_backend_name)(gpu)).to_string_lossy();
eprintln!("[llmman] mediagen: using {name} backend");
let ctx = Ctx::new(api, (api.ggml_graph_overhead_custom)(1, false) + 1024, true)?;
let gf = (api.ggml_new_graph_custom)(ctx.raw, 1, false);
if gf.is_null() {
return Err(anyhow!("ggml_new_graph_custom failed"));
}
probe = Some((ctx, gf));
(api.ggml_backend_get_default_buffer_type)(gpu)
} else {
eprintln!("[llmman] mediagen: using CPU backend");
(api.ggml_backend_get_default_buffer_type)(cpu)
};
api.set_n_threads(cpu, n_threads);
let mut be = Backend {
api,
gpu,
cpu,
weight_buft,
sched: ptr::null_mut(),
max_nodes,
probe,
gpu_failed: Cell::new(false),
abort: None,
};
be.release_compute()?;
Ok(be)
}
}
pub fn weight_buft(&self) -> GgmlBackendBuft {
self.weight_buft
}
pub fn max_nodes(&self) -> usize {
self.max_nodes
}
pub fn gpu_memory(&self) -> Option<(usize, usize)> {
(!self.gpu.is_null()).then(|| self.dev_memory(self.gpu))
}
pub fn weight_memory(&self) -> (usize, usize) {
self.dev_memory(if self.gpu.is_null() {
self.cpu
} else {
self.gpu
})
}
fn dev_memory(&self, backend: GgmlBackend) -> (usize, usize) {
let (mut free, mut total) = (0usize, 0usize);
unsafe {
let dev = (self.api.ggml_backend_get_device)(backend);
(self.api.ggml_backend_dev_memory)(dev, &mut free, &mut total);
}
(free, total)
}
pub fn gpu_name(&self) -> String {
if self.gpu.is_null() {
return String::new();
}
unsafe { std::ffi::CStr::from_ptr((self.api.ggml_backend_name)(self.gpu)) }
.to_string_lossy()
.into_owned()
}
pub fn supports_direct_conv(&self) -> bool {
let (Some(conv), Some(supports)) = (
self.api.ggml_conv_2d_direct,
self.api.ggml_backend_supports_op,
) else {
return false;
};
if self.gpu.is_null() {
return false;
}
let Ok(ctx) = Ctx::new(self.api, 16 * 1024, true) else {
return false;
};
let k = ctx.new_tensor(ffi::ty::F16, &[3, 3, 8, 8]);
let x = ctx.new_tensor(ffi::ty::F32, &[16, 16, 8, 2]);
let y = unsafe { conv(ctx.raw, k.as_ptr(), x.as_ptr(), 1, 1, 1, 1, 1, 1) };
unsafe { supports(self.gpu, y) }
}
pub fn release_compute(&mut self) -> Result<()> {
unsafe {
if !self.sched.is_null() {
(self.api.ggml_backend_sched_free)(self.sched);
self.sched = ptr::null_mut();
}
if self.gpu_failed.replace(false) && !self.gpu.is_null() {
(self.api.ggml_backend_free)(self.gpu);
self.gpu = init_gpu(self.api);
if self.gpu.is_null() {
return Err(anyhow!(
"failed to recreate the GPU backend after a failure"
));
}
eprintln!("[llmman] mediagen: recreated the GPU backend after a failure");
}
let mut ptrs = Vec::new();
if !self.gpu.is_null() {
ptrs.push(self.gpu);
}
ptrs.push(self.cpu);
let mut bufts: Vec<GgmlBackendBuft> = ptrs
.iter()
.map(|&b| (self.api.ggml_backend_get_default_buffer_type)(b))
.collect();
self.sched = (self.api.ggml_backend_sched_new)(
ptrs.as_mut_ptr(),
bufts.as_mut_ptr(),
ptrs.len() as c_int,
self.max_nodes,
false,
true,
);
}
if self.sched.is_null() {
return Err(anyhow!("ggml_backend_sched_new failed"));
}
Ok(())
}
pub fn read_f32(&self, t: Tensor<'_>) -> Vec<f32> {
read_f32(self.api, t)
}
pub fn set_abort_flag(&mut self, flag: Arc<AtomicBool>) {
let data = Arc::as_ptr(&flag) as *mut std::ffi::c_void;
self.abort = Some(flag);
self.api
.set_abort_callback(self.cpu, Some(abort_requested), data);
}
pub fn cancelled(&self) -> bool {
self.abort
.as_ref()
.is_some_and(|f| f.load(Ordering::Relaxed))
}
fn check_gpu(&self) -> Result<()> {
let Some((_, gf)) = &self.probe else {
return Ok(());
};
if self.gpu.is_null() {
return Ok(());
}
let status = unsafe { (self.api.ggml_backend_graph_compute)(self.gpu, *gf) };
if status != ffi::GGML_STATUS_SUCCESS {
self.gpu_failed.set(true);
return Err(anyhow!(
"the GPU failed to compute the graph (out of memory?), status {status}"
));
}
Ok(())
}
}
unsafe extern "C" fn abort_requested(data: *mut std::ffi::c_void) -> bool {
(*(data as *const AtomicBool)).load(Ordering::Relaxed)
}
fn read_f32(api: &Api, t: Tensor<'_>) -> Vec<f32> {
let n = unsafe { (api.ggml_nelements)(t.as_ptr()) } as usize;
let mut out = vec![0f32; n];
unsafe { (api.ggml_backend_tensor_get)(t.as_ptr(), out.as_mut_ptr() as *mut _, 0, n * 4) };
out
}
fn init_gpu(api: &Api) -> GgmlBackend {
unsafe {
let gpu = (api.ggml_backend_init_by_type)(ffi::GGML_BACKEND_DEVICE_TYPE_GPU, ptr::null());
if !gpu.is_null() {
return gpu;
}
(api.ggml_backend_init_by_type)(ffi::GGML_BACKEND_DEVICE_TYPE_IGPU, ptr::null())
}
}
impl Drop for Backend {
fn drop(&mut self) {
unsafe {
if !self.sched.is_null() {
(self.api.ggml_backend_sched_free)(self.sched);
}
self.probe = None;
if !self.gpu.is_null() {
(self.api.ggml_backend_free)(self.gpu);
}
(self.api.ggml_backend_free)(self.cpu);
}
}
}
pub struct Ctx {
pub(super) api: &'static Api,
pub(super) raw: *mut GgmlContext,
}
impl Drop for Ctx {
fn drop(&mut self) {
unsafe { (self.api.ggml_free)(self.raw) }
}
}
macro_rules! ops {
($($name:ident => $sym:ident($($arg:ident: $ty:ty),*);)*) => {
impl<'o> Graph<'o> {
$(pub fn $name(&'o self, $($arg: $ty),*) -> Tensor<'o> {
unsafe {
Tensor::from_raw((self.ctx.api.$sym)(self.ctx.raw, $($arg.into()),*), self)
}
})*
}
};
}
ops! {
add => ggml_add(a: Tensor<'o>, b: Tensor<'o>);
sub => ggml_sub(a: Tensor<'o>, b: Tensor<'o>);
mul => ggml_mul(a: Tensor<'o>, b: Tensor<'o>);
scale => ggml_scale(a: Tensor<'o>, s: f32);
scale_bias => ggml_scale_bias(a: Tensor<'o>, s: f32, b: f32);
mul_mat => ggml_mul_mat(a: Tensor<'o>, b: Tensor<'o>);
rms_norm => ggml_rms_norm(a: Tensor<'o>, eps: f32);
norm => ggml_norm(a: Tensor<'o>, eps: f32);
silu => ggml_silu(a: Tensor<'o>);
relu => ggml_relu(a: Tensor<'o>);
gelu => ggml_gelu(a: Tensor<'o>);
sigmoid => ggml_sigmoid(a: Tensor<'o>);
tanh => ggml_tanh(a: Tensor<'o>);
sin => ggml_sin(a: Tensor<'o>);
sqr => ggml_sqr(a: Tensor<'o>);
sqrt => ggml_sqrt(a: Tensor<'o>);
log => ggml_log(a: Tensor<'o>);
clamp => ggml_clamp(a: Tensor<'o>, min: f32, max: f32);
reshape_2d => ggml_reshape_2d(a: Tensor<'o>, ne0: i64, ne1: i64);
reshape_3d => ggml_reshape_3d(a: Tensor<'o>, ne0: i64, ne1: i64, ne2: i64);
reshape_4d => ggml_reshape_4d(a: Tensor<'o>, ne0: i64, ne1: i64, ne2: i64, ne3: i64);
view_2d => ggml_view_2d(a: Tensor<'o>, ne0: i64, ne1: i64, nb1: usize, offset: usize);
view_3d => ggml_view_3d(a: Tensor<'o>, ne0: i64, ne1: i64, ne2: i64, nb1: usize, nb2: usize, offset: usize);
view_4d => ggml_view_4d(a: Tensor<'o>, ne0: i64, ne1: i64, ne2: i64, ne3: i64, nb1: usize, nb2: usize, nb3: usize, offset: usize);
permute => ggml_permute(a: Tensor<'o>, a0: c_int, a1: c_int, a2: c_int, a3: c_int);
transpose => ggml_transpose(a: Tensor<'o>);
cont => ggml_cont(a: Tensor<'o>);
cont_2d => ggml_cont_2d(a: Tensor<'o>, ne0: i64, ne1: i64);
concat => ggml_concat(a: Tensor<'o>, b: Tensor<'o>, dim: c_int);
cast => ggml_cast(a: Tensor<'o>, ty: u32);
get_rows => ggml_get_rows(a: Tensor<'o>, b: Tensor<'o>);
repeat => ggml_repeat(a: Tensor<'o>, b: Tensor<'o>);
pad => ggml_pad(a: Tensor<'o>, p0: c_int, p1: c_int, p2: c_int, p3: c_int);
pad_ext => ggml_pad_ext(a: Tensor<'o>, lp0: c_int, rp0: c_int, lp1: c_int, rp1: c_int, lp2: c_int, rp2: c_int, lp3: c_int, rp3: c_int);
conv_1d => ggml_conv_1d(a: Tensor<'o>, b: Tensor<'o>, s0: c_int, p0: c_int, d0: c_int);
conv_2d => ggml_conv_2d(a: Tensor<'o>, b: Tensor<'o>, s0: c_int, s1: c_int, p0: c_int, p1: c_int, d0: c_int, d1: c_int);
conv_transpose_1d => ggml_conv_transpose_1d(a: Tensor<'o>, b: Tensor<'o>, s0: c_int, p0: c_int, d0: c_int);
conv_2d_dw_direct => ggml_conv_2d_dw_direct(a: Tensor<'o>, b: Tensor<'o>, stride0: c_int, stride1: c_int, pad0: c_int, pad1: c_int, dilation0: c_int, dilation1: c_int);
ssm_conv => ggml_ssm_conv(sx: Tensor<'o>, c: Tensor<'o>);
interpolate => ggml_interpolate(a: Tensor<'o>, ne0: i64, ne1: i64, ne2: i64, ne3: i64, mode: u32);
timestep_embedding => ggml_timestep_embedding(t: Tensor<'o>, dim: c_int, max_period: c_int);
}
fn opt_ptr(t: Option<Tensor<'_>>) -> ffi::TensorPtr {
t.map_or(ptr::null_mut(), Tensor::as_ptr)
}
impl<'o> Graph<'o> {
pub fn flash_attn_ext(
&'o self,
q: Tensor<'o>,
k: Tensor<'o>,
v: Tensor<'o>,
mask: Option<Tensor<'o>>,
scale: f32,
max_bias: f32,
logit_softcap: f32,
) -> Tensor<'o> {
unsafe {
Tensor::from_raw(
(self.ctx.api.ggml_flash_attn_ext)(
self.ctx.raw,
q.as_ptr(),
k.as_ptr(),
v.as_ptr(),
opt_ptr(mask),
scale,
max_bias,
logit_softcap,
),
self,
)
}
}
pub fn soft_max_ext(
&'o self,
a: Tensor<'o>,
mask: Option<Tensor<'o>>,
scale: f32,
max_bias: f32,
) -> Tensor<'o> {
unsafe {
Tensor::from_raw(
(self.ctx.api.ggml_soft_max_ext)(
self.ctx.raw,
a.as_ptr(),
opt_ptr(mask),
scale,
max_bias,
),
self,
)
}
}
}
impl Ctx {
pub(super) fn new(api: &'static Api, mem_size: usize, no_alloc: bool) -> Result<Ctx> {
let raw = unsafe {
(api.ggml_init)(ffi::GgmlInitParams {
mem_size,
mem_buffer: ptr::null_mut(),
no_alloc,
})
};
if raw.is_null() {
return Err(anyhow!("ggml_init failed"));
}
Ok(Ctx { api, raw })
}
pub(super) fn new_tensor(&self, ty: u32, ne: &[i64]) -> Tensor<'_> {
unsafe {
Tensor::from_raw(
(self.api.ggml_new_tensor)(self.raw, ty, ne.len() as c_int, ne.as_ptr()),
self,
)
}
}
pub(super) fn set_name(&self, t: Tensor<'_>, name: &str) {
let name = CString::new(name).unwrap_or_default();
unsafe { (self.api.ggml_set_name)(t.as_ptr(), name.as_ptr()) };
}
}
impl<'o> Graph<'o> {
pub fn api(&'o self) -> &'static Api {
self.ctx.api
}
pub fn new_tensor(&'o self, ty: u32, ne: &[i64]) -> Tensor<'o> {
unsafe { Tensor::from_raw(self.ctx.new_tensor(ty, ne).as_ptr(), self) }
}
pub fn set_name(&'o self, t: Tensor<'o>, name: &str) {
self.ctx.set_name(t, name);
}
pub fn modulate(
&'o self,
x: Tensor<'o>,
scale: Tensor<'o>,
shift: Option<Tensor<'o>>,
) -> Tensor<'o> {
let y = self.mul(x, self.scale_bias(scale, 1.0, 1.0));
match shift {
Some(s) => self.add(y, s),
None => y,
}
}
pub fn linear(&'o self, x: Tensor<'o>, w: Tensor<'o>, b: Option<Tensor<'o>>) -> Tensor<'o> {
let y = self.mul_mat(w, x);
match b {
Some(b) => self.add(y, b),
None => y,
}
}
pub fn prec_f32(&'o self, t: Tensor<'o>) -> Tensor<'o> {
unsafe { (self.ctx.api.ggml_prec_set_acc)(t.as_ptr(), ffi::GGML_PREC_F32) };
t
}
pub fn set_rows(&'o self, a: Tensor<'o>, b: Tensor<'o>, c: Tensor<'o>) -> Option<Tensor<'o>> {
let f = self.ctx.api.ggml_set_rows?;
Some(unsafe { Tensor::from_raw(f(self.ctx.raw, a.as_ptr(), b.as_ptr(), c.as_ptr()), self) })
}
pub fn conv_2d_direct(
&'o self,
k: Tensor<'o>,
x: Tensor<'o>,
s: i32,
pad: i32,
) -> Option<Tensor<'o>> {
let f = self.ctx.api.ggml_conv_2d_direct?;
Some(unsafe {
Tensor::from_raw(
f(self.ctx.raw, k.as_ptr(), x.as_ptr(), s, s, pad, pad, 1, 1),
self,
)
})
}
}
pub struct Resident {
ctx: Ctx,
buf: ffi::GgmlBackendBuffer,
tensors: Vec<ffi::TensorPtr>,
}
unsafe impl Send for Resident {}
impl Resident {
pub fn new(be: &Backend, ty: u32, shapes: &[&[i64]]) -> Result<Resident> {
let api = be.api;
let ctx = Ctx::new(
api,
unsafe { (api.ggml_tensor_overhead)() } * (shapes.len() + 1),
true,
)?;
let tensors = shapes
.iter()
.map(|ne| ctx.new_tensor(ty, ne).as_ptr())
.collect();
let buf =
unsafe { (api.ggml_backend_alloc_ctx_tensors_from_buft)(ctx.raw, be.weight_buft) };
if buf.is_null() {
return Err(anyhow!(
"allocating {} resident tensors failed",
shapes.len()
));
}
Ok(Resident { ctx, buf, tensors })
}
pub fn clear(&self) {
unsafe { (self.ctx.api.ggml_backend_buffer_clear)(self.buf, 0) }
}
pub fn tensor(&self, i: usize) -> Tensor<'_> {
unsafe { Tensor::from_raw(self.tensors[i], self) }
}
pub fn len(&self) -> usize {
self.tensors.len()
}
pub fn is_empty(&self) -> bool {
self.tensors.is_empty()
}
}
impl Drop for Resident {
fn drop(&mut self) {
unsafe { (self.ctx.api.ggml_backend_buffer_free)(self.buf) }
}
}
pub struct Graph<'o> {
ctx: Ctx,
gf: *mut ffi::GgmlCgraph,
inputs: RefCell<Vec<(ffi::TensorPtr, Vec<u8>)>>,
outputs: RefCell<Vec<ffi::TensorPtr>>,
owners: PhantomData<fn(&'o ()) -> &'o ()>,
}
impl<'o> Graph<'o> {
pub fn new(api: &'static Api, max_nodes: usize) -> Result<Graph<'o>> {
let mem = unsafe {
(api.ggml_tensor_overhead)() * max_nodes
+ (api.ggml_graph_overhead_custom)(max_nodes, false)
};
let ctx = Ctx::new(api, mem, true)?;
let gf = unsafe { (api.ggml_new_graph_custom)(ctx.raw, max_nodes, false) };
if gf.is_null() {
return Err(anyhow!("ggml_new_graph_custom failed"));
}
Ok(Graph {
ctx,
gf,
inputs: RefCell::new(Vec::new()),
outputs: RefCell::new(Vec::new()),
owners: PhantomData,
})
}
pub fn input_f32(&'o self, ne: &[i64], data: &[f32], name: &str) -> Tensor<'o> {
let bytes: Vec<u8> = data.iter().flat_map(|v| v.to_le_bytes()).collect();
self.add_input(ffi::ty::F32, ne, bytes, name)
}
pub fn input_i32(&'o self, ne: &[i64], data: &[i32], name: &str) -> Tensor<'o> {
let bytes: Vec<u8> = data.iter().flat_map(|v| v.to_le_bytes()).collect();
self.add_input(ffi::ty::I32, ne, bytes, name)
}
fn add_input(&'o self, ty: u32, ne: &[i64], bytes: Vec<u8>, name: &str) -> Tensor<'o> {
let t = self.new_tensor(ty, ne);
self.set_name(t, name);
unsafe { (self.ctx.api.ggml_set_input)(t.as_ptr()) };
debug_assert_eq!(
unsafe { (self.ctx.api.ggml_nbytes)(t.as_ptr()) },
bytes.len()
);
self.inputs.borrow_mut().push((t.as_ptr(), bytes));
t
}
pub fn input(&'o self, i: usize) -> Tensor<'o> {
let ptr = self.inputs.borrow()[i].0;
unsafe { Tensor::from_raw(ptr, self) }
}
pub fn n_inputs(&'o self) -> usize {
self.inputs.borrow().len()
}
pub fn update_input(&'o self, i: usize, bytes: &[u8]) {
self.fill_input(i, bytes.len(), |slot| slot.copy_from_slice(bytes));
}
fn fill_input(&'o self, i: usize, len: usize, fill: impl FnOnce(&mut [u8])) {
let mut inputs = self.inputs.borrow_mut();
let slot = &mut inputs.get_mut(i).expect("no such graph input").1;
assert_eq!(slot.len(), len, "input size changed");
fill(slot);
}
pub fn update_input_f32(&'o self, i: usize, data: &[f32]) {
self.update_words(i, data.iter().map(|v| v.to_le_bytes()));
}
pub fn update_input_i32(&'o self, i: usize, data: &[i32]) {
self.update_words(i, data.iter().map(|v| v.to_le_bytes()));
}
fn update_words(&'o self, i: usize, words: impl ExactSizeIterator<Item = [u8; 4]>) {
self.fill_input(i, words.len() * 4, |slot| {
for (dst, w) in slot.as_chunks_mut::<4>().0.iter_mut().zip(words) {
*dst = w;
}
});
}
pub fn store(&'o self, src: Tensor<'o>, dst: Tensor<'o>) {
self.mark_output(unsafe {
Tensor::from_raw(
(self.ctx.api.ggml_cpy)(self.ctx.raw, src.as_ptr(), dst.as_ptr()),
self,
)
});
}
pub fn mark_output(&'o self, t: Tensor<'o>) -> usize {
unsafe {
(self.ctx.api.ggml_set_output)(t.as_ptr());
(self.ctx.api.ggml_build_forward_expand)(self.gf, t.as_ptr());
}
let mut outputs = self.outputs.borrow_mut();
outputs.push(t.as_ptr());
outputs.len() - 1
}
pub fn output(&'o self, i: usize) -> Tensor<'o> {
let ptr = self.outputs.borrow()[i];
unsafe { Tensor::from_raw(ptr, self) }
}
pub fn measure(&'o self, be: &Backend) -> Result<usize> {
let api = self.ctx.api;
unsafe {
let buft = (api.ggml_backend_get_default_buffer_type)(be.cpu);
let galloc = (api.ggml_gallocr_new)(buft);
if galloc.is_null() {
return Err(anyhow!("ggml_gallocr_new failed"));
}
let ok = (api.ggml_gallocr_reserve)(galloc, self.gf);
let size = if ok {
(api.ggml_gallocr_get_buffer_size)(galloc, 0)
} else {
usize::MAX
};
(api.ggml_gallocr_free)(galloc);
Ok(size)
}
}
pub fn compute(&'o self, be: &Backend) -> Result<()> {
let api = self.ctx.api;
unsafe {
(api.ggml_backend_sched_reset)(be.sched);
if !(api.ggml_backend_sched_alloc_graph)(be.sched, self.gf) {
return Err(anyhow!(
"failed to allocate a graph of {} nodes",
(api.ggml_graph_n_nodes)(self.gf)
));
}
for (t, bytes) in self.inputs.borrow().iter() {
(api.ggml_backend_tensor_set)(*t, bytes.as_ptr() as *const _, 0, bytes.len());
}
let status = (api.ggml_backend_sched_graph_compute)(be.sched, self.gf);
if status == ffi::GGML_STATUS_ABORTED {
return Err(anyhow!("graph compute cancelled"));
}
if status != ffi::GGML_STATUS_SUCCESS {
be.gpu_failed.set(!be.gpu.is_null());
return Err(anyhow!("graph compute failed with status {status}"));
}
}
be.check_gpu()
}
pub fn output_f32(&'o self, t: Tensor<'o>) -> Vec<f32> {
read_f32(self.ctx.api, t)
}
}