trt-runner 0.1.0

Minimal TensorRT 10 inference runner: deserialize an engine, bind I/O tensors, enqueue V3. No bindings generator, no runtime dependencies.
Documentation
//! CUDA plumbing: the device context, a stream, and the two buffer kinds a runner needs.
//!
//! Everything here goes through the CUDA runtime API (`cudaSetDevice`, `cudaMalloc`, `cudaHostAlloc`).
//! That is deliberate: `cudaSetDevice` makes the device's primary context current on the calling
//! thread, so the crate cannot get into the state that driver-API bindings do, where a retained
//! primary context is held but never activated and every later call fails with
//! `CUDA_ERROR_INVALID_CONTEXT`.

use std::os::raw::c_void;

use crate::error::{cuda_check, Error};
use crate::ffi;
use crate::Result;

/// Number of CUDA devices visible to the process.
pub fn device_count() -> Result<i32> {
    let mut count = 0i32;
    cuda_check(
        unsafe { ffi::trt_cuda_device_count(&mut count) },
        "cudaGetDeviceCount",
    )?;
    Ok(count)
}

/// Activates `device` on the calling thread. Call this once per thread before touching any other
/// function in this crate, including on threads that only run inference.
pub fn set_device(device: i32) -> Result<()> {
    cuda_check(unsafe { ffi::trt_cuda_set_device(device) }, "cudaSetDevice")
}

/// A CUDA stream. Dropping a stream that this crate created destroys it; the legacy default stream
/// is never owned and therefore never destroyed.
#[derive(Debug)]
pub struct Stream {
    raw: *mut c_void,
    owned: bool,
}

impl Stream {
    /// A new non-blocking stream. Non-blocking keeps the stream from synchronising with the legacy
    /// default stream, which is what you want when preprocessing and inference overlap.
    pub fn new() -> Result<Self> {
        Self::create(1)
    }

    /// A new stream that does synchronise with the legacy default stream.
    pub fn blocking() -> Result<Self> {
        Self::create(0)
    }

    fn create(non_blocking: i32) -> Result<Self> {
        let mut raw: *mut c_void = std::ptr::null_mut();
        cuda_check(
            unsafe { ffi::trt_cuda_stream_create(&mut raw, non_blocking) },
            "cudaStreamCreate",
        )?;
        Ok(Self { raw, owned: true })
    }

    /// The legacy default stream. Cheap, but remember that it implicitly synchronises with every
    /// blocking stream and with synchronous memory copies.
    pub const fn default_stream() -> Self {
        Self {
            raw: std::ptr::null_mut(),
            owned: false,
        }
    }

    /// The `cudaStream_t` value, for handing to foreign code.
    pub fn as_raw(&self) -> *mut c_void {
        self.raw
    }

    /// Blocks until every operation queued on this stream has finished.
    pub fn synchronize(&self) -> Result<()> {
        cuda_check(
            unsafe { ffi::trt_cuda_stream_sync(self.raw) },
            "cudaStreamSynchronize",
        )
    }
}

impl Default for Stream {
    fn default() -> Self {
        Self::default_stream()
    }
}

impl Drop for Stream {
    fn drop(&mut self) {
        if self.owned && !self.raw.is_null() {
            // Destroying a stream cannot meaningfully fail here; the worst case is a leaked stream
            // at process exit.
            let _ = cuda_check(
                unsafe { ffi::trt_cuda_stream_destroy(self.raw) },
                "cudaStreamDestroy",
            );
        }
    }
}

// Streams are handles; the caller is responsible for the CUDA context being current on whichever
// thread uses them.
unsafe impl Send for Stream {}

/// Memory on the device. Sizes are in bytes throughout the crate.
#[derive(Debug)]
pub struct DeviceBuffer {
    ptr: *mut c_void,
    bytes: usize,
}

impl DeviceBuffer {
    /// Allocates `bytes` of device memory. The contents are undefined; use
    /// [`DeviceBuffer::zeroed`] when the network must see zeroes.
    pub fn new(bytes: usize) -> Result<Self> {
        let mut ptr: *mut c_void = std::ptr::null_mut();
        cuda_check(
            unsafe { ffi::trt_cuda_malloc(&mut ptr, bytes) },
            "cudaMalloc",
        )?;
        Ok(Self { ptr, bytes })
    }

    /// Allocates and zero-fills, waiting for the fill so the buffer is definitely clean on return.
    pub fn zeroed(bytes: usize, stream: &Stream) -> Result<Self> {
        let buffer = Self::new(bytes)?;
        cuda_check(
            unsafe { ffi::trt_cuda_zero(buffer.ptr, bytes, stream.as_raw()) },
            "cudaMemsetAsync",
        )?;
        stream.synchronize()?;
        Ok(buffer)
    }

    /// Size of the allocation in bytes.
    pub fn size_bytes(&self) -> usize {
        self.bytes
    }

    /// The device pointer, for `bind_raw` or for foreign code.
    pub fn as_ptr(&self) -> *const c_void {
        self.ptr
    }

    /// Mutable device pointer.
    pub fn as_mut_ptr(&mut self) -> *mut c_void {
        self.ptr
    }

    /// Queues a host to device copy. Safe to use straight before [`crate::Context::enqueue`]: work on
    /// one stream runs in order.
    pub fn copy_from_host(&mut self, src: &[u8], stream: &Stream) -> Result<()> {
        if src.len() > self.bytes {
            return Err(Error::SizeMismatch {
                tensor: "device buffer".to_string(),
                expected: self.bytes,
                found: src.len(),
            });
        }
        cuda_check(
            unsafe {
                ffi::trt_cuda_copy_h2d(
                    self.ptr,
                    src.as_ptr() as *const c_void,
                    src.len(),
                    stream.as_raw(),
                )
            },
            "cudaMemcpyAsync host to device",
        )
    }

    /// Queues a device to host copy **without waiting for it**. Reading `dst` before the stream is
    /// synchronised is a data race that usually shows up as a buffer of zeroes, which is why the
    /// ordinary [`DeviceBuffer::copy_to_host`] synchronises for you.
    pub fn copy_to_host_async(&self, dst: &mut [u8], stream: &Stream) -> Result<()> {
        if dst.len() > self.bytes {
            return Err(Error::SizeMismatch {
                tensor: "device buffer".to_string(),
                expected: self.bytes,
                found: dst.len(),
            });
        }
        cuda_check(
            unsafe {
                ffi::trt_cuda_copy_d2h(
                    dst.as_mut_ptr() as *mut c_void,
                    self.ptr,
                    dst.len(),
                    stream.as_raw(),
                )
            },
            "cudaMemcpyAsync device to host",
        )
    }

    /// Copies into `dst` and blocks until the copy has landed. `stream` must be the stream the
    /// inference was enqueued on, otherwise the data may not exist yet.
    pub fn copy_to_host(&self, dst: &mut [u8], stream: &Stream) -> Result<()> {
        self.copy_to_host_async(dst, stream)?;
        stream.synchronize()
    }
}

impl Drop for DeviceBuffer {
    fn drop(&mut self) {
        let _ = cuda_check(unsafe { ffi::trt_cuda_free(self.ptr) }, "cudaFree");
    }
}

// See `Stream`: valid on whatever thread holds the CUDA context.
unsafe impl Send for DeviceBuffer {}

/// Page-locked host memory. Copies to and from this are what actually stay asynchronous; a plain
/// `Vec<f32>` makes the driver stage the copy and quietly serialise.
#[derive(Debug)]
pub struct PinnedBuffer {
    ptr: *mut c_void,
    bytes: usize,
}

impl PinnedBuffer {
    /// Allocates `bytes` of page-locked host memory.
    pub fn new(bytes: usize) -> Result<Self> {
        let mut ptr: *mut c_void = std::ptr::null_mut();
        cuda_check(
            unsafe { ffi::trt_cuda_host_malloc(&mut ptr, bytes) },
            "cudaHostAlloc",
        )?;
        Ok(Self { ptr, bytes })
    }

    /// Size of the allocation in bytes.
    pub fn size_bytes(&self) -> usize {
        self.bytes
    }

    /// The contents as bytes.
    pub fn bytes(&self) -> &[u8] {
        unsafe { std::slice::from_raw_parts(self.ptr as *const u8, self.bytes) }
    }

    /// The contents as mutable bytes, which is how a CPU side writer feeds the network.
    pub fn bytes_mut(&mut self) -> &mut [u8] {
        unsafe { std::slice::from_raw_parts_mut(self.ptr as *mut u8, self.bytes) }
    }

    /// The same memory as `f32`. `cudaHostAlloc` returns memory aligned to at least 16 bytes, and a
    /// float tensor is always a whole number of floats, so the only check needed is the length.
    ///
    /// # Panics
    /// If the buffer length is not a multiple of four.
    pub fn f32s(&self) -> &[f32] {
        assert_eq!(
            self.bytes % std::mem::size_of::<f32>(),
            0,
            "buffer is not a whole number of f32"
        );
        unsafe {
            std::slice::from_raw_parts(
                self.ptr as *const f32,
                self.bytes / std::mem::size_of::<f32>(),
            )
        }
    }

    /// Mutable counterpart of [`PinnedBuffer::f32s`].
    ///
    /// # Panics
    /// If the buffer length is not a multiple of four.
    pub fn f32s_mut(&mut self) -> &mut [f32] {
        assert_eq!(
            self.bytes % std::mem::size_of::<f32>(),
            0,
            "buffer is not a whole number of f32"
        );
        unsafe {
            std::slice::from_raw_parts_mut(
                self.ptr as *mut f32,
                self.bytes / std::mem::size_of::<f32>(),
            )
        }
    }
}

impl Drop for PinnedBuffer {
    fn drop(&mut self) {
        let _ = cuda_check(unsafe { ffi::trt_cuda_host_free(self.ptr) }, "cudaFreeHost");
    }
}

// See `Stream`: valid on whatever thread holds the CUDA context.
unsafe impl Send for PinnedBuffer {}