native-onnx 0.2.0

ONNX inference with system CUDA and TensorRT, reusable buffers, and explicit backend selection
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

//! Coordinate ORT global capture with native setup, teardown and inference.
//! Shared access permits concurrent replay; exclusive access protects capture.
use crate::Result;
use std::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};

// CaptureModeGlobal can affect other host threads, even on another device.
// This gate covers this crate's sessions only, not external CUDA consumers.
static EXECUTION: RwLock<()> = RwLock::new(());

pub(crate) fn exclusive() -> Result<RwLockWriteGuard<'static, ()>> {
    EXECUTION.write().map_err(|_| {
        anyhow::anyhow!(
            "CUDA capture coordinator poisoned by an earlier panic; restart the process"
        )
    })
}

pub(crate) fn shared() -> Result<RwLockReadGuard<'static, ()>> {
    EXECUTION.read().map_err(|_| {
        anyhow::anyhow!(
            "CUDA capture coordinator poisoned by an earlier panic; restart the process"
        )
    })
}

pub(crate) fn teardown() -> RwLockWriteGuard<'static, ()> {
    // Destructors must still free resources after a Rust panic.
    EXECUTION
        .write()
        .unwrap_or_else(std::sync::PoisonError::into_inner)
}