#include "shim.h"
#include <NvInfer.h>
#include <cuda_runtime_api.h>
#include <algorithm>
#include <cstdio>
#include <cstring>
#include <string>
#if NV_TENSORRT_MAJOR < 10
#error "trt-runner requires TensorRT 10 or newer (enqueueV3 and the tensor-name I/O API)"
#endif
namespace {
thread_local std::string g_last_error;
void record(const char* message) {
if (message != nullptr) {
g_last_error.assign(message);
}
}
void clear_error() { g_last_error.clear(); }
class Logger : public nvinfer1::ILogger {
public:
explicit Logger(int32_t verbosity) : verbosity_(verbosity) {}
void log(nvinfer1::ILogger::Severity severity, nvinfer1::AsciiChar const* message) noexcept override {
if (severity <= nvinfer1::ILogger::Severity::kERROR) {
record(message);
}
if (static_cast<int32_t>(severity) <= verbosity_ && message != nullptr) {
std::fprintf(stderr, "[trt-runner] %s\n", message);
}
}
private:
int32_t verbosity_;
};
struct RuntimeHolder {
Logger logger;
nvinfer1::IRuntime* runtime{nullptr};
explicit RuntimeHolder(int32_t verbosity) : logger(verbosity) {}
};
constexpr int32_t kUnsupportedDtype = -1;
int32_t dtype_code(nvinfer1::DataType type) {
using nvinfer1::DataType;
switch (type) {
case DataType::kFLOAT: return 0;
case DataType::kHALF: return 1;
case DataType::kINT8: return 2;
case DataType::kINT32: return 3;
case DataType::kBOOL: return 4;
case DataType::kUINT8: return 5;
case DataType::kFP8: return 6;
case DataType::kBF16: return 7;
case DataType::kINT64: return 8;
case DataType::kINT4: return 9;
default: return kUnsupportedDtype;
}
}
int32_t cuda_ok(cudaError_t error, const char* what) {
if (error == cudaSuccess) {
return 0;
}
record(what);
g_last_error.append(": ").append(cudaGetErrorString(error));
return static_cast<int32_t>(error);
}
}
extern "C" {
const char* trt_last_error(void) { return g_last_error.c_str(); }
const char* trt_cuda_error_string(int code) { return cudaGetErrorString(static_cast<cudaError_t>(code)); }
int32_t trt_version_major(void) { return NV_TENSORRT_MAJOR; }
int32_t trt_version_minor(void) { return NV_TENSORRT_MINOR; }
int32_t trt_version_patch(void) { return NV_TENSORRT_PATCH; }
void* trt_runtime_create(int32_t verbosity) {
clear_error();
auto* holder = new (std::nothrow) RuntimeHolder(verbosity);
if (holder == nullptr) {
record("out of memory creating the runtime holder");
return nullptr;
}
holder->runtime = nvinfer1::createInferRuntime(holder->logger);
if (holder->runtime == nullptr) {
record("createInferRuntime failed; is libnvinfer the same version as the headers?");
delete holder;
return nullptr;
}
return holder;
}
void trt_runtime_destroy(void* runtime) {
auto* holder = static_cast<RuntimeHolder*>(runtime);
if (holder == nullptr) {
return;
}
delete holder->runtime; delete holder;
}
void* trt_runtime_deserialize(void* runtime, const void* data, size_t size) {
clear_error();
auto* holder = static_cast<RuntimeHolder*>(runtime);
if (holder == nullptr || holder->runtime == nullptr) {
record("deserialize called with a null runtime");
return nullptr;
}
nvinfer1::ICudaEngine* engine = holder->runtime->deserializeCudaEngine(data, size);
if (engine == nullptr) {
record("deserializeCudaEngine failed: engines are tied to the GPU, the driver and the exact "
"TensorRT build that produced them");
}
return engine;
}
void trt_engine_destroy(void* engine) { delete static_cast<nvinfer1::ICudaEngine*>(engine); }
int32_t trt_engine_num_io(void* engine) {
return static_cast<nvinfer1::ICudaEngine*>(engine)->getNbIOTensors();
}
const char* trt_engine_io_name(void* engine, int32_t index) {
auto* impl = static_cast<nvinfer1::ICudaEngine*>(engine);
if (index < 0 || index >= impl->getNbIOTensors()) {
record("io tensor index out of range");
return nullptr;
}
return impl->getIOTensorName(index);
}
int32_t trt_engine_io_mode(void* engine, const char* name) {
switch (static_cast<nvinfer1::ICudaEngine*>(engine)->getTensorIOMode(name)) {
case nvinfer1::TensorIOMode::kNONE: return 0;
case nvinfer1::TensorIOMode::kINPUT: return 1;
case nvinfer1::TensorIOMode::kOUTPUT: return 2;
default: return -1;
}
}
int32_t trt_engine_io_dtype(void* engine, const char* name) {
int32_t code = dtype_code(static_cast<nvinfer1::ICudaEngine*>(engine)->getTensorDataType(name));
if (code == kUnsupportedDtype) {
record("tensor has a data type this crate cannot size; open an issue with the engine layout");
}
return code;
}
int32_t trt_engine_io_dims(void* engine, const char* name, int64_t* out, int32_t capacity) {
nvinfer1::Dims dims = static_cast<nvinfer1::ICudaEngine*>(engine)->getTensorShape(name);
if (dims.nbDims < 0) {
record("tensor has no shape");
return -1;
}
if (out == nullptr || capacity < dims.nbDims) {
record("dimension buffer is too small");
return -1;
}
std::copy_n(dims.d, dims.nbDims, out);
return dims.nbDims;
}
void* trt_context_create(void* engine) {
clear_error();
nvinfer1::IExecutionContext* context = static_cast<nvinfer1::ICudaEngine*>(engine)->createExecutionContext();
if (context == nullptr) {
record("createExecutionContext failed");
}
return context;
}
void trt_context_destroy(void* context) { delete static_cast<nvinfer1::IExecutionContext*>(context); }
int32_t trt_context_set_profile(void* context, int32_t index, void* stream) {
clear_error();
auto* impl = static_cast<nvinfer1::IExecutionContext*>(context);
return impl->setOptimizationProfileAsync(index, static_cast<cudaStream_t>(stream)) ? 0 : -1;
}
int32_t trt_context_set_shape(void* context, const char* name, const int64_t* dims, int32_t count) {
clear_error();
nvinfer1::Dims shape{};
shape.nbDims = count;
std::copy_n(dims, std::min(count, nvinfer1::Dims::MAX_DIMS), shape.d);
auto* impl = static_cast<nvinfer1::IExecutionContext*>(context);
return impl->setInputShape(name, shape) ? 0 : -1;
}
int32_t trt_context_bind(void* context, const char* name, const void* address) {
clear_error();
auto* impl = static_cast<nvinfer1::IExecutionContext*>(context);
return impl->setTensorAddress(name, const_cast<void*>(address)) ? 0 : -1;
}
int32_t trt_context_enqueue(void* context, void* stream) {
clear_error();
auto* impl = static_cast<nvinfer1::IExecutionContext*>(context);
return impl->enqueueV3(static_cast<cudaStream_t>(stream)) ? 0 : -1;
}
int32_t trt_cuda_device_count(int32_t* out) {
int count = 0;
int32_t code = cuda_ok(cudaGetDeviceCount(&count), "cudaGetDeviceCount");
if (code == 0) {
*out = count;
}
return code;
}
int32_t trt_cuda_set_device(int32_t device) {
return cuda_ok(cudaSetDevice(device), "cudaSetDevice");
}
int32_t trt_cuda_malloc(void** out, size_t bytes) { return cuda_ok(cudaMalloc(out, bytes), "cudaMalloc"); }
int32_t trt_cuda_free(void* ptr) { return cuda_ok(cudaFree(ptr), "cudaFree"); }
int32_t trt_cuda_host_malloc(void** out, size_t bytes) {
return cuda_ok(cudaHostAlloc(out, bytes, cudaHostAllocDefault), "cudaHostAlloc");
}
int32_t trt_cuda_host_free(void* ptr) { return cuda_ok(cudaFreeHost(ptr), "cudaFreeHost"); }
int32_t trt_cuda_stream_create(void** out, int32_t non_blocking) {
cudaError_t error = non_blocking != 0
? cudaStreamCreateWithFlags(reinterpret_cast<cudaStream_t*>(out), cudaStreamNonBlocking)
: cudaStreamCreate(reinterpret_cast<cudaStream_t*>(out));
return cuda_ok(error, "cudaStreamCreate");
}
int32_t trt_cuda_stream_destroy(void* stream) {
return cuda_ok(cudaStreamDestroy(static_cast<cudaStream_t>(stream)), "cudaStreamDestroy");
}
int32_t trt_cuda_stream_sync(void* stream) {
return cuda_ok(cudaStreamSynchronize(static_cast<cudaStream_t>(stream)), "cudaStreamSynchronize");
}
int32_t trt_cuda_copy_h2d(void* device, const void* host, size_t bytes, void* stream) {
return cuda_ok(cudaMemcpyAsync(device, host, bytes, cudaMemcpyHostToDevice, static_cast<cudaStream_t>(stream)),
"cudaMemcpyAsync host to device");
}
int32_t trt_cuda_copy_d2h(void* host, const void* device, size_t bytes, void* stream) {
return cuda_ok(cudaMemcpyAsync(host, device, bytes, cudaMemcpyDeviceToHost, static_cast<cudaStream_t>(stream)),
"cudaMemcpyAsync device to host");
}
int32_t trt_cuda_zero(void* device, size_t bytes, void* stream) {
return cuda_ok(cudaMemsetAsync(device, 0, bytes, static_cast<cudaStream_t>(stream)), "cudaMemsetAsync");
}
}