trt-runner 0.1.0

Minimal TensorRT 10 inference runner: deserialize an engine, bind I/O tensors, enqueue V3. No bindings generator, no runtime dependencies.
Documentation
#include "shim.h"

#include <NvInfer.h>
#include <cuda_runtime_api.h>

#include <algorithm>
#include <cstdio>
#include <cstring>
#include <string>

#if NV_TENSORRT_MAJOR < 10
#error "trt-runner requires TensorRT 10 or newer (enqueueV3 and the tensor-name I/O API)"
#endif

namespace {

thread_local std::string g_last_error;

void record(const char* message) {
    if (message != nullptr) {
        g_last_error.assign(message);
    }
}

void clear_error() { g_last_error.clear(); }

/// Forwards TensorRT diagnostics to the caller. Errors are always kept so a failed enqueueV3 can be
/// explained; the rest is printed only when the verbosity asked for it.
class Logger : public nvinfer1::ILogger {
public:
    explicit Logger(int32_t verbosity) : verbosity_(verbosity) {}

    void log(nvinfer1::ILogger::Severity severity, nvinfer1::AsciiChar const* message) noexcept override {
        if (severity <= nvinfer1::ILogger::Severity::kERROR) {
            record(message);
        }
        if (static_cast<int32_t>(severity) <= verbosity_ && message != nullptr) {
            std::fprintf(stderr, "[trt-runner] %s\n", message);
        }
    }

private:
    int32_t verbosity_;
};

/// The logger has to stay alive for every object created through the runtime.
struct RuntimeHolder {
    Logger logger;
    nvinfer1::IRuntime* runtime{nullptr};
    explicit RuntimeHolder(int32_t verbosity) : logger(verbosity) {}
};

constexpr int32_t kUnsupportedDtype = -1;

int32_t dtype_code(nvinfer1::DataType type) {
    using nvinfer1::DataType;
    switch (type) {
    case DataType::kFLOAT: return 0;
    case DataType::kHALF: return 1;
    case DataType::kINT8: return 2;
    case DataType::kINT32: return 3;
    case DataType::kBOOL: return 4;
    case DataType::kUINT8: return 5;
    case DataType::kFP8: return 6;
    case DataType::kBF16: return 7;
    case DataType::kINT64: return 8;
    case DataType::kINT4: return 9;
    default: return kUnsupportedDtype;
    }
}

int32_t cuda_ok(cudaError_t error, const char* what) {
    if (error == cudaSuccess) {
        return 0;
    }
    record(what);
    g_last_error.append(": ").append(cudaGetErrorString(error));
    return static_cast<int32_t>(error);
}

} // namespace

extern "C" {

const char* trt_last_error(void) { return g_last_error.c_str(); }

const char* trt_cuda_error_string(int code) { return cudaGetErrorString(static_cast<cudaError_t>(code)); }

int32_t trt_version_major(void) { return NV_TENSORRT_MAJOR; }
int32_t trt_version_minor(void) { return NV_TENSORRT_MINOR; }
int32_t trt_version_patch(void) { return NV_TENSORRT_PATCH; }

void* trt_runtime_create(int32_t verbosity) {
    clear_error();
    auto* holder = new (std::nothrow) RuntimeHolder(verbosity);
    if (holder == nullptr) {
        record("out of memory creating the runtime holder");
        return nullptr;
    }
    holder->runtime = nvinfer1::createInferRuntime(holder->logger);
    if (holder->runtime == nullptr) {
        record("createInferRuntime failed; is libnvinfer the same version as the headers?");
        delete holder;
        return nullptr;
    }
    return holder;
}

void trt_runtime_destroy(void* runtime) {
    auto* holder = static_cast<RuntimeHolder*>(runtime);
    if (holder == nullptr) {
        return;
    }
    delete holder->runtime; // TensorRT 10 replaced destroy() with a public virtual destructor
    delete holder;
}

void* trt_runtime_deserialize(void* runtime, const void* data, size_t size) {
    clear_error();
    auto* holder = static_cast<RuntimeHolder*>(runtime);
    if (holder == nullptr || holder->runtime == nullptr) {
        record("deserialize called with a null runtime");
        return nullptr;
    }
    nvinfer1::ICudaEngine* engine = holder->runtime->deserializeCudaEngine(data, size);
    if (engine == nullptr) {
        record("deserializeCudaEngine failed: engines are tied to the GPU, the driver and the exact "
               "TensorRT build that produced them");
    }
    return engine;
}

void trt_engine_destroy(void* engine) { delete static_cast<nvinfer1::ICudaEngine*>(engine); }

int32_t trt_engine_num_io(void* engine) {
    return static_cast<nvinfer1::ICudaEngine*>(engine)->getNbIOTensors();
}

const char* trt_engine_io_name(void* engine, int32_t index) {
    auto* impl = static_cast<nvinfer1::ICudaEngine*>(engine);
    if (index < 0 || index >= impl->getNbIOTensors()) {
        record("io tensor index out of range");
        return nullptr;
    }
    return impl->getIOTensorName(index);
}

int32_t trt_engine_io_mode(void* engine, const char* name) {
    switch (static_cast<nvinfer1::ICudaEngine*>(engine)->getTensorIOMode(name)) {
    case nvinfer1::TensorIOMode::kNONE: return 0;
    case nvinfer1::TensorIOMode::kINPUT: return 1;
    case nvinfer1::TensorIOMode::kOUTPUT: return 2;
    default: return -1;
    }
}

int32_t trt_engine_io_dtype(void* engine, const char* name) {
    int32_t code = dtype_code(static_cast<nvinfer1::ICudaEngine*>(engine)->getTensorDataType(name));
    if (code == kUnsupportedDtype) {
        record("tensor has a data type this crate cannot size; open an issue with the engine layout");
    }
    return code;
}

int32_t trt_engine_io_dims(void* engine, const char* name, int64_t* out, int32_t capacity) {
    nvinfer1::Dims dims = static_cast<nvinfer1::ICudaEngine*>(engine)->getTensorShape(name);
    if (dims.nbDims < 0) {
        record("tensor has no shape");
        return -1;
    }
    if (out == nullptr || capacity < dims.nbDims) {
        record("dimension buffer is too small");
        return -1;
    }
    std::copy_n(dims.d, dims.nbDims, out);
    return dims.nbDims;
}

void* trt_context_create(void* engine) {
    clear_error();
    nvinfer1::IExecutionContext* context = static_cast<nvinfer1::ICudaEngine*>(engine)->createExecutionContext();
    if (context == nullptr) {
        record("createExecutionContext failed");
    }
    return context;
}

void trt_context_destroy(void* context) { delete static_cast<nvinfer1::IExecutionContext*>(context); }

int32_t trt_context_set_profile(void* context, int32_t index, void* stream) {
    clear_error();
    auto* impl = static_cast<nvinfer1::IExecutionContext*>(context);
    return impl->setOptimizationProfileAsync(index, static_cast<cudaStream_t>(stream)) ? 0 : -1;
}

int32_t trt_context_set_shape(void* context, const char* name, const int64_t* dims, int32_t count) {
    clear_error();
    nvinfer1::Dims shape{};
    shape.nbDims = count;
    std::copy_n(dims, std::min(count, nvinfer1::Dims::MAX_DIMS), shape.d);
    auto* impl = static_cast<nvinfer1::IExecutionContext*>(context);
    return impl->setInputShape(name, shape) ? 0 : -1;
}

int32_t trt_context_bind(void* context, const char* name, const void* address) {
    clear_error();
    auto* impl = static_cast<nvinfer1::IExecutionContext*>(context);
    return impl->setTensorAddress(name, const_cast<void*>(address)) ? 0 : -1;
}

int32_t trt_context_enqueue(void* context, void* stream) {
    clear_error();
    auto* impl = static_cast<nvinfer1::IExecutionContext*>(context);
    return impl->enqueueV3(static_cast<cudaStream_t>(stream)) ? 0 : -1;
}

int32_t trt_cuda_device_count(int32_t* out) {
    int count = 0;
    int32_t code = cuda_ok(cudaGetDeviceCount(&count), "cudaGetDeviceCount");
    if (code == 0) {
        *out = count;
    }
    return code;
}

int32_t trt_cuda_set_device(int32_t device) {
    /* cudaSetDevice makes that device's primary context current on the calling thread, which is what
     * every other call here assumes. Forgetting it is the usual cause of invalid-context errors. */
    return cuda_ok(cudaSetDevice(device), "cudaSetDevice");
}

int32_t trt_cuda_malloc(void** out, size_t bytes) { return cuda_ok(cudaMalloc(out, bytes), "cudaMalloc"); }

int32_t trt_cuda_free(void* ptr) { return cuda_ok(cudaFree(ptr), "cudaFree"); }

int32_t trt_cuda_host_malloc(void** out, size_t bytes) {
    return cuda_ok(cudaHostAlloc(out, bytes, cudaHostAllocDefault), "cudaHostAlloc");
}

int32_t trt_cuda_host_free(void* ptr) { return cuda_ok(cudaFreeHost(ptr), "cudaFreeHost"); }

int32_t trt_cuda_stream_create(void** out, int32_t non_blocking) {
    cudaError_t error = non_blocking != 0
        ? cudaStreamCreateWithFlags(reinterpret_cast<cudaStream_t*>(out), cudaStreamNonBlocking)
        : cudaStreamCreate(reinterpret_cast<cudaStream_t*>(out));
    return cuda_ok(error, "cudaStreamCreate");
}

int32_t trt_cuda_stream_destroy(void* stream) {
    return cuda_ok(cudaStreamDestroy(static_cast<cudaStream_t>(stream)), "cudaStreamDestroy");
}

int32_t trt_cuda_stream_sync(void* stream) {
    return cuda_ok(cudaStreamSynchronize(static_cast<cudaStream_t>(stream)), "cudaStreamSynchronize");
}

int32_t trt_cuda_copy_h2d(void* device, const void* host, size_t bytes, void* stream) {
    return cuda_ok(cudaMemcpyAsync(device, host, bytes, cudaMemcpyHostToDevice, static_cast<cudaStream_t>(stream)),
                   "cudaMemcpyAsync host to device");
}

int32_t trt_cuda_copy_d2h(void* host, const void* device, size_t bytes, void* stream) {
    return cuda_ok(cudaMemcpyAsync(host, device, bytes, cudaMemcpyDeviceToHost, static_cast<cudaStream_t>(stream)),
                   "cudaMemcpyAsync device to host");
}

int32_t trt_cuda_zero(void* device, size_t bytes, void* stream) {
    return cuda_ok(cudaMemsetAsync(device, 0, bytes, static_cast<cudaStream_t>(stream)), "cudaMemsetAsync");
}

} // extern "C"