use super::{InferenceRequest, InferenceResponse, LocalInferenceClient, ModelInfo};
use crate::error::{IronError, Result};
use async_trait::async_trait;
#[cfg(feature = "remote-backends")]
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone)]
pub struct TensorRtClient {
triton_url: String,
#[cfg(feature = "remote-backends")]
client: reqwest::Client,
}
impl TensorRtClient {
pub fn new(triton_url: &str) -> Result<Self> {
#[cfg(feature = "remote-backends")]
{
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(60))
.build()
.map_err(|e| IronError::Network(e.to_string()))?;
Ok(Self {
triton_url: triton_url.to_string(),
client,
})
}
#[cfg(not(feature = "remote-backends"))]
{
Err(IronError::NotImplemented(
"TensorRT-LLM client requires 'remote-backends' feature".to_string(),
))
}
}
#[allow(clippy::should_implement_trait)]
pub fn default() -> Result<Self> {
Self::new("localhost:8001")
}
pub fn check_gpu_support() -> bool {
#[cfg(feature = "nvidia")]
{
use crate::gpu::GpuCollection;
if let Ok(gpus) = GpuCollection::auto_detect() {
return !gpus.is_empty();
}
}
false
}
}
#[async_trait]
impl LocalInferenceClient for TensorRtClient {
fn name(&self) -> &str {
"TensorRT-LLM"
}
async fn is_available(&self) -> bool {
#[cfg(feature = "remote-backends")]
{
let url = format!("http://{}/v2/health/ready", self.triton_url);
let healthy = self
.client
.get(&url)
.send()
.await
.is_ok_and(|response| response.status().is_success());
healthy && Self::check_gpu_support()
}
#[cfg(not(feature = "remote-backends"))]
false
}
async fn list_models(&self) -> Result<Vec<ModelInfo>> {
#[cfg(feature = "remote-backends")]
{
let url = format!("http://{}/v2/models", self.triton_url);
let response = self
.client
.get(&url)
.send()
.await
.map_err(|e| IronError::Network(e.to_string()))?;
if !response.status().is_success() {
return Err(IronError::Agent(
"Failed to list models from Triton".to_string(),
));
}
let text = response
.text()
.await
.map_err(|e| IronError::Agent(e.to_string()))?;
if let Ok(repo) = serde_json::from_str::<TritonModelRepository>(&text) {
return Ok(repo
.models
.into_iter()
.map(|m| ModelInfo {
name: m.name,
size: None,
family: None,
parameter_count: None,
quantization: None,
})
.collect());
}
Err(IronError::Agent(
"Triton model repository response did not match the expected shape".to_string(),
))
}
#[cfg(not(feature = "remote-backends"))]
Err(IronError::NotImplemented(
"TensorRT-LLM client requires 'remote-backends' feature".to_string(),
))
}
async fn generate(&self, request: InferenceRequest) -> Result<InferenceResponse> {
#[cfg(feature = "remote-backends")]
{
use std::time::Instant;
let start = Instant::now();
let triton_request = TritonInferRequest {
inputs: vec![
TritonInput {
name: "text_input".to_string(),
shape: vec![1, 1],
datatype: "BYTES".to_string(),
data: vec![request.prompt.clone()],
},
TritonInput {
name: "max_tokens".to_string(),
shape: vec![1, 1],
datatype: "INT32".to_string(),
data: vec![request.max_tokens.unwrap_or(256).to_string()],
},
],
outputs: vec![TritonOutput {
name: "text_output".to_string(),
}],
};
let url = format!(
"http://{}/v2/models/{}/infer",
self.triton_url, request.model
);
let response = self
.client
.post(&url)
.json(&triton_request)
.send()
.await
.map_err(|e| IronError::Network(e.to_string()))?;
if !response.status().is_success() {
let status = response.status();
let body = response.text().await.unwrap_or_default();
return Err(IronError::Agent(format!(
"Triton inference failed ({}): {}",
status, body
)));
}
let triton_response: TritonInferResponse = response
.json()
.await
.map_err(|e| IronError::Agent(format!("Failed to parse Triton response: {}", e)))?;
let text = triton_response
.outputs
.iter()
.find(|o| o.name == "text_output")
.and_then(|o| o.data.first().cloned())
.ok_or_else(|| {
IronError::Agent("Triton response has no text_output".to_string())
})?;
Ok(InferenceResponse {
text,
model: request.model,
tokens_generated: None, duration_ms: start.elapsed().as_millis() as u64,
truncated: None,
})
}
#[cfg(not(feature = "remote-backends"))]
Err(IronError::NotImplemented(
"TensorRT-LLM client requires 'remote-backends' feature".to_string(),
))
}
async fn model_info(&self, model_name: &str) -> Result<ModelInfo> {
self.list_models()
.await?
.into_iter()
.find(|m| m.name == model_name)
.ok_or_else(|| {
IronError::Agent(format!("Triton is not serving a model named {model_name}"))
})
}
}
#[cfg(feature = "remote-backends")]
#[derive(Debug, Serialize)]
struct TritonInferRequest {
inputs: Vec<TritonInput>,
outputs: Vec<TritonOutput>,
}
#[cfg(feature = "remote-backends")]
#[derive(Debug, Serialize)]
struct TritonInput {
name: String,
shape: Vec<usize>,
datatype: String,
data: Vec<String>,
}
#[cfg(feature = "remote-backends")]
#[derive(Debug, Serialize)]
struct TritonOutput {
name: String,
}
#[cfg(feature = "remote-backends")]
#[derive(Debug, Deserialize)]
struct TritonInferResponse {
outputs: Vec<TritonOutputData>,
}
#[cfg(feature = "remote-backends")]
#[derive(Debug, Deserialize)]
struct TritonOutputData {
name: String,
data: Vec<String>,
}
#[cfg(feature = "remote-backends")]
#[derive(Debug, Deserialize)]
struct TritonModelRepository {
models: Vec<TritonModel>,
}
#[cfg(feature = "remote-backends")]
#[derive(Debug, Deserialize)]
struct TritonModel {
name: String,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_tensorrt_client_creation() {
let result = TensorRtClient::new("localhost:8001");
#[cfg(feature = "remote-backends")]
assert!(result.is_ok());
#[cfg(not(feature = "remote-backends"))]
assert!(result.is_err());
}
#[test]
fn test_gpu_support_check() {
let has_gpu = TensorRtClient::check_gpu_support();
println!("NVIDIA GPU support: {}", has_gpu);
}
}