triton_client::inference::grpc_inference_service_client

Struct GrpcInferenceServiceClient

Source 
pub struct GrpcInferenceServiceClient<T> { /* private fields */ }
Expand description

@@ @@.. cpp:var:: service InferenceService @@ @@ Inference Server GRPC endpoints. @@

Implementations§

Source§

impl<T> GrpcInferenceServiceClient<T>
where T: GrpcService<BoxBody>, T::Error: Into<StdError>, T::ResponseBody: Body<Data = Bytes> + Send + 'static, <T::ResponseBody as Body>::Error: Into<StdError> + Send,

Source

pub fn new(inner: T) -> Self

Source

pub fn with_interceptor<F>( inner: T, interceptor: F, ) -> GrpcInferenceServiceClient<InterceptedService<T, F>>
where F: Interceptor, T::ResponseBody: Default, T: Service<Request<BoxBody>, Response = Response<<T as GrpcService<BoxBody>>::ResponseBody>>, <T as Service<Request<BoxBody>>>::Error: Into<StdError> + Send + Sync,

Source

pub fn send_gzip(self) -> Self

Compress requests with gzip.

This requires the server to support it otherwise it might respond with an error.

Source

pub fn accept_gzip(self) -> Self

Enable decompressing responses with gzip.

Source

pub async fn server_live( &mut self, request: impl IntoRequest<ServerLiveRequest>, ) -> Result<Response<ServerLiveResponse>, Status>

@@ .. cpp:var:: rpc ServerLive(ServerLiveRequest) returns @@ (ServerLiveResponse) @@ @@ Check liveness of the inference server. @@

Source

pub async fn server_ready( &mut self, request: impl IntoRequest<ServerReadyRequest>, ) -> Result<Response<ServerReadyResponse>, Status>

@@ .. cpp:var:: rpc ServerReady(ServerReadyRequest) returns @@ (ServerReadyResponse) @@ @@ Check readiness of the inference server. @@

Source

pub async fn model_ready( &mut self, request: impl IntoRequest<ModelReadyRequest>, ) -> Result<Response<ModelReadyResponse>, Status>

@@ .. cpp:var:: rpc ModelReady(ModelReadyRequest) returns @@ (ModelReadyResponse) @@ @@ Check readiness of a model in the inference server. @@

Source

pub async fn server_metadata( &mut self, request: impl IntoRequest<ServerMetadataRequest>, ) -> Result<Response<ServerMetadataResponse>, Status>

@@ .. cpp:var:: rpc ServerMetadata(ServerMetadataRequest) returns @@ (ServerMetadataResponse) @@ @@ Get server metadata. @@

Source

pub async fn model_metadata( &mut self, request: impl IntoRequest<ModelMetadataRequest>, ) -> Result<Response<ModelMetadataResponse>, Status>

@@ .. cpp:var:: rpc ModelMetadata(ModelMetadataRequest) returns @@ (ModelMetadataResponse) @@ @@ Get model metadata. @@

Source

pub async fn model_infer( &mut self, request: impl IntoRequest<ModelInferRequest>, ) -> Result<Response<ModelInferResponse>, Status>

@@ .. cpp:var:: rpc ModelInfer(ModelInferRequest) returns @@ (ModelInferResponse) @@ @@ Perform inference using a specific model. @@

Source

pub async fn model_stream_infer( &mut self, request: impl IntoStreamingRequest<Message = ModelInferRequest>, ) -> Result<Response<Streaming<ModelStreamInferResponse>>, Status>

@@ .. cpp:var:: rpc ModelStreamInfer(stream ModelInferRequest) returns @@ (stream ModelStreamInferResponse) @@ @@ Perform streaming inference. @@

Source

pub async fn model_config( &mut self, request: impl IntoRequest<ModelConfigRequest>, ) -> Result<Response<ModelConfigResponse>, Status>

@@ .. cpp:var:: rpc ModelConfig(ModelConfigRequest) returns @@ (ModelConfigResponse) @@ @@ Get model configuration. @@

Source

pub async fn model_statistics( &mut self, request: impl IntoRequest<ModelStatisticsRequest>, ) -> Result<Response<ModelStatisticsResponse>, Status>

@@ .. cpp:var:: rpc ModelStatistics( @@ ModelStatisticsRequest) @@ returns (ModelStatisticsResponse) @@ @@ Get the cumulative inference statistics for a model. @@

Source

pub async fn repository_index( &mut self, request: impl IntoRequest<RepositoryIndexRequest>, ) -> Result<Response<RepositoryIndexResponse>, Status>

@@ .. cpp:var:: rpc RepositoryIndex(RepositoryIndexRequest) returns @@ (RepositoryIndexResponse) @@ @@ Get the index of model repository contents. @@

Source

pub async fn repository_model_load( &mut self, request: impl IntoRequest<RepositoryModelLoadRequest>, ) -> Result<Response<RepositoryModelLoadResponse>, Status>

@@ .. cpp:var:: rpc RepositoryModelLoad(RepositoryModelLoadRequest) returns @@ (RepositoryModelLoadResponse) @@ @@ Load or reload a model from a repository. @@

Source

pub async fn repository_model_unload( &mut self, request: impl IntoRequest<RepositoryModelUnloadRequest>, ) -> Result<Response<RepositoryModelUnloadResponse>, Status>

@@ .. cpp:var:: rpc RepositoryModelUnload(RepositoryModelUnloadRequest) @@ returns (RepositoryModelUnloadResponse) @@ @@ Unload a model. @@

Source

pub async fn system_shared_memory_status( &mut self, request: impl IntoRequest<SystemSharedMemoryStatusRequest>, ) -> Result<Response<SystemSharedMemoryStatusResponse>, Status>

@@ .. cpp:var:: rpc SystemSharedMemoryStatus( @@ SystemSharedMemoryStatusRequest) @@ returns (SystemSharedMemoryStatusRespose) @@ @@ Get the status of all registered system-shared-memory regions. @@

Source

pub async fn system_shared_memory_register( &mut self, request: impl IntoRequest<SystemSharedMemoryRegisterRequest>, ) -> Result<Response<SystemSharedMemoryRegisterResponse>, Status>

@@ .. cpp:var:: rpc SystemSharedMemoryRegister( @@ SystemSharedMemoryRegisterRequest) @@ returns (SystemSharedMemoryRegisterResponse) @@ @@ Register a system-shared-memory region. @@

Source

pub async fn system_shared_memory_unregister( &mut self, request: impl IntoRequest<SystemSharedMemoryUnregisterRequest>, ) -> Result<Response<SystemSharedMemoryUnregisterResponse>, Status>

@@ .. cpp:var:: rpc SystemSharedMemoryUnregister( @@ SystemSharedMemoryUnregisterRequest) @@ returns (SystemSharedMemoryUnregisterResponse) @@ @@ Unregister a system-shared-memory region. @@

Source

pub async fn cuda_shared_memory_status( &mut self, request: impl IntoRequest<CudaSharedMemoryStatusRequest>, ) -> Result<Response<CudaSharedMemoryStatusResponse>, Status>

@@ .. cpp:var:: rpc CudaSharedMemoryStatus( @@ CudaSharedMemoryStatusRequest) @@ returns (CudaSharedMemoryStatusRespose) @@ @@ Get the status of all registered CUDA-shared-memory regions. @@

Source

pub async fn cuda_shared_memory_register( &mut self, request: impl IntoRequest<CudaSharedMemoryRegisterRequest>, ) -> Result<Response<CudaSharedMemoryRegisterResponse>, Status>

@@ .. cpp:var:: rpc CudaSharedMemoryRegister( @@ CudaSharedMemoryRegisterRequest) @@ returns (CudaSharedMemoryRegisterResponse) @@ @@ Register a CUDA-shared-memory region. @@

Source

pub async fn cuda_shared_memory_unregister( &mut self, request: impl IntoRequest<CudaSharedMemoryUnregisterRequest>, ) -> Result<Response<CudaSharedMemoryUnregisterResponse>, Status>

@@ .. cpp:var:: rpc CudaSharedMemoryUnregister( @@ CudaSharedMemoryUnregisterRequest) @@ returns (CudaSharedMemoryUnregisterResponse) @@ @@ Unregister a CUDA-shared-memory region. @@

Source

pub async fn trace_setting( &mut self, request: impl IntoRequest<TraceSettingRequest>, ) -> Result<Response<TraceSettingResponse>, Status>

@@ .. cpp:var:: rpc TraceSetting(TraceSettingRequest) @@ returns (TraceSettingResponse) @@ @@ Update and get the trace setting of the Triton server. @@

Trait Implementations§

Source§

impl<T: Clone> Clone for GrpcInferenceServiceClient<T>

Source§

fn clone(&self) -> GrpcInferenceServiceClient<T>

Returns a duplicate of the value. Read more
1.0.0 · Source§

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source. Read more
Source§

impl<T: Debug> Debug for GrpcInferenceServiceClient<T>

Source§

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter. Read more

Auto Trait Implementations§

Blanket Implementations§

Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
Source§

impl<T> CloneToUninit for T
where T: Clone,

Source§

unsafe fn clone_to_uninit(&self, dest: *mut u8)

🔬This is a nightly-only experimental API. (clone_to_uninit)
Performs copy-assignment from self to dest. Read more
Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

Source§

impl<T> Instrument for T

Source§

fn instrument(self, span: Span) -> Instrumented<Self>

Instruments this type with the provided Span, returning an Instrumented wrapper. Read more
Source§

fn in_current_span(self) -> Instrumented<Self>

Instruments this type with the current Span, returning an Instrumented wrapper. Read more
Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T> IntoRequest<T> for T

Source§

fn into_request(self) -> Request<T>

Wrap the input message T in a tonic::Request
Source§

impl<T> ToOwned for T
where T: Clone,

Source§

type Owned = T

The resulting type after obtaining ownership.
Source§

fn to_owned(&self) -> T

Creates owned data from borrowed data, usually by cloning. Read more
Source§

fn clone_into(&self, target: &mut T)

Uses borrowed data to replace owned data, usually by cloning. Read more
Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = Infallible

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.
Source§

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

Source§

fn vzip(self) -> V

Source§

impl<T> WithSubscriber for T

Source§

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>
where S: Into<Dispatch>,

Attaches the provided Subscriber to this type, returning a WithDispatch wrapper. Read more
Source§

fn with_current_subscriber(self) -> WithDispatch<Self>

Attaches the current default Subscriber to this type, returning a WithDispatch wrapper. Read more