inference 0.3.0

A crate for managing the machine learning inference process
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

//! Triton gRPC requests related to the server's health and readiness to serve a model

use tonic::{Request, Status};
use tracing::{event, instrument, Level};

use crate::TritonClient;

use crate::inference::{ServerLiveRequest, ServerReadyRequest};
use crate::inference::{ServerMetadataRequest, ServerMetadataResponse};

impl TritonClient {
    /// Is an inference server live at the provided client?
    ///  
    /// RPC ServerLive
    #[instrument]
    pub async fn is_server_live(&self) -> Result<bool, Status> {
        let request = Request::new(ServerLiveRequest {});
        match self.client().server_live(request).await {
            Ok(r) => {
                // if you get a valid response, server can still be live or dead
                let server_liveliness = r.get_ref().live;
                event!(Level::INFO, server_live = ?r);
                Ok(server_liveliness)
            }
            Err(s) => {
                // if you don't get a valid response, the server isn't live by default
                event!(Level::ERROR, "failed to query server liveliness, {s}");
                Err(s)
            }
        }
    }

    /// Is an inference server ready at the provided client?
    ///
    /// RPC ServerReady
    #[instrument]
    pub async fn is_server_ready(&self) -> Result<bool, Status> {
        let request = Request::new(ServerReadyRequest {});
        match self.client().server_ready(request).await {
            Ok(r) => {
                let readiness = r.get_ref().ready;
                event!(Level::INFO, server_ready = ?r);
                Ok(readiness)
            }
            Err(s) => {
                event!(Level::ERROR, "failed to query server readiness, {s}");
                Err(s)
            }
        }
    }

    /// Get the server metadata for the provided client
    ///
    /// RPC ServerMetadata
    #[instrument]
    pub async fn server_metadata(&self) -> Result<ServerMetadataResponse, Status> {
        let request = Request::new(ServerMetadataRequest {});
        match self.client().server_metadata(request).await {
            Ok(r) => {
                event!(Level::INFO, server_metadata = ?r);
                Ok(r.into_inner())
            }
            Err(s) => {
                event!(Level::ERROR, "failed to get server metadata, {s}");
                Err(s)
            }
        }
    }
}