Struct ModelInstanceGroup

pub struct ModelInstanceGroup {
    pub name: String,
    pub kind: i32,
    pub count: i32,
    pub rate_limiter: Option<ModelRateLimiter>,
    pub gpus: Vec<i32>,
    pub secondary_devices: Vec<SecondaryDevice>,
    pub profile: Vec<String>,
    pub passive: bool,
    pub host_policy: String,
}

@@ @@.. cpp:var:: message ModelInstanceGroup @@ @@ A group of one or more instances of a model and resources made @@ available for those instances. @@

Fields

name: String

@@ .. cpp:var:: string name @@ @@ Optional name of this group of instances. If not specified the @@ name will be formed as _. The name of @@ individual instances will be further formed by a unique instance @@ number and GPU index: @@

kind: i32

@@ .. cpp:var:: Kind kind @@ @@ The kind of this instance group. Default is KIND_AUTO. If @@ KIND_AUTO or KIND_GPU then both ‘count’ and ‘gpu’ are valid and @@ may be specified. If KIND_CPU or KIND_MODEL only ‘count’ is valid @@ and ‘gpu’ cannot be specified. @@

count: i32

@@ .. cpp:var:: int32 count @@ @@ For a group assigned to GPU, the number of instances created for @@ each GPU listed in ‘gpus’. For a group assigned to CPU the number @@ of instances created. Default is 1.

rate_limiter: Option<ModelRateLimiter>

@@ .. cpp:var:: ModelRateLimiter rate_limiter @@ @@ The rate limiter specific settings to be associated with this @@ instance group. Optional, if not specified no rate limiting @@ will be applied to this instance group. @@

gpus: Vec<i32>

@@ .. cpp:var:: int32 gpus (repeated) @@ @@ GPU(s) where instances should be available. For each GPU listed, @@ ‘count’ instances of the model will be available. Setting ‘gpus’ @@ to empty (or not specifying at all) is eqivalent to listing all @@ available GPUs. @@

secondary_devices: Vec<SecondaryDevice>

@@ .. cpp:var:: SecondaryDevice secondary_devices (repeated) @@ @@ Secondary devices that are required by instances specified by this @@ instance group. Optional. @@

profile: Vec<String>

@@ .. cpp:var:: string profile (repeated) @@ @@ For TensorRT models containing multiple optimization profile, this @@ parameter specifies a set of optimization profiles available to this @@ instance group. The inference server will choose the optimal profile @@ based on the shapes of the input tensors. This field should lie @@ between 0 and - 1 @@ and be specified only for TensorRT backend, otherwise an error will @@ be generated. If not specified, the server will select the first @@ optimization profile by default. @@

passive: bool

@@ .. cpp:var:: bool passive @@ @@ Whether the instances within this instance group will be accepting @@ inference requests from the scheduler. If true, the instances will @@ not be added to the scheduler. Default value is false. @@

host_policy: String

@@ .. cpp:var:: string host_policy @@ @@ The host policy name that the instance to be associated with. @@ The default value is set to reflect the device kind of the instance, @@ for instance, KIND_CPU is “cpu”, KIND_MODEL is “model” and @@ KIND_GPU is “gpu_<gpu_id>”. @@

Implementations

impl ModelInstanceGroup

pub fn kind(&self) -> Kind

Returns the enum value of kind, or the default if the field is set to an invalid enum value.

pub fn set_kind(&mut self, value: Kind)

Sets kind to the provided enum value.

Trait Implementations

impl Clone for ModelInstanceGroup

fn clone(&self) -> ModelInstanceGroup

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for ModelInstanceGroup

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for ModelInstanceGroup

fn default() -> Self

Returns the “default value” for a type.

impl Message for ModelInstanceGroup

fn encoded_len(&self) -> usize

Returns the encoded length of the message without a length delimiter.

fn clear(&mut self)

Clears the message, resetting all fields to their default.

fn encode<B>(&self, buf: &mut B) -> Result<(), EncodeError>

where B: BufMut, Self: Sized,

Encodes the message to a buffer.

fn encode_to_vec(&self) -> Vec<u8>

where Self: Sized,

Encodes the message to a newly allocated buffer.

fn encode_length_delimited<B>(&self, buf: &mut B) -> Result<(), EncodeError>

where B: BufMut, Self: Sized,

Encodes the message with a length-delimiter to a buffer.

fn encode_length_delimited_to_vec(&self) -> Vec<u8>

where Self: Sized,

Encodes the message with a length-delimiter to a newly allocated buffer.

fn decode<B>(buf: B) -> Result<Self, DecodeError>

where B: Buf, Self: Default,

Decodes an instance of the message from a buffer.

fn decode_length_delimited<B>(buf: B) -> Result<Self, DecodeError>

where B: Buf, Self: Default,

Decodes a length-delimited instance of the message from the buffer.

fn merge<B>(&mut self, buf: B) -> Result<(), DecodeError>

where B: Buf, Self: Sized,

Decodes an instance of the message from a buffer, and merges it into self.

fn merge_length_delimited<B>(&mut self, buf: B) -> Result<(), DecodeError>

where B: Buf, Self: Sized,

Decodes a length-delimited instance of the message from buffer, and merges it into self.

impl PartialEq for ModelInstanceGroup

fn eq(&self, other: &ModelInstanceGroup) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl StructuralPartialEq for ModelInstanceGroup