Struct ModelDynamicBatching 

pub struct ModelDynamicBatching {
    pub preferred_batch_size: Vec<i32>,
    pub max_queue_delay_microseconds: u64,
    pub preserve_ordering: bool,
    pub priority_levels: u64,
    pub default_priority_level: u64,
    pub default_queue_policy: Option<ModelQueuePolicy>,
    pub priority_queue_policy: HashMap<u64, ModelQueuePolicy>,
}

@@ @@.. cpp:var:: message ModelDynamicBatching @@ @@ Dynamic batching configuration. These settings control how dynamic @@ batching operates for the model. @@

Fields

preferred_batch_size: Vec<i32>

@@ .. cpp:var:: int32 preferred_batch_size (repeated) @@ @@ Preferred batch sizes for dynamic batching. If a batch of one of @@ these sizes can be formed it will be executed immediately. If @@ not specified a preferred batch size will be chosen automatically @@ based on model and GPU characteristics. @@

max_queue_delay_microseconds: u64

@@ .. cpp:var:: uint64 max_queue_delay_microseconds @@ @@ The maximum time, in microseconds, a request will be delayed in @@ the scheduling queue to wait for additional requests for @@ batching. Default is 0. @@

preserve_ordering: bool

@@ .. cpp:var:: bool preserve_ordering @@ @@ Should the dynamic batcher preserve the ordering of responses to @@ match the order of requests received by the scheduler. Default is @@ false. If true, the responses will be returned in the same order as @@ the order of requests sent to the scheduler. If false, the responses @@ may be returned in arbitrary order. This option is specifically @@ needed when a sequence of related inference requests (i.e. inference @@ requests with the same correlation ID) are sent to the dynamic @@ batcher to ensure that the sequence responses are in the correct @@ order. @@

priority_levels: u64

@@ .. cpp:var:: uint64 priority_levels @@ @@ The number of priority levels to be enabled for the model, @@ the priority level starts from 1 and 1 is the highest priority. @@ Requests are handled in priority order with all priority 1 requests @@ processed before priority 2, all priority 2 requests processed before @@ priority 3, etc. Requests with the same priority level will be @@ handled in the order that they are received. @@

default_priority_level: u64

@@ .. cpp:var:: uint64 default_priority_level @@ @@ The priority level used for requests that don’t specify their @@ priority. The value must be in the range [ 1, ‘priority_levels’ ]. @@

default_queue_policy: Option<ModelQueuePolicy>

@@ .. cpp:var:: ModelQueuePolicy default_queue_policy @@ @@ The default queue policy used for requests that don’t require @@ priority handling and requests that specify priority levels where @@ there is no specific policy given. If not specified, a policy with @@ default field values will be used. @@

priority_queue_policy: HashMap<u64, ModelQueuePolicy>

@@ .. cpp:var:: map<uint64, ModelQueuePolicy> priority_queue_policy @@ @@ Specify the queue policy for the priority level. The default queue @@ policy will be used if a priority level doesn’t specify a queue @@ policy. @@

Trait Implementations

impl Clone for ModelDynamicBatching

fn clone(&self) -> ModelDynamicBatching

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for ModelDynamicBatching

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for ModelDynamicBatching

fn default() -> Self

Returns the “default value” for a type.

impl Message for ModelDynamicBatching

fn encoded_len(&self) -> usize

Returns the encoded length of the message without a length delimiter.

fn clear(&mut self)

Clears the message, resetting all fields to their default.

fn encode(&self, buf: &mut impl BufMut) -> Result<(), EncodeError>

where Self: Sized,

Encodes the message to a buffer.

fn encode_to_vec(&self) -> Vec<u8>

where Self: Sized,

Encodes the message to a newly allocated buffer.

fn encode_length_delimited( &self, buf: &mut impl BufMut, ) -> Result<(), EncodeError>

where Self: Sized,

Encodes the message with a length-delimiter to a buffer.

fn encode_length_delimited_to_vec(&self) -> Vec<u8>

where Self: Sized,

Encodes the message with a length-delimiter to a newly allocated buffer.

fn decode(buf: impl Buf) -> Result<Self, DecodeError>

where Self: Default,

Decodes an instance of the message from a buffer.

fn decode_length_delimited(buf: impl Buf) -> Result<Self, DecodeError>

where Self: Default,

Decodes a length-delimited instance of the message from the buffer.

fn merge(&mut self, buf: impl Buf) -> Result<(), DecodeError>

where Self: Sized,

Decodes an instance of the message from a buffer, and merges it into self.

fn merge_length_delimited(&mut self, buf: impl Buf) -> Result<(), DecodeError>

where Self: Sized,

Decodes a length-delimited instance of the message from buffer, and merges it into self.

impl PartialEq for ModelDynamicBatching

fn eq(&self, other: &ModelDynamicBatching) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl StructuralPartialEq for ModelDynamicBatching