Struct ModelSequenceBatching 

pub struct ModelSequenceBatching {
    pub max_sequence_idle_microseconds: u64,
    pub control_input: Vec<ControlInput>,
    pub state: Vec<State>,
    pub iterative_sequence: bool,
    pub strategy_choice: Option<StrategyChoice>,
}

@@ @@.. cpp:var:: message ModelSequenceBatching @@ @@ Sequence batching configuration. These settings control how sequence @@ batching operates for the model. @@

Fields

max_sequence_idle_microseconds: u64

@@ .. cpp:var:: uint64 max_sequence_idle_microseconds @@ @@ The maximum time, in microseconds, that a sequence is allowed to @@ be idle before it is aborted. The inference server considers a @@ sequence idle when it does not have any inference request queued @@ for the sequence. If this limit is exceeded, the inference server @@ will free the sequence slot allocated by the sequence and make it @@ available for another sequence. If not specified (or specified as @@ zero) a default value of 1000000 (1 second) is used. @@

control_input: Vec<ControlInput>

@@ .. cpp:var:: ControlInput control_input (repeated) @@ @@ The model input(s) that the server should use to communicate @@ sequence start, stop, ready and similar control values to the @@ model. @@

state: Vec<State>

@@ .. cpp:var:: State state (repeated) @@ @@ The optional state that can be stored in Triton for performing @@ inference requests on a sequence. Each sequence holds an implicit @@ state local to itself. The output state tensor provided by the @@ model in ‘output_name’ field of the current inference request will @@ be transferred as an input tensor named ‘input_name’ in the next @@ request of the same sequence. The input state of the first request @@ in the sequence contains garbage data. @@

iterative_sequence: bool

@@ .. cpp:var:: bool iterative_sequence @@ @@ Requests for iterative sequences are processed over a number @@ of iterations. An iterative sequence is initiated by a single @@ request and is “rescheduled” by the model until completion. @@ Requests for inflight requests will be batched together @@ and can complete independently. Note this feature @@ requires backend support. Default value is false.

strategy_choice: Option<StrategyChoice>

@@ .. cpp:var:: oneof strategy_choice @@ @@ The strategy used by the sequence batcher. Default strategy @@ is ‘direct’. @@

Trait Implementations

impl Clone for ModelSequenceBatching

fn clone(&self) -> ModelSequenceBatching

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for ModelSequenceBatching

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for ModelSequenceBatching

fn default() -> Self

Returns the “default value” for a type.

impl Message for ModelSequenceBatching

fn encoded_len(&self) -> usize

Returns the encoded length of the message without a length delimiter.

fn clear(&mut self)

Clears the message, resetting all fields to their default.

fn encode(&self, buf: &mut impl BufMut) -> Result<(), EncodeError>

where Self: Sized,

Encodes the message to a buffer.

fn encode_to_vec(&self) -> Vec<u8>

where Self: Sized,

Encodes the message to a newly allocated buffer.

fn encode_length_delimited( &self, buf: &mut impl BufMut, ) -> Result<(), EncodeError>

where Self: Sized,

Encodes the message with a length-delimiter to a buffer.

fn encode_length_delimited_to_vec(&self) -> Vec<u8>

where Self: Sized,

Encodes the message with a length-delimiter to a newly allocated buffer.

fn decode(buf: impl Buf) -> Result<Self, DecodeError>

where Self: Default,

Decodes an instance of the message from a buffer.

fn decode_length_delimited(buf: impl Buf) -> Result<Self, DecodeError>

where Self: Default,

Decodes a length-delimited instance of the message from the buffer.

fn merge(&mut self, buf: impl Buf) -> Result<(), DecodeError>

where Self: Sized,

Decodes an instance of the message from a buffer, and merges it into self.

fn merge_length_delimited(&mut self, buf: impl Buf) -> Result<(), DecodeError>

where Self: Sized,

Decodes a length-delimited instance of the message from buffer, and merges it into self.

impl PartialEq for ModelSequenceBatching

fn eq(&self, other: &ModelSequenceBatching) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl StructuralPartialEq for ModelSequenceBatching