Struct ExecutorCapabilities 

pub struct ExecutorCapabilities {
    pub max_batch_size: usize,
    pub max_sequence_length: usize,
    pub attention_mechanisms: Vec<AttentionType>,
    pub supports_dynamic_batching: bool,
    pub supports_continuous_batching: bool,
    pub supports_speculative_decoding: bool,
    pub supports_tensor_parallelism: bool,
    pub supports_pipeline_parallelism: bool,
    pub supported_dtypes: Vec<DataType>,
    pub supported_devices: Vec<Device>,
    pub memory_requirements: MemoryRequirements,
}

Executor capabilities and configuration

Fields

max_batch_size: usize

Maximum supported batch size

max_sequence_length: usize

Maximum sequence length

attention_mechanisms: Vec<AttentionType>

Supported attention mechanisms

supports_dynamic_batching: bool

Whether executor supports dynamic batching

supports_continuous_batching: bool

Whether executor supports continuous batching

supports_speculative_decoding: bool

Whether executor supports speculative decoding

supports_tensor_parallelism: bool

Whether executor supports tensor parallelism

supports_pipeline_parallelism: bool

Whether executor supports pipeline parallelism

supported_dtypes: Vec<DataType>

Supported data types

supported_devices: Vec<Device>

Supported devices

memory_requirements: MemoryRequirements

Memory requirements estimation

Trait Implementations

impl Clone for ExecutorCapabilities

fn clone(&self) -> ExecutorCapabilities

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for ExecutorCapabilities

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for ExecutorCapabilities

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Serialize for ExecutorCapabilities

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.