Struct InferenceRequest 

pub struct InferenceRequest {
    pub id: RequestId,
    pub prompt: String,
    pub model_id: ModelId,
    pub sampling_params: SamplingParams,
    pub stream: bool,
    pub priority: Priority,
    pub client_id: Option<ClientId>,
    pub session_id: Option<SessionId>,
    pub created_at: DateTime<Utc>,
    pub metadata: HashMap<String, Value>,
}

Inference request

Fields

id: RequestId

Unique request identifier

prompt: String

Input prompt text

model_id: ModelId

Model to use for inference

sampling_params: SamplingParams

Sampling parameters

stream: bool

Whether to stream response

priority: Priority

Request priority

client_id: Option<ClientId>

Client identifier

session_id: Option<SessionId>

Session identifier for stateful interactions

created_at: DateTime<Utc>

Request creation timestamp

metadata: HashMap<String, Value>

Additional metadata

Implementations

impl InferenceRequest

pub fn new( prompt: impl Into<String>, model_id: impl Into<ModelId>, ) -> InferenceRequest

Create a new inference request

pub fn with_sampling_params(self, params: SamplingParams) -> InferenceRequest

Set sampling parameters

pub fn with_stream(self, stream: bool) -> InferenceRequest

Enable streaming

pub fn with_priority(self, priority: Priority) -> InferenceRequest

Set priority

pub fn with_client_id(self, client_id: impl Into<ClientId>) -> InferenceRequest

Set client ID

pub fn with_session_id(self, session_id: SessionId) -> InferenceRequest

Set session ID

pub fn with_metadata( self, key: impl Into<String>, value: Value, ) -> InferenceRequest

Add metadata

Trait Implementations

impl Clone for InferenceRequest

fn clone(&self) -> InferenceRequest

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for InferenceRequest

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for InferenceRequest

fn deserialize<__D>( __deserializer: __D, ) -> Result<InferenceRequest, <__D as Deserializer<'de>>::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Serialize for InferenceRequest

fn serialize<__S>( &self, __serializer: __S, ) -> Result<<__S as Serializer>::Ok, <__S as Serializer>::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.