Struct InferenceRequest 

pub struct InferenceRequest {
    pub id: RequestId,
    pub prompt: String,
    pub model_id: ModelId,
    pub sampling_params: SamplingParams,
    pub stream: bool,
    pub priority: Priority,
    pub client_id: Option<ClientId>,
    pub session_id: Option<SessionId>,
    pub created_at: DateTime<Utc>,
    pub metadata: HashMap<String, Value>,
}

Inference request

Fields

id: RequestId

Unique request identifier

prompt: String

Input prompt text

model_id: ModelId

Model to use for inference

sampling_params: SamplingParams

Sampling parameters

stream: bool

Whether to stream response

priority: Priority

Request priority

client_id: Option<ClientId>

Client identifier

session_id: Option<SessionId>

Session identifier for stateful interactions

created_at: DateTime<Utc>

Request creation timestamp

metadata: HashMap<String, Value>

Additional metadata

Implementations

impl InferenceRequest

pub fn new(prompt: impl Into<String>, model_id: impl Into<ModelId>) -> Self

Create a new inference request

pub fn with_sampling_params(self, params: SamplingParams) -> Self

Set sampling parameters

pub fn with_stream(self, stream: bool) -> Self

Enable streaming

pub fn with_priority(self, priority: Priority) -> Self

Set priority

pub fn with_client_id(self, client_id: impl Into<ClientId>) -> Self

Set client ID

pub fn with_session_id(self, session_id: SessionId) -> Self

Set session ID

pub fn with_metadata(self, key: impl Into<String>, value: Value) -> Self

Add metadata

Trait Implementations

impl Clone for InferenceRequest

fn clone(&self) -> InferenceRequest

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for InferenceRequest

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for InferenceRequest

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Serialize for InferenceRequest

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.