flyllm 0.4.1

A Rust library for unifying LLM backends as an abstraction layer with load balancing.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

use std::collections::HashMap;
use serde::{Serialize, Deserialize};
use serde_json::{json, Value};

/// Definition of a task that can be routed to specific providers
///
/// Tasks represent specialized capabilities or configurations that
/// certain providers might be better suited for. Each task can have
/// associated parameters that affect how the request is processed.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TaskDefinition {
    pub name: String,
    pub parameters: HashMap<String, serde_json::Value>,
}

impl TaskDefinition {
    /// Creates a new TaskDefinition with the given name.
    pub fn new(name: impl Into<String>) -> Self {
        TaskDefinition {
            name: name.into(),
            parameters: HashMap::new(),
        }
    }

    /// Adds or updates a parameter for this task definition.
    /// Accepts any value that can be converted into a serde_json::Value.
    pub fn with_param(mut self, key: impl Into<String>, value: impl Into<Value>) -> Self {
        self.parameters.insert(key.into(), value.into());
        self
    }

    /// Sets the `max_tokens` parameter to a given value.
    pub fn with_max_tokens(self, tokens: u32) -> Self {
        self.with_param("max_tokens", json!(tokens))
    }

    /// Sets the `temperature` parameter to a given value.
    pub fn with_temperature(self, temp: f32) -> Self {
        self.with_param("temperature", json!(temp))
    }
}