latch-core 0.2.0

Neutral shared types and configuration for the Latch gateway SDK
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152

use serde::{Deserialize, Serialize};
use std::time::SystemTime;

// ── Observation types ──

#[derive(Clone, Debug)]
pub struct RequestObservation {
    pub endpoint_id: String,
    pub pool_id: String,
    pub started_at: SystemTime,
    pub success: bool,
    pub error: Option<ObservationError>,
    pub was_retry: bool,
    pub latency: LatencyBreakdown,
    pub tokens: TokenStats,
    pub stream: Option<StreamMetrics>,
}

#[derive(Clone, Debug)]
pub struct LatencyBreakdown {
    pub total_ms: u64,
    pub ttft_ms: Option<u64>,
}

#[derive(Clone, Debug, Default)]
pub struct TokenStats {
    pub input: u64,
    pub output: u64,
}

#[derive(Clone, Debug)]
pub struct StreamMetrics {
    pub ttft_ms: u64,
    pub tokens_per_second: f64,
    pub max_inter_chunk_ms: u64,
    pub broken: bool,
}

#[derive(Clone, Debug)]
pub enum ObservationError {
    Timeout,
    HttpStatus(u16),
    ConnectionFailed,
    RateLimited,
    StreamBroken,
    EmptyResponse,
    Truncated,
    MalformedResponse,
    Other(String),
}

// ── Scoring config ──

#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ScoreConfig {
    pub window_size: usize,
    pub decay_period_secs: u64,
    pub baseline_score: f32,
    pub availability_weight: f32,
    pub latency_weight: f32,
    pub quality_weight: f32,
    pub cost_weight: f32,
    pub good_ttft_ms: u64,
    pub acceptable_ttft_ms: u64,
    pub good_tps: f64,
    pub max_error_rate: f32,
    pub max_truncation_rate: f32,
    pub max_empty_response_rate: f32,
}

impl Default for ScoreConfig {
    fn default() -> Self {
        Self {
            window_size: 100,
            decay_period_secs: 300,
            baseline_score: 60.0,
            availability_weight: 0.35,
            latency_weight: 0.25,
            quality_weight: 0.25,
            cost_weight: 0.15,
            good_ttft_ms: 500,
            acceptable_ttft_ms: 2000,
            good_tps: 50.0,
            max_error_rate: 0.20,
            max_truncation_rate: 0.10,
            max_empty_response_rate: 0.05,
        }
    }
}

// ── Scoring results ──

#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct EndpointScore {
    pub endpoint_id: String,
    pub pool_id: String,
    pub score: f32,
    pub tier: ScoreTier,
    pub observation_count: usize,
    pub last_updated: SystemTime,
    pub breakdown: ScoreBreakdown,
}

#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub enum ScoreTier {
    Poor,    // < 40
    Bronze,  // 40-69
    Silver,  // 70-89
    Gold,    // >= 90
}

// Note: ScoreTier ordering: Poor < Bronze < Silver < Gold
impl PartialOrd for ScoreTier {
    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
        Some(self.cmp(other))
    }
}

impl Ord for ScoreTier {
    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
        let self_val = match self {
            ScoreTier::Poor => 0,
            ScoreTier::Bronze => 1,
            ScoreTier::Silver => 2,
            ScoreTier::Gold => 3,
        };
        let other_val = match other {
            ScoreTier::Poor => 0,
            ScoreTier::Bronze => 1,
            ScoreTier::Silver => 2,
            ScoreTier::Gold => 3,
        };
        self_val.cmp(&other_val)
    }
}

#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ScoreBreakdown {
    pub availability: f32,
    pub latency: f32,
    pub quality: f32,
    pub cost: f32,
    pub penalty: f32,
}

#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct RankingResult {
    pub ranked_endpoints: Vec<EndpointScore>,
    pub recommended: Option<EndpointScore>,
    pub recommended_fallback: Option<EndpointScore>,
    pub excluded: Vec<EndpointScore>,
}