llm 1.3.8

A Rust library unifying multiple LLM backends.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

use std::time::Instant;

#[derive(Debug, Clone)]
pub enum AppStatus {
    Idle,
    Thinking,
    Streaming,
    Error(String),
}

impl AppStatus {
    pub fn is_busy(&self) -> bool {
        matches!(self, AppStatus::Thinking | AppStatus::Streaming)
    }
}

#[derive(Debug, Clone, Default)]
pub struct StatusMetrics {
    started_at: Option<Instant>,
    tokens: Option<u32>,
    last_reported_sec: Option<u64>,
}

impl StatusMetrics {
    pub fn start(&mut self) {
        if self.started_at.is_none() {
            self.started_at = Some(Instant::now());
        }
        self.tokens = None;
        self.last_reported_sec = None;
    }

    pub fn stop(&mut self) {
        self.started_at = None;
        self.tokens = None;
        self.last_reported_sec = None;
    }

    pub fn update_tokens(&mut self, tokens: u32) {
        self.tokens = Some(tokens);
    }

    pub fn elapsed_ms(&self) -> Option<u128> {
        self.started_at.map(|at| at.elapsed().as_millis())
    }

    pub fn tokens(&self) -> Option<u32> {
        self.tokens
    }

    pub fn should_redraw(&mut self) -> bool {
        let Some(started_at) = self.started_at else {
            return false;
        };
        let secs = started_at.elapsed().as_secs();
        if self.last_reported_sec == Some(secs) {
            return false;
        }
        self.last_reported_sec = Some(secs);
        true
    }
}