llmtop 0.1.0

Realtime TUI monitor for local LLM servers (ollama, llama.cpp). The only GPU monitor that knows what model is running and how much each token costs you in energy and dollar-equivalent.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133

mod app;
mod collectors;
mod config;
mod pricing;
mod proxy;
mod ui;

use clap::Parser;
use color_eyre::eyre::Result;
use crossterm::{
    event::{DisableMouseCapture, EnableMouseCapture},
    execute,
    terminal::{EnterAlternateScreen, LeaveAlternateScreen, disable_raw_mode, enable_raw_mode},
};
use ratatui::{Terminal, prelude::CrosstermBackend};
use std::{io, time::Duration};
use tokio::sync::mpsc;

use crate::{app::App, config::Cli};

pub enum AppEvent {
    Input(crossterm::event::Event),
    Tick,
    Hardware(collectors::HardwareSnapshot),
    Ollama(Vec<collectors::ModelInfo>),
}

#[tokio::main]
async fn main() -> Result<()> {
    color_eyre::install()?;
    let cli = Cli::parse();

    enable_raw_mode()?;
    let mut stdout = io::stdout();
    execute!(stdout, EnterAlternateScreen, EnableMouseCapture)?;
    let backend = CrosstermBackend::new(stdout);
    let mut terminal = Terminal::new(backend)?;

    let result = run(&mut terminal, cli).await;

    disable_raw_mode()?;
    execute!(
        terminal.backend_mut(),
        LeaveAlternateScreen,
        DisableMouseCapture
    )?;
    terminal.show_cursor()?;

    result
}

async fn run(terminal: &mut Terminal<CrosstermBackend<io::Stdout>>, cli: Cli) -> Result<()> {
    let (tx, mut rx) = mpsc::unbounded_channel::<AppEvent>();
    let mut app = App::new(cli);

    // Input task — blocking crossterm reads in spawn_blocking.
    let input_tx = tx.clone();
    tokio::task::spawn_blocking(move || {
        loop {
            if crossterm::event::poll(Duration::from_millis(200)).unwrap_or(false)
                && let Ok(ev) = crossterm::event::read()
                && input_tx.send(AppEvent::Input(ev)).is_err()
            {
                break;
            }
        }
    });

    // Tick — redraw heartbeat.
    let tick_tx = tx.clone();
    tokio::spawn(async move {
        let mut interval = tokio::time::interval(Duration::from_millis(250));
        loop {
            interval.tick().await;
            if tick_tx.send(AppEvent::Tick).is_err() {
                break;
            }
        }
    });

    // Hardware poll (Day-1 stub).
    let hw_tx = tx.clone();
    tokio::spawn(async move {
        let mut interval = tokio::time::interval(Duration::from_millis(500));
        loop {
            interval.tick().await;
            let snap = collectors::poll_hardware().await;
            if hw_tx.send(AppEvent::Hardware(snap)).is_err() {
                break;
            }
        }
    });

    // Optional proxy: when --proxy is set, intercept generate/chat for tok/s.
    let proxy_port = app.cli.proxy_port();
    let sink = proxy_port.map(|_| proxy::new_sink());
    if let (Some(port), Some(s)) = (proxy_port, sink.clone()) {
        let upstream = app.cli.ollama_url.clone();
        tokio::spawn(async move {
            let _ = proxy::run(port, upstream, s).await;
        });
    }

    // Ollama poll.
    let ollama_url = app.cli.ollama_url.clone();
    let sink_for_poll = sink.clone();
    tokio::spawn(async move {
        let mut interval = tokio::time::interval(Duration::from_secs(1));
        loop {
            interval.tick().await;
            let models = collectors::poll_ollama(&ollama_url, sink_for_poll.as_ref()).await;
            if tx.send(AppEvent::Ollama(models)).is_err() {
                break;
            }
        }
    });

    terminal.draw(|f| ui::draw(f, &app))?;
    while let Some(ev) = rx.recv().await {
        match ev {
            AppEvent::Input(input) => {
                if app.handle_input(input) {
                    break;
                }
            }
            AppEvent::Tick => {}
            AppEvent::Hardware(snap) => app.update_hardware(snap),
            AppEvent::Ollama(models) => app.update_ollama(models),
        }
        terminal.draw(|f| ui::draw(f, &app))?;
    }
    Ok(())
}