whisper-nvidia-cli 0.1.0

One-shot agent-native local NVIDIA GPU speech-to-text CLI powered by whisper.cpp
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
//! NVIDIA device and CUDA toolkit discovery. No product env is read.
//!
//! Free memory is `Option<u64>` on purpose. A plain `0` cannot tell "the card
//! has nothing free" from "the probe could not measure", and a resource gate
//! that reads the unknown case as unlimited is the permissive default.

use std::path::PathBuf;
use std::process::Command;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, OnceLock};

use crate::error::Error;

/// Snapshot used by doctor and the VRAM gate.
#[derive(Debug, Clone)]
pub struct GpuInfo {
    /// True when at least one NVIDIA device answered the probe.
    pub present: bool,
    /// Marketing name of the first device.
    pub name: String,
    /// Installed NVIDIA driver version.
    pub driver: String,
    /// Free device memory, or `None` when the probe could not measure it.
    pub free_bytes: Option<u64>,
    /// Total device memory, or `None` when the probe could not measure it.
    pub total_bytes: Option<u64>,
    /// Number of devices the probe saw.
    pub count: u32,
    /// True when a CUDA toolkit was found on the host.
    pub toolkit: bool,
    /// CUDA compiler version, or `None` when `nvcc` is absent.
    pub nvcc: Option<String>,
    /// Compute capability of the first device, or `None` when unmeasured.
    pub cc: Option<String>,
}

/// Raw result of one `nvidia-smi` query.
struct SmiReading {
    present: bool,
    name: String,
    driver: String,
    free_bytes: Option<u64>,
    total_bytes: Option<u64>,
    count: u32,
}

impl SmiReading {
    fn absent() -> Self {
        Self {
            present: false,
            name: String::new(),
            driver: String::new(),
            free_bytes: None,
            total_bytes: None,
            count: 0,
        }
    }
}

/// Probe the host, or report the interrupt that cut the probe short.
///
/// An interrupted probe returns the same "absent" reading as a host with no
/// card, and the caller then reports 70 for a run the operator cancelled. That
/// is the same confusion as answering "model missing" for a model that is
/// present but unreadable: two different causes collapsed into one answer, and
/// the wrong one is the one that gets acted on.
///
/// Every caller of [`probe`] wants this, so the check lives here rather than
/// being repeated — and remembered — at each call site.
pub fn probe_checked(device: u32) -> Result<GpuInfo, Error> {
    let info = probe(device);
    if let Some(err) = crate::signal::shutdown_error() {
        return Err(err);
    }
    Ok(info)
}

/// Probe the host. `nvidia-smi` and `nvcc` are OS tools, not product env.
pub fn probe(device: u32) -> GpuInfo {
    let toolkit = which::which("nvcc").ok();
    let cc = which::which("cc")
        .ok()
        .or_else(|| which::which("cl").ok())
        .or_else(|| which::which("gcc").ok());
    let smi = query_smi(device);
    GpuInfo {
        present: smi.present,
        name: smi.name,
        driver: smi.driver,
        free_bytes: smi.free_bytes,
        total_bytes: smi.total_bytes,
        count: smi.count,
        toolkit: toolkit.is_some(),
        nvcc: toolkit.map(|p| p.display().to_string()),
        cc: cc.map(|p| p.display().to_string()),
    }
}

/// The compiled-in poll interval, in milliseconds.
const DEFAULT_PROBE_POLL_MS: u64 = 25;

/// How often a probe is checked for exit and for a pending interrupt, in
/// milliseconds.
///
/// Published like the memory cap, and for the same reason: the value belongs to
/// the invocation, while the loop that spends it sits two private call frames
/// below the store that holds it. The initial value is the same 25 ms the
/// literal used, so a host that never publishes anything polls as it always did.
static PROBE_POLL_MS: AtomicU64 = AtomicU64::new(DEFAULT_PROBE_POLL_MS);

/// Publish the operator's probe poll interval, in milliseconds.
///
/// Zero keeps the default rather than disabling the sleep: a poll of zero is a
/// busy loop, and this knob exists to lower the cost of the wait, not to raise
/// it to a full core.
pub fn publish_probe_poll_ms(ms: u64) {
    PROBE_POLL_MS.store(ms, Ordering::SeqCst);
}

/// The interval the probe loop sleeps for between checks.
fn probe_poll() -> std::time::Duration {
    let ms = PROBE_POLL_MS.load(Ordering::SeqCst);
    std::time::Duration::from_millis(if ms == 0 { DEFAULT_PROBE_POLL_MS } else { ms })
}

/// The operator's `nvidia-smi` path. Empty, or unpublished, means search PATH.
static SMI_PATH: OnceLock<String> = OnceLock::new();

/// Publish the operator's `nvidia-smi` path.
///
/// Empty means "find it", a non-empty value means "use this one" — the same
/// contract the helper path already has. A driver installed under its own
/// prefix is invisible to a PATH search, and `doctor` then reports a card the
/// host really has as absent, with no way for the operator to say otherwise.
///
/// The first publish of an invocation wins, and it happens before any probe.
pub fn publish_nvidia_smi_path(path: String) {
    let _ = SMI_PATH.set(path);
}

/// Run a host probe so that an interrupt can still end the run.
///
/// `Command::output()` blocks in a way no signal handler reaches. That is
/// tolerable for a probe that answers in milliseconds and intolerable for one
/// that hangs, which is what a wedged NVIDIA driver produces — the exact
/// condition the probe exists to report on. Measured with a hanging stand-in:
/// the first interrupt did nothing at all, and the probe's own child outlived
/// the process.
///
/// Polling instead of blocking means the cooperative shutdown reaches here, and
/// spawning through [`crate::signal::spawn_tracked`] means a forced exit kills
/// the probe together with anything it started.
fn run_probe(mut cmd: Command) -> std::io::Result<std::process::Output> {
    let (id, child) = crate::signal::spawn_tracked(&mut cmd)?;
    loop {
        let finished = {
            let mut guard = child.lock().unwrap_or_else(|e| e.into_inner());
            guard.try_wait()?.is_some()
        };
        if finished {
            break;
        }
        if crate::signal::is_shutdown_requested() || crate::signal::deadline_exceeded() {
            let mut guard = child.lock().unwrap_or_else(|e| e.into_inner());
            crate::signal::kill_child_tree(&mut guard);
            crate::signal::release_child(id);
            return Err(std::io::Error::other("probe interrupted"));
        }
        std::thread::sleep(probe_poll());
    }
    crate::signal::release_child(id);
    // The handle is uniquely held again now that tracking released its clone.
    let child = Arc::try_unwrap(child)
        .map_err(|_| std::io::Error::other("probe handle still shared"))?
        .into_inner()
        .unwrap_or_else(|e| e.into_inner());
    child.wait_with_output()
}

fn query_smi(device: u32) -> SmiReading {
    // A configured path is used exactly as given. Pointing at a driver the
    // PATH cannot see is the entire reason the key exists, so second-guessing
    // the operator here would give the key nothing left to do.
    let configured = SMI_PATH
        .get()
        .map(String::as_str)
        .unwrap_or_default()
        .trim();
    let bin = if configured.is_empty() {
        let Ok(found) = which::which("nvidia-smi") else {
            return SmiReading::absent();
        };
        found
    } else {
        PathBuf::from(configured)
    };
    let mut cmd = Command::new(&bin);
    cmd.args([
        "--query-gpu=name,driver_version,memory.free,memory.total",
        "--format=csv,noheader,nounits",
    ])
    .stdout(std::process::Stdio::piped())
    .stderr(std::process::Stdio::null());
    let Ok(out) = run_probe(cmd) else {
        return SmiReading::absent();
    };
    if !out.status.success() {
        return SmiReading::absent();
    }
    let text = String::from_utf8_lossy(&out.stdout);
    let lines: Vec<&str> = text.lines().filter(|l| !l.trim().is_empty()).collect();
    let count = lines.len() as u32;
    if count == 0 {
        return SmiReading::absent();
    }
    // A requested index past the end is reported by `count`; the caller
    // rejects it. Clamping here only keeps the probe from panicking.
    let idx = (device as usize).min(lines.len() - 1);
    let parts: Vec<&str> = lines[idx].split(',').map(str::trim).collect();
    if parts.len() < 4 {
        return SmiReading {
            present: true,
            name: lines[idx].to_string(),
            driver: String::new(),
            free_bytes: None,
            total_bytes: None,
            count,
        };
    }
    SmiReading {
        present: true,
        name: parts[0].to_string(),
        driver: parts[1].to_string(),
        free_bytes: mib_to_bytes(parts[2]),
        total_bytes: mib_to_bytes(parts[3]),
        count,
    }
}

fn mib_to_bytes(raw: &str) -> Option<u64> {
    raw.parse::<u64>().ok().map(|mib| mib * 1024 * 1024)
}

/// Refuse a model that cannot fit `concurrency` copies in free VRAM.
///
/// Unmeasurable free memory fails closed: the alternative is loading a model
/// that may not fit and killing the card mid-batch.
pub fn ensure_vram(info: &GpuInfo, need_bytes: u64, concurrency: usize) -> Result<(), Error> {
    if !info.present {
        return Err(Error::GpuMissing);
    }
    let Some(free) = info.free_bytes else {
        return Err(Error::VramUnknown);
    };
    let need = need_bytes.saturating_mul(concurrency.max(1) as u64);
    if need > free {
        return Err(Error::VramInsufficient {
            need_bytes: need,
            have_bytes: free,
        });
    }
    Ok(())
}

/// Suggest the largest catalog model that fits free VRAM.
///
/// Returns `None` when free memory is unknown: suggesting the largest model
/// on an unmeasured card is a guess dressed as advice.
pub fn suggest_model(
    free_bytes: Option<u64>,
    catalog: &[crate::catalog::ModelEntry],
) -> Option<&str> {
    let free = free_bytes?;
    catalog
        .iter()
        .rev()
        .find(|m| m.vram_bytes <= free)
        .map(|m| m.name.as_str())
}

/// Reject a device index the host does not have.
pub fn ensure_device(info: &GpuInfo, device: u32) -> Result<(), Error> {
    if !info.present {
        return Err(Error::GpuMissing);
    }
    if device >= info.count {
        return Err(Error::UnknownDevice(device.to_string()));
    }
    Ok(())
}

/// Resident set size of this process, or `None` where it cannot be measured.
///
/// Field two of `/proc/self/statm` is the resident page count on Linux, and
/// `GetProcessMemoryInfo` reports the working set on Windows. Windows used to
/// fall into the `None` arm, which meant `--max-memory` was accepted there,
/// documented there, and protected nothing there — a guarantee that does not
/// hold and does not announce itself is worse than one that is simply absent.
///
/// A platform with neither still returns `None`, and the caller treats that the
/// same way it treats unknown VRAM: never as "no memory is being used". Use
/// [`memory_cap_is_measurable`] to tell the two situations apart.
pub fn process_rss_bytes() -> Option<u64> {
    #[cfg(target_os = "linux")]
    {
        let raw = std::fs::read_to_string("/proc/self/statm").ok()?;
        let pages: u64 = raw.split_whitespace().nth(1)?.parse().ok()?;
        // SAFETY: `sysconf` reads a static system value and returns -1 on
        // failure, which the check below rejects before any multiplication.
        let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) };
        if page <= 0 {
            return None;
        }
        Some(pages.saturating_mul(page as u64))
    }
    #[cfg(windows)]
    {
        use windows_sys::Win32::System::ProcessStatus::{
            GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS,
        };
        use windows_sys::Win32::System::Threading::GetCurrentProcess;

        let mut counters = PROCESS_MEMORY_COUNTERS {
            cb: std::mem::size_of::<PROCESS_MEMORY_COUNTERS>() as u32,
            ..unsafe { std::mem::zeroed() }
        };
        // SAFETY: `GetCurrentProcess` returns a pseudo-handle that needs no
        // close, and `counters` is a live, correctly sized struct whose `cb`
        // field declares that size to the callee, which is what the API
        // requires. A zero return means failure and is rejected below.
        let ok = unsafe {
            GetProcessMemoryInfo(
                GetCurrentProcess(),
                &raw mut counters,
                std::mem::size_of::<PROCESS_MEMORY_COUNTERS>() as u32,
            )
        };
        if ok == 0 {
            return None;
        }
        Some(counters.WorkingSetSize as u64)
    }
    #[cfg(not(any(target_os = "linux", windows)))]
    {
        None
    }
}

/// Whether this build can measure its own resident memory at all.
///
/// The cap degrades to "satisfied" when the reading is unavailable, so an agent
/// that relies on `--max-memory` has to be able to ask whether the number
/// behind it exists. `doctor` publishes this so the answer is readable from the
/// envelope instead of guessed from the platform.
pub fn memory_cap_is_measurable() -> bool {
    cfg!(any(target_os = "linux", windows))
}

/// Where the resident-memory reading comes from on this build.
///
/// Stable for a given binary, which is what lets it sit in the `doctor`
/// envelope: FR-023 asks that the same argv produce the same stdout, so a
/// diagnostic may report a capability but never a live measurement.
pub fn memory_cap_source() -> &'static str {
    if cfg!(target_os = "linux") {
        "procfs"
    } else if cfg!(windows) {
        "GetProcessMemoryInfo"
    } else {
        "none"
    }
}

/// Refuse to continue once this process is over the operator's memory cap.
///
/// The cap used to be compared against the catalog size of the model, which is
/// a prediction made before anything was allocated. A cap the operator sets to
/// protect a shared machine has to be measured against what the process is
/// actually holding, or a long recording blows past it untouched.
///
/// A cap of zero disables the check. An unmeasurable RSS is reported as
/// satisfied rather than as a failure: refusing every run on a platform with
/// no measurement would make the flag unusable there.
pub fn ensure_process_memory(max_bytes: u64) -> Result<(), Error> {
    if max_bytes == 0 {
        return Ok(());
    }
    let Some(rss) = process_rss_bytes() else {
        return Ok(());
    };
    if rss > max_bytes {
        return Err(Error::MemoryCap);
    }
    Ok(())
}

/// The invocation's process memory cap in bytes. Zero means no cap.
///
/// Published like the deadline, and for the same reason: the cap belongs to the
/// invocation rather than to any one stage, and the stage that actually spends
/// the memory is a decode loop five call frames below the flag that set it.
static MEMORY_CAP: AtomicU64 = AtomicU64::new(0);

/// Publish the cap every stage can consult.
pub fn publish_memory_cap(bytes: u64) {
    MEMORY_CAP.store(bytes, Ordering::SeqCst);
}

/// True once this process is holding more than the operator allowed.
///
/// Checked at the same polling points as the deadline. A cap enforced only at
/// stage boundaries is a post-mortem, not a guard: measured on a twenty minute
/// wav under a 50 MiB cap, the process peaked at 123 MiB — 2.46 times the cap —
/// and refused only after the whole allocation had already landed on the host.
/// The flag exists so the host survives, so the check has to run while the
/// memory is still being taken.
///
/// An unmeasurable RSS reads as satisfied, matching [`ensure_process_memory`]:
/// refusing every run on a platform with no measurement makes the flag unusable
/// there.
pub fn memory_cap_exceeded() -> bool {
    let cap = MEMORY_CAP.load(Ordering::SeqCst);
    if cap == 0 {
        return false;
    }
    process_rss_bytes().is_some_and(|rss| rss > cap)
}

/// Refuse an allocation the cap cannot afford, before it is taken.
///
/// Polling covers loops. It cannot cover a `collect` or a `clone` that takes
/// its memory in one step: by the time the next check runs, the host has
/// already paid. Those sites know their own cost in advance — a sample count
/// times four bytes — so the honest guard is to ask whether the cap can afford
/// it rather than to notice afterwards that it could not.
///
/// This is the general form of a defect found seven times in this codebase: a
/// capability applied at whichever points someone thought of, so every new
/// allocation is born outside it. Declaring the cost at the allocation makes
/// the guard travel with the thing it guards.
///
/// A cap of zero disables the check, and an unmeasurable RSS reads as
/// satisfied, matching [`ensure_process_memory`] and [`memory_cap_exceeded`].
pub fn ensure_headroom(bytes: u64) -> Result<(), Error> {
    let cap = MEMORY_CAP.load(Ordering::SeqCst);
    if cap == 0 {
        return Ok(());
    }
    let Some(rss) = process_rss_bytes() else {
        return Ok(());
    };
    if rss.saturating_add(bytes) > cap {
        return Err(Error::MemoryCap);
    }
    Ok(())
}

/// Drop the CUDA visibility variable, and say so on stderr.
///
/// FR-023 says the product "ignora a variável de sistema e avisa no stderr".
/// Warning alone did only half of that: the variable stayed in the process
/// environment, and the CUDA runtime inside the C engine reads it to RENUMBER
/// the devices. With `CUDA_VISIBLE_DEVICES=1` set, device 0 chosen from the
/// persisted config would have addressed the host's second card while every
/// envelope kept reporting `device: 0` — the config appeared to win, and the
/// environment actually did. The requirement's success condition is "o device
/// persistido é o usado", which cannot hold while the variable survives.
///
/// Removing it here, once, before any probe or engine call, is what makes the
/// warning true. The invocation is single-threaded at this point, which is the
/// condition `remove_var` cares about.
pub fn warn_ignored_cuda_env(ui: crate::i18n::UiLang) {
    if std::env::var_os("CUDA_VISIBLE_DEVICES").is_some() {
        std::env::remove_var("CUDA_VISIBLE_DEVICES");
        eprintln!("{}", crate::i18n::tr(ui, "warn_cuda_env"));
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::catalog::Catalog;

    fn info(present: bool, free: Option<u64>, count: u32) -> GpuInfo {
        GpuInfo {
            present,
            name: "test".into(),
            driver: "0".into(),
            free_bytes: free,
            total_bytes: free,
            count,
            toolkit: true,
            nvcc: None,
            cc: None,
        }
    }

    #[test]
    fn suggest_picks_largest_that_fits() {
        let c = Catalog::seeds("https://example.invalid/x");
        let name = suggest_model(Some(1300 * 1024 * 1024), &c.models).expect("fits");
        assert_eq!(name, "small");
    }

    #[test]
    fn unknown_free_does_not_suggest_largest() {
        let c = Catalog::seeds("https://example.invalid/x");
        assert_eq!(suggest_model(None, &c.models), None);
    }

    #[test]
    fn unknown_free_does_not_skip_gate() {
        let err = ensure_vram(&info(true, None, 1), 1024, 1).expect_err("must refuse");
        assert!(matches!(err, Error::VramUnknown));
        assert_eq!(err.exit_code(), 70);
    }

    #[test]
    fn tiny_card_refuses_large_batch() {
        let one_gib = 1024 * 1024 * 1024;
        assert!(ensure_vram(&info(true, Some(one_gib), 1), one_gib / 2, 1).is_ok());
        let err = ensure_vram(&info(true, Some(one_gib), 1), one_gib / 2, 8)
            .expect_err("batch must not fit");
        assert_eq!(err.exit_code(), 70);
    }

    #[test]
    fn missing_card_is_gpu_missing() {
        assert!(matches!(
            ensure_vram(&info(false, None, 0), 1, 1),
            Err(Error::GpuMissing)
        ));
    }

    #[test]
    fn device_past_the_end_is_unknown_device() {
        assert!(ensure_device(&info(true, Some(1), 1), 0).is_ok());
        let err = ensure_device(&info(true, Some(1), 1), 3).expect_err("must refuse");
        assert!(matches!(err, Error::UnknownDevice(_)));
        assert_eq!(err.exit_code(), 78);
    }
}