use std::cell::RefCell;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use eframe::egui;
use crate::jobs::{kind, JobHandle, JobSink};
use super::facett_theme::{Theme, AMBER, RED};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Backend {
Candle,
Ollama,
Ort,
}
impl Backend {
pub fn as_str(self) -> &'static str {
match self {
Backend::Candle => "candle",
Backend::Ollama => "ollama",
Backend::Ort => "ort",
}
}
pub fn label(self) -> &'static str {
match self {
Backend::Candle => "Candle",
Backend::Ollama => "Ollama",
Backend::Ort => "ONNX Runtime",
}
}
}
pub struct ModelSpec {
pub id: &'static str,
pub backend: Backend,
pub note: &'static str,
}
pub const CATALOG: &[ModelSpec] = &[
ModelSpec { id: "llama3.2", backend: Backend::Ollama, note: "3B general chat model" },
ModelSpec { id: "qwen2.5-coder:7b", backend: Backend::Ollama, note: "code-specialized model" },
ModelSpec { id: "nomic-embed-text", backend: Backend::Ollama, note: "text embeddings" },
ModelSpec {
id: "jinaai/jina-embeddings-v2-base-code",
backend: Backend::Candle,
note: "code embeddings (HuggingFace)",
},
];
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Device {
Cpu,
Cuda,
Rocm,
}
impl Device {
pub const ALL: [Device; 3] = [Device::Cpu, Device::Cuda, Device::Rocm];
pub fn as_str(self) -> &'static str {
match self {
Device::Cpu => "cpu",
Device::Cuda => "cuda",
Device::Rocm => "rocm",
}
}
pub fn label(self) -> &'static str {
match self {
Device::Cpu => "CPU",
Device::Cuda => "CUDA (NVIDIA)",
Device::Rocm => "ROCm (AMD)",
}
}
}
#[derive(Clone, Debug, PartialEq)]
pub enum DlStatus {
Downloading,
Done,
Failed(String),
Cancelled,
}
impl DlStatus {
pub fn as_str(&self) -> &'static str {
match self {
DlStatus::Downloading => "downloading",
DlStatus::Done => "done",
DlStatus::Failed(_) => "failed",
DlStatus::Cancelled => "cancelled",
}
}
pub fn is_terminal(&self) -> bool {
!matches!(self, DlStatus::Downloading)
}
}
#[derive(Clone, Debug)]
pub struct DownloadReq {
pub id: String,
pub backend: Backend,
pub device: Device,
pub dest: PathBuf,
}
pub trait DownloadProc: Send {
fn progress(&self) -> Option<f32>;
fn status(&self) -> DlStatus;
fn cancel(&mut self);
}
pub trait ModelDownloader: Send + Sync {
fn start(&self, req: &DownloadReq) -> Result<Box<dyn DownloadProc>, String>;
}
pub struct OllamaPullDownloader;
struct CmdState {
frac: Option<f32>,
status: DlStatus,
pid: i32,
cancelled: bool,
}
struct CmdProc {
state: Arc<Mutex<CmdState>>,
}
impl DownloadProc for CmdProc {
fn progress(&self) -> Option<f32> {
self.state.lock().unwrap().frac
}
fn status(&self) -> DlStatus {
self.state.lock().unwrap().status.clone()
}
fn cancel(&mut self) {
let mut g = self.state.lock().unwrap();
g.cancelled = true;
if !g.status.is_terminal() {
unsafe { libc::kill(g.pid, libc::SIGTERM) };
}
}
}
fn parse_percent(line: &str) -> Option<f32> {
for tok in line.split_whitespace().rev() {
if let Some(num) = tok.strip_suffix('%') {
if let Ok(pct) = num.parse::<f32>() {
return Some((pct / 100.0).clamp(0.0, 1.0));
}
}
}
None
}
impl ModelDownloader for OllamaPullDownloader {
fn start(&self, req: &DownloadReq) -> Result<Box<dyn DownloadProc>, String> {
if req.backend != Backend::Ollama {
return Err(format!(
"downloading a {} model (HuggingFace hf-hub) is not wired yet โ a follow-up; \
pick an Ollama model, or the fake downloader drives this in tests.",
req.backend.label()
));
}
std::fs::create_dir_all(&req.dest).ok();
let mut child = std::process::Command::new("ollama")
.arg("pull")
.arg(&req.id)
.env("OLLAMA_MODELS", &req.dest)
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.map_err(|e| format!("spawn `ollama pull {}`: {e}", req.id))?;
let pid = child.id() as i32;
let state = Arc::new(Mutex::new(CmdState {
frac: None,
status: DlStatus::Downloading,
pid,
cancelled: false,
}));
let stderr = child.stderr.take();
let worker = Arc::clone(&state);
std::thread::spawn(move || {
use std::io::{BufRead, BufReader};
if let Some(err) = stderr {
for line in BufReader::new(err).lines().map_while(Result::ok) {
if let Some(f) = parse_percent(&line) {
worker.lock().unwrap().frac = Some(f);
}
}
}
let ok = child.wait().map(|s| s.success()).unwrap_or(false);
let mut g = worker.lock().unwrap();
if g.cancelled {
g.status = DlStatus::Cancelled;
} else if ok {
g.frac = Some(1.0);
g.status = DlStatus::Done;
} else {
g.status = DlStatus::Failed("`ollama pull` exited nonzero".into());
}
});
Ok(Box::new(CmdProc { state }))
}
}
pub struct FakeDownloader {
steps: u32,
fail: Option<String>,
seen: Mutex<Vec<DownloadReq>>,
}
impl FakeDownloader {
pub fn new(steps: u32) -> Self {
Self { steps: steps.max(1), fail: None, seen: Mutex::new(Vec::new()) }
}
pub fn failing(reason: impl Into<String>) -> Self {
Self { steps: 1, fail: Some(reason.into()), seen: Mutex::new(Vec::new()) }
}
pub fn requests(&self) -> Vec<DownloadReq> {
self.seen.lock().unwrap().clone()
}
}
struct FakeProc {
tick: Mutex<u32>,
steps: u32,
cancelled: Mutex<bool>,
}
impl DownloadProc for FakeProc {
fn progress(&self) -> Option<f32> {
let mut t = self.tick.lock().unwrap();
if *t < self.steps && !*self.cancelled.lock().unwrap() {
*t += 1;
}
Some((*t as f32 / self.steps as f32).clamp(0.0, 1.0))
}
fn status(&self) -> DlStatus {
if *self.cancelled.lock().unwrap() {
return DlStatus::Cancelled;
}
if *self.tick.lock().unwrap() >= self.steps {
DlStatus::Done
} else {
DlStatus::Downloading
}
}
fn cancel(&mut self) {
*self.cancelled.lock().unwrap() = true;
}
}
impl ModelDownloader for FakeDownloader {
fn start(&self, req: &DownloadReq) -> Result<Box<dyn DownloadProc>, String> {
self.seen.lock().unwrap().push(req.clone());
if let Some(reason) = &self.fail {
return Err(reason.clone());
}
Ok(Box::new(FakeProc {
tick: Mutex::new(0),
steps: self.steps,
cancelled: Mutex::new(false),
}))
}
}
#[derive(Clone, Debug)]
pub struct GpuProbe {
pub onnxruntime: bool,
pub cuda_driver: bool,
pub rocm: bool,
pub cuda_pinned: bool,
pub rocm_pinned: bool,
pub gen_candle: bool,
pub gen_ollama: bool,
pub embed_ort: bool,
}
fn gpu_probe() -> GpuProbe {
#[cfg(feature = "embed-ort")]
let onnxruntime = crate::vector::cuda::onnxruntime_dylib().is_some();
#[cfg(not(feature = "embed-ort"))]
let onnxruntime = false;
#[cfg(feature = "embed-ort")]
let cuda_driver = crate::vector::cuda::driver_present();
#[cfg(not(feature = "embed-ort"))]
let cuda_driver = false;
#[cfg(feature = "embed-ort-rocm")]
let rocm = crate::vector::rocm::available();
#[cfg(not(feature = "embed-ort-rocm"))]
let rocm = false;
GpuProbe {
onnxruntime,
cuda_driver,
rocm,
cuda_pinned: Path::new("/opt/nornir/cuda").exists(),
rocm_pinned: Path::new("/opt/nornir/rocm").exists(),
gen_candle: cfg!(feature = "gen-candle"),
gen_ollama: cfg!(feature = "gen-ollama"),
embed_ort: cfg!(feature = "embed-ort"),
}
}
struct DownloadJob {
job_id: String,
model: String,
backend: Backend,
device: Device,
status: DlStatus,
frac: Option<f32>,
proc: Option<Box<dyn DownloadProc>>,
handle: Option<JobHandle>,
}
pub struct ModelsTab {
downloader: Arc<dyn ModelDownloader>,
sink: JobSink,
workspace: String,
dest: PathBuf,
device: Device,
selected: usize,
jobs: Vec<DownloadJob>,
gpu: GpuProbe,
theme: Theme,
pending_cancel: RefCell<Vec<String>>,
pending_download: RefCell<Vec<usize>>,
}
impl ModelsTab {
pub fn new(sink: JobSink, workspace: impl Into<String>) -> Self {
Self::with_downloader(Arc::new(OllamaPullDownloader), sink, workspace)
}
pub fn with_downloader(
downloader: Arc<dyn ModelDownloader>,
sink: JobSink,
workspace: impl Into<String>,
) -> Self {
Self {
downloader,
sink,
workspace: workspace.into(),
dest: default_models_dir(),
device: Device::Cpu,
selected: 0,
jobs: Vec::new(),
gpu: gpu_probe(),
theme: Theme::default(),
pending_cancel: RefCell::new(Vec::new()),
pending_download: RefCell::new(Vec::new()),
}
}
pub fn set_palette(&mut self, t: Theme) {
self.theme = t;
}
pub fn set_downloader_for_test(&mut self, downloader: Arc<dyn ModelDownloader>) {
self.downloader = downloader;
}
pub fn download(&mut self, idx: usize) -> String {
let spec = &CATALOG[idx];
let device = self.device;
let req = DownloadReq {
id: spec.id.to_string(),
backend: spec.backend,
device,
dest: self.dest.join(spec.backend.as_str()),
};
let handle = JobHandle::start(
self.sink.clone(),
kind::MODEL_DOWNLOAD,
spec.id,
&self.workspace,
serde_json::json!({ "backend": spec.backend.as_str(), "device": device.as_str() }),
);
let job_id = handle.job_id().to_string();
match self.downloader.start(&req) {
Ok(proc) => {
nornir_testmatrix::functional_status(
"nornir/viz/models",
&format!("download_{}", spec.id),
true,
&format!("download '{}' ({}) started on {}", spec.id, spec.backend.as_str(), device.as_str()),
);
self.jobs.push(DownloadJob {
job_id: job_id.clone(),
model: spec.id.to_string(),
backend: spec.backend,
device,
status: DlStatus::Downloading,
frac: None,
proc: Some(proc),
handle: Some(handle),
});
}
Err(reason) => {
nornir_testmatrix::functional_status(
"nornir/viz/models",
&format!("download_{}", spec.id),
false,
&format!("download '{}' FAILED to start: {reason}", spec.id),
);
handle.fail(&anyhow::anyhow!("{reason}"));
self.jobs.push(DownloadJob {
job_id: job_id.clone(),
model: spec.id.to_string(),
backend: spec.backend,
device,
status: DlStatus::Failed(reason),
frac: None,
proc: None,
handle: None,
});
}
}
job_id
}
pub fn poll(&mut self) {
for job in &mut self.jobs {
if job.status.is_terminal() {
continue;
}
let Some(proc) = job.proc.as_ref() else { continue };
job.frac = proc.progress();
let status = proc.status();
if status != job.status {
job.status = status.clone();
}
if status.is_terminal() {
Self::terminate_job(job, &status);
}
}
}
pub fn cancel(&mut self, job_id: &str) -> bool {
let Some(job) = self.jobs.iter_mut().find(|j| j.job_id == job_id) else {
return false;
};
if job.status.is_terminal() {
return false;
}
if let Some(proc) = job.proc.as_mut() {
proc.cancel();
}
job.status = DlStatus::Cancelled;
Self::terminate_job(job, &DlStatus::Cancelled);
true
}
fn terminate_job(job: &mut DownloadJob, status: &DlStatus) {
job.proc = None;
let Some(handle) = job.handle.take() else { return };
match status {
DlStatus::Done => handle.finish(
serde_json::json!({ "model": job.model, "backend": job.backend.as_str() }),
&job.model,
),
DlStatus::Failed(e) => handle.fail(&anyhow::anyhow!("{e}")),
DlStatus::Cancelled => handle.fail_with_detail(
serde_json::json!({ "cancelled": true, "model": job.model }),
),
DlStatus::Downloading => {}
}
}
pub fn set_device_for_test(&mut self, device: Device) {
self.device = device;
}
pub fn download_for_test(&mut self, idx: usize) -> String {
self.download(idx)
}
pub fn poll_for_test(&mut self) {
self.poll();
}
pub fn cancel_for_test(&mut self, job_id: &str) -> bool {
self.cancel(job_id)
}
pub fn state_json(&self) -> serde_json::Value {
serde_json::json!({
"dest": self.dest.display().to_string(),
"device": self.device.as_str(),
"selected": self.selected,
"selected_model": CATALOG.get(self.selected).map(|s| s.id),
"catalog": CATALOG.iter().map(|s| serde_json::json!({
"id": s.id,
"backend": s.backend.as_str(),
"note": s.note,
})).collect::<Vec<_>>(),
"gpu": {
"onnxruntime_present": self.gpu.onnxruntime,
"cuda_driver": self.gpu.cuda_driver,
"cuda_pinned": self.gpu.cuda_pinned,
"rocm": self.gpu.rocm,
"rocm_pinned": self.gpu.rocm_pinned,
"gen_candle": self.gpu.gen_candle,
"gen_ollama": self.gpu.gen_ollama,
"embed_ort": self.gpu.embed_ort,
},
"job_count": self.jobs.len(),
"active_downloads": self.jobs.iter().filter(|j| !j.status.is_terminal()).count(),
"jobs": self.jobs.iter().map(|j| serde_json::json!({
"job_id": j.job_id,
"model": j.model,
"backend": j.backend.as_str(),
"device": j.device.as_str(),
"status": j.status.as_str(),
"progress": j.frac,
"error": match &j.status { DlStatus::Failed(e) => Some(e.clone()), _ => None },
})).collect::<Vec<_>>(),
"palette": self.theme.name,
})
}
pub fn draw(&mut self, ui: &mut egui::Ui) {
self.poll();
if self.jobs.iter().any(|j| !j.status.is_terminal()) {
ui.ctx().request_repaint();
}
let theme = self.theme;
ui.horizontal(|ui| {
ui.heading("๐ง Models");
});
ui.label(
"browse known LLM / embedding models, pick a device, and download one \
as a nornir job (progress + cancel below).",
);
let mut backends = Vec::new();
if self.gpu.gen_candle {
backends.push("candle");
}
if self.gpu.gen_ollama {
backends.push("ollama");
}
if self.gpu.embed_ort {
backends.push("ort");
}
ui.horizontal_wrapped(|ui| {
ui.colored_label(theme.text_dim, "backends:");
ui.label(if backends.is_empty() { "none".into() } else { backends.join(" ยท ") });
ui.separator();
let (ort_col, ort_txt) = if self.gpu.onnxruntime {
(theme.accent, "libonnxruntime: present")
} else {
(theme.text_dim, "libonnxruntime: not found")
};
ui.colored_label(ort_col, ort_txt);
if self.gpu.cuda_driver {
ui.colored_label(theme.accent, "ยท CUDA driver");
}
if self.gpu.rocm {
ui.colored_label(theme.accent, "ยท ROCm");
}
});
ui.horizontal(|ui| {
ui.colored_label(theme.text_dim, "device:");
for dev in Device::ALL {
let avail = match dev {
Device::Cpu => true,
Device::Cuda => self.gpu.cuda_driver || self.gpu.cuda_pinned,
Device::Rocm => self.gpu.rocm || self.gpu.rocm_pinned,
};
let mut label = dev.label().to_string();
if !avail && dev != Device::Cpu {
label.push_str(" (not detected)");
}
ui.selectable_value(&mut self.device, dev, label);
}
});
ui.small(format!("models dir: {}", self.dest.display()));
ui.separator();
egui::Grid::new("models_catalog_grid")
.striped(true)
.num_columns(3)
.spacing([18.0, 6.0])
.show(ui, |ui| {
ui.strong("model");
ui.strong("backend");
ui.strong("");
ui.end_row();
for (idx, spec) in CATALOG.iter().enumerate() {
ui.vertical(|ui| {
if ui.selectable_label(self.selected == idx, spec.id).clicked() {
self.selected = idx;
}
ui.small(spec.note);
});
ui.label(spec.backend.label());
if ui.button(format!("โฌ Download {}", spec.id)).clicked() {
self.pending_download.borrow_mut().push(idx);
}
ui.end_row();
}
});
if !self.jobs.is_empty() {
ui.separator();
ui.strong("downloads");
for job in &self.jobs {
ui.horizontal(|ui| {
let frac = job.frac.unwrap_or(0.0);
ui.add(
egui::ProgressBar::new(frac)
.desired_width(200.0)
.text(format!("{} ยท {}", job.model, job.status.as_str())),
);
match &job.status {
DlStatus::Downloading => {
if ui.button("โ Cancel").clicked() {
self.pending_cancel.borrow_mut().push(job.job_id.clone());
}
}
DlStatus::Failed(e) => {
ui.colored_label(RED, e);
}
DlStatus::Cancelled => {
ui.colored_label(AMBER, "cancelled");
}
DlStatus::Done => {
ui.colored_label(theme.accent, "โ");
}
}
});
}
}
let downloads: Vec<usize> = self.pending_download.borrow_mut().drain(..).collect();
for idx in downloads {
self.download(idx);
}
let cancels: Vec<String> = self.pending_cancel.borrow_mut().drain(..).collect();
for id in cancels {
self.cancel(&id);
}
}
}
fn default_models_dir() -> PathBuf {
if let Ok(d) = std::env::var("NORNIR_MODELS_DIR") {
return PathBuf::from(d);
}
if let Ok(home) = std::env::var("HOME") {
return PathBuf::from(home).join(".nornir").join("models");
}
std::env::temp_dir().join("nornir-models")
}