Skip to main content

rigspark_runtime/
ollama_installed.rs

1use serde::{Deserialize, Serialize};
2use serde_json::Value;
3use sha2::{Digest, Sha256};
4use std::{
5    collections::BTreeMap,
6    path::{Path, PathBuf},
7};
8use tokio_util::sync::CancellationToken;
9
10#[derive(Debug, thiserror::Error)]
11pub enum OllamaError {
12    #[error("invalid Ollama input or response: {0}")]
13    Invalid(&'static str),
14    #[error("installed model integrity verification failed: {0}")]
15    Integrity(&'static str),
16    #[error("Ollama request failed: {0}")]
17    Request(String),
18}
19#[derive(Debug, Clone, PartialEq, Serialize)]
20#[serde(rename_all = "camelCase")]
21pub struct InstalledModel {
22    pub id: String,
23    pub digest: String,
24    pub size_bytes: u64,
25    pub quant: Option<String>,
26    pub context_length: Option<u32>,
27    pub kv_bytes_per_token: Option<u64>,
28    pub capabilities: Vec<String>,
29}
30pub fn size_installed(
31    model: &InstalledModel,
32    hardware: &rigspark_core::sizing::Hardware,
33    context: Option<u32>,
34) -> Result<Value, OllamaError> {
35    crate::hardware::validate_hardware(hardware).map_err(|_| OllamaError::Invalid("hardware"))?;
36    if context.is_some_and(|value| !(1..=10000000).contains(&value)) {
37        return Err(OllamaError::Invalid("context"));
38    }
39    let (kind, capacity) = rigspark_core::sizing::memory_capacity(hardware);
40    let usable = capacity * 0.85;
41    let weights_fit = model.size_bytes as f64 <= usable;
42    let required = context
43        .zip(model.kv_bytes_per_token)
44        .map(|(context, kv)| model.size_bytes as f64 + kv as f64 * f64::from(context));
45    let over_cap = context
46        .zip(model.context_length)
47        .is_some_and(|(context, cap)| context > cap);
48    let fit = if over_cap || !weights_fit || required.is_some_and(|required| required > usable) {
49        "no"
50    } else if required.is_none() {
51        "unknown"
52    } else {
53        "yes"
54    };
55    let mut value =
56        serde_json::to_value(model).map_err(|_| OllamaError::Invalid("installed model"))?;
57    for (key, entry) in [
58        ("context", serde_json::json!(context)),
59        ("fit", serde_json::json!(fit)),
60        ("weightsFit", serde_json::json!(weights_fit)),
61        ("requiredBytes", serde_json::json!(required)),
62        ("usableBytes", serde_json::json!(usable)),
63        ("memoryKind", serde_json::json!(kind)),
64        ("evidence", serde_json::json!("local-runtime-metadata")),
65        ("throughput", serde_json::json!("unknown")),
66    ] {
67        value[key] = entry;
68    }
69    Ok(value)
70}
71fn digest(value: &str) -> bool {
72    value.len() == 64
73        && value
74            .bytes()
75            .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
76}
77pub(crate) fn model_path(root: &Path, id: &str) -> Result<PathBuf, OllamaError> {
78    let mut segments: Vec<_> = id.split('/').collect();
79    if segments.len() > 3 {
80        return Err(OllamaError::Invalid("installed model path"));
81    }
82    let last = segments.pop().ok_or(OllamaError::Invalid("model id"))?;
83    let parts: Vec<_> = last.split(':').collect();
84    if parts.len() > 2 {
85        return Err(OllamaError::Invalid("model tag"));
86    }
87    let name = parts[0];
88    let tag = parts.get(1).copied().unwrap_or("latest");
89    let namespace = segments.pop().unwrap_or("library");
90    let registry = segments.pop().unwrap_or("registry.ollama.ai");
91    let safe = |value: &str| {
92        !value.is_empty()
93            && value.as_bytes()[0].is_ascii_alphanumeric()
94            && value.bytes().all(|byte| {
95                byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._-".contains(&byte)
96            })
97    };
98    if ![registry, namespace, name, tag].into_iter().all(safe) {
99        return Err(OllamaError::Invalid("installed model path"));
100    }
101    Ok(root
102        .join("manifests")
103        .join(registry)
104        .join(namespace)
105        .join(name)
106        .join(tag))
107}
108pub fn parse_inventory(value: Value) -> Result<Vec<InstalledModel>, OllamaError> {
109    #[derive(Deserialize)]
110    struct Details {
111        quantization_level: Option<String>,
112    }
113    #[derive(Deserialize)]
114    struct Entry {
115        name: String,
116        digest: String,
117        size: u64,
118        remote_host: Option<String>,
119        remote_model: Option<String>,
120        details: Option<Details>,
121    }
122    #[derive(Deserialize)]
123    struct Inventory {
124        models: Vec<Entry>,
125    }
126    let inventory: Inventory =
127        serde_json::from_value(value).map_err(|_| OllamaError::Invalid("inventory"))?;
128    if inventory.models.len() > 10000 {
129        return Err(OllamaError::Invalid("inventory exceeds model limit"));
130    }
131    let mut result = Vec::new();
132    for entry in inventory.models {
133        if entry.name.is_empty()
134            || entry.name.len() > 256
135            || !digest(&entry.digest)
136            || entry.size == 0
137            || entry.size > 9007199254740991
138            || entry
139                .details
140                .as_ref()
141                .and_then(|details| details.quantization_level.as_ref())
142                .is_some_and(|value| value.len() > 100)
143        {
144            return Err(OllamaError::Invalid("inventory entry"));
145        }
146        if entry
147            .remote_host
148            .as_ref()
149            .is_some_and(|value| !value.is_empty())
150            || entry
151                .remote_model
152                .as_ref()
153                .is_some_and(|value| !value.is_empty())
154        {
155            continue;
156        }
157        model_path(Path::new(""), &entry.name)?;
158        result.push(InstalledModel {
159            id: entry.name,
160            digest: entry.digest,
161            size_bytes: entry.size,
162            quant: entry.details.and_then(|details| details.quantization_level),
163            context_length: None,
164            kv_bytes_per_token: None,
165            capabilities: Vec::new(),
166        });
167    }
168    Ok(result)
169}
170#[derive(Deserialize)]
171pub struct Metadata {
172    #[serde(default)]
173    pub model_info: BTreeMap<String, Value>,
174    #[serde(default)]
175    pub capabilities: Vec<String>,
176    pub parameters: Option<String>,
177    pub remote_host: Option<String>,
178}
179pub fn parse_metadata(value: Value) -> Result<Metadata, OllamaError> {
180    let metadata: Metadata =
181        serde_json::from_value(value).map_err(|_| OllamaError::Invalid("model metadata"))?;
182    if metadata
183        .remote_host
184        .as_ref()
185        .is_some_and(|value| !value.is_empty())
186        || metadata.capabilities.len() > 100
187        || metadata.capabilities.iter().any(|value| value.len() > 100)
188        || metadata
189            .parameters
190            .as_ref()
191            .is_some_and(|value| value.len() > 100000)
192    {
193        return Err(OllamaError::Invalid("local model metadata"));
194    }
195    Ok(metadata)
196}
197pub fn inspect_metadata(
198    mut model: InstalledModel,
199    value: Value,
200) -> Result<InstalledModel, OllamaError> {
201    let metadata = parse_metadata(value)?;
202    let family = metadata
203        .model_info
204        .get("general.architecture")
205        .and_then(Value::as_str)
206        .unwrap_or("");
207    let number = |key: &str, maximum: u64| {
208        metadata
209            .model_info
210            .get(&format!("{family}.{key}"))
211            .and_then(Value::as_u64)
212            .filter(|value| *value > 0 && *value <= maximum)
213    };
214    model.context_length = number("context_length", 10000000).map(|value| value as u32);
215    if ["llama", "qwen2"].contains(&family) {
216        model.kv_bytes_per_token = (|| {
217            let layers = number("block_count", 100000)?;
218            let heads = number("attention.head_count_kv", 100000)?;
219            let key = number("attention.key_length", 100000)?;
220            let value = number("attention.value_length", 100000)?;
221            layers
222                .checked_mul(heads)?
223                .checked_mul(key + value)?
224                .checked_mul(2)
225        })();
226    }
227    model.capabilities = metadata.capabilities;
228    Ok(model)
229}
230#[derive(Deserialize)]
231#[serde(rename_all = "camelCase")]
232struct Layer {
233    digest: String,
234    size: u64,
235    media_type: String,
236}
237#[derive(Deserialize)]
238#[serde(rename_all = "camelCase")]
239struct Manifest {
240    schema_version: u8,
241    config: Layer,
242    layers: Vec<Layer>,
243}
244fn contained(root: &Path, path: &Path) -> Result<(), OllamaError> {
245    let canonical = path
246        .canonicalize()
247        .map_err(|_| OllamaError::Integrity("missing local content"))?;
248    if !canonical.starts_with(root) {
249        return Err(OllamaError::Integrity("local path escaped model root"));
250    }
251    Ok(())
252}
253pub async fn verify_manifest(
254    root: &Path,
255    id: &str,
256    expected_manifest: &str,
257    expected_weight: Option<&str>,
258    expected_bytes: Option<u64>,
259    cancel: &CancellationToken,
260) -> Result<(), OllamaError> {
261    verify_catalog_manifest(
262        root,
263        id,
264        expected_manifest,
265        expected_weight,
266        expected_bytes,
267        &[],
268        cancel,
269    )
270    .await
271}
272
273const EMBEDDED_MANIFEST: &str = "application/vnd.docker.distribution.manifest.v2+json";
274
275/// Digests that identify a local Ollama manifest, the primary one first. In Ollama 0.40+'s
276/// content-addressed store that is the manifest blob's digest; in the older store it is the
277/// manifest file's SHA-256, followed by any embedded upstream manifest whose blob verifies.
278pub fn manifest_digests(root: &Path, id: &str) -> Result<Vec<String>, OllamaError> {
279    Ok(read_manifest(root, id)?.1)
280}
281
282/// The local manifest for `id` and the digests that identify it.
283fn read_manifest(root: &Path, id: &str) -> Result<(String, Vec<String>), OllamaError> {
284    let root = root
285        .canonicalize()
286        .map_err(|_| OllamaError::Integrity("model root unavailable"))?;
287    if let Some(found) = read_v2_manifest(&root, id)? {
288        return Ok(found);
289    }
290    let path = model_path(&root, id)?;
291    contained(&root, &path)?;
292    let raw = crate::state::secure_read(&path, 4 * 1024 * 1024, false)
293        .map_err(|_| OllamaError::Integrity("manifest unavailable"))?;
294    let digests = identities(&root, &raw);
295    Ok((raw, digests))
296}
297
298/// Ollama 0.40+ links `manifests-v2/<registry>/<namespace>/<name>/<tag>` to the manifest's
299/// content-addressed blob. Only a link to exactly `blobs/sha256-<hex>` whose content hashes
300/// to `<hex>` is accepted.
301fn read_v2_manifest(root: &Path, id: &str) -> Result<Option<(String, Vec<String>)>, OllamaError> {
302    let v1 = model_path(root, id)?;
303    let relative = v1
304        .strip_prefix(root.join("manifests"))
305        .map_err(|_| OllamaError::Invalid("installed model path"))?;
306    let mut parts: Vec<_> = relative.components().map(|part| part.as_os_str()).collect();
307    if parts
308        .first()
309        .is_some_and(|registry| *registry == "registry.ollama.ai")
310    {
311        parts[0] = std::ffi::OsStr::new("ollama.com");
312    }
313    let link = parts
314        .iter()
315        .fold(root.join("manifests-v2"), |path, part| path.join(part));
316    let target = match std::fs::read_link(&link) {
317        Ok(target) => target,
318        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
319        Err(_) => return Err(OllamaError::Integrity("manifest link unreadable")),
320    };
321    let target = target.to_string_lossy();
322    let hex = target
323        .strip_prefix("../../../../blobs/sha256-")
324        .filter(|hex| digest(hex))
325        .ok_or(OllamaError::Integrity("manifest link escapes blobs"))?;
326    let blob = root.join("blobs").join(format!("sha256-{hex}"));
327    contained(root, &blob)?;
328    let raw = crate::state::secure_read(&blob, 4 * 1024 * 1024, false)
329        .map_err(|_| OllamaError::Integrity("manifest unavailable"))?;
330    if format!("{:x}", Sha256::digest(raw.as_bytes())) != hex {
331        return Err(OllamaError::Integrity(
332            "manifest does not match its address",
333        ));
334    }
335    Ok(Some((raw, vec![hex.to_owned()])))
336}
337
338fn identities(root: &Path, raw: &str) -> Vec<String> {
339    let mut digests = vec![format!("{:x}", Sha256::digest(raw.as_bytes()))];
340    let Ok(manifest) = serde_json::from_str::<Manifest>(raw) else {
341        return digests;
342    };
343    for layer in manifest
344        .layers
345        .iter()
346        .filter(|layer| layer.media_type == EMBEDDED_MANIFEST && layer.size <= 64 * 1024)
347    {
348        let Some(hex) = layer
349            .digest
350            .strip_prefix("sha256:")
351            .filter(|hex| digest(hex))
352        else {
353            continue;
354        };
355        let blob = root.join("blobs").join(format!("sha256-{hex}"));
356        let verified = contained(root, &blob).is_ok()
357            && crate::state::secure_read(&blob, 64 * 1024, false)
358                .is_ok_and(|text| format!("{:x}", Sha256::digest(text.as_bytes())) == hex);
359        if verified && !digests.iter().any(|known| known == hex) {
360            digests.push(hex.to_owned());
361        }
362    }
363    digests
364}
365
366/// A manifest's config is a few hundred bytes of JSON; anything larger is not a config.
367const MAX_CONFIG_BYTES: u64 = 64 * 1024;
368
369pub async fn verify_catalog_manifest(
370    root: &Path,
371    id: &str,
372    expected_manifest: &str,
373    expected_weight: Option<&str>,
374    expected_bytes: Option<u64>,
375    projectors: &[rigspark_core::sizing::ProjectorArtifact],
376    cancel: &CancellationToken,
377) -> Result<(), OllamaError> {
378    if !digest(expected_manifest)
379        || expected_weight.is_some_and(|value| !digest(value))
380        || expected_bytes.is_some_and(|value| value == 0 || value > 9007199254740991)
381    {
382        return Err(OllamaError::Invalid("expected integrity"));
383    }
384    if cancel.is_cancelled() {
385        return Err(OllamaError::Request("cancelled".into()));
386    }
387    let root = root
388        .canonicalize()
389        .map_err(|_| OllamaError::Integrity("model root unavailable"))?;
390    let (raw, digests) = read_manifest(&root, id)?;
391    if !digests.iter().any(|known| known == expected_manifest) {
392        return Err(OllamaError::Integrity("manifest digest mismatch"));
393    }
394    let manifest: Manifest =
395        serde_json::from_str(&raw).map_err(|_| OllamaError::Invalid("manifest"))?;
396    if manifest.schema_version != 2 || manifest.layers.is_empty() || manifest.layers.len() > 10000 {
397        return Err(OllamaError::Invalid("manifest layers"));
398    }
399    let weights: Vec<_> = manifest
400        .layers
401        .iter()
402        .filter(|layer| layer.media_type == "application/vnd.ollama.image.model")
403        .collect();
404    if !projectors.is_empty() {
405        let actual: Vec<_> = manifest
406            .layers
407            .iter()
408            .filter(|layer| layer.media_type == "application/vnd.ollama.image.projector")
409            .collect();
410        let mut digests = std::collections::HashSet::new();
411        if projectors.len() > 16
412            || actual.len() != projectors.len()
413            || projectors.iter().any(|expected| {
414                !digest(&expected.sha256)
415                    || expected.bytes == 0
416                    || !digests.insert(&expected.sha256)
417                    || !actual.iter().any(|layer| {
418                        layer.digest == format!("sha256:{}", expected.sha256)
419                            && layer.size == expected.bytes
420                    })
421            })
422        {
423            return Err(OllamaError::Integrity("catalog projector mismatch"));
424        }
425    }
426    if weights.is_empty() {
427        return Err(OllamaError::Integrity("manifest has no model weights"));
428    }
429    if expected_weight.is_some_and(|expected| {
430        !weights
431            .iter()
432            .any(|layer| layer.digest == format!("sha256:{expected}"))
433    }) {
434        return Err(OllamaError::Integrity("catalog digest mismatch"));
435    }
436    let total = manifest
437        .layers
438        .iter()
439        .filter(|layer| {
440            layer.media_type == "application/vnd.ollama.image.model"
441                || layer.media_type == "application/vnd.ollama.image.projector"
442        })
443        .try_fold(0u64, |total, layer| total.checked_add(layer.size))
444        .ok_or(OllamaError::Invalid("weight size overflow"))?;
445    if expected_bytes.is_some_and(|expected| total < expected.div_ceil(2)) {
446        return Err(OllamaError::Integrity("catalog size floor"));
447    }
448    for (index, layer) in std::iter::once(&manifest.config)
449        .chain(&manifest.layers)
450        .enumerate()
451    {
452        let sha = layer
453            .digest
454            .strip_prefix("sha256:")
455            .filter(|value| digest(value))
456            .ok_or(OllamaError::Invalid("blob digest"))?;
457        if layer.size > 9007199254740991 {
458            return Err(OllamaError::Invalid("blob size"));
459        }
460        let blob = root.join("blobs").join(format!("sha256-{sha}"));
461        contained(&root, &blob)?;
462        let (actual, bytes) = crate::acquire::hash_file(&blob, cancel)
463            .await
464            .map_err(|_| OllamaError::Integrity("blob unavailable or cancelled"))?;
465        // The digest binds every blob's content. Ollama's registry has published manifests
466        // whose config entry lists the wrong size with the right digest, so a small config is
467        // accepted on its digest alone; weights and every other layer need both to match.
468        let config_ok = index == 0 && bytes <= MAX_CONFIG_BYTES;
469        if actual != sha || (bytes != layer.size && !config_ok) {
470            return Err(OllamaError::Integrity("blob digest or size mismatch"));
471        }
472    }
473    if read_manifest(&root, id)?.0 != raw {
474        return Err(OllamaError::Integrity(
475            "manifest changed during verification",
476        ));
477    }
478    Ok(())
479}
480
481#[async_trait::async_trait]
482pub trait InstalledVerifier: Send + Sync {
483    async fn verify(
484        &self,
485        id: &str,
486        digest: &str,
487        cancel: &CancellationToken,
488    ) -> Result<(), OllamaError>;
489}
490pub struct LocalVerifier {
491    pub root: PathBuf,
492}
493#[async_trait::async_trait]
494impl InstalledVerifier for LocalVerifier {
495    async fn verify(
496        &self,
497        id: &str,
498        digest: &str,
499        cancel: &CancellationToken,
500    ) -> Result<(), OllamaError> {
501        verify_manifest(&self.root, id, digest, None, None, cancel).await
502    }
503}
504pub struct InstalledModels<'runtime> {
505    transport: &'runtime dyn crate::http::Transport,
506    verifier: &'runtime dyn InstalledVerifier,
507}
508impl<'runtime> InstalledModels<'runtime> {
509    pub fn new(
510        transport: &'runtime dyn crate::http::Transport,
511        verifier: &'runtime dyn InstalledVerifier,
512    ) -> Self {
513        Self {
514            transport,
515            verifier,
516        }
517    }
518    async fn request(
519        &self,
520        endpoint: &str,
521        path: &str,
522        body: Option<Value>,
523        cancel: &CancellationToken,
524    ) -> Result<Value, OllamaError> {
525        let request = crate::http::Request::new(endpoint, path, body, None)
526            .map_err(|error| OllamaError::Request(error.to_string()))?;
527        crate::http::read_json(self.transport, request, cancel, 4 * 1024 * 1024)
528            .await
529            .map_err(|error| OllamaError::Request(error.to_string()))
530    }
531    pub async fn list(
532        &self,
533        endpoint: &str,
534        cancel: &CancellationToken,
535    ) -> Result<Vec<InstalledModel>, OllamaError> {
536        parse_inventory(self.request(endpoint, "/api/tags", None, cancel).await?)
537    }
538    pub async fn inspect(
539        &self,
540        endpoint: &str,
541        id: &str,
542        cancel: &CancellationToken,
543    ) -> Result<InstalledModel, OllamaError> {
544        model_path(Path::new(""), id)?;
545        let selected = exact(self.list(endpoint, cancel).await?, id)?;
546        inspect_metadata(
547            selected,
548            self.request(
549                endpoint,
550                "/api/show",
551                Some(serde_json::json!({"model":id})),
552                cancel,
553            )
554            .await?,
555        )
556    }
557    pub async fn activate(
558        &self,
559        endpoint: &str,
560        model: &InstalledModel,
561        context: Option<u32>,
562        cancel: &CancellationToken,
563    ) -> Result<String, OllamaError> {
564        self.activate_with_id(
565            endpoint,
566            model,
567            context,
568            &uuid::Uuid::new_v4().to_string(),
569            cancel,
570        )
571        .await
572    }
573    pub async fn activate_with_id(
574        &self,
575        endpoint: &str,
576        model: &InstalledModel,
577        context: Option<u32>,
578        unique: &str,
579        cancel: &CancellationToken,
580    ) -> Result<String, OllamaError> {
581        crate::state::loopback(endpoint).map_err(|_| OllamaError::Invalid("endpoint"))?;
582        model_path(Path::new(""), &model.id)?;
583        if !digest(&model.digest)
584            || context.is_some_and(|context| !(1..=10000000).contains(&context))
585        {
586            return Err(OllamaError::Invalid("activation metadata"));
587        }
588        let variant = context.map(|context| format!("llmup-context-{unique}:{context}"));
589        if let Some(variant) = &variant {
590            model_path(Path::new(""), variant)?;
591            if variant.len() > 256 {
592                return Err(OllamaError::Invalid("variant name"));
593            }
594        }
595        self.verifier
596            .verify(&model.id, &model.digest, cancel)
597            .await?;
598        if exact(self.list(endpoint, cancel).await?, &model.id)?.digest != model.digest {
599            return Err(OllamaError::Integrity("source manifest changed; retry"));
600        }
601        let Some(context) = context else {
602            return Ok(model.id.clone());
603        };
604        let variant = variant.ok_or(OllamaError::Invalid("variant name"))?;
605        let created=self.request(endpoint,"/api/create",Some(serde_json::json!({"model":variant,"from":model.id,"parameters":{"num_ctx":context},"stream":false})),cancel).await?;
606        if created.get("status").and_then(Value::as_str) != Some("success") {
607            return Err(OllamaError::Request(
608                "context variant creation failed".into(),
609            ));
610        }
611        let metadata = parse_metadata(
612            self.request(
613                endpoint,
614                "/api/show",
615                Some(serde_json::json!({"model":variant})),
616                cancel,
617            )
618            .await?,
619        )?;
620        if !metadata.parameters.as_ref().is_some_and(|parameters| {
621            parameters.lines().any(|line| {
622                let mut parts = line.split_whitespace();
623                parts.next() == Some("num_ctx")
624                    && parts.next().and_then(|value| value.parse::<u32>().ok()) == Some(context)
625            })
626        }) {
627            return Err(OllamaError::Integrity(
628                "runtime did not retain requested context",
629            ));
630        }
631        let installed = exact(self.list(endpoint, cancel).await?, &variant)?;
632        self.verifier
633            .verify(&installed.id, &installed.digest, cancel)
634            .await?;
635        if exact(self.list(endpoint, cancel).await?, &model.id)?.digest != model.digest {
636            return Err(OllamaError::Integrity(
637                "source changed during context activation",
638            ));
639        }
640        Ok(variant)
641    }
642}
643fn exact(models: Vec<InstalledModel>, id: &str) -> Result<InstalledModel, OllamaError> {
644    let mut matching = models.into_iter().filter(|model| model.id == id);
645    let selected = matching
646        .next()
647        .ok_or(OllamaError::Invalid("model not installed locally"))?;
648    if matching.next().is_some() {
649        return Err(OllamaError::Invalid("ambiguous installed model"));
650    }
651    Ok(selected)
652}