modelc 0.1.8

Rust CLI that compiles LLM weights (GGUF, Safetensors, ONNX, PyTorch) into a single .modelc artifact and serves a local OpenAI-compatible inference API with Metal GPU and CPU SIMD acceleration.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105

use std::collections::HashMap;

use anyhow::Result;

mod common;

use modelc::model::{DataType, Model, TensorData};

/// Build a minimal safetensors file containing a single f32 tensor.
fn make_minimal_lora_safetensors() -> Vec<u8> {
    use safetensors::tensor::TensorView;

    // Tensor shape: [2, 2] = 4 f32 values = 16 bytes.
    let shape: Vec<usize> = vec![2, 2];
    let data: Vec<u8> = vec![1.0f32, 0.0, 0.0, 1.0]
        .into_iter()
        .flat_map(|f| f.to_le_bytes())
        .collect();

    let tv = TensorView::new(safetensors::Dtype::F32, shape, &data).unwrap();
    let mut metadata: HashMap<String, TensorView> = HashMap::new();
    metadata.insert("weight".to_string(), tv);

    safetensors::serialize(&metadata, &None).unwrap()
}

#[test]
fn lora_apply_changes_base_tensor() -> Result<()> {
    let mut model = Model {
        name: "base".to_string(),
        architecture: "mlp".to_string(),
        metadata: HashMap::new(),
        tensors: {
            let mut t = HashMap::new();
            t.insert(
                "weight".to_string(),
                TensorData {
                    shape: vec![2, 2],
                    dtype: DataType::F32,
                    data: vec![0.0f32, 0.0, 0.0, 0.0]
                        .into_iter()
                        .flat_map(|f| f.to_le_bytes())
                        .collect(),
                },
            );
            t
        },
    };

    // Build a minimal safetensors file with a single tensor.
    let lora_buf = make_minimal_lora_safetensors();
    let temp_dir = common::tempdir()?;
    let lora_path = temp_dir.path().join("adapter.safetensors");
    std::fs::write(&lora_path, &lora_buf)?;

    modelc::lora::apply_lora(&mut model, &lora_path, 1.0)?;

    let updated = &model.tensors["weight"];
    let values: Vec<f32> = updated
        .data
        .chunks_exact(4)
        .map(|c| f32::from_le_bytes(c.try_into().unwrap()))
        .collect();

    // Base was all zeros, LoRA adds scaled values.
    // The first element should be non-zero after LoRA application.
    assert!(
        values[0].abs() > 0.001,
        "expected LoRA to modify weight, got {:?}",
        values
    );
    Ok(())
}

#[test]
fn lora_apply_skips_non_f32() -> Result<()> {
    let mut model = Model {
        name: "base".to_string(),
        architecture: "mlp".to_string(),
        metadata: HashMap::new(),
        tensors: {
            let mut t = HashMap::new();
            t.insert(
                "weight".to_string(),
                TensorData {
                    shape: vec![2, 2],
                    dtype: DataType::I8,
                    data: vec![0, 0, 0, 0],
                },
            );
            t
        },
    };

    let lora_buf = make_minimal_lora_safetensors();
    let temp_dir = common::tempdir()?;
    let lora_path = temp_dir.path().join("adapter.safetensors");
    std::fs::write(&lora_path, &lora_buf)?;

    modelc::lora::apply_lora(&mut model, &lora_path, 1.0)?;

    // I8 tensor should remain unchanged.
    assert_eq!(model.tensors["weight"].dtype, DataType::I8);
    Ok(())
}