shimmy 1.3.3

Lightweight 5MB Ollama alternative with native SafeTensors support. No Python dependencies, 2x faster loading.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78

// Create a test SafeTensors file for testing native loading

use std::fs;
use std::path::Path;

fn create_minimal_safetensors() -> Vec<u8> {
    // Create a minimal valid SafeTensors format
    // SafeTensors format: 8-byte header (length) + JSON metadata + tensor data
    let metadata = r#"{"embed_tokens.weight":{"dtype":"F32","shape":[2,2],"data_offsets":[0,16]}}"#;
    let metadata_bytes = metadata.as_bytes();
    let metadata_len = metadata_bytes.len() as u64;

    let mut data = Vec::new();
    data.extend_from_slice(&metadata_len.to_le_bytes());
    data.extend_from_slice(metadata_bytes);

    // Add tensor data (4 x 4 bytes for 2x2 F32 matrix)
    let tensor_data = [1.0f32, 0.5f32, 0.25f32, 0.125f32];
    for value in tensor_data {
        data.extend_from_slice(&value.to_le_bytes());
    }

    data
}

fn create_config_json() -> String {
    r#"{
  "model_type": "test_model",
  "vocab_size": 1000,
  "hidden_size": 64,
  "num_hidden_layers": 2,
  "max_position_embeddings": 128
}"#
    .to_string()
}

fn create_tokenizer_json() -> String {
    r#"{
  "model": {
    "type": "BPE",
    "vocab": {
      "<s>": 0,
      "</s>": 1,
      "<unk>": 2,
      "hello": 3,
      "world": 4,
      "test": 5,
      " ": 6
    }
  }
}"#
    .to_string()
}

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let test_dir = Path::new("test-safetensors-model");
    fs::create_dir_all(test_dir)?;

    // Create SafeTensors file
    let safetensors_data = create_minimal_safetensors();
    fs::write(test_dir.join("model.safetensors"), safetensors_data)?;
    println!("Created: {}/model.safetensors", test_dir.display());

    // Create config.json
    fs::write(test_dir.join("config.json"), create_config_json())?;
    println!("Created: {}/config.json", test_dir.display());

    // Create tokenizer.json
    fs::write(test_dir.join("tokenizer.json"), create_tokenizer_json())?;
    println!("Created: {}/tokenizer.json", test_dir.display());

    println!("\nTest SafeTensors model created successfully!");
    println!("You can now test with:");
    println!("  cargo run -- probe test-safetensors-model");
    println!("  cargo run -- generate test-safetensors-model --prompt \"Hello world\"");

    Ok(())
}