lattice-inference 0.2.2

Pure Rust transformer inference engine — safetensors loading, SIMD matmul, BGE/Qwen3 embeddings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

{
  "version": 1,
  "created": "2026-03-28",
  "device": "Apple M2 Max",
  "commit": "e6b2a5c0b",
  "metrics": {
    "gdn_ref_us_per_step": {
      "value": 1856.26,
      "unit": "us",
      "description": "GatedDeltaNet reference step latency"
    },
    "gdn_fused_us_per_step": {
      "value": 1689.67,
      "unit": "us",
      "description": "GatedDeltaNet fused step latency"
    },
    "gdn_fused_speedup": {
      "value": 1.10,
      "unit": "x",
      "description": "Fused / reference speedup ratio"
    },
    "llm_tok_per_sec": {
      "value": 4.18,
      "unit": "tok/s",
      "description": "Qwen3.5-2B decode throughput (greedy, 20 tokens)"
    },
    "llm_ttft_ms": {
      "value": 956.05,
      "unit": "ms",
      "description": "Time to first token (short prompt)"
    },
    "llm_prefill_ms_per_tok": {
      "value": 191.21,
      "unit": "ms/tok",
      "description": "Qwen3.5-2B prefill latency per prompt token"
    },
    "llm_total_ms_20tok": {
      "value": 4780.27,
      "unit": "ms",
      "description": "Total time to generate 20 tokens from short prompt"
    },
    "llm_load_ms": {
      "value": 10072.0,
      "unit": "ms",
      "description": "Model weight loading time"
    },
    "embed_short_ms": {
      "value": 10.50,
      "unit": "ms",
      "description": "Qwen3-0.6B encode 'hello world'"
    },
    "embed_medium_ms": {
      "value": 18.60,
      "unit": "ms",
      "description": "Qwen3-0.6B encode ~250 char text"
    },
    "embed_long_ms": {
      "value": 330.33,
      "unit": "ms",
      "description": "Qwen3-0.6B encode ~4500 char text"
    }
  }
}