met 1.0.0-beta-preview

Metrale Engine: pure Rust LLM inference server (HTTP API, scheduler, TUI, CLI).
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122

# scenario verify_k3
bind_gpu_to_thread()
create_stream() -> 1
create_event() -> 2
adapter_id_for(-1)
adapter_id_for(-1)
num_free_blocks() -> 1000
-- tick 1
alloc_sequence() -> slot 0
adapter_id_for(-1)
acquire_adapter_slot(-1)
ep_broadcast_cmd_for_seq(seq=0, cmd=0xfffffff0)
ep_broadcast_cmd(0x5)
ep_broadcast_cmd(0x0)
ep_broadcast_cmd(0x5)
ep_broadcast_tokens(n=5)
ep_sync_vision_embeds(n=5)
prefill(n=5, s1@0:len=5, stream=0)
argmax_on_device(row=0, stream=0) -> 10
mtp_slot_draft_capacity(0)
sync_secondary()
ep_broadcast_cmd_for_seq(seq=0, cmd=0xa)
decode(token=10, s1@0:len=6, stream=0)
argmax_on_device(row=0, stream=0) -> 11
save_hidden_for_mtp(0, stream=0)
run_mtp_propose_multi(token=11, pos=6, n=2, s1@0:len=6, stream=0, mask=false) -> [12, 13]
start_checkpoint_async(s1@0:len=6)
num_free_blocks() -> 999
-- tick 2
mtp_slot_draft_capacity(0)
sync_secondary()
ep_broadcast_cmd_for_seq(seq=0, cmd=0xfffffff3)
ep_broadcast_cmd(0xb)
ep_broadcast_cmd(0xc)
ep_broadcast_cmd(0xd)
decode_verify_graphed_k3(tokens=[11, 12, 13], s1@0:len=9) -> [12, 13, 14]
ep_broadcast_cmd(0x2)
commit_accepted_prefix(s1@0:len=9, na=3, k=3)
save_hidden_for_mtp(2, stream=0)
trim_proposer_state(s1@0:len=9, na=2, stream=0)
run_mtp_propose_multi(token=14, pos=9, n=2, s1@0:len=9, stream=0, mask=false) -> [15, 16]
num_free_blocks() -> 999
-- tick 3
mtp_slot_draft_capacity(0)
sync_secondary()
ep_broadcast_cmd_for_seq(seq=0, cmd=0xfffffff3)
ep_broadcast_cmd(0xe)
ep_broadcast_cmd(0xf)
ep_broadcast_cmd(0x10)
decode_verify_graphed_k3(tokens=[14, 15, 16], s1@0:len=12) -> [15, 16, 17]
ep_broadcast_cmd(0x2)
commit_accepted_prefix(s1@0:len=12, na=3, k=3)
save_hidden_for_mtp(2, stream=0)
trim_proposer_state(s1@0:len=12, na=2, stream=0)
run_mtp_propose_multi(token=17, pos=12, n=2, s1@0:len=12, stream=0, mask=false) -> [19, 19]
num_free_blocks() -> 999
-- tick 4
mtp_slot_draft_capacity(0)
sync_secondary()
ep_broadcast_cmd_for_seq(seq=0, cmd=0xfffffff3)
ep_broadcast_cmd(0x11)
ep_broadcast_cmd(0x13)
ep_broadcast_cmd(0x13)
decode_verify_graphed_k3(tokens=[17, 19, 19], s1@0:len=15) -> [18, 1, 1]
ep_broadcast_cmd(0x0)
trim_proposer_state(s1@0:len=13, na=0, stream=0)
commit_accepted_prefix(s1@0:len=13, na=1, k=3)
save_hidden_for_mtp(0, stream=0)
run_mtp_propose_multi(token=18, pos=13, n=2, s1@0:len=13, stream=0, mask=false) -> [19, 20]
num_free_blocks() -> 999
-- tick 5
mtp_slot_draft_capacity(0)
sync_secondary()
ep_broadcast_cmd_for_seq(seq=0, cmd=0xfffffff3)
ep_broadcast_cmd(0x12)
ep_broadcast_cmd(0x13)
ep_broadcast_cmd(0x14)
decode_verify_graphed_k3(tokens=[18, 19, 20], s1@0:len=16) -> [19, 20, 21]
ep_broadcast_cmd(0x2)
commit_accepted_prefix(s1@0:len=16, na=3, k=3)
save_hidden_for_mtp(2, stream=0)
trim_proposer_state(s1@0:len=16, na=2, stream=0)
run_mtp_propose_multi(token=21, pos=16, n=2, s1@0:len=16, stream=0, mask=false) -> [23, 23]
num_free_blocks() -> 998
-- tick 6
mtp_slot_draft_capacity(0)
sync_secondary()
ep_broadcast_cmd_for_seq(seq=0, cmd=0xfffffff3)
ep_broadcast_cmd(0x15)
ep_broadcast_cmd(0x17)
ep_broadcast_cmd(0x17)
decode_verify_graphed_k3(tokens=[21, 23, 23], s1@0:len=19) -> [22, 1, 1]
ep_broadcast_cmd(0x0)
trim_proposer_state(s1@0:len=17, na=0, stream=0)
commit_accepted_prefix(s1@0:len=17, na=1, k=3)
save_hidden_for_mtp(0, stream=0)
run_mtp_propose_multi(token=22, pos=17, n=2, s1@0:len=17, stream=0, mask=false) -> [23, 63]
num_free_blocks() -> 998
-- tick 7
mtp_slot_draft_capacity(0)
sync_secondary()
ep_broadcast_cmd_for_seq(seq=0, cmd=0xfffffff3)
ep_broadcast_cmd(0x16)
ep_broadcast_cmd(0x17)
ep_broadcast_cmd(0x3f)
decode_verify_graphed_k3(tokens=[22, 23, 63], s1@0:len=20) -> [23, 63, 63]
ep_broadcast_cmd(0x2)
fx apply ReleaseSeq{cache=true, finish_sequence}
cache_sequence(s1@0:len=20)
free_sequence(s1@0:len=20, blocks=2)
ep_broadcast_cmd_for_seq(seq=0, cmd=0xfffffff1)
num_free_blocks() -> 1000
-- tick 8
fx apply EpShutdown
ep_broadcast_cmd_for_seq(seq=0, cmd=0xffffffff)
fx apply Quiesce{stream=0}
synchronize(stream=0)
fx apply Quiesce{stream=1}
synchronize(stream=1)
fx teardown
teardown()
out s1: T10 T11 T12 T13 T14 T15 T16 T17 T18 T19 T20 T21 T22 T23 Done(stop, n=15, think=0, cached=0, acc=8, guard=None)