met 1.0.0-beta-preview

Metrale Engine: pure Rust LLM inference server (HTTP API, scheduler, TUI, CLI).
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

# scenario mixed_prefill
bind_gpu_to_thread()
create_stream() -> 1
create_event() -> 2
adapter_id_for(-1)
adapter_id_for(-1)
num_free_blocks() -> 1000
-- tick 1
alloc_sequence_for(budget=13) -> slot 0
adapter_id_for(-1)
acquire_adapter_slot(-1)
ep_broadcast_cmd_for_seq(seq=0, cmd=0xfffffff0)
ep_broadcast_cmd(0x4)
ep_broadcast_cmd(0x0)
ep_broadcast_cmd(0x4)
ep_broadcast_tokens(n=4)
ep_sync_vision_embeds(n=4)
prefill_chunk(start=0, len=4, last=true, s1@0:len=4, stream=1)
record_event(event=2, stream=1)
stream_wait_event(stream=0, event=2)
argmax_on_device(row=0, stream=0) -> 10
fx launch Decode{n=1, ctx=Unified, readback=Argmax}
decode_batch(tokens=[10], seqs=["s1@0:len=4"], stream=0)
commit_ctx(s1@0:len=5, n=1, base=4, row=0)
argmax_batch(row=0, n=1, stream=0) -> [11]
fx apply MarconiCheckpoint
decode_marconi_checkpoint(s1@0:len=5)
num_free_blocks() -> 999
-- tick 2
fx launch Decode{n=1, ctx=Unified, readback=Argmax}
decode_batch(tokens=[11], seqs=["s1@0:len=5"], stream=0)
commit_ctx(s1@0:len=6, n=1, base=5, row=0)
argmax_batch(row=0, n=1, stream=0) -> [12]
fx apply MarconiCheckpoint
decode_marconi_checkpoint(s1@0:len=6)
adapter_id_for(-1)
adapter_id_for(-1)
num_free_blocks() -> 999
-- tick 3
alloc_sequence_for(budget=24) -> slot 1
adapter_id_for(-1)
acquire_adapter_slot(-1)
ep_broadcast_cmd_for_seq(seq=1, cmd=0xfffffff0)
ep_broadcast_cmd(0x8)
ep_broadcast_cmd(0x0)
ep_broadcast_cmd(0x14)
ep_broadcast_tokens(n=20)
ep_sync_vision_embeds(n=20)
prefill_chunk(start=0, len=8, last=false, s2@1:len=8, stream=1)
record_event(event=2, stream=1)
stream_wait_event(stream=0, event=2)
poll_innerq()
mixed_forward(decode=[12], prefill=s2@1:len=16 start=8 len=8 last=false, stream=1)
record_event(event=2, stream=1)
stream_wait_event(stream=0, event=2)
argmax_batch(row=0, n=1, stream=0) -> [13]
fx apply MarconiCheckpoint
decode_marconi_checkpoint(s1@0:len=7)
num_free_blocks() -> 997
-- tick 4
poll_innerq()
mixed_forward(decode=[13], prefill=s2@1:len=20 start=16 len=4 last=true, stream=1)
record_event(event=2, stream=1)
stream_wait_event(stream=0, event=2)
argmax_on_device(row=1, stream=0) -> 20
record_event(event=2, stream=1)
stream_wait_event(stream=0, event=2)
argmax_batch(row=0, n=1, stream=0) -> [14]
fx apply MarconiCheckpoint
decode_marconi_checkpoint(s1@0:len=8)
num_free_blocks() -> 997
-- tick 5
fx launch Decode{n=2, ctx=Unified, readback=Argmax}
decode_batch(tokens=[14, 20], seqs=["s1@0:len=8", "s2@1:len=20"], stream=0)
commit_ctx(s1@0:len=9, n=1, base=8, row=0)
commit_ctx(s2@1:len=21, n=1, base=20, row=1)
argmax_batch(row=0, n=2, stream=0) -> [15, 21]
fx apply MarconiCheckpoint
decode_marconi_checkpoint(s1@0:len=9)
fx apply MarconiCheckpoint
decode_marconi_checkpoint(s2@1:len=21)
num_free_blocks() -> 997
-- tick 6
fx launch Decode{n=2, ctx=Unified, readback=Argmax}
decode_batch(tokens=[15, 21], seqs=["s1@0:len=9", "s2@1:len=21"], stream=0)
commit_ctx(s1@0:len=10, n=1, base=9, row=0)
commit_ctx(s2@1:len=22, n=1, base=21, row=1)
argmax_batch(row=0, n=2, stream=0) -> [16, 22]
fx apply MarconiCheckpoint
decode_marconi_checkpoint(s1@0:len=10)
fx apply MarconiCheckpoint
decode_marconi_checkpoint(s2@1:len=22)
num_free_blocks() -> 997
-- tick 7
fx launch Decode{n=2, ctx=Unified, readback=Argmax}
decode_batch(tokens=[16, 22], seqs=["s1@0:len=10", "s2@1:len=22"], stream=0)
commit_ctx(s1@0:len=11, n=1, base=10, row=0)
commit_ctx(s2@1:len=23, n=1, base=22, row=1)
argmax_batch(row=0, n=2, stream=0) -> [17, 63]
fx apply MarconiCheckpoint
decode_marconi_checkpoint(s1@0:len=11)
fx apply ReleaseSeq{cache=true, finish_sequence}
cache_sequence(s2@1:len=23)
free_sequence(s2@1:len=23, blocks=2)
ep_broadcast_cmd_for_seq(seq=1, cmd=0xfffffff1)
num_free_blocks() -> 999
-- tick 8
fx launch Decode{n=1, ctx=Unified, readback=Argmax}
decode_batch(tokens=[17], seqs=["s1@0:len=11"], stream=0)
commit_ctx(s1@0:len=12, n=1, base=11, row=0)
argmax_batch(row=0, n=1, stream=0) -> [63]
fx apply ReleaseSeq{cache=true, finish_sequence}
cache_sequence(s1@0:len=12)
free_sequence(s1@0:len=12, blocks=1)
ep_broadcast_cmd_for_seq(seq=0, cmd=0xfffffff1)
num_free_blocks() -> 1000
-- tick 9
fx apply EpShutdown
ep_broadcast_cmd_for_seq(seq=0, cmd=0xffffffff)
fx apply Quiesce{stream=0}
synchronize(stream=0)
fx apply Quiesce{stream=1}
synchronize(stream=1)
fx teardown
teardown()
out s1: T10 T11 T12 T13 T14 T15 T16 T17 Done(stop, n=9, think=0, cached=0, acc=0, guard=None)
out s2: T20 T21 T22 Done(stop, n=4, think=0, cached=0, acc=0, guard=None)