met 1.0.0-beta-preview

Metrale Engine: pure Rust LLM inference server (HTTP API, scheduler, TUI, CLI).
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

// SPDX-License-Identifier: MIT OR Apache-2.0

//! 2026-09-25: Start this tick's admitted requests, continue in-progress
//! prefills, and the idle-tick requeue resume.
//!
//! Owner: scheduler.
//! Invariants: none beyond the types.

use super::*;

impl SchedulerCore {
    pub(super) fn start_prefills(&mut self, new_reqs: Vec<InferenceRequest>) -> LaneVerdict {
        let Self {
            ctx: sched,
            eos_tokens,
            grammar_engine,
            active,
            prefilling,
            ..
        } = self;
        let chunked = self.chunked;
        let always_mixed = self.always_mixed;
        let max_prefill_tokens = self.max_prefill_tokens;
        let max_batch_tokens = self.max_batch_tokens;
        let prefill_stream = self.prefill_stream;
        let prefill_event = self.prefill_event;
        let spontaneous_think_budget = self.spontaneous_think_budget;
        let think_end_token = self.think_end_token;
        let think_start_token = self.think_start_token;
        let tool_call_start_token = self.tool_call_start_token;
        let tool_call_end_token = self.tool_call_end_token;
        let t_loop = sched.io.clock.now();
        start_new_requests(
            sched.io.dev.model(),
            sched,
            new_reqs,
            chunked,
            always_mixed,
            max_prefill_tokens,
            max_batch_tokens,
            eos_tokens,
            prefill_stream,
            prefill_event,
            grammar_engine,
            spontaneous_think_budget,
            think_end_token,
            think_start_token,
            tool_call_start_token,
            tool_call_end_token,
            active,
            prefilling,
        );
        sched.io.tel.mark(mtp_timing::Phase::LoopAdmit, t_loop);

        LaneVerdict::Proceed
    }

    pub(super) fn continue_prefills(&mut self) -> LaneVerdict {
        let Self {
            ctx: sched,
            policy,
            active,
            prefilling,
            preempted,
            did_mixed_step,
            ..
        } = self;
        let max_prefill_tokens = self.max_prefill_tokens;
        let max_batch_tokens = self.max_batch_tokens;
        let always_mixed = self.always_mixed;
        let prefill_stream = self.prefill_stream;
        let prefill_event = self.prefill_event;
        let use_mtp = self.use_mtp;
        let use_self_speculative = self.use_self_speculative;
        let use_ngram_speculative = self.use_ngram_speculative;
        let think_end_token = self.think_end_token;
        let think_start_token = self.think_start_token;
        let code_fence_token = self.code_fence_token;
        let tool_call_start_token = self.tool_call_start_token;
        let tool_call_end_token = self.tool_call_end_token;
        let adaptive_sampling = self.adaptive_sampling;
        let max_batch_size = self.max_batch_size;
        let block_size = self.block_size;
        let t_loop = sched.io.clock.now();
        *did_mixed_step = continue_in_progress_prefills(
            sched.io.dev.model(),
            &**policy,
            active,
            prefilling,
            max_prefill_tokens,
            max_batch_tokens,
            always_mixed,
            prefill_stream,
            prefill_event,
            use_mtp,
            use_self_speculative,
            use_ngram_speculative,
            think_end_token,
            think_start_token,
            code_fence_token,
            tool_call_start_token,
            tool_call_end_token,
            adaptive_sampling,
            sched,
        );
        sched.io.tel.mark(mtp_timing::Phase::LoopPrefill, t_loop);

        if active.is_empty() {
            // 2026-09-25: `SkipRest` below also skips the end-of-tick resume, so give
            // requeued sequences their resume here first.
            if !preempted.is_empty() {
                preempt::resume_preempted_seqs(
                    sched.io.dev.model(),
                    &sched.io,
                    active,
                    preempted,
                    max_batch_size,
                    block_size,
                );
            }
            if active.is_empty() {
                return LaneVerdict::SkipRest;
            }
        }

        LaneVerdict::Proceed
    }
}