met 1.0.0-beta-preview

Metrale Engine: pure Rust LLM inference server (HTTP API, scheduler, TUI, CLI).
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

// SPDX-License-Identifier: MIT OR Apache-2.0

//! 2026-09-25: Tests that `request_start`, the TTFT origin, is taken before
//! grammar compilation. A parser records when its compile begins; the
//! deferred `start_chunked_prefill` runs twice with the same grammar
//! engine, and no GPU work or sleep is involved.
//!
//! Owner: scheduler.
//! Invariants: none beyond the types.

use std::sync::{Arc, Mutex};
use std::time::Instant;

use crate::api::InferenceRequest;
use crate::api::inference_types::GrammarSpec;
use crate::grammar::{GrammarEngine, GrammarError};
use crate::tool_parser::{
    IncomingToolCall, PromptLevers, ToolCallParser, ToolChoice, ToolDefinition,
};
use metrale_grammar::CompiledGrammar;

struct ObservedParser(Arc<Mutex<Option<Instant>>>);

impl ToolCallParser for ObservedParser {
    fn name(&self) -> &str {
        "observed-grammar"
    }
    fn system_prompt(&self, _: &[ToolDefinition], _: &ToolChoice, _: &PromptLevers) -> String {
        unreachable!("request is already tokenized")
    }
    fn format_tool_calls(&self, _: &[IncomingToolCall]) -> String {
        unreachable!("request is already tokenized")
    }
    fn compile_tool_grammar(
        &self,
        engine: &mut GrammarEngine,
        _: &[ToolDefinition],
        _: bool,
    ) -> Option<Result<CompiledGrammar, GrammarError>> {
        *self.0.lock().unwrap() = Some(Instant::now());
        Some(engine.compile_ebnf("root ::= \"x\"", "root"))
    }
}

fn request(grammar_spec: Option<GrammarSpec>) -> InferenceRequest {
    super::test_support::blocking_request(grammar_spec)
}

#[test]
fn deferred_prefill_clock_includes_cold_and_cached_grammar_preparation() {
    let marker = Arc::new(Mutex::new(None));
    let mut engine = Some(GrammarEngine::new(&["x".to_string()], &[]).unwrap());
    for cached in [false, true] {
        let spec = GrammarSpec::ToolCall {
            tools: vec![],
            parser: Arc::new(ObservedParser(Arc::clone(&marker))),
            use_triggers: true,
        };
        let result = super::prefill_a_step::start_chunked_prefill(
            &super::sched_ctx::SchedCtx::for_test(),
            None,
            None,
            None,
            None,
            &super::lifecycle_tests::StubModel,
            request(Some(spec)),
            &[],
            1,
            0,
            0,
            &mut engine,
            Some(0),
            true,
            None,
            None,
        )
        .unwrap();
        let super::emit_step::StartPrefillResult::InProgress(prefill) = result else {
            panic!("deferred prefill unexpectedly performed inference");
        };
        assert!(
            prefill.grammar_state.is_some(),
            "real grammar must be prepared"
        );
        let compiled_at = marker.lock().unwrap().unwrap();
        assert!(
            prefill.request_start <= compiled_at,
            "TTFT origin excluded grammar preparation (cached={cached})"
        );
    }
}