dynamo-mocker 1.5.0

Mock LLM scheduler and KV manager for testing
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Engine-neutral metrics exposed by the Mocker scheduler boundary.

/// Latest observable state for one Mocker scheduler rank.
#[derive(Clone, Default, Debug, PartialEq)]
pub struct MockerMetrics {
    pub dp_rank: dynamo_kv_router::protocols::DpRank,
    pub active_decode_blocks: u64,
    pub total_blocks: u64,
    pub gpu_cache_usage_perc: f64,
    pub running_requests: u64,
    pub waiting_requests: u64,
    pub vllm_preemptions_total: u64,
    pub sglang_cache_hit_tokens: u64,
    pub sglang_cache_total_tokens: u64,
}

impl MockerMetrics {
    pub fn new(
        dp_rank: dynamo_kv_router::protocols::DpRank,
        active_decode_blocks: u64,
        total_blocks: u64,
    ) -> Self {
        Self::from_parts(dp_rank, active_decode_blocks, total_blocks, 0, 0, 0, 0, 0)
    }

    #[allow(clippy::too_many_arguments)]
    pub fn from_parts(
        dp_rank: dynamo_kv_router::protocols::DpRank,
        active_decode_blocks: u64,
        total_blocks: u64,
        running_requests: u64,
        waiting_requests: u64,
        vllm_preemptions_total: u64,
        sglang_cache_hit_tokens: u64,
        sglang_cache_total_tokens: u64,
    ) -> Self {
        let gpu_cache_usage_perc = if total_blocks == 0 {
            0.0
        } else {
            active_decode_blocks as f64 / total_blocks as f64
        };
        Self {
            dp_rank,
            active_decode_blocks,
            total_blocks,
            gpu_cache_usage_perc,
            running_requests,
            waiting_requests,
            vllm_preemptions_total,
            sglang_cache_hit_tokens,
            sglang_cache_total_tokens,
        }
    }
}