skippy-server 0.76.1

Embedded Skippy staged runtime server
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132

use super::*;
use crate::frontend::generation::EmbeddedFusedFirstDecode;
use std::net::TcpStream;
use std::time::Instant;

pub(super) struct EmbeddedPrefixRestore {
    pub(super) allowed: bool,
    pub(super) chain_cache_restored: bool,
    pub(super) chain_restored_tokens: usize,
    pub(super) chain_cache_stats: StageReplyStats,
    pub(super) fused_first_decode: Option<EmbeddedFusedFirstDecode>,
}

impl StageOpenAiBackend {
    #[allow(clippy::too_many_arguments)]
    pub(super) fn restore_embedded_prefix(
        &self,
        request: &EmbeddedStageZeroGeneration<'_>,
        session_key: &str,
        downstream: &mut TcpStream,
        prefill_tokens: &[i32],
        wire_sampling: Option<WireSamplingConfig>,
        cache_stats: &mut GenerationCacheStats,
    ) -> OpenAiResult<EmbeddedPrefixRestore> {
        let allowed = !request.native_mtp_enabled;
        let started = Instant::now();
        let mut chain_cache_restored = false;
        let mut chain_restored_tokens = 0usize;
        let mut chain_cache_stats = StageReplyStats::default();
        let mut fused_first_decode = None;

        if !allowed && self.kv.is_some() {
            let mut attrs = self.openai_attrs(request.ids);
            attrs.insert(
                "skippy.kv.decision".to_string(),
                json!("bypass_native_mtp_sidecar"),
            );
            attrs.insert(
                "skippy.kv.prompt_token_count".to_string(),
                json!(prefill_tokens.len()),
            );
            self.telemetry
                .emit("stage.openai_kv_lookup_decision", attrs);
        }

        if allowed && request.max_tokens > 0 && request.draft.is_none() {
            let current = *request
                .prompt_token_ids
                .last()
                .expect("checked non-empty prompt");
            if let Some(cached) =
                self.try_restore_embedded_split_exact_replay(request, session_key, downstream)?
            {
                chain_cache_restored = true;
                chain_restored_tokens = request
                    .prompt_token_ids
                    .len()
                    .saturating_add(cached.predicted_tokens.len().saturating_sub(1));
                chain_cache_stats = cached.reply_stats;
                cache_stats.cached_prompt_tokens = saturating_u32(request.prompt_token_ids.len());
                cache_stats.matched_prefix_tokens = saturating_u32(request.prompt_token_ids.len());
                cache_stats.suffix_prefill_tokens = 0;
                cache_stats.status = "hit";
                cache_stats.hit_kind = Some("chain_exact_replay");
                fused_first_decode = Some(cached);
            } else if let Some(cached) = self.try_restore_embedded_split_full_prompt_first_token(
                request,
                session_key,
                downstream,
            )? {
                chain_cache_restored = true;
                chain_restored_tokens = request.prompt_token_ids.len();
                chain_cache_stats = cached.reply_stats;
                cache_stats.cached_prompt_tokens = saturating_u32(request.prompt_token_ids.len());
                cache_stats.matched_prefix_tokens = saturating_u32(request.prompt_token_ids.len());
                cache_stats.suffix_prefill_tokens = 0;
                cache_stats.status = "hit";
                cache_stats.hit_kind = Some("chain_full_prompt_first_token");
                fused_first_decode = Some(cached);
            } else if let Some(fused) = self.try_restore_embedded_split_prefill_and_decode(
                request,
                session_key,
                downstream,
                prefill_tokens,
                current,
                wire_sampling,
            )? {
                chain_cache_restored = true;
                chain_restored_tokens = prefill_tokens.len();
                chain_cache_stats = fused.reply_stats;
                cache_stats.cached_prompt_tokens = saturating_u32(prefill_tokens.len());
                cache_stats.matched_prefix_tokens = saturating_u32(prefill_tokens.len());
                cache_stats.suffix_prefill_tokens = 0;
                cache_stats.status = "hit";
                cache_stats.hit_kind = Some("chain_fused_exact_prefix");
                fused_first_decode = Some(fused);
            }
        }

        if !chain_cache_restored
            && allowed
            && let Some(restore) = self.try_restore_embedded_split_prefill(
                request,
                session_key,
                downstream,
                prefill_tokens,
            )?
        {
            chain_restored_tokens = restore.restored_tokens;
            chain_cache_restored = chain_restored_tokens >= prefill_tokens.len();
            chain_cache_stats = restore.stats;
            cache_stats.cached_prompt_tokens = saturating_u32(chain_restored_tokens);
            cache_stats.matched_prefix_tokens = saturating_u32(chain_restored_tokens);
            cache_stats.suffix_prefill_tokens =
                saturating_u32(prefill_tokens.len().saturating_sub(chain_restored_tokens));
            cache_stats.status = "hit";
            cache_stats.hit_kind = Some("chain_prefix");
        }

        if cache_stats.cached_prompt_tokens > 0 {
            cache_stats.restore_ms = started.elapsed().as_secs_f64() * 1_000.0;
        }

        Ok(EmbeddedPrefixRestore {
            allowed,
            chain_cache_restored,
            chain_restored_tokens,
            chain_cache_stats,
            fused_first_decode,
        })
    }
}