skippy-server 0.76.1

Embedded Skippy staged runtime server
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

use super::*;

pub(super) use super::{
    admission::{GenerationTokenBudget, GenerationTokenBudgetRequest},
    iteration_scheduler::IterationScheduler,
    prefill::{
        EmbeddedPrefillDrain, PrefillChunkObservation, PrefillChunkPolicy, PrefillChunkPolicyArgs,
        PrefillChunkSchedule,
    },
};
pub(super) use crate::binary_transport::WireCondition;
pub(super) use crate::kv_integration::PrefillKvIdentity;
pub(super) use crate::kv_integration::{KvStageIntegration, proactive_eviction_attrs};
pub(super) use crate::runtime_state::load_runtime;
pub(super) use crate::telemetry::Telemetry;
pub(super) use anyhow::{Context as _, Result, anyhow, bail};
pub(super) use async_trait::async_trait;
pub(super) use axum::{http::StatusCode, response::IntoResponse};
pub(super) use base64::Engine as _;
pub(super) use openai_frontend::{
    AssistantMessage, ChatCompletionChoice, ChatCompletionRequest, ChatCompletionResponse,
    ChatCompletionStream, ChatHookAction, ChatHookOutcome, CompactionConfig, CompletionRequest,
    CompletionResponse, CompletionStream, FinishReason, GuardrailMode, GuardrailPolicy,
    MessageContent, ModelObject, OpenAiBackend, OpenAiError, OpenAiRequestContext, OpenAiResult,
    Usage, apply_chat_hook_outcome,
};
pub(super) use serde_json::{Value, json};
pub(super) use skippy_metrics::attr as attr_key;
pub(super) use skippy_protocol::{
    LoadMode, MessageBase, PeerConfig, SCHEMA_VERSION, StageConfig, StageKvCacheConfig,
    StageKvCacheMode, StageKvCachePayload,
    binary::{LLAMA_TOKEN_NULL, StageReplyStats, WireMessageKind, write_stage_message},
};
pub(super) use skippy_runtime::{
    ChatReasoningFormat, ChatTemplateOptions, GenerationSignalWindow, ModelInfo,
};
pub(super) use std::io::Cursor;
pub(super) use std::{
    env, fs,
    net::SocketAddr,
    path::{Path, PathBuf},
    sync::{
        Arc, Mutex,
        atomic::{AtomicU64, AtomicUsize, Ordering},
    },
    time::Duration,
};
pub(super) use tokio::sync::Semaphore;

mod chat_stream_deltas;
mod draft_runtime;
mod generation;
mod guardrails;
mod multimodal;
mod prefill;
mod prefix_cache;
mod prompting;
mod request;
mod support;
mod wire_messages;