sipp-rs 0.1.0

Unified Rust library for extensible Sipp inference
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

//! Tests the `runtime::inference_runtime::runtime` module in `sipp`.
//!
//! Covers deterministic inference-runtime helpers, state transitions, and error paths while avoiding native model execution unless a test is explicitly ignored.

use std::collections::HashSet;

use crate::engine::protocol::{ModelClass, PoolingType};
use crate::native_bridge::{NativeRuntimeHandle, SamplerHandle};
use crate::runtime::config::{NativeRuntimeConfig, ResolvedRuntimeLimits};
use crate::runtime::inference_runtime::capabilities::RuntimeModelCapabilities;
use crate::runtime::llama::LlamaBatchBuilder;
use crate::runtime::metrics::RuntimeObservabilityMetrics;
use crate::runtime::request::RequestQueue;
use crate::runtime::scheduler::{BatchPlanner, SamplerCacheKey, SharedBatchPlan, SlotScheduler};
use crate::runtime::session::KvCacheManager;

use super::InferenceRuntime;

pub(crate) fn test_runtime(config: NativeRuntimeConfig) -> InferenceRuntime {
    InferenceRuntime {
        config,
        resolved_limits: ResolvedRuntimeLimits::default(),
        capabilities: RuntimeModelCapabilities {
            class: ModelClass::DecoderOnly,
            embedding_dimensions: 1,
            pooling_type: PoolingType::None,
            decoder_start_token: None,
            has_chat_template: false,
            embedding_context: false,
        },
        native_runtime: NativeRuntimeHandle::empty_for_tests(),
        _residency_lease: None,
        last_runtime_observability: RuntimeObservabilityMetrics::default(),
        has_last_runtime_observability: false,
        kv_cache: KvCacheManager::default(),
        request_queue: RequestQueue::new(),
        slot_scheduler: SlotScheduler::default(),
        batch_planner: BatchPlanner,
        shared_batch_builder: LlamaBatchBuilder::default(),
        next_request_id: 1,
        model_fingerprint: 0,
        committed_observability_request_ids: HashSet::new(),
        scratch_decode_ready_slots: Vec::new(),
        scratch_prefill_ready_slots: Vec::new(),
        scratch_logits_contributions: Vec::new(),
        scratch_embedding_read_slots: Vec::new(),
        scratch_encoder_slots: Vec::new(),
        scratch_plan: SharedBatchPlan::default(),
        scratch_token_piece: Vec::new(),
        total_decode_ms: 0.0,
        total_prefill_ms: 0.0,
        total_input_tokens: 0,
        total_output_tokens: 0,
        total_cache_hits: 0,
        total_prefill_tokens: 0,
        sampler_pool: std::collections::HashMap::<SamplerCacheKey, Vec<SamplerHandle>>::new(),
        resident_backend_samplers: std::collections::HashMap::new(),
    }
}