sipp-rs 0.1.0

Unified Rust library for extensible Sipp inference
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

use crate::engine::protocol::{EngineEvent, EngineState, EngineStatus, ModelState};
use crate::runtime::InferenceRuntime;

use crate::runtime::numeric::unix_time_ms;

use super::stats::{engine_stats_from_runtime, read_backend_info};
use super::EngineEventSubscribers;

/////////////////////////////////////////////////////////////////////////////////
/// TESTS
/////////////////////////////////////////////////////////////////////////////////

#[cfg(test)]
#[path = "../../tests/engine/driver/events_tests.rs"]
mod events_tests;

/////////////////////////////////////////////////////////////////////////////////
/// SRC
/////////////////////////////////////////////////////////////////////////////////

pub(super) fn build_engine_state_with_status(
    runtime: &InferenceRuntime,
    model_state: &ModelState,
    status: Option<EngineStatus>,
) -> EngineState {
    EngineState {
        status: status.unwrap_or_else(|| default_runtime_status(runtime)),
        model: Some(model_state.clone()),
        backend: read_backend_info(),
        runtime: Some(runtime.resolved_limits.clone()),
        requests: Vec::new(),
        stats: runtime
            .try_get_runtime_observability()
            .map(engine_stats_from_runtime)
            .unwrap_or_default(),
        updated_at_unix_ms: unix_time_ms(),
    }
}

fn default_runtime_status(runtime: &InferenceRuntime) -> EngineStatus {
    if runtime.is_ready() {
        EngineStatus::Ready
    } else {
        EngineStatus::Error
    }
}

pub(super) fn emit_state_event(
    runtime: &InferenceRuntime,
    model_state: &ModelState,
    event_subscribers: &EngineEventSubscribers,
    status: EngineStatus,
) {
    emit_event(
        event_subscribers,
        EngineEvent::State(Box::new(build_engine_state_with_status(
            runtime,
            model_state,
            Some(status),
        ))),
    );
}

pub(super) fn emit_event(event_subscribers: &EngineEventSubscribers, event: EngineEvent) {
    let Ok(mut subscribers) = event_subscribers.lock() else {
        return;
    };
    subscribers.retain(|subscriber| subscriber.send(event.clone()).is_ok());
}