ferrox-server 0.14.0

OpenAI-compatible HTTP server for the Ferrox inference engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

//! What the worker counts as it runs, and the snapshot `/metrics` and
//! `/admin/stats` read it through.

use std::sync::atomic::{AtomicU64, AtomicUsize};

/// Counters the worker keeps as it runs, exposed through
/// `ContinuousBatcher::stats` so prefill/decode interleaving is
/// *observable* rather than merely intended.
#[derive(Default)]
pub(super) struct Counters {
    pub(super) prefill_chunks: AtomicU64,
    pub(super) prefill_tokens: AtomicU64,
    pub(super) decode_steps: AtomicU64,
    /// High-water mark of KV blocks actually held by in-flight work.
    ///
    /// Deliberately measured from the rows and prefills themselves
    /// rather than derived from `BlockBudget::free`: a ledger-derived
    /// peak cannot exceed the budget however broken admission is, so it
    /// would be a gauge that reports the invariant instead of checking
    /// it.
    pub(super) peak_blocks: AtomicUsize,
}

/// A snapshot of the worker's counters.
#[derive(Debug, Clone, Copy, Default, serde::Serialize)]
pub struct BatcherStats {
    /// `PrefillState::step_chunk` calls the worker has made.
    pub prefill_chunks: u64,
    /// Prompt tokens run through prefill.
    pub prefill_tokens: u64,
    /// Batched decode steps (one per tick that had an active row,
    /// regardless of how many rows that step covered).
    pub decode_steps: u64,
    /// Jobs currently waiting for admission.
    pub queue_depth: usize,
    /// Jobs refused because the queue was full.
    pub queue_rejected: u64,
    /// Total KV blocks in the budget; 0 when none is configured.
    pub kv_blocks_total: usize,
    /// KV blocks not currently reserved by an in-flight request.
    pub kv_blocks_free: usize,
    /// Token positions per KV block.
    pub kv_block_size: usize,
    /// Jobs refused because they exceed the whole KV block budget.
    /// Distinct from `queue_rejected`, which is momentary pressure.
    pub kv_rejected_too_large: u64,
    /// Jobs refused for asking for more context than one request may
    /// have. Distinct again: the fix is in the request, not the box.
    pub kv_rejected_context_length: u64,
    /// Most KV blocks ever held by in-flight work at one moment,
    /// counted from the rows themselves.
    pub kv_blocks_peak: usize,
    /// Requests the scheduler actually stopped because they were
    /// cancelled.
    pub aborted: u64,
}