ferrox-server 0.21.0

OpenAI-compatible HTTP server for the Ferrox inference engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

//! Best-effort JSON-object response mode: mask logits to JSON-safe token
//! pieces during sampling, then validate the final text parses as a JSON
//! object. Not grammar-complete — see docs/API.md.

use axum::http::StatusCode;
use axum::Json;

use crate::ApiError;

/// Characters allowed in a decoded token piece when `json_object` mode is on.
fn json_safe_char(c: char) -> bool {
    matches!(
        c,
        '{' | '}'
            | '['
            | ']'
            | '"'
            | ':'
            | ','
            | '.'
            | '-'
            | '+'
            | ' '
            | '\t'
            | '\n'
            | '\r'
            | '\\'
            | '/'
    ) || c.is_ascii_digit()
        || c.is_ascii_alphabetic()
        || c == '_'
}

pub(crate) fn piece_is_json_safe(piece: &str) -> bool {
    piece.chars().all(json_safe_char)
}

/// Zero logits for vocab entries whose decoded piece is not JSON-safe.
pub(crate) fn mask_logits_for_json(logits: &mut [f32], decode_token: impl Fn(usize) -> String) {
    for (i, s) in logits.iter_mut().enumerate() {
        if !piece_is_json_safe(&decode_token(i)) {
            *s = f32::NEG_INFINITY;
        }
    }
}

pub(crate) fn validate_json_object_output(text: &str) -> Result<(), ApiError> {
    let trimmed = text.trim();
    let value: serde_json::Value = serde_json::from_str(trimmed).map_err(|e| {
        (
            StatusCode::BAD_REQUEST,
            Json(serde_json::json!({
                "error": {
                    "message": format!(
                        "response_format json_object: model output is not valid JSON ({e})"
                    )
                }
            })),
        )
    })?;
    if !value.is_object() {
        return Err((
            StatusCode::BAD_REQUEST,
            Json(serde_json::json!({
                "error": {
                    "message": "response_format json_object: model output must be a JSON object"
                }
            })),
        ));
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn json_safe_chars_allow_object_skeleton() {
        assert!(piece_is_json_safe("{\"key\": 1}"));
        assert!(!piece_is_json_safe("<html>"));
    }
}