infraqueue-model-prefetch 0.1.1

Model prefetcher for INFRAQUEUE AI
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

use serde_json::{Map, Value};

/// Enhances an OpenAI-compatible request body with prefetch/caching hints based on backend.
/// Currently, for llama.cpp HTTP server (commonly http://127.0.0.1:8080), we set `cache_prompt = true`
/// to enable prompt prefill caching across related requests.
pub fn enhance_body_for_prefetch(base_url: Option<&str>, body: &mut Map<String, Value>) {
    if let Some(b) = base_url {
        let lb = b.to_lowercase();
        let looks_like_llamacpp = lb.contains("127.0.0.1:8080")
            || lb.contains("localhost:8080")
            || lb.contains("llama.cpp")
            || lb.contains("llamacpp");
        if looks_like_llamacpp {
            body.entry("cache_prompt".to_string())
                .or_insert(Value::Bool(true));
        }
    }
}