infraqueue_model_prefetch/lib.rs
1use serde_json::{Map, Value};
2
3/// Enhances an OpenAI-compatible request body with prefetch/caching hints based on backend.
4/// Currently, for llama.cpp HTTP server (commonly http://127.0.0.1:8080), we set `cache_prompt = true`
5/// to enable prompt prefill caching across related requests.
6pub fn enhance_body_for_prefetch(base_url: Option<&str>, body: &mut Map<String, Value>) {
7 if let Some(b) = base_url {
8 let lb = b.to_lowercase();
9 let looks_like_llamacpp = lb.contains("127.0.0.1:8080")
10 || lb.contains("localhost:8080")
11 || lb.contains("llama.cpp")
12 || lb.contains("llamacpp");
13 if looks_like_llamacpp {
14 body.entry("cache_prompt".to_string())
15 .or_insert(Value::Bool(true));
16 }
17 }
18}