dwctl 8.75.1

The Doubleword Control Layer - A self-hostable observability and analytics platform for LLM applications
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
//! Anthropic Messages request -> OpenAI Chat Completions request (JSON).

use serde_json::{Value, json};

use super::model::{Content, ContentBlock, ImageSource, InputMessage, MessagesRequest, System, Tool, ToolResultContent};
use crate::inference::translation::TranslationError;

/// Translate an Anthropic Messages request into a Chat Completions request body. `cache_enabled`
/// gates emission of the top-level automatic-caching marker (see below).
pub fn to_chat_completions(req: MessagesRequest, cache_enabled: bool) -> Result<Value, TranslationError> {
    let mut messages: Vec<Value> = Vec::new();

    if let Some(system) = &req.system
        && let Some(msg) = system_to_message(system)
    {
        messages.push(msg);
    }

    for m in &req.messages {
        convert_message(m, &mut messages)?;
    }

    let mut out = serde_json::Map::new();
    out.insert("model".into(), json!(req.model));
    out.insert("max_tokens".into(), json!(req.max_tokens));
    out.insert("messages".into(), Value::Array(messages));

    if req.stream {
        out.insert("stream".into(), json!(true));
        // Ask for a usage row on the final chunk so the response carries usage.
        out.insert("stream_options".into(), json!({ "include_usage": true }));
    }
    if let Some(t) = req.temperature {
        out.insert("temperature".into(), json!(t));
    }
    if let Some(p) = req.top_p {
        out.insert("top_p".into(), json!(p));
    }
    if let Some(k) = req.top_k {
        // OpenAI has no standard `top_k`; forwarded as an additive field that
        // top_k-aware backends (vLLM / sglang) honour and others ignore. Dropping
        // it would silently change sampling for clients that set it.
        out.insert("top_k".into(), json!(k));
    }
    if let Some(s) = &req.stop_sequences {
        out.insert("stop".into(), json!(s));
    }
    if let Some(tools) = &req.tools
        && !tools.is_empty()
    {
        out.insert("tools".into(), Value::Array(tools.iter().map(tool_to_openai).collect()));
    }
    if let Some(tc) = &req.tool_choice
        && let Some(v) = tool_choice_to_openai(tc)
    {
        out.insert("tool_choice".into(), v);
    }
    if let Some(effort) = thinking_to_reasoning_effort(req.thinking.as_ref()) {
        out.insert("reasoning_effort".into(), json!(effort));
    }
    // Only forward `service_tier` to engage dwctl's flex tier. Anthropic's
    // standard values (`auto`, `standard_only`) describe priority-vs-standard,
    // which dwctl routing ignores - and `standard_only` is not a valid OpenAI
    // value, so forwarding it could 400 a downstream provider. `flex` is a
    // dwctl-specific opt-in that the inference middleware routes to handle_flex.
    if req.service_tier.as_deref() == Some("flex") {
        out.insert("service_tier".into(), json!("flex"));
    }
    // Top-level automatic-caching marker: an internal signal the cache middleware reads (to
    // synthesize a breakpoint on the last block) AND strips before the upstream call. Emit it ONLY
    // when caching is enabled — that middleware is wired on the same `cache.enabled` flag, so with
    // caching off it wouldn't be there to strip the field, and emitting an unknown top-level field
    // would leak to OpenAI-compatible upstreams (which may reject it) for no benefit. A `null` is
    // "no marker". Explicit block-level `cache_control` is already preserved by the content-part
    // converters (and behaves the same as today when caching is off).
    if cache_enabled
        && let Some(cc) = &req.cache_control
        && !cc.is_null()
    {
        out.insert("cache_control".into(), cc.clone());
    }

    Ok(Value::Object(out))
}

/// Anthropic `thinking` config -> OpenAI `reasoning_effort` bucket. Only enabled
/// thinking maps; absent or disabled leaves the backend at its default.
fn thinking_to_reasoning_effort(thinking: Option<&Value>) -> Option<&'static str> {
    let t = thinking?;
    if t.get("type").and_then(Value::as_str) != Some("enabled") {
        return None;
    }
    let budget = t.get("budget_tokens").and_then(Value::as_u64).unwrap_or(0);
    Some(if budget <= 2048 {
        "low"
    } else if budget <= 8192 {
        "medium"
    } else {
        "high"
    })
}

/// Anthropic top-level `system` -> a leading OpenAI system message. Text blocks
/// keep `cache_control` by emitting array-form content parts.
fn system_to_message(system: &System) -> Option<Value> {
    match system {
        System::Text(t) if !t.is_empty() => Some(json!({ "role": "system", "content": t })),
        System::Text(_) => None,
        System::Blocks(blocks) => {
            let parts: Vec<Value> = blocks.iter().filter_map(text_block_to_part).collect();
            if parts.is_empty() {
                None
            } else {
                Some(json!({ "role": "system", "content": parts }))
            }
        }
    }
}

fn convert_message(m: &InputMessage, out: &mut Vec<Value>) -> Result<(), TranslationError> {
    match m.role.as_str() {
        "assistant" => out.push(convert_assistant(&m.content)),
        "user" => convert_user(&m.content, out),
        other => return Err(TranslationError::BadRequest(format!("unsupported message role: {other}"))),
    }
    Ok(())
}

/// Assistant turn: text -> `content`, `tool_use` blocks -> `tool_calls`.
///
/// A `cache_control` marker on an assistant text block is PRESERVED: the SDK advances a breakpoint
/// onto recent turns, so dropping it here (as we used to) meant the growing conversation never
/// cached. We emit the marked message as a single-text-part array carrying the marker (the same
/// shape a native OpenAI-ingress request may send; the cache layer reads it, then strips it before
/// the upstream call). A single text part is deliberate: with the marker stripped it hashes
/// identically to the plain-string form, so the prefix stays stable as the marker moves off this
/// message on the next turn — the read chain keeps matching.
fn convert_assistant(content: &Content) -> Value {
    let mut text = String::new();
    let mut tool_calls: Vec<Value> = Vec::new();
    // The last `cache_control` seen on a text block (the breakpoint to preserve).
    let mut marker: Option<Value> = None;

    match content {
        Content::Text(t) => text.push_str(t),
        Content::Blocks(blocks) => {
            for b in blocks {
                match b {
                    ContentBlock::Text { text: t, cache_control } => {
                        text.push_str(t);
                        // An explicit `null` is "no marker" (matching parse/strip) — don't let it
                        // flip the message to array form.
                        if let Some(cc) = cache_control
                            && !cc.is_null()
                        {
                            marker = Some(cc.clone());
                        }
                    }
                    ContentBlock::ToolUse { id, name, input, .. } => {
                        tool_calls.push(json!({
                            "id": id,
                            "type": "function",
                            "function": {
                                "name": name,
                                "arguments": serde_json::to_string(input).unwrap_or_else(|_| "{}".into()),
                            }
                        }));
                    }
                    // Images / tool_results are not expected on an assistant turn.
                    _ => {}
                }
            }
        }
    }

    let mut msg = serde_json::Map::new();
    msg.insert("role".into(), json!("assistant"));
    // OpenAI permits null content when tool_calls are present.
    if text.is_empty() {
        msg.insert("content".into(), Value::Null);
    } else if let Some(cc) = marker {
        msg.insert("content".into(), json!([{ "type": "text", "text": text, "cache_control": cc }]));
    } else {
        msg.insert("content".into(), json!(text));
    }
    if !tool_calls.is_empty() {
        msg.insert("tool_calls".into(), Value::Array(tool_calls));
    }
    Value::Object(msg)
}

/// User turn: `tool_result` blocks -> `tool` messages (emitted first, so they
/// follow the prior assistant `tool_calls`); text/image -> a `user` message.
///
/// Anthropic allows `tool_result` and text/image to coexist in one message;
/// OpenAI does not, so they are split into separate messages here. This is the
/// standard OpenAI conversation shape and has been validated against the target
/// backends.
fn convert_user(content: &Content, out: &mut Vec<Value>) {
    match content {
        Content::Text(t) => out.push(json!({ "role": "user", "content": t })),
        Content::Blocks(blocks) => {
            let mut tool_messages: Vec<Value> = Vec::new();
            let mut user_parts: Vec<Value> = Vec::new();

            for b in blocks {
                match b {
                    ContentBlock::Text { .. } => {
                        if let Some(part) = text_block_to_part(b) {
                            user_parts.push(part);
                        }
                    }
                    ContentBlock::Image { source, cache_control } => {
                        let mut part = json!({ "type": "image_url", "image_url": { "url": image_source_to_url(source) } });
                        if let Some(cc) = cache_control {
                            part["cache_control"] = cc.clone();
                        }
                        user_parts.push(part);
                    }
                    ContentBlock::ToolResult {
                        tool_use_id,
                        content,
                        cache_control,
                        ..
                    } => {
                        // In an agent loop the newest block before a model call is usually the
                        // tool_result, so this is where the SDK's advancing breakpoint most often
                        // lands. Preserve a marker as a single-text-part array (same rationale as
                        // convert_assistant: stripped, it hashes like the plain string, so the read
                        // chain stays stable as the marker advances). Unmarked → plain string, as before.
                        let result_text = tool_result_to_text(content);
                        // An explicit `null` is "no marker" (matching parse/strip) → plain string, as before.
                        let content_val = match cache_control {
                            Some(cc) if !cc.is_null() => json!([{ "type": "text", "text": result_text, "cache_control": cc }]),
                            _ => json!(result_text),
                        };
                        tool_messages.push(json!({
                            "role": "tool",
                            "tool_call_id": tool_use_id,
                            "content": content_val,
                        }));
                    }
                    _ => {}
                }
            }

            out.append(&mut tool_messages);
            if !user_parts.is_empty() {
                out.push(json!({ "role": "user", "content": user_parts }));
            }
        }
    }
}

/// A `text` content block -> an OpenAI `text` content part, preserving any
/// `cache_control` marker. Returns `None` for non-text blocks.
fn text_block_to_part(block: &ContentBlock) -> Option<Value> {
    if let ContentBlock::Text { text, cache_control } = block {
        let mut part = json!({ "type": "text", "text": text });
        if let Some(cc) = cache_control {
            part["cache_control"] = cc.clone();
        }
        Some(part)
    } else {
        None
    }
}

fn image_source_to_url(source: &ImageSource) -> String {
    match source {
        ImageSource::Url { url } => url.clone(),
        // Inline base64 becomes a data URI; the downstream image_normalizer
        // only acts on http(s) URLs and leaves data URIs alone.
        ImageSource::Base64 { media_type, data } => format!("data:{media_type};base64,{data}"),
    }
}

fn tool_result_to_text(content: &Option<ToolResultContent>) -> String {
    match content {
        Some(ToolResultContent::Text(t)) => t.clone(),
        Some(ToolResultContent::Blocks(blocks)) => blocks
            .iter()
            .filter_map(|b| b.get("text").and_then(|t| t.as_str()))
            .collect::<Vec<_>>()
            .join(""),
        None => String::new(),
    }
}

fn tool_to_openai(t: &Tool) -> Value {
    let mut tool = json!({
        "type": "function",
        "function": {
            "name": t.name,
            "description": t.description,
            "parameters": t.input_schema.clone().unwrap_or_else(|| json!({ "type": "object" })),
        }
    });
    // Carry a tool-level cache breakpoint through for the downstream classifier.
    if let Some(cc) = &t.cache_control {
        tool["cache_control"] = cc.clone();
    }
    tool
}

/// Anthropic `tool_choice` -> OpenAI `tool_choice`.
fn tool_choice_to_openai(tc: &Value) -> Option<Value> {
    match tc.get("type").and_then(|v| v.as_str()) {
        Some("auto") => Some(json!("auto")),
        Some("any") => Some(json!("required")),
        Some("tool") => tc
            .get("name")
            .and_then(|n| n.as_str())
            .map(|name| json!({ "type": "function", "function": { "name": name } })),
        _ => None,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    // Translate with caching enabled (the prod configuration); pass `false` explicitly where the
    // cache-disabled behaviour is under test.
    fn translate(v: Value) -> Value {
        to_chat_completions(serde_json::from_value::<MessagesRequest>(v).unwrap(), true).unwrap()
    }

    #[test]
    fn top_level_cache_control_forwarded_for_automatic_caching() {
        // With caching enabled, the automatic-caching marker must survive translation onto the Chat
        // Completions body so the cache layer (which reads it) can synthesize a last-block breakpoint.
        let out = translate(json!({
            "model": "m",
            "max_tokens": 16,
            "cache_control": {"type": "ephemeral", "ttl": "1h"},
            "messages": [{"role": "user", "content": "hi"}]
        }));
        assert_eq!(out["cache_control"], json!({"type": "ephemeral", "ttl": "1h"}));
    }

    #[test]
    fn top_level_cache_control_absent_or_null_not_forwarded() {
        let absent = translate(json!({
            "model": "m", "max_tokens": 16,
            "messages": [{"role": "user", "content": "hi"}]
        }));
        assert!(absent.get("cache_control").is_none(), "no marker → not emitted");

        let null = translate(json!({
            "model": "m", "max_tokens": 16,
            "cache_control": null,
            "messages": [{"role": "user", "content": "hi"}]
        }));
        assert!(null.get("cache_control").is_none(), "null marker → not emitted");
    }

    #[test]
    fn top_level_cache_control_not_emitted_when_caching_disabled() {
        // With caching OFF the middleware isn't in the stack to strip it, so the field must NOT be
        // emitted — otherwise it leaks an unknown top-level field to the upstream.
        let req = serde_json::from_value::<MessagesRequest>(json!({
            "model": "m", "max_tokens": 16,
            "cache_control": {"type": "ephemeral"},
            "messages": [{"role": "user", "content": "hi"}]
        }))
        .unwrap();
        let out = to_chat_completions(req, false).unwrap();
        assert!(out.get("cache_control").is_none(), "caching disabled → top-level marker dropped");
    }

    #[test]
    fn assistant_cache_control_preserved_as_array_content() {
        // An advancing marker on an assistant text block must survive translation (it used to be
        // flattened away), carried on a single text part so the cache layer can read it.
        let out = translate(json!({
            "model": "m", "max_tokens": 16,
            "messages": [
                { "role": "user", "content": "hi" },
                { "role": "assistant", "content": [
                    { "type": "text", "text": "let me think", "cache_control": { "type": "ephemeral" } }
                ]}
            ]
        }));
        let asst = &out["messages"][1];
        assert_eq!(asst["role"], "assistant");
        assert_eq!(asst["content"][0]["type"], "text");
        assert_eq!(asst["content"][0]["text"], "let me think");
        assert_eq!(asst["content"][0]["cache_control"]["type"], "ephemeral");
    }

    #[test]
    fn assistant_without_marker_stays_string() {
        // No marker → byte-identical to today (plain string content), so unmarked turns are untouched.
        let out = translate(json!({
            "model": "m", "max_tokens": 16,
            "messages": [ { "role": "assistant", "content": [ { "type": "text", "text": "plain" } ] } ]
        }));
        assert_eq!(out["messages"][0]["content"], "plain");
    }

    #[test]
    fn tool_result_cache_control_preserved_as_array_content() {
        // The agent-loop case: the marker rides the tool_result (the newest block before a model
        // call). It must survive onto the translated tool message.
        let out = translate(json!({
            "model": "m", "max_tokens": 16,
            "messages": [
                { "role": "assistant", "content": [ { "type": "tool_use", "id": "tu_1", "name": "wx", "input": {} } ] },
                { "role": "user", "content": [
                    { "type": "tool_result", "tool_use_id": "tu_1", "content": "sunny",
                      "cache_control": { "type": "ephemeral", "ttl": "1h" } }
                ]}
            ]
        }));
        let tool = &out["messages"][1];
        assert_eq!(tool["role"], "tool");
        assert_eq!(tool["tool_call_id"], "tu_1");
        assert_eq!(tool["content"][0]["text"], "sunny");
        assert_eq!(tool["content"][0]["cache_control"]["ttl"], "1h");
    }

    #[test]
    fn explicit_null_cache_control_is_not_a_marker() {
        // `cache_control: null` is "no marker" everywhere else (parse/strip) — it must NOT flip the
        // message to array form on assistant or tool_result.
        let out = translate(json!({
            "model": "m", "max_tokens": 16,
            "messages": [
                { "role": "assistant", "content": [ { "type": "text", "text": "hi", "cache_control": null } ] },
                { "role": "user", "content": [
                    { "type": "tool_result", "tool_use_id": "tu_1", "content": "sunny", "cache_control": null }
                ]}
            ]
        }));
        assert_eq!(out["messages"][0]["content"], "hi", "null marker → assistant stays a string");
        assert_eq!(out["messages"][1]["content"], "sunny", "null marker → tool_result stays a string");
    }

    #[test]
    fn tool_result_without_marker_stays_string() {
        let out = translate(json!({
            "model": "m", "max_tokens": 16,
            "messages": [
                { "role": "assistant", "content": [ { "type": "tool_use", "id": "tu_1", "name": "wx", "input": {} } ] },
                { "role": "user", "content": [ { "type": "tool_result", "tool_use_id": "tu_1", "content": "sunny" } ] }
            ]
        }));
        assert_eq!(out["messages"][1]["content"], "sunny", "unmarked tool_result stays a plain string");
    }
}