switchyard-translation 0.3.0

Pure Rust request, response, and stream translation for Switchyard
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Shared streaming codec contracts, registry, and stream state.

use std::collections::BTreeMap;
use std::sync::Arc;

use serde::{Deserialize, Serialize};
use serde_json::{Map, Value, json};

use crate::LlmResponseChunk;
use crate::codecs::anthropic::AnthropicMessagesStreamCodec;
use crate::codecs::openai_chat::OpenAiChatStreamCodec;
use crate::codecs::responses::OpenAiResponsesStreamCodec;
use crate::engine::{FormatRegistry, TranslationEngine};
use crate::error::{Result, TranslationError};
use crate::format::{FormatId, WireFormat};
use crate::llm::Usage;

/// Mutable state accumulated while translating one streaming response.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct StreamTranslationState {
    pub source: Option<FormatId>,
    pub target: Option<FormatId>,
    /// Model name observed on the source provider stream.
    pub model: Option<String>,
    /// Message/response ID observed on the source provider stream.
    pub message_id: Option<String>,
    /// Model that served the call, exposed to the client in place of the id the
    /// source stream reported. `None` falls back to [`Self::model`].
    pub target_model: Option<String>,
    /// Optional message/response ID the target stream should expose to the client.
    pub target_message_id: Option<String>,
    pub saw_message_start: bool,
    pub emitted_message_start: bool,
    pub finished: bool,
    /// Set once an in-band error event was emitted; the encoder then emits nothing further.
    pub errored: bool,
    pub usage: Usage,

    pub(crate) output_tokens_seen: u64,
    pub(crate) saw_backend_usage: bool,
    pub(crate) stop_reason: Option<String>,
    /// Refusal metadata carried by the source, replayed instead of a synthesized
    /// object so a named policy category and its explanation are not flattened away.
    pub(crate) stop_details: Option<Value>,
    pub(crate) emitted_message_delta: bool,

    pub(crate) next_content_index: usize,
    pub(crate) text_block_index: Option<usize>,
    pub(crate) text_block_started: bool,
    pub(crate) emitted_content_block: bool,
    pub(crate) tool_states: BTreeMap<usize, StreamToolState>,
    #[serde(default)]
    pub(crate) pending_chat_tool_names: BTreeMap<usize, String>,
    #[serde(default)]
    pub(crate) active_anthropic_tool: Option<usize>,
    #[serde(default)]
    pub(crate) deferred_anthropic_tools: Vec<usize>,
    /// Empty initial inputs remain placeholders until their blocks close without arguments.
    #[serde(default)]
    pub(crate) empty_anthropic_tool_inputs: std::collections::BTreeSet<usize>,
    /// Reasoning text observed while DECODING, per output index, so a completed item
    /// that repeats already-streamed text is not decoded twice.
    pub(crate) decoded_reasoning: BTreeMap<usize, String>,
    /// Output indexes whose encrypted reasoning payload was already decoded.
    pub(crate) decoded_reasoning_encrypted: std::collections::BTreeSet<usize>,
    /// Set once a tool call was observed while DECODING, so a terminal event that names no
    /// stop reason can still report tool use.
    pub(crate) decoded_tool_call: bool,
    /// Text decoded from Responses, keyed by output index and then content index.
    #[serde(default)]
    pub(crate) decoded_response_text: BTreeMap<usize, BTreeMap<usize, String>>,

    pub(crate) response_created: bool,
    pub(crate) response_text_started: bool,
    pub(crate) response_text_output_index: Option<usize>,
    pub(crate) response_text: String,
    /// Reasoning items being ENCODED, keyed by source content index. A response can carry
    /// several reasoning items (GPT-5 emits one ahead of each tool call); each becomes its
    /// own output item so none of their encrypted payloads is lost.
    pub(crate) response_reasoning: BTreeMap<usize, ResponseReasoningState>,
    /// Source indexes whose provider reasoning item id was already announced while DECODING.
    pub(crate) decoded_reasoning_ids: std::collections::BTreeSet<usize>,
    pub(crate) next_response_output_index: usize,
    pub(crate) response_sequence_number: u64,
    pub(crate) next_chat_tool_index: usize,

    pub(crate) reasoning_block_index: Option<usize>,
    pub(crate) reasoning_block_started: bool,
}

// One Responses reasoning output item under construction by the encoder.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub(crate) struct ResponseReasoningState {
    /// Set once the item's `added` event was emitted.
    pub(crate) started: bool,
    /// Set once the item's summary part opened, which happens on the first text delta. An
    /// encrypted-only item never opens one, so it never has to close one either.
    pub(crate) summary_started: bool,
    pub(crate) output_index: Option<usize>,
    /// Provider item id the encrypted reasoning was issued under. Used as the emitted item id
    /// so the client's replay verifies upstream; `None` falls back to a synthesized id.
    pub(crate) item_id: Option<String>,
    pub(crate) text: String,
    /// Opaque `encrypted_content` carried by a Responses reasoning item. Kept verbatim so
    /// the emitted item stays replayable by the client even when no plaintext streamed.
    pub(crate) encrypted: Option<String>,
    #[serde(default)]
    pub(crate) anthropic_signature: Option<String>,
}

// Tracks an in-progress streamed tool call across provider-specific deltas.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub(crate) struct StreamToolState {
    pub(crate) id: Option<String>,
    pub(crate) name: Option<String>,
    pub(crate) arguments: String,
    /// Arguments observed while DECODING the source stream.
    ///
    /// Separate from `arguments`, which encoders accumulate. A decoder that
    /// deduplicates against `arguments` only works when one state performs
    /// both halves of the translation; when a caller buffers a stream with its
    /// own state and encodes later, the field is empty and the duplicate is
    /// emitted.
    pub(crate) decoded_arguments: String,
    #[serde(default)]
    pub(crate) has_decoded_identity: bool,
    pub(crate) pending_arguments: String,
    pub(crate) started: bool,
    pub(crate) content_index: Option<usize>,
    /// Position in the OpenAI Chat `tool_calls` array, assigned when ENCODING. Chat numbers
    /// tool calls on their own, while Anthropic and Responses index the whole content array,
    /// so the source index cannot be reused.
    pub(crate) chat_tool_index: Option<usize>,
    pub(crate) response_output_index: Option<usize>,
    pub(crate) response_item_id: Option<String>,
}

impl StreamTranslationState {
    /// Creates stream state with source and target formats already attached.
    pub fn new(source: impl Into<FormatId>, target: impl Into<FormatId>) -> Self {
        Self {
            source: Some(source.into()),
            target: Some(target.into()),
            ..Self::default()
        }
    }
}

/// Registry-backed streaming translator.
#[derive(Default)]
pub struct StreamTranslationEngine {
    engine: TranslationEngine,
}

/// Codec contract for one provider streaming event format.
pub trait StreamCodec: Send + Sync {
    /// Returns the stream format handled by this codec.
    fn format(&self) -> FormatId;

    /// Decodes one provider event into zero or more neutral events.
    fn decode_event(
        &self,
        state: &mut StreamTranslationState,
        event: &Value,
    ) -> Vec<LlmResponseChunk>;

    /// Encodes one neutral event into zero or more provider events.
    fn encode_event(
        &self,
        state: &mut StreamTranslationState,
        event: LlmResponseChunk,
    ) -> Vec<Value>;

    /// Advances encoder state after an exact same-format event replay.
    ///
    /// Exact replay returns the preserved provider JSON instead of the JSON emitted by
    /// [`Self::encode_event`]. The encoder must nevertheless observe the normalized chunks so
    /// [`Self::finish`] can close an incomplete stream without duplicating an already replayed
    /// terminal event. Codecs whose terminal state cannot be inferred from `MessageStop` alone
    /// may override this hook and inspect `raw`.
    fn observe_replayed_event(
        &self,
        state: &mut StreamTranslationState,
        _raw: &Value,
        normalized: Vec<LlmResponseChunk>,
    ) {
        let replayed_terminal = normalized
            .iter()
            .any(|chunk| matches!(chunk, LlmResponseChunk::MessageStop { .. }));
        for chunk in normalized {
            drop(self.encode_event(state, chunk));
        }
        if replayed_terminal {
            state.finished = true;
        }
    }

    /// Emits any terminal provider events needed after the source stream ends.
    ///
    /// This is intentionally required on every codec. Some target formats
    /// need explicit terminal events after the source closes (for example,
    /// Anthropic ``message_delta``/``message_stop`` or Responses
    /// ``response.completed``). Formats that have no source-close work should
    /// return an empty vector explicitly so the no-op behavior is a conscious
    /// codec-level choice.
    fn finish(&self, state: &mut StreamTranslationState) -> Vec<Value>;
}

/// Registry mapping stream wire formats to stream codecs.
#[derive(Default)]
pub struct StreamCodecRegistry {
    codecs: BTreeMap<FormatId, Arc<dyn StreamCodec>>,
}

impl StreamCodecRegistry {
    /// Creates an empty stream codec registry.
    pub fn new() -> Self {
        Self::default()
    }

    /// Creates a registry populated with built-in stream codecs.
    pub fn with_builtins() -> Self {
        let mut registry = Self::new();
        registry.register(OpenAiChatStreamCodec);
        registry.register(AnthropicMessagesStreamCodec);
        registry.register(OpenAiResponsesStreamCodec);
        registry
    }

    /// Registers or replaces a stream codec for its declared format.
    pub fn register(&mut self, codec: impl StreamCodec + 'static) {
        self.codecs.insert(codec.format(), Arc::new(codec));
    }

    /// Looks up a stream codec by format identifier.
    pub fn codec(&self, format: impl Into<FormatId>) -> Result<Arc<dyn StreamCodec>> {
        let format = format.into();
        self.codecs.get(&format).cloned().ok_or_else(|| {
            TranslationError::Other(format!("no stream codec registered for {format}"))
        })
    }
}

impl StreamTranslationEngine {
    /// Creates a streaming engine from an explicit codec registry.
    pub fn new(registry: StreamCodecRegistry) -> Self {
        Self {
            engine: TranslationEngine::with_registries(FormatRegistry::with_builtins(), registry),
        }
    }

    /// Translates one source provider event into target provider events.
    pub fn translate_event(
        &self,
        state: &mut StreamTranslationState,
        source: impl Into<FormatId>,
        target: impl Into<FormatId>,
        event: &Value,
    ) -> Result<Vec<Value>> {
        self.engine.translate_event(state, source, target, event)
    }

    /// Finishes target-provider stream emission after the source stream closes.
    pub fn finish(
        &self,
        state: &mut StreamTranslationState,
        target: impl Into<FormatId>,
    ) -> Result<Vec<Value>> {
        self.engine.finish_stream(state, target)
    }

    /// Convenience helper using built-in codecs and error events instead of `Result`.
    pub fn translate_event_with_builtins(
        state: &mut StreamTranslationState,
        source: impl Into<FormatId>,
        target: impl Into<FormatId>,
        event: &Value,
    ) -> Vec<Value> {
        Self::default()
            .translate_event(state, source, target, event)
            .unwrap_or_else(|error| vec![json!({"error": {"message": error.to_string()}})])
    }
}

/// Decodes one provider stream event with the built-in codec registry.
pub fn decode_stream_event(
    state: &mut StreamTranslationState,
    source: impl Into<FormatId>,
    event: &Value,
) -> Vec<LlmResponseChunk> {
    let source = source.into();
    StreamCodecRegistry::with_builtins()
        .codec(source)
        .map(|codec| codec.decode_event(state, event))
        .unwrap_or_else(|error| {
            // A missing codec is a translation-side failure, not something the
            // upstream sent, so it decodes to `DecodeError` rather than `StreamError`.
            vec![LlmResponseChunk::DecodeError {
                message: error.to_string(),
            }]
        })
}

pub(crate) fn encode_response_stream_event(
    state: &mut StreamTranslationState,
    target_codec: &dyn StreamCodec,
    target: &FormatId,
    event: crate::LlmResponseStreamEvent,
) -> Vec<Value> {
    if state.errored {
        return Vec::new();
    }
    let (preservation, normalized) = event.into_parts();
    if let Some(preservation) = preservation {
        let (source, raw) = preservation.into_parts();
        if let Err(error) = super::responses::validate_stream_output(&source, target, &raw) {
            return target_codec.encode_event(
                state,
                LlmResponseChunk::DecodeError {
                    message: error.to_string(),
                },
            );
        }
        // Invalid protocol data must become an error frame, not be replayed as ordinary data.
        let has_decode_error = normalized
            .iter()
            .any(|chunk| matches!(chunk, LlmResponseChunk::DecodeError { .. }));
        if &source == target && !has_decode_error {
            // Exact replay bypasses the target encoder's emitted JSON, but the encoder must
            // still observe every normalized chunk. Otherwise `finish` starts from empty state:
            // a clean EOF after a nonterminal provider event can omit or synthesize malformed
            // terminal events, while a replayed terminal can be emitted twice.
            target_codec.observe_replayed_event(state, &raw, normalized);
            return vec![raw];
        }
    }

    normalized
        .into_iter()
        .flat_map(|chunk| target_codec.encode_event(state, chunk))
        .collect()
}

/// Encodes one neutral stream event with the built-in codec registry.
pub fn encode_stream_event(
    state: &mut StreamTranslationState,
    target: impl Into<FormatId>,
    event: LlmResponseChunk,
) -> Vec<Value> {
    StreamCodecRegistry::with_builtins()
        .codec(target)
        .map(|codec| codec.encode_event(state, event))
        .unwrap_or_else(|error| vec![json!({"error": {"message": error.to_string()}})])
}

// Records source-provider identity carried by decoded stream events.
pub(crate) fn record_source_identity(
    state: &mut StreamTranslationState,
    id: Option<String>,
    model: Option<String>,
) {
    if id.is_some() {
        state.message_id = id;
    }
    if model.is_some() {
        state.model = model;
    }
}

// Returns the served model when supplied, otherwise the upstream model.
pub(crate) fn target_model_or_source_model(state: &StreamTranslationState) -> String {
    state
        .target_model
        .clone()
        .or_else(|| state.model.clone())
        .unwrap_or_else(|| "unknown".to_string())
}

// Returns the target/client ID when supplied, otherwise the upstream ID.
pub(crate) fn target_message_id_or_source_message_id(
    state: &StreamTranslationState,
) -> Option<&str> {
    state
        .target_message_id
        .as_deref()
        .or(state.message_id.as_deref())
}

// Checks whether the current source format matches a built-in format.
pub(crate) fn state_source_is(state: &StreamTranslationState, format: WireFormat) -> bool {
    let format_id: FormatId = format.into();
    match &state.source {
        Some(source) => source == &format_id,
        None => false,
    }
}

// Reads a non-empty string field from an event object.
pub(crate) fn string_field(object: &Map<String, Value>, key: &str) -> Option<String> {
    object
        .get(key)
        .and_then(Value::as_str)
        .filter(|value| !value.is_empty())
        .map(ToOwned::to_owned)
}