Skip to main content

cpa_exec/
codex_duplex.rs

1//! Response steering: a full-duplex Codex WebSocket (codex_websockets_duplex.go).
2//!
3//! With `response-steering` on, a downstream Responses WebSocket whose turn runs on a
4//! Codex credential in WebSocket mode keeps that upstream socket for the rest of the
5//! connection. Once the first `response.created` reached the client, the client's later
6//! frames go straight to the socket: `response.steer` raw, `response.create` and
7//! `response.append` shaped like turns. A response terminal is not a stream terminal:
8//! accepted steering may start an automatic successor or wait for tool results. The
9//! stream ends with the socket, and the socket with the stream. There is no redial,
10//! credential switch, local acknowledgement or replay here; only upstream owns a
11//! steering submission.
12//!
13//! Go runs a writer and a reader goroutine around shared state. Here one task owns the
14//! state and selects fairly between upstream events, client frames and delivery; a
15//! separate writer task owns the socket's writes, so a write blocked in the network never
16//! stops upstream events (Go's writer blocking in `writeCodexWebsocketMessage`). Writer
17//! waits (`waitFor`, queued creates) become state the task re-checks after every step.
18
19use std::collections::{HashMap, VecDeque};
20use std::sync::Arc;
21
22use bytes::Bytes;
23use cpa_core::credential::Credential;
24use cpa_core::exec::{ExecError, ExecRequest, ExecStream, FailureScope};
25use futures_util::{FutureExt, StreamExt};
26use tokio::sync::{OwnedMutexGuard, mpsc};
27
28use super::{Read, Replay, Turn, Upstream, request_frame, transport, ws_error};
29use crate::codex_json::{set_raw, set_str};
30use crate::codex_request::{self as request, Call, Settings, View};
31use crate::codex_response as response;
32
33/// Bound on queued and outstanding `response.create` requests.
34const MAX_PENDING: usize = 16;
35/// Responses whose settings are retained for automatic successors and appends.
36const RETAINED_RESPONSES: usize = 16;
37/// Frames accepted for the writer and not yet written. Client frames are read only while
38/// there is room, so a stalled upstream write backpressures the client as Go's does.
39const WRITE_QUEUE: usize = 16;
40/// Local items (events, rejections) waiting for the consumer. Upstream is read only when
41/// none wait, as Go's reader blocks on its unbuffered `out`.
42const OUTBOX: usize = 16;
43
44type Enabled = dyn Fn(&str) -> bool + Send + Sync;
45
46/// The queue of a downstream connection's client frames, in arrival order. A bounded
47/// channel backpressures the client; the connection's own loop reads it between turns.
48pub type ClientFrames = Arc<tokio::sync::Mutex<mpsc::Receiver<Bytes>>>;
49
50/// A downstream connection's later client frames (Go `WebsocketInput`) and the live
51/// account check for its bound credential (Go `WebsocketAuthCheck`).
52pub struct SteeringInput {
53    frames: ClientFrames,
54    enabled: Box<Enabled>,
55}
56
57impl SteeringInput {
58    /// `frames` is the connection's frame queue, shared with its loop as Go hands both
59    /// the same channel; a duplex turn holds it for the rest of the connection.
60    /// `enabled(credential_id)` says whether the bound credential may still send.
61    pub fn new(frames: ClientFrames, enabled: impl Fn(&str) -> bool + Send + Sync + 'static) -> Self {
62        Self {
63            frames,
64            enabled: Box::new(enabled),
65        }
66    }
67}
68
69/// What the duplex reads of one request's settings (Go `codexWebsocketPrepared`).
70#[derive(Clone, Default)]
71pub(super) struct Prepared {
72    /// Raw `instructions` of the shaped body (Go `clientBody`).
73    instructions: Option<String>,
74    /// Raw `instructions` of the client payload (Go `originalPayload`); retained
75    /// response settings drop it.
76    original_instructions: Option<String>,
77    /// The multi-agent v2 optimization renamed the `collaboration` namespace.
78    optimized: bool,
79    /// The request already used the upstream names (`multiAgentV2Conflict`).
80    conflict: bool,
81    /// Native Codex request: completion output stays as upstream sent it.
82    native: bool,
83    /// The request's reasoning replay scope (`replayScope`).
84    replay: Replay,
85}
86
87impl Prepared {
88    pub(super) fn new(
89        body: &str,
90        original: &[u8],
91        optimized: bool,
92        conflict: bool,
93        native: bool,
94        replay: Replay,
95    ) -> Self {
96        let raw = |json: &str| {
97            let value = gjson::get(json, "instructions");
98            value.exists().then(|| value.json().to_owned())
99        };
100        Self {
101            instructions: raw(body),
102            original_instructions: raw(&String::from_utf8_lossy(original)),
103            optimized,
104            conflict,
105            native,
106            replay,
107        }
108    }
109
110    /// The settings a response retains (Go keeps `reasoning` and `instructions`, never
111    /// request history or headers; `reasoning` only feeds per-response usage here).
112    fn retained(&self) -> Self {
113        Self {
114            original_instructions: None,
115            ..self.clone()
116        }
117    }
118
119    fn instructions(&self) -> Option<&str> {
120        self.instructions.as_deref().or(self.original_instructions.as_deref())
121    }
122}
123
124/// The duplex stream's state: Go's writer and reader goroutines' shared metadata.
125/// An upstream message whose observation is running (Go's reader inside
126/// `EmitWebSocketResponseEvent`): the rest of its handling waits for it.
127struct Observing {
128    observation: futures_util::future::BoxFuture<'static, ()>,
129    payload: String,
130    kind: String,
131    establishing: bool,
132}
133
134pub(super) struct Duplex {
135    turn: Turn,
136    /// While set, no further upstream message is read; client input and the writer
137    /// keep going, as Go's writer goroutine does while the reader waits.
138    observing: Option<Observing>,
139    /// A writer-side failure raised while a message was observed. Go's writer only
140    /// records its error; the reader finishes the observer and its message first, then
141    /// reports it. Client input and writes stop meanwhile.
142    failing: Option<ExecError>,
143    frames: OwnedMutexGuard<mpsc::Receiver<Bytes>>,
144    input: Arc<SteeringInput>,
145    /// The bootstrap request: the template for later creates (Go `req`, `opts`).
146    req: ExecRequest,
147    credential: Credential,
148    proxy: crate::proxy::Proxy,
149    cpa_session: Option<String>,
150    settings: Settings,
151    exec_session: String,
152    initial: Prepared,
153    /// The bootstrap request's client model (Go `initial.originalPayload` `model`).
154    initial_model: String,
155    /// Creates sent upstream and not yet started or rejected, oldest first.
156    pending: VecDeque<Prepared>,
157    /// Parents of steers written and not yet acknowledged.
158    unacknowledged: Vec<String>,
159    /// Accepted steer id to parent response.
160    accepted: HashMap<String, String>,
161    current: Prepared,
162    response_id: String,
163    response_settings: HashMap<String, Prepared>,
164    response_order: VecDeque<String>,
165    /// Settings pinned by in-flight steering, independent of the retention window.
166    steering_settings: HashMap<String, Prepared>,
167    /// The response waiting for required input (`response.steer.pending`).
168    waiting_parent: String,
169    /// Upstream started a successor on its own.
170    automatic_active: bool,
171    /// Client creates waiting until steering settles.
172    queued_creates: VecDeque<String>,
173    /// A steer whose parent settings are unknown waits for pending creates to start.
174    deferred_steer: Option<String>,
175    /// Client frames are read only after the first `response.created` was delivered.
176    input_ready: bool,
177    first_response: bool,
178    response_active: bool,
179    /// Items for the consumer, in order; `out` takes one at a time.
180    outbox: VecDeque<Result<Bytes, ExecError>>,
181    out: mpsc::Sender<Result<Bytes, ExecError>>,
182    /// The writer task's queue and handle; the task ends with the first write error.
183    writes: mpsc::Sender<Outgoing>,
184    writer: tokio::task::JoinHandle<Result<(), ExecError>>,
185    done: bool,
186}
187
188impl Drop for Duplex {
189    /// The socket ends with the stream; the handler already knows (no notify). Closing
190    /// it releases a writer blocked in the network, which then stops (Go joins it).
191    fn drop(&mut self) {
192        self.turn
193            .invalidate(&transport("codex websockets executor: duplex closed"), false);
194        self.writer.abort();
195    }
196}
197
198/// One frame for the writer task.
199struct Outgoing {
200    frame: String,
201    /// A create's multi-agent v2 state `(optimized, conflict)`; steers carry none.
202    namespace: Option<(bool, bool)>,
203}
204
205/// The network half of Go's writer goroutine: frames in order. A create updates the
206/// socket's namespace state only when it reaches the writer (`setMultiAgentV2Optimized`),
207/// so a create still queued behind a blocked write cannot change how the running
208/// response's events are restored. Then the live account check (`WebsocketAuthEnabled`)
209/// right before `writeCodexWebsocketMessage`.
210async fn write_loop(
211    conn: Arc<Upstream>,
212    input: Arc<SteeringInput>,
213    credential: String,
214    wire: crate::codex_capture::Wire,
215    mut queue: mpsc::Receiver<Outgoing>,
216) -> Result<(), ExecError> {
217    while let Some(Outgoing { frame, namespace }) = queue.recv().await {
218        if let Some((optimized, conflict)) = namespace {
219            conn.note_multi_agent(optimized, conflict);
220        }
221        if !(input.enabled)(&credential) {
222            return Err(local("websocket credential is no longer enabled"));
223        }
224        wire.ws_frame(&conn.target.url, frame.as_bytes());
225        conn.send(frame).await?;
226    }
227    Ok(())
228}
229
230/// A failure of the bound socket: request-scoped, so the credential is not cooled
231/// (Go `codexDuplexConnectionError`). Status and body are kept for the handler.
232fn connection(mut error: ExecError) -> ExecError {
233    error.scope = FailureScope::Request;
234    error
235}
236
237fn local(message: &str) -> ExecError {
238    connection(transport(message))
239}
240
241enum Step {
242    Upstream(Option<Read>),
243    Observed,
244    Client(Option<Bytes>),
245    Writer(Result<Result<(), ExecError>, tokio::task::JoinError>),
246}
247
248impl Duplex {
249    /// `streamCodexDuplex`, after the bootstrap `response.create` was written.
250    pub(super) async fn start(
251        turn: Turn,
252        input: Arc<SteeringInput>,
253        initial: Prepared,
254        req: ExecRequest,
255        view: &View<'_>,
256        settings: &Settings,
257        exec_session: &str,
258    ) -> ExecStream {
259        // Turns on a session are serialised, so no other stream holds the frames.
260        let frames = input.frames.clone().lock_owned().await;
261        let initial_model = gjson::get(&String::from_utf8_lossy(&req.original_body), "model")
262            .str()
263            .to_owned();
264        let (writes, queue) = mpsc::channel(WRITE_QUEUE);
265        let writer = tokio::spawn(write_loop(
266            turn.conn.clone(),
267            input.clone(),
268            view.credential.id.clone(),
269            turn.wire.clone(),
270            queue,
271        ));
272        // Go's `out` is unbuffered; one slot lets the task hand over and keep reading.
273        let (out, chunks) = mpsc::channel(1);
274        let duplex = Self {
275            turn,
276            observing: None,
277            failing: None,
278            frames,
279            input,
280            req,
281            credential: view.credential.clone(),
282            proxy: view.proxy.clone(),
283            cpa_session: view.session.clone(),
284            settings: settings.clone(),
285            exec_session: exec_session.to_owned(),
286            current: initial.clone(),
287            pending: VecDeque::from([initial.clone()]),
288            initial,
289            initial_model,
290            unacknowledged: Vec::new(),
291            accepted: HashMap::new(),
292            response_id: String::new(),
293            response_settings: HashMap::new(),
294            response_order: VecDeque::new(),
295            steering_settings: HashMap::new(),
296            waiting_parent: String::new(),
297            automatic_active: false,
298            queued_creates: VecDeque::new(),
299            deferred_steer: None,
300            input_ready: false,
301            first_response: true,
302            response_active: false,
303            outbox: VecDeque::new(),
304            out,
305            writes,
306            writer,
307            done: false,
308        };
309        // The task runs whether or not the consumer polls, as Go's goroutines do, and
310        // ends when the consumer drops the stream.
311        tokio::spawn(duplex.run());
312        futures_util::stream::unfold(chunks, |mut chunks| async move {
313            chunks.recv().await.map(|item| (item, chunks))
314        })
315        .boxed()
316    }
317
318    async fn run(mut self) {
319        self.serve().await;
320        // An observation that already started finishes before the turn (and its
321        // session) is released.
322        if let Some(observing) = self.observing.take() {
323            observing.observation.await;
324        }
325    }
326
327    async fn serve(&mut self) {
328        loop {
329            if self.input_ready && !self.done && self.failing.is_none() {
330                self.progress().await;
331            }
332            let deliver = !self.outbox.is_empty();
333            if self.done && !deliver {
334                return;
335            }
336            // After progress(): a failure it raised stops client input at once.
337            let live = !self.done && self.failing.is_none();
338            let writable = self.writes.capacity() > 0;
339            let read_client =
340                live && self.input_ready && self.deferred_steer.is_none() && writable && self.outbox.len() < OUTBOX;
341            let await_room = live && self.input_ready && !writable && !self.writes.is_closed();
342            let observing = self.observing.is_some();
343            let pending_observation = &mut self.observing;
344            let step = tokio::select! {
345                permit = self.out.reserve(), if deliver => match permit {
346                    Ok(permit) => {
347                        permit.send(self.outbox.pop_front().expect("an item to deliver"));
348                        continue;
349                    }
350                    Err(_) => return,
351                },
352                () = self.out.closed(), if !deliver => return,
353                read = self.turn.rx.recv(), if !self.done && !deliver && !observing => Step::Upstream(read),
354                () = async {
355                    if let Some(observing) = pending_observation.as_mut() {
356                        (&mut observing.observation).await;
357                    }
358                }, if observing => Step::Observed,
359                frame = self.frames.recv(), if read_client => Step::Client(frame),
360                ended = &mut self.writer, if live => Step::Writer(ended),
361                // The writer freed room: re-check progress and client reads.
362                _ = self.writes.reserve(), if await_room => continue,
363            };
364            match step {
365                Step::Upstream(read) => self.upstream(read),
366                Step::Observed => {
367                    let Observing {
368                        payload,
369                        kind,
370                        establishing,
371                        ..
372                    } = self.observing.take().expect("an observation finished");
373                    self.observed(&payload, &kind, establishing);
374                    // The deferred failure follows the message, unless the message
375                    // itself ended the stream.
376                    if let Some(error) = self.failing.take()
377                        && !self.done
378                    {
379                        self.fail(error);
380                    }
381                }
382                Step::Client(frame) => self.client(frame).await,
383                Step::Writer(ended) => self.fail(match ended {
384                    Ok(Err(error)) => error,
385                    Ok(Ok(())) | Err(_) => local("websocket writer stopped"),
386                }),
387            }
388        }
389    }
390
391    /// Go's reader reports the writer's failure in place of its own read error (the
392    /// writer's cancel is what ended the read).
393    fn writer_error(&mut self) -> Option<ExecError> {
394        if self.done || !self.writer.is_finished() {
395            return None;
396        }
397        match (&mut self.writer).now_or_never()? {
398            Ok(Err(error)) => Some(error),
399            Ok(Ok(())) | Err(_) => None,
400        }
401    }
402
403    fn fail(&mut self, error: ExecError) {
404        if self.observing.is_some() {
405            self.failing.get_or_insert(error);
406            return;
407        }
408        self.outbox.push_back(Err(connection(error)));
409        self.done = true;
410    }
411
412    fn enabled(&self) -> bool {
413        (self.input.enabled)(&self.credential.id)
414    }
415
416    /// Go's local `reject`: an `error` event; the socket stays usable.
417    fn reject(&mut self, message: &str) {
418        let message = cpa_common::json::quote(message);
419        let mut payload = br#"{"error":{"message":"#.to_vec();
420        payload.extend_from_slice(&message);
421        payload.extend_from_slice(br#","type":"invalid_request_error"},"status":400,"type":"error"}"#);
422        self.outbox.push_back(Ok(Bytes::from(payload)));
423    }
424
425    /// `readyForCreate`: no unacknowledged steer, no automatic successor running, and
426    /// every accepted steer waits on the response the create continues.
427    fn ready_for_create(&self) -> bool {
428        self.unacknowledged.is_empty()
429            && !self.automatic_active
430            && self.accepted.values().all(|parent| *parent == self.waiting_parent)
431    }
432
433    /// Writer progress after a state change: a deferred steer once pending creates have
434    /// started, then queued creates while steering allows them (`flushPendingCreates`).
435    /// Each step needs room in the write queue (Go's writer is blocked meanwhile).
436    async fn progress(&mut self) {
437        if let Some(steer) = self.deferred_steer.take() {
438            if !self.pending.is_empty() || self.writes.capacity() == 0 {
439                self.deferred_steer = Some(steer);
440                return;
441            }
442            self.steer(steer);
443        }
444        while !self.done
445            && self.failing.is_none()
446            && !self.queued_creates.is_empty()
447            && self.ready_for_create()
448            && self.writes.capacity() > 0
449            && self.outbox.len() < OUTBOX
450        {
451            let payload = self.queued_creates.pop_front().expect("non-empty");
452            self.create(payload).await;
453        }
454    }
455
456    /// Hands a frame to the writer task; callers checked for room.
457    fn write(&mut self, frame: String, namespace: Option<(bool, bool)>) {
458        match self.writes.try_send(Outgoing { frame, namespace }) {
459            Ok(()) => {}
460            // The writer stopped on an error it reports through its handle.
461            Err(mpsc::error::TrySendError::Closed(_)) => {}
462            Err(mpsc::error::TrySendError::Full(_)) => self.fail(local("websocket write queue is full")),
463        }
464    }
465
466    /// One client frame (the writer's input loop).
467    async fn client(&mut self, frame: Option<Bytes>) {
468        let Some(frame) = frame else {
469            // The downstream reader stopped: Go's writer fails with `context.Canceled`,
470            // a connection error that does not cool the credential.
471            self.fail(local("context canceled"));
472            return;
473        };
474        if !self.enabled() {
475            // A fresh connection can select an enabled credential; this frame is neither
476            // sent on a disabled account nor replayed elsewhere.
477            self.fail(local("websocket credential is no longer enabled"));
478            return;
479        }
480        if !cpa_common::json::std_valid(&frame) {
481            self.reject("invalid websocket request JSON");
482            return;
483        }
484        // ponytail: invalid UTF-8 inside JSON strings is replaced; Go forwards the bytes.
485        let payload = String::from_utf8_lossy(&frame).into_owned();
486        let kind = gjson::get(&payload, "type").str().to_owned();
487        match kind.as_str() {
488            "response.steer" => {
489                let parent = gjson::get(&payload, "previous_response_id").str().to_owned();
490                if !self.response_settings.contains_key(&parent) && !self.pending.is_empty() {
491                    // `waitFor(len(pending) == 0)`: its settings may belong to a create
492                    // upstream has not started yet.
493                    self.deferred_steer = Some(payload);
494                    return;
495                }
496                self.steer(payload);
497            }
498            "response.create" | "response.append" => {
499                if self.queued_creates.is_empty() && self.ready_for_create() {
500                    self.create(payload).await;
501                } else if self.queued_creates.len() >= MAX_PENDING {
502                    self.fail(local("too many outstanding response.create requests"));
503                } else {
504                    self.queued_creates.push_back(payload);
505                }
506            }
507            other => self.reject(&format!("unsupported websocket request type: {other}")),
508        }
509    }
510
511    /// A steer goes upstream as the client sent it plus the payload rules: no create
512    /// translation or defaults (SIWC credentials lose `configuration_update` items);
513    /// upstream validates unknown fields and input.
514    fn steer(&mut self, mut payload: String) {
515        if crate::codex_siwc::enabled(&self.credential) {
516            let stripped = cpa_common::thinking::strip_configuration_update_items(payload.as_bytes());
517            if stripped != payload.as_bytes() {
518                payload = String::from_utf8_lossy(&stripped).into_owned();
519            }
520        }
521        let payload = request::steer_rules(&self.req, &self.settings, payload);
522        let parent = gjson::get(&payload, "previous_response_id").str().to_owned();
523        if let Some(settings) = self.response_settings.get(&parent).cloned() {
524            self.steering_settings.insert(parent.clone(), settings);
525        }
526        self.unacknowledged.push(parent);
527        self.write(payload, None);
528    }
529
530    /// `processCreatePayload`.
531    async fn create(&mut self, mut payload: String) {
532        let append = gjson::get(&payload, "type").str() == "response.append";
533        let mut parent = gjson::get(&payload, "previous_response_id").str().trim().to_owned();
534        if append && parent.is_empty() && !self.response_id.is_empty() {
535            parent.clone_from(&self.response_id);
536            payload = set_str(&payload, "previous_response_id", &parent);
537        }
538        if !self.accepted.is_empty() && parent != self.waiting_parent {
539            self.reject("response.create must continue the response waiting for required input");
540            return;
541        }
542        let model = gjson::get(&payload, "model").str().trim().to_owned();
543        if !model.is_empty() && model != self.req.model && model != self.initial_model {
544            self.fail(ExecError::replay_required());
545            return;
546        }
547        if model.is_empty() {
548            let model = if self.req.model.is_empty() {
549                self.initial_model.trim()
550            } else {
551                self.req.model.as_str()
552            };
553            payload = set_str(&payload, "model", model);
554        }
555        if append && !gjson::get(&payload, "instructions").exists() {
556            let target = self.response_settings.get(&parent).unwrap_or(&self.initial);
557            if let Some(instructions) = target.instructions().or(self.initial.instructions()) {
558                payload = set_raw(&payload, "instructions", instructions);
559            }
560        }
561        let mut next = self.req.clone();
562        next.body = Bytes::from(payload);
563        next.original_body = next.body.clone();
564        let view = View {
565            proxy: self.proxy.clone(),
566            session: self.cpa_session.clone(),
567            ..View::new(&self.credential)
568        };
569        let request::Shaped {
570            body,
571            optimized,
572            conflict,
573            finalize,
574        } = match request::shape(&next, &view, &self.settings, Call::Websocket) {
575            Ok(shaped) => shaped,
576            Err(error) => {
577                self.fail(error);
578                return;
579            }
580        };
581        let (body, replay) = Replay::apply(&self.initial.replay.cache, &next, body).await;
582        // ponytail: Go starts a usage record per response; this stream reports every
583        // response's events to the bootstrap attempt's record and keeps its request.
584        let (body, _) = request::prompt_cache(&next, body, Some(&self.exec_session), true);
585        let body = super::turn_body(&view, &next, &self.settings, &finalize, body);
586        // Go also fails when the URL differs from the bootstrap's; it cannot here, since
587        // both come from the same credential.
588        let prepared = Prepared::new(
589            &body,
590            &next.original_body,
591            optimized,
592            conflict,
593            request::is_native(&next),
594            replay,
595        );
596        if self.pending.len() >= MAX_PENDING {
597            self.fail(local("too many outstanding response.create requests"));
598            return;
599        }
600        self.pending.push_back(prepared);
601        self.write(request_frame(body), Some((optimized, conflict)));
602    }
603
604    /// `restoreMultiAgent` for an event of `prepared`'s request on this socket.
605    fn restores(&self, prepared: &Prepared) -> bool {
606        self.turn.conn.restores(prepared.optimized, prepared.conflict)
607    }
608
609    /// `releaseSteeringSettings`.
610    fn release_steering_settings(&mut self, parent: &str) {
611        let in_flight = self.unacknowledged.iter().any(|p| p == parent) || self.accepted.values().any(|p| p == parent);
612        if !in_flight {
613            self.steering_settings.remove(parent);
614        }
615    }
616
617    /// One upstream frame (the reader goroutine).
618    fn upstream(&mut self, read: Option<Read>) {
619        let text = match read {
620            Some(Read::Text(text)) => text,
621            // Go's duplex wraps the raw read error, not `mapCodexWebsocketReadError`'s 413.
622            Some(Read::Failed(error)) => {
623                let error = self.writer_error().or_else(|| self.turn.loss()).unwrap_or(error);
624                return self.fail(error);
625            }
626            None => {
627                let error = self
628                    .writer_error()
629                    .or_else(|| self.turn.loss())
630                    .unwrap_or_else(|| transport("codex websockets executor: session read channel closed"));
631                return self.fail(error);
632            }
633        };
634        let payload = text.trim();
635        if payload.is_empty() {
636            return;
637        }
638        let kind = gjson::get(payload, "type").str().to_owned();
639        let establishing = self.first_response && kind == "response.created";
640        if kind == "response.created" && !self.response_created(payload) {
641            return;
642        }
643        self.turn.observe(payload);
644        match self.turn.wire.ws_response_observation(payload) {
645            Some(observation) => {
646                self.observing = Some(Observing {
647                    observation,
648                    payload: payload.to_owned(),
649                    kind,
650                    establishing,
651                });
652            }
653            None => self.observed(payload, &kind, establishing),
654        }
655    }
656
657    /// The rest of an upstream message's handling, once observers saw it.
658    fn observed(&mut self, payload: &str, kind: &str, establishing: bool) {
659        // Steering acknowledgements, pending notices and failures are opaque: IDs, input,
660        // sequence numbers and event types reach the client byte for byte.
661        if kind.starts_with("response.steer.") {
662            self.steer_event(kind, payload);
663            self.outbox.push_back(Ok(Bytes::from(payload.to_owned())));
664            return;
665        }
666        let failure = kind == "error" || kind == "response.failed";
667        let cooling = self.turn.cooling;
668        let classify = |payload: &str| {
669            ws_error(payload, cooling).or_else(|| response::terminal_failure(payload, cooling).map(|(e, _)| e))
670        };
671        if !self.first_response && failure {
672            // Account health does not depend on which queued request failed: the stream
673            // ends with the original classification and is not replayed elsewhere.
674            if let Some(error) = classify(payload).filter(|e| matches!(e.status, 401 | 403 | 429)) {
675                self.outbox.push_back(Ok(Bytes::from(payload.to_owned())));
676                self.outbox.push_back(Err(error));
677                self.done = true;
678                return;
679            }
680        }
681        // The request this event belongs to: a rejected pending create, else the running one.
682        let mut rejected = None;
683        if !self.first_response && failure {
684            let mut failed = gjson::get(payload, "response.id").str().to_owned();
685            if failed.is_empty() {
686                failed = gjson::get(payload, "response_id").str().to_owned();
687            }
688            // A failure of the running response must not consume a queued create; a
689            // rejection before `response.created` belongs to the oldest pending create.
690            let current_failure = !failed.is_empty() && failed == self.response_id;
691            let ambiguous = failed.is_empty()
692                && ((!self.pending.is_empty() && self.response_active) || !self.unacknowledged.is_empty());
693            if ambiguous {
694                // Guessing could corrupt either request: keep the event, end the socket.
695                self.outbox.push_back(Ok(Bytes::from(payload.to_owned())));
696                self.fail(local(
697                    "cannot associate websocket failure with a response or pending create",
698                ));
699                return;
700            }
701            match self.pending.pop_front() {
702                Some(create) if !current_failure => rejected = Some(create),
703                popped => {
704                    if let Some(popped) = popped {
705                        self.pending.push_front(popped);
706                    }
707                    self.response_active = false;
708                    self.automatic_active = false;
709                }
710            }
711        }
712        let event = rejected.as_ref().unwrap_or(&self.current);
713        let payload = response::restore(payload, self.restores(event));
714        // Every rejection clears its own request's replay; only the first fails the
715        // stream (and can enter bootstrap retry), later ones are events the client may
716        // correct on this socket.
717        let terminal = classify(&payload);
718        if let Some(error) = &terminal {
719            event.replay.clear(&payload, cooling);
720            self.turn.wire.ws_exec_error("upstream_error", error);
721        }
722        if self.first_response
723            && let Some(error) = terminal
724        {
725            self.outbox.push_back(Err(error));
726            self.done = true;
727            return;
728        }
729        let mut out = payload.into_owned();
730        if kind == "response.output_item.done" {
731            self.turn.items.collect(&out);
732        }
733        if matches!(kind, "response.completed" | "response.done" | "response.incomplete") {
734            self.response_active = false;
735            self.automatic_active = false;
736            out = response::normalize_completion(out);
737            if !self.current.native {
738                out = self.turn.items.patch(out);
739            }
740            if kind != "response.incomplete" {
741                self.current.replay.completed(&out);
742            }
743        }
744        self.outbox
745            .push_back(Ok(Bytes::from(response::ensure_usage_details(out))));
746        if establishing {
747            // Delivered before any locally generated error, so the handler also sees the
748            // bootstrap succeed first.
749            self.input_ready = true;
750        }
751    }
752
753    /// `response.created`: which request it serves and what it retains. False ends the
754    /// stream (an automatic successor without retained parent settings).
755    fn response_created(&mut self, payload: &str) -> bool {
756        let mut parent = gjson::get(payload, "response.previous_response_id").str().to_owned();
757        if parent.is_empty() {
758            parent.clone_from(&self.response_id);
759        }
760        let automatic = !self.first_response && self.pending.is_empty();
761        if automatic {
762            let settings = self
763                .steering_settings
764                .get(&parent)
765                .or_else(|| self.response_settings.get(&parent))
766                .cloned();
767            let Some(settings) = settings else {
768                self.fail(local("automatic successor has no retained parent settings"));
769                return false;
770            };
771            self.current = settings;
772        }
773        self.accepted.retain(|_, target| *target != parent);
774        self.waiting_parent.clear();
775        self.automatic_active = automatic;
776        if let Some(next) = self.pending.pop_front() {
777            self.current = next;
778        }
779        self.response_id = gjson::get(payload, "response.id").str().to_owned();
780        self.response_settings
781            .insert(self.response_id.clone(), self.current.retained());
782        self.response_order.push_back(self.response_id.clone());
783        if self.response_order.len() > RETAINED_RESPONSES
784            && let Some(oldest) = self.response_order.pop_front()
785        {
786            self.response_settings.remove(&oldest);
787        }
788        self.release_steering_settings(&parent);
789        self.first_response = false;
790        self.response_active = true;
791        self.turn.items = Default::default();
792        true
793    }
794
795    /// `response.steer.*`: acknowledgement bookkeeping.
796    fn steer_event(&mut self, kind: &str, payload: &str) {
797        let id = gjson::get(payload, "steer.id").str().to_owned();
798        let mut parent = gjson::get(payload, "steer.previous_response_id").str().to_owned();
799        if parent.is_empty() {
800            parent.clone_from(&self.response_id);
801        }
802        let consume = |unacknowledged: &mut Vec<String>| {
803            if let Some(i) = unacknowledged.iter().position(|t| *t == parent || t.is_empty()) {
804                unacknowledged.remove(i);
805            }
806        };
807        match kind {
808            "response.steer.accepted" => {
809                consume(&mut self.unacknowledged);
810                self.accepted.insert(id, parent);
811            }
812            "response.steer.failed" => {
813                if self.accepted.remove(&id).is_none() {
814                    consume(&mut self.unacknowledged);
815                }
816                self.release_steering_settings(&parent);
817            }
818            // Tool results may already wait in the queue; no automatic successor can start
819            // until an explicit continuation supplies them.
820            "response.steer.pending" => self.waiting_parent = parent,
821            _ => {}
822        }
823    }
824}