Skip to main content

tea_kernel/
model_turn.rs

1use std::collections::BTreeMap;
2
3use futures_util::StreamExt;
4use tea_model::{
5    HostedToolCompleted, ModelCompletion, ModelEvent, ModelFailure, ModelFailureCode,
6    ModelProvider, ModelSourceCitation, ModelStreamIndex, ModelStreamValidator, ProviderToolCallId,
7    ToolCallCompleted,
8};
9use tea_protocol::{
10    AgentEvent, CanonicalMessage, ContentBlock, EventDelta, HostedToolActivity, MessageId,
11    RetryClass, RunId, SourceCitation, StopReason, ToolCallId, TurnId,
12};
13
14use crate::approval::add_duration;
15use crate::observe::EventEmitter;
16use crate::retry::ModelRetryPolicy;
17use crate::{KernelError, KernelErrorCode, KernelIdSource, TurnRequestSnapshot};
18
19#[derive(Debug)]
20pub(crate) struct ModelTurnOutput {
21    pub(crate) message: CanonicalMessage,
22    pub(crate) tool_calls: Vec<CompletedToolCall>,
23    pub(crate) completion: ModelCompletion,
24    pub(crate) output_bytes: usize,
25}
26
27#[derive(Debug)]
28pub(crate) struct CompletedToolCall {
29    pub(crate) tool_call_id: ToolCallId,
30    pub(crate) tool_name: String,
31    pub(crate) arguments: serde_json::Value,
32}
33
34#[derive(Debug)]
35enum PendingContent {
36    Text(String),
37    Thinking(String),
38    Tool {
39        tool_call_id: ToolCallId,
40        completed: Option<ToolCallCompleted>,
41    },
42    HostedTool {
43        tool_call_id: ToolCallId,
44        completed: Option<HostedToolCompleted>,
45    },
46    Citation(ModelSourceCitation),
47}
48
49#[allow(clippy::too_many_arguments)]
50#[allow(clippy::too_many_lines)]
51pub(crate) async fn stream_turn(
52    provider: &dyn ModelProvider,
53    snapshot: TurnRequestSnapshot,
54    ids: &dyn KernelIdSource,
55    emitter: &mut EventEmitter<'_>,
56    run_id: RunId,
57    turn_id: TurnId,
58    cancellation: tea_control::CancellationScope,
59    clock: &dyn crate::KernelClock,
60    deadline: tea_protocol::ProtocolTimestamp,
61    output_budget: usize,
62    retry_policy: ModelRetryPolicy,
63) -> Result<ModelTurnOutput, KernelError> {
64    let message_id = ids.next_message_id()?;
65    let hosted_tool_names = snapshot.hosted_tool_names().to_vec();
66    let request = snapshot.into_request();
67    let mut attempt: u32 = 0;
68    loop {
69        let mut stream = provider.stream(request.clone(), cancellation.clone());
70        let mut validator = ModelStreamValidator::new();
71        let mut content = Vec::new();
72        let mut tool_positions = BTreeMap::new();
73        let mut hosted_positions = BTreeMap::new();
74        let mut hosted_provider_ids = BTreeMap::new();
75        let mut completion = None;
76        let mut output_bytes = 0usize;
77        let attempt_failure: Option<ModelFailure> = loop {
78            let event = tokio::select! {
79                biased;
80                () = cancellation.cancelled() => {
81                    return Err(KernelError::new(
82                        KernelErrorCode::Cancelled,
83                        "model request was cancelled",
84                    ));
85                }
86                () = clock.sleep_until(deadline) => {
87                    return Err(KernelError::new(
88                        KernelErrorCode::LimitExceeded,
89                        "run deadline was reached during model streaming",
90                    ));
91                }
92                event = stream.next() => event,
93            };
94            let Some(event) = event else { break None };
95            validator.observe(&event).map_err(|error| {
96                KernelError::new(KernelErrorCode::ModelFailure, error.to_string())
97            })?;
98            match event {
99                ModelEvent::TextDelta(delta) => {
100                    output_bytes = add_output(output_bytes, delta.as_str().len(), output_budget)?;
101                    let index = append_text(&mut content, delta.as_str(), false);
102                    emitter
103                        .emit(
104                            Some(run_id),
105                            Some(turn_id),
106                            AgentEvent::MessageDelta {
107                                message_id,
108                                content_index: to_content_index(index)?,
109                                delta: EventDelta::TextDelta {
110                                    text: delta.as_str().to_owned(),
111                                },
112                            },
113                        )
114                        .await?;
115                }
116                ModelEvent::ThinkingDelta(delta) => {
117                    output_bytes = add_output(output_bytes, delta.as_str().len(), output_budget)?;
118                    let index = append_text(&mut content, delta.as_str(), true);
119                    emitter
120                        .emit(
121                            Some(run_id),
122                            Some(turn_id),
123                            AgentEvent::MessageDelta {
124                                message_id,
125                                content_index: to_content_index(index)?,
126                                delta: EventDelta::ThinkingDelta {
127                                    text: delta.as_str().to_owned(),
128                                },
129                            },
130                        )
131                        .await?;
132                }
133                ModelEvent::ToolCallStarted(call) => {
134                    if hosted_tool_names
135                        .iter()
136                        .any(|name| name.as_str() == call.tool_name())
137                    {
138                        return Err(model_contract(
139                            "provider returned a client tool call that was not projected",
140                        ));
141                    }
142                    let position = content.len();
143                    let tool_call_id = ids.next_tool_call_id()?;
144                    if tool_positions.insert(call.index(), position).is_some() {
145                        return Err(model_contract("model reused a tool-call index"));
146                    }
147                    content.push(PendingContent::Tool {
148                        tool_call_id,
149                        completed: None,
150                    });
151                }
152                ModelEvent::Started(_) | ModelEvent::ToolArgumentsDelta(_) => {}
153                ModelEvent::ToolCallCompleted(call) => {
154                    complete_tool(&mut content, &tool_positions, call)?;
155                }
156                ModelEvent::HostedToolStarted(call) => {
157                    if !hosted_tool_names
158                        .iter()
159                        .any(|name| name.as_str() == call.tool_name())
160                    {
161                        return Err(model_contract(
162                            "provider returned a hosted tool activity that was not projected",
163                        ));
164                    }
165                    let position = content.len();
166                    let tool_call_id = ids.next_tool_call_id()?;
167                    if hosted_positions.insert(call.index(), position).is_some()
168                        || hosted_provider_ids
169                            .insert(call.provider_call_id().clone(), tool_call_id)
170                            .is_some()
171                    {
172                        return Err(model_contract("model reused a hosted tool identity"));
173                    }
174                    content.push(PendingContent::HostedTool {
175                        tool_call_id,
176                        completed: None,
177                    });
178                    emitter
179                        .emit(
180                            Some(run_id),
181                            Some(turn_id),
182                            AgentEvent::HostedToolStarted {
183                                tool_call_id,
184                                tool_name: call.tool_name().to_owned(),
185                            },
186                        )
187                        .await?;
188                }
189                ModelEvent::HostedToolCompleted(call) => {
190                    let block_bytes = hosted_block_bytes(&content, &hosted_positions, &call)?;
191                    output_bytes = add_output(output_bytes, block_bytes, output_budget)?;
192                    let source_count = u32::try_from(call.sources().len())
193                        .map_err(|_| model_contract("hosted source count exceeded event bounds"))?;
194                    let observation = AgentEvent::HostedToolCompleted {
195                        tool_call_id: *hosted_provider_ids
196                            .get(call.provider_call_id())
197                            .ok_or_else(|| {
198                                model_contract("completed hosted tool has no identity")
199                            })?,
200                        tool_name: call.tool_name().to_owned(),
201                        arguments: call.arguments().clone(),
202                        outcome: call.outcome().clone(),
203                        source_count,
204                    };
205                    complete_hosted_tool(&mut content, &hosted_positions, call)?;
206                    emitter
207                        .emit(Some(run_id), Some(turn_id), observation)
208                        .await?;
209                }
210                ModelEvent::SourceCitation(citation) => {
211                    let canonical = canonical_citation(&citation, &hosted_provider_ids)?;
212                    let block = ContentBlock::citation(canonical);
213                    let block_bytes = serde_json::to_vec(&block)
214                        .map_err(|_| model_contract("model citation could not be measured"))?
215                        .len();
216                    output_bytes = add_output(output_bytes, block_bytes, output_budget)?;
217                    content.push(PendingContent::Citation(citation));
218                }
219                ModelEvent::Completed(value) => completion = Some(value),
220                ModelEvent::Failed(failure) => break Some(failure),
221            }
222        };
223        let Some(failure) = attempt_failure else {
224            validator.finish().map_err(|error| {
225                KernelError::new(KernelErrorCode::ModelFailure, error.to_string())
226            })?;
227            let completion =
228                completion.ok_or_else(|| model_contract("model stream has no completion"))?;
229            return finish_turn(message_id, clock.now()?, content, completion, output_bytes);
230        };
231        // Non-retryable failures terminate immediately.
232        if failure.code() == ModelFailureCode::Cancelled || failure.retry() == RetryClass::Never {
233            let code = if failure.code() == ModelFailureCode::Cancelled {
234                KernelErrorCode::Cancelled
235            } else {
236                KernelErrorCode::ModelFailure
237            };
238            if failure.code() == ModelFailureCode::Cancelled {
239                return Err(KernelError::new(code, "model request was cancelled"));
240            }
241            if failure.is_safe_diagnostic() {
242                return Err(KernelError::provider_failure(code, failure.message()));
243            }
244            return Err(KernelError::new(code, "model provider request failed"));
245        }
246        // Retryable failure within the policy.
247        if attempt + 1 >= retry_policy.max_attempts() {
248            if failure.is_safe_diagnostic() {
249                return Err(KernelError::provider_failure(
250                    KernelErrorCode::RetryExhausted,
251                    format!("model retry policy was exhausted: {}", failure.message()),
252                ));
253            }
254            return Err(KernelError::new(
255                KernelErrorCode::RetryExhausted,
256                "model retry policy was exhausted",
257            ));
258        }
259        let retry_attempt = attempt + 1;
260        let max_retries = retry_policy.max_attempts().saturating_sub(1);
261        let delay = retry_policy.delay_for_failure(&failure, retry_attempt);
262        let delay_ms = u64::try_from(delay.as_millis())
263            .map_err(|_| model_contract("model retry delay exceeded event bounds"))?;
264        emitter
265            .emit(
266                Some(run_id),
267                Some(turn_id),
268                AgentEvent::ModelRetryScheduled {
269                    message_id,
270                    attempt: retry_attempt,
271                    max_retries,
272                    delay_ms,
273                },
274            )
275            .await?;
276        let wake = add_duration(clock.now()?, delay)?;
277        tokio::select! {
278            biased;
279            () = cancellation.cancelled() => {
280                return Err(KernelError::new(
281                    KernelErrorCode::Cancelled,
282                    "model request was cancelled during retry backoff",
283                ));
284            }
285            () = clock.sleep_until(wake) => {}
286        }
287        attempt = retry_attempt;
288        emitter
289            .emit(
290                Some(run_id),
291                Some(turn_id),
292                AgentEvent::ModelRetryStarted {
293                    message_id,
294                    attempt,
295                    max_retries,
296                },
297            )
298            .await?;
299    }
300}
301
302fn finish_turn(
303    message_id: MessageId,
304    timestamp: tea_protocol::ProtocolTimestamp,
305    pending: Vec<PendingContent>,
306    completion: ModelCompletion,
307    output_bytes: usize,
308) -> Result<ModelTurnOutput, KernelError> {
309    let hosted_ids = pending
310        .iter()
311        .filter_map(|block| match block {
312            PendingContent::HostedTool {
313                tool_call_id,
314                completed: Some(call),
315            } => Some((call.provider_call_id().clone(), *tool_call_id)),
316            PendingContent::Text(_)
317            | PendingContent::Thinking(_)
318            | PendingContent::Tool { .. }
319            | PendingContent::HostedTool {
320                completed: None, ..
321            }
322            | PendingContent::Citation(_) => None,
323        })
324        .collect::<BTreeMap<_, _>>();
325    let mut content = Vec::with_capacity(pending.len());
326    let mut tool_calls = Vec::new();
327    for block in pending {
328        match block {
329            PendingContent::Text(text) => content.push(ContentBlock::text(text)?),
330            PendingContent::Thinking(text) => content.push(ContentBlock::thinking(text)?),
331            PendingContent::Tool {
332                tool_call_id,
333                completed: Some(call),
334            } => {
335                let tool_name = call.tool_name().to_owned();
336                let arguments = call.arguments().clone();
337                content.push(ContentBlock::tool_call_with_provider_id(
338                    tool_call_id,
339                    call.provider_call_id().as_str(),
340                    tool_name.clone(),
341                    arguments.clone(),
342                )?);
343                tool_calls.push(CompletedToolCall {
344                    tool_call_id,
345                    tool_name,
346                    arguments,
347                });
348            }
349            PendingContent::Tool {
350                completed: None, ..
351            } => return Err(model_contract("model tool call is incomplete")),
352            PendingContent::HostedTool {
353                tool_call_id,
354                completed: Some(call),
355            } => {
356                let activity = HostedToolActivity::new(
357                    tool_call_id,
358                    call.provider_call_id().as_str(),
359                    call.tool_name(),
360                    call.arguments().clone(),
361                    call.outcome().clone(),
362                    call.sources().to_vec(),
363                    call.continuation().cloned(),
364                )?;
365                content.push(ContentBlock::hosted_tool(activity));
366            }
367            PendingContent::HostedTool {
368                completed: None, ..
369            } => return Err(model_contract("model hosted tool activity is incomplete")),
370            PendingContent::Citation(citation) => {
371                content.push(ContentBlock::citation(canonical_citation(
372                    &citation,
373                    &hosted_ids,
374                )?));
375            }
376        }
377    }
378    if content.is_empty() {
379        return Err(model_contract("model completion contains no content"));
380    }
381    let has_tools = !tool_calls.is_empty();
382    if has_tools != matches!(completion.stop_reason(), StopReason::ToolUse) {
383        return Err(model_contract(
384            "model stop reason does not match completed tool calls",
385        ));
386    }
387    let message = CanonicalMessage::assistant(
388        message_id,
389        content,
390        completion.stop_reason().clone(),
391        timestamp,
392    )?;
393    Ok(ModelTurnOutput {
394        message,
395        tool_calls,
396        completion,
397        output_bytes,
398    })
399}
400
401fn append_text(content: &mut Vec<PendingContent>, delta: &str, thinking: bool) -> usize {
402    let last_matches = matches!(
403        content.last(),
404        Some(PendingContent::Thinking(_)) if thinking
405    ) || matches!(content.last(), Some(PendingContent::Text(_)) if !thinking);
406    if last_matches {
407        match content.last_mut() {
408            Some(PendingContent::Text(text) | PendingContent::Thinking(text)) => {
409                text.push_str(delta);
410            }
411            Some(
412                PendingContent::Tool { .. }
413                | PendingContent::HostedTool { .. }
414                | PendingContent::Citation(_),
415            )
416            | None => {}
417        }
418        content.len() - 1
419    } else {
420        content.push(if thinking {
421            PendingContent::Thinking(delta.to_owned())
422        } else {
423            PendingContent::Text(delta.to_owned())
424        });
425        content.len() - 1
426    }
427}
428
429fn complete_tool(
430    content: &mut [PendingContent],
431    positions: &BTreeMap<ModelStreamIndex, usize>,
432    call: ToolCallCompleted,
433) -> Result<(), KernelError> {
434    let position = positions
435        .get(&call.index())
436        .copied()
437        .ok_or_else(|| model_contract("completed tool call has no start"))?;
438    match content.get_mut(position) {
439        Some(PendingContent::Tool { completed, .. }) if completed.is_none() => {
440            *completed = Some(call);
441            Ok(())
442        }
443        Some(
444            PendingContent::Text(_)
445            | PendingContent::Thinking(_)
446            | PendingContent::Tool { .. }
447            | PendingContent::HostedTool { .. }
448            | PendingContent::Citation(_),
449        )
450        | None => Err(model_contract("completed tool call has invalid position")),
451    }
452}
453
454fn complete_hosted_tool(
455    content: &mut [PendingContent],
456    positions: &BTreeMap<ModelStreamIndex, usize>,
457    call: HostedToolCompleted,
458) -> Result<(), KernelError> {
459    let position = positions
460        .get(&call.index())
461        .copied()
462        .ok_or_else(|| model_contract("completed hosted tool has no start"))?;
463    match content.get_mut(position) {
464        Some(PendingContent::HostedTool { completed, .. }) if completed.is_none() => {
465            *completed = Some(call);
466            Ok(())
467        }
468        Some(
469            PendingContent::Text(_)
470            | PendingContent::Thinking(_)
471            | PendingContent::Tool { .. }
472            | PendingContent::HostedTool { .. }
473            | PendingContent::Citation(_),
474        )
475        | None => Err(model_contract("completed hosted tool has invalid position")),
476    }
477}
478
479fn hosted_block_bytes(
480    content: &[PendingContent],
481    positions: &BTreeMap<ModelStreamIndex, usize>,
482    call: &HostedToolCompleted,
483) -> Result<usize, KernelError> {
484    let position = positions
485        .get(&call.index())
486        .copied()
487        .ok_or_else(|| model_contract("completed hosted tool has no start"))?;
488    let Some(PendingContent::HostedTool { tool_call_id, .. }) = content.get(position) else {
489        return Err(model_contract("hosted tool has invalid position"));
490    };
491    let activity = HostedToolActivity::new(
492        *tool_call_id,
493        call.provider_call_id().as_str(),
494        call.tool_name(),
495        call.arguments().clone(),
496        call.outcome().clone(),
497        call.sources().to_vec(),
498        call.continuation().cloned(),
499    )?;
500    serde_json::to_vec(&ContentBlock::hosted_tool(activity))
501        .map(|bytes| bytes.len())
502        .map_err(|_| model_contract("hosted tool activity could not be measured"))
503}
504
505fn canonical_citation(
506    citation: &ModelSourceCitation,
507    hosted_ids: &BTreeMap<ProviderToolCallId, ToolCallId>,
508) -> Result<SourceCitation, KernelError> {
509    let mut canonical = citation.citation().clone();
510    if let Some(provider_call_id) = citation.provider_call_id() {
511        let tool_call_id = hosted_ids
512            .get(provider_call_id)
513            .copied()
514            .ok_or_else(|| model_contract("citation references an unknown hosted tool"))?;
515        canonical = canonical.with_tool_call_id(tool_call_id);
516    }
517    Ok(canonical)
518}
519
520fn add_output(current: usize, delta: usize, output_budget: usize) -> Result<usize, KernelError> {
521    let total = current.checked_add(delta).ok_or_else(|| {
522        KernelError::new(
523            KernelErrorCode::LimitExceeded,
524            "assistant output byte count overflowed",
525        )
526    })?;
527    if total > output_budget {
528        return Err(KernelError::new(
529            KernelErrorCode::LimitExceeded,
530            "assistant output byte limit was reached",
531        ));
532    }
533    Ok(total)
534}
535
536fn to_content_index(value: usize) -> Result<u32, KernelError> {
537    u32::try_from(value).map_err(|_| model_contract("assistant content index is out of range"))
538}
539
540fn model_contract(message: &'static str) -> KernelError {
541    KernelError::new(KernelErrorCode::ModelFailure, message)
542}
543
544impl From<tea_protocol::ContentValidationError> for KernelError {
545    fn from(error: tea_protocol::ContentValidationError) -> Self {
546        Self::new(KernelErrorCode::ModelFailure, error.to_string())
547    }
548}
549
550impl From<tea_protocol::MessageValidationError> for KernelError {
551    fn from(error: tea_protocol::MessageValidationError) -> Self {
552        Self::new(KernelErrorCode::ModelFailure, error.to_string())
553    }
554}
555
556impl From<tea_protocol::ExternalContentError> for KernelError {
557    fn from(error: tea_protocol::ExternalContentError) -> Self {
558        Self::new(KernelErrorCode::ModelFailure, error.to_string())
559    }
560}