1#![allow(missing_debug_implementations)]
38
39use std::{
40 collections::VecDeque,
41 sync::{Arc, Mutex},
42 time::Duration,
43};
44
45use serde::{Deserialize, Serialize};
46use temporalio_macros::{activities, workflow, workflow_methods};
47use temporalio_sdk::{
48 ActivityOptions, ChildWorkflowOptions, SyncWorkflowContext, WorkflowContext,
49 WorkflowContextView, WorkflowResult,
50 activities::{ActivityContext, ActivityError},
51 error::ApplicationFailure,
52};
53use thiserror::Error;
54use tokio::sync::mpsc;
55
56use tracing::Instrument as _;
57
58use crate::{
59 agent::AgentRegistry,
60 guardrail::{GuardrailVerdict, RunBudget},
61 handoff::{Handoff, HandoffContext},
62 invocation::{Admission, InvocationId, InvocationRegistry, ToolCallResult},
63 provider::{
64 Provider, ProviderLimitStatus, SessionDirective, TurnError, TurnEvent, TurnEventSink,
65 TurnIdentity, TurnOutcome, TurnRequest, TurnUsage,
66 },
67 telemetry,
68 tool::{ToolContext, ToolFailure, ToolPolicy},
69};
70
71#[derive(Debug, Clone, Serialize, Deserialize)]
73pub struct RunInput {
74 pub agent: String,
76 pub prompt: String,
78 pub config: RunConfig,
80 #[serde(default, skip_serializing_if = "Option::is_none")]
82 pub handoff: Option<HandoffContext>,
83}
84
85#[derive(Debug, Clone, Serialize, Deserialize)]
87#[non_exhaustive]
88pub struct RunConfig {
89 pub budget: RunBudget,
91 pub interactive: bool,
94 pub turn_timeout: Duration,
96 pub turn_heartbeat_timeout: Duration,
99 pub turn_max_attempts: u32,
101}
102
103impl Default for RunConfig {
104 fn default() -> Self {
105 Self {
106 budget: RunBudget::default(),
107 interactive: false,
108 turn_timeout: Duration::from_secs(600),
109 turn_heartbeat_timeout: Duration::from_secs(120),
110 turn_max_attempts: 3,
111 }
112 }
113}
114
115impl RunConfig {
118 pub fn with_budget(mut self, budget: RunBudget) -> Self {
120 self.budget = budget;
121 self
122 }
123
124 pub fn with_interactive(mut self, interactive: bool) -> Self {
126 self.interactive = interactive;
127 self
128 }
129
130 pub fn with_turn_timeout(mut self, timeout: Duration) -> Self {
132 self.turn_timeout = timeout;
133 self
134 }
135
136 pub fn with_turn_heartbeat_timeout(mut self, timeout: Duration) -> Self {
138 self.turn_heartbeat_timeout = timeout;
139 self
140 }
141
142 pub fn with_turn_max_attempts(mut self, attempts: u32) -> Self {
144 self.turn_max_attempts = attempts;
145 self
146 }
147}
148
149#[derive(Debug, Clone, Serialize, Deserialize)]
151#[non_exhaustive]
152pub struct TurnRecord {
153 pub turn: u32,
155 pub input: String,
157 pub text: String,
159 pub session_id: String,
161 pub usage: TurnUsage,
163 #[serde(default, skip_serializing_if = "Option::is_none")]
166 pub limit: Option<ProviderLimitStatus>,
167}
168
169#[derive(Debug, Clone, Default, Serialize, Deserialize)]
171#[serde(default)]
172#[non_exhaustive]
173pub struct RunUsage {
174 pub total_cost_usd: f64,
176 pub input_tokens: u64,
178 pub output_tokens: u64,
180 pub cached_input_tokens: u64,
185 pub cache_creation_input_tokens: u64,
187 pub reasoning_output_tokens: u64,
189 pub turns_with_unknown_tokens: u32,
191 pub turns_with_unknown_cost: u32,
193}
194
195impl RunUsage {
196 fn absorb_turn(&mut self, turn: &TurnUsage) {
197 match turn.total_cost_usd {
198 Some(cost) => self.total_cost_usd += cost,
199 None => self.turns_with_unknown_cost += 1,
200 }
201 match (turn.input_tokens, turn.output_tokens) {
202 (Some(input), Some(output)) => {
203 self.input_tokens += input;
204 self.output_tokens += output;
205 }
206 _ => {
207 self.turns_with_unknown_tokens += 1;
211 }
212 }
213 self.cached_input_tokens += turn.cached_input_tokens.unwrap_or(0);
214 self.cache_creation_input_tokens += turn.cache_creation_input_tokens.unwrap_or(0);
215 self.reasoning_output_tokens += turn.reasoning_output_tokens.unwrap_or(0);
216 }
217
218 fn absorb_run(&mut self, child: &Self) {
219 self.total_cost_usd += child.total_cost_usd;
220 self.input_tokens += child.input_tokens;
221 self.output_tokens += child.output_tokens;
222 self.cached_input_tokens += child.cached_input_tokens;
223 self.cache_creation_input_tokens += child.cache_creation_input_tokens;
224 self.reasoning_output_tokens += child.reasoning_output_tokens;
225 self.turns_with_unknown_tokens += child.turns_with_unknown_tokens;
226 self.turns_with_unknown_cost += child.turns_with_unknown_cost;
227 }
228
229 pub fn from_turns<'turn>(turns: impl IntoIterator<Item = &'turn TurnRecord>) -> Self {
234 let mut usage = Self::default();
235 for record in turns {
236 usage.absorb_turn(&record.usage);
237 }
238 usage
239 }
240
241 pub fn total_tokens(&self) -> u64 {
244 self.input_tokens.saturating_add(self.output_tokens)
245 }
246}
247
248#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
250#[non_exhaustive]
251pub enum RunEnd {
252 Completed,
254 ConversationEnded,
256 BudgetExceeded {
259 cap: BudgetCap,
261 },
262 GuardrailBlocked {
264 guardrail: String,
266 reason: String,
268 },
269}
270
271#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
273pub enum BudgetCap {
274 Turns {
276 spent: u32,
278 cap: u32,
280 },
281 TotalTokens {
283 spent: u64,
285 cap: u64,
287 },
288 CostUsd {
290 spent: f64,
292 cap: f64,
294 },
295}
296
297#[derive(Debug, Clone, Serialize, Deserialize)]
299#[non_exhaustive]
300pub struct RunOutput {
301 pub text: String,
303 pub session_id: Option<String>,
306 pub usage: RunUsage,
308 pub turns: u32,
310 pub end: RunEnd,
312}
313
314#[workflow]
317#[derive(Debug, Default)]
318pub struct AgentRun {
319 registry: Arc<AgentRegistry>,
322 pending_messages: VecDeque<String>,
324 conversation_ended: bool,
326 transcript: Vec<TurnRecord>,
328 agent_name: Option<String>,
331 invocations: InvocationRegistry,
333 effective_budget: Option<RunBudget>,
335 delegated: Vec<RunOutput>,
338}
339
340impl AgentRun {
341 pub fn with_registry(registry: Arc<AgentRegistry>) -> Self {
343 Self {
344 registry,
345 ..Self::default()
346 }
347 }
348}
349
350#[workflow_methods]
351impl AgentRun {
352 #[run]
355 pub async fn run(
356 ctx: &mut WorkflowContext<Self>,
357 input: RunInput,
358 ) -> WorkflowResult<RunOutput> {
359 let registry = ctx.state(|state| state.registry.clone());
360 let agent = registry.get(&input.agent).map_err(|error| {
361 temporalio_workflow::WorkflowTermination::failed_application(
362 ApplicationFailure::builder(anyhow::anyhow!(error))
363 .type_name("odori::run::unknown_agent".to_owned())
364 .non_retryable(true)
365 .build(),
366 )
367 })?;
368
369 let effective_budget = input.config.budget.intersect(agent.budget());
370 ctx.state_mut(|state| {
371 state.agent_name = Some(input.agent.clone());
372 state.effective_budget = Some(effective_budget.clone());
373 });
374
375 let mut usage = RunUsage::default();
376 let mut budget_turns = 0_u32;
377 let mut delegated_cursor = 0_usize;
378 let end;
379
380 if let Some((name, reason)) = trip(agent.input_guardrails(), &input.prompt) {
382 return Ok(RunOutput {
383 text: String::new(),
384 session_id: None,
385 usage,
386 turns: 0,
387 end: RunEnd::GuardrailBlocked {
388 guardrail: name,
389 reason,
390 },
391 });
392 }
393
394 let mut next_input = Some(input.prompt.clone());
395 loop {
396 let turn_index = ctx.state(|state| state.transcript.len() as u32);
397
398 if let Some(cap) = pre_turn_budget_cap(&effective_budget, &usage, budget_turns) {
400 end = RunEnd::BudgetExceeded { cap };
401 break;
402 }
403
404 let Some(turn_input) = next_input.take() else {
405 unreachable!("loop continues only with an input queued");
406 };
407
408 let session = match ctx.state(|state| {
411 state
412 .transcript
413 .last()
414 .map(|record| record.session_id.clone())
415 }) {
416 None => SessionDirective::Start,
417 Some(parent) => SessionDirective::ResumeForked { session_id: parent },
418 };
419
420 let activity_input = TurnActivityInput {
421 agent: input.agent.clone(),
422 turn: turn_index,
423 input: turn_input.clone(),
424 session,
425 };
426 let outcome: TurnActivityOutput = ctx
427 .execute_activity(
428 TurnActivities::execute_turn,
429 activity_input,
430 turn_activity_options(&input.config),
431 )
432 .await?;
433
434 usage.absorb_turn(&outcome.usage);
435 budget_turns = budget_turns.saturating_add(1);
436 let delegated = ctx.state(|state| state.delegated[delegated_cursor..].to_vec());
437 delegated_cursor += delegated.len();
438 for child in delegated {
439 usage.absorb_run(&child.usage);
440 budget_turns = budget_turns.saturating_add(child.turns);
441 }
442 let record = TurnRecord {
443 turn: turn_index,
444 input: turn_input,
445 text: outcome.text.clone(),
446 session_id: outcome.session_id.clone(),
447 usage: outcome.usage.clone(),
448 limit: outcome.limit.clone(),
449 };
450 ctx.state_mut(|state| state.transcript.push(record));
451
452 if let Some(cap) = post_turn_budget_cap(&effective_budget, &usage, budget_turns) {
456 end = RunEnd::BudgetExceeded { cap };
457 break;
458 }
459
460 if let Some((name, reason)) = trip(agent.output_guardrails(), &outcome.text) {
462 end = RunEnd::GuardrailBlocked {
463 guardrail: name,
464 reason,
465 };
466 break;
467 }
468
469 if !input.config.interactive {
470 end = RunEnd::Completed;
471 break;
472 }
473
474 ctx.wait_condition(|state| {
478 state.conversation_ended || !state.pending_messages.is_empty()
479 })
480 .await?;
481 next_input = ctx.state_mut(|state| state.pending_messages.pop_front());
482 if next_input.is_none() {
483 end = RunEnd::ConversationEnded;
484 break;
485 }
486 }
487
488 let (text, session_id) = ctx.state(|state| {
489 (
490 state
491 .transcript
492 .last()
493 .map(|record| record.text.clone())
494 .unwrap_or_default(),
495 state
496 .transcript
497 .last()
498 .map(|record| record.session_id.clone()),
499 )
500 });
501 Ok(RunOutput {
502 text,
503 session_id,
504 usage,
505 turns: budget_turns,
506 end,
507 })
508 }
509
510 #[signal]
512 pub fn user_message(&mut self, _ctx: &mut SyncWorkflowContext<Self>, message: String) {
513 self.pending_messages.push_back(message);
514 }
515
516 #[signal]
518 pub fn end_conversation(&mut self, _ctx: &mut SyncWorkflowContext<Self>) {
519 self.conversation_ended = true;
520 }
521
522 #[query]
524 pub fn transcript(&self, _ctx: &WorkflowContextView) -> Vec<TurnRecord> {
525 self.transcript.clone()
526 }
527
528 #[update]
536 pub async fn tool_invoked(
537 ctx: &mut WorkflowContext<Self>,
538 invocation: ToolInvocation,
539 ) -> ToolInvocationReply {
540 if invocation.identity.call_id.is_empty() {
542 return ToolInvocationReply::Rejected(InvocationRejection::InvalidCallId);
543 }
544 let current_turn = ctx.state(|state| state.transcript.len() as u32);
545 if invocation.identity.turn > current_turn {
546 return ToolInvocationReply::Rejected(InvocationRejection::UnknownTurn);
547 }
548 let agent = {
549 let registry = ctx.state(|state| state.registry.clone());
550 let Some(name) = ctx.state(|state| state.agent_name.clone()) else {
551 return ToolInvocationReply::Rejected(InvocationRejection::UnknownTurn);
552 };
553 match registry.get(&name) {
554 Ok(agent) => agent,
555 Err(_) => return ToolInvocationReply::Rejected(InvocationRejection::UnknownTool),
556 }
557 };
558 let tool_policy = agent
559 .tools()
560 .iter()
561 .find(|tool| tool.name() == invocation.tool)
562 .map(|tool| tool.policy().clone());
563 let handoff = agent
564 .handoffs()
565 .iter()
566 .find(|handoff| handoff.tool_name() == invocation.tool)
567 .cloned();
568 if tool_policy.is_none() && handoff.is_none() {
569 return ToolInvocationReply::Rejected(InvocationRejection::UnknownTool);
570 }
571
572 let admission = ctx.state_mut(|state| {
573 state
574 .invocations
575 .admit(&invocation.identity, &invocation.tool)
576 });
577 match admission {
578 Admission::Recorded(result) => ToolInvocationReply::Completed(result),
579 Admission::Fenced => ToolInvocationReply::Rejected(InvocationRejection::Fenced),
580 Admission::AwaitExisting => {
581 let turn = invocation.identity.turn;
582 let call_id = invocation.identity.call_id.clone();
583 if ctx
584 .wait_condition(|state| state.invocations.recorded(turn, &call_id).is_some())
585 .await
586 .is_err()
587 {
588 return ToolInvocationReply::Rejected(InvocationRejection::RunCancelled);
589 }
590 let result = ctx.state(|state| state.invocations.recorded(turn, &call_id).cloned());
591 match result {
592 Some(result) => ToolInvocationReply::Completed(result),
593 None => ToolInvocationReply::Rejected(InvocationRejection::RunCancelled),
594 }
595 }
596 Admission::Execute(ticket) => {
597 let result = match (tool_policy, handoff) {
598 (Some(policy), _) => {
599 let outcome: Result<ToolCallResult, _> = ctx
600 .execute_activity(
601 ToolActivities::execute_tool,
602 ExecuteToolInput {
603 agent: agent.name().to_owned(),
604 tool: invocation.tool.clone(),
605 arguments: invocation.arguments.clone(),
606 identity: invocation.identity.clone(),
607 },
608 tool_activity_options(&policy),
609 )
610 .await;
611 match outcome {
616 Ok(result) => result,
617 Err(error) => {
618 ToolCallResult::error(format!("tool execution failed: {error}"))
619 }
620 }
621 }
622 (None, Some(handoff)) => {
623 execute_handoff(ctx, agent.name(), &handoff, &invocation).await
624 }
625 (None, None) => unreachable!("tool kind validated above"),
626 };
627 ctx.state_mut(|state| state.invocations.complete(ticket, result.clone()));
628 ToolInvocationReply::Completed(result)
629 }
630 }
631 }
632}
633
634async fn execute_handoff(
635 ctx: &mut WorkflowContext<AgentRun>,
636 source_agent: &str,
637 handoff: &Handoff,
638 invocation: &ToolInvocation,
639) -> ToolCallResult {
640 let Some(child_input) = invocation
641 .arguments
642 .get("input")
643 .and_then(serde_json::Value::as_str)
644 .map(str::to_owned)
645 else {
646 return ToolCallResult::error("handoff requires a string `input` argument");
647 };
648
649 let registry = ctx.state(|state| state.registry.clone());
650 if registry.get(handoff.target()).is_err() {
651 return ToolCallResult::error(format!(
652 "handoff target agent {:?} is not registered",
653 handoff.target()
654 ));
655 }
656
657 let info = ctx.info();
658 let workflow_id = info.workflow_id().to_owned();
659 let run_id = info.run_id().to_owned();
660 let effective_budget = ctx
661 .state(|state| state.effective_budget.clone())
662 .unwrap_or_default();
663 let (spent, spent_turns) = ctx.state(parent_recorded_spend);
664 let Some(child_budget) = remaining_budget(&effective_budget, &spent, spent_turns, true) else {
665 return ToolCallResult::error("handoff blocked: parent run budget has no remaining turn");
666 };
667 let context = HandoffContext {
668 source_agent: source_agent.to_owned(),
669 target_agent: handoff.target().to_owned(),
670 input: child_input.clone(),
671 parent_workflow_id: workflow_id.clone(),
672 parent_run_id: run_id,
673 parent_turn: invocation.identity.turn,
674 call_id: invocation.identity.call_id.clone(),
675 };
676 let child_id = handoff_workflow_id(
677 &workflow_id,
678 invocation.identity.turn,
679 &invocation.identity.call_id,
680 );
681 let child_config = RunConfig::default().with_budget(child_budget);
682 let started = match ctx
683 .start_child_workflow(
684 AgentRun::run,
685 RunInput {
686 agent: handoff.target().to_owned(),
687 prompt: child_input,
688 config: child_config,
689 handoff: Some(context),
690 },
691 ChildWorkflowOptions::workflow_id(child_id),
692 )
693 .await
694 {
695 Ok(started) => started,
696 Err(error) => {
697 return ToolCallResult::error(format!("handoff child failed to start: {error}"));
698 }
699 };
700 let output = match started.result().await {
701 Ok(output) => output,
702 Err(error) => {
703 return ToolCallResult::error(format!("handoff child failed: {error}"));
704 }
705 };
706 let result = match &output.end {
707 RunEnd::Completed | RunEnd::ConversationEnded => ToolCallResult::text(output.text.clone()),
708 RunEnd::BudgetExceeded { cap } => {
709 ToolCallResult::error(format!("handoff child budget exceeded: {cap:?}"))
710 }
711 RunEnd::GuardrailBlocked { guardrail, reason } => ToolCallResult::error(format!(
712 "handoff child guardrail {guardrail:?} blocked the request: {reason}"
713 )),
714 };
715 ctx.state_mut(|state| state.delegated.push(output));
716 result
717}
718
719fn handoff_workflow_id(parent: &str, turn: u32, call_id: &str) -> String {
720 let safe_call_id: String = call_id
721 .chars()
722 .map(|character| {
723 if character.is_ascii_alphanumeric() || matches!(character, '-' | '_') {
724 character
725 } else {
726 '_'
727 }
728 })
729 .take(80)
730 .collect();
731 format!("{parent}-handoff-{turn}-{safe_call_id}")
732}
733
734fn parent_recorded_spend(state: &AgentRun) -> (RunUsage, u32) {
735 let mut usage = RunUsage::default();
736 for turn in &state.transcript {
737 usage.absorb_turn(&turn.usage);
738 }
739 let mut turns = state.transcript.len() as u32;
740 for child in &state.delegated {
741 usage.absorb_run(&child.usage);
742 turns = turns.saturating_add(child.turns);
743 }
744 (usage, turns)
745}
746
747fn remaining_budget(
748 budget: &RunBudget,
749 spent: &RunUsage,
750 spent_turns: u32,
751 reserve_parent_turn: bool,
752) -> Option<RunBudget> {
753 let reserved = u32::from(reserve_parent_turn);
754 let max_turns = match budget.max_turns {
755 Some(cap) => {
756 let remaining = cap.saturating_sub(spent_turns.saturating_add(reserved));
757 if remaining == 0 {
758 return None;
759 }
760 Some(remaining)
761 }
762 None => None,
763 };
764 Some(RunBudget {
765 max_turns,
766 max_total_tokens: budget
767 .max_total_tokens
768 .map(|cap| cap.saturating_sub(spent.total_tokens())),
769 max_cost_usd: budget
770 .max_cost_usd
771 .map(|cap| (cap - spent.total_cost_usd).max(0.0)),
772 })
773}
774
775fn pre_turn_budget_cap(budget: &RunBudget, usage: &RunUsage, turns: u32) -> Option<BudgetCap> {
776 if let Some(cap) = budget.max_turns
777 && turns >= cap
778 {
779 return Some(BudgetCap::Turns { spent: turns, cap });
780 }
781 if let Some(cap) = budget.max_total_tokens
782 && usage.total_tokens() >= cap
783 {
784 return Some(BudgetCap::TotalTokens {
785 spent: usage.total_tokens(),
786 cap,
787 });
788 }
789 if let Some(cap) = budget.max_cost_usd
790 && usage.total_cost_usd >= cap
791 {
792 return Some(BudgetCap::CostUsd {
793 spent: usage.total_cost_usd,
794 cap,
795 });
796 }
797 None
798}
799
800fn post_turn_budget_cap(budget: &RunBudget, usage: &RunUsage, turns: u32) -> Option<BudgetCap> {
801 if let Some(cap) = budget.max_turns
802 && turns > cap
803 {
804 return Some(BudgetCap::Turns { spent: turns, cap });
805 }
806 if let Some(cap) = budget.max_total_tokens
807 && usage.total_tokens() > cap
808 {
809 return Some(BudgetCap::TotalTokens {
810 spent: usage.total_tokens(),
811 cap,
812 });
813 }
814 if let Some(cap) = budget.max_cost_usd
815 && usage.total_cost_usd > cap
816 {
817 return Some(BudgetCap::CostUsd {
818 spent: usage.total_cost_usd,
819 cap,
820 });
821 }
822 None
823}
824
825fn turn_activity_options(config: &RunConfig) -> ActivityOptions {
826 ActivityOptions::with_start_to_close_timeout(config.turn_timeout)
827 .heartbeat_timeout(config.turn_heartbeat_timeout)
828 .retry_policy(
829 temporalio_common::RetryPolicy::builder()
830 .maximum_attempts(config.turn_max_attempts)
831 .build(),
832 )
833 .build()
834}
835
836fn tool_activity_options(policy: &ToolPolicy) -> ActivityOptions {
837 let close_timeouts = match policy.schedule_to_close {
838 Some(schedule_to_close) => {
839 temporalio_common::ActivityCloseTimeouts::ScheduleAndStartToClose {
840 schedule_to_close,
841 start_to_close: policy.start_to_close,
842 }
843 }
844 None => temporalio_common::ActivityCloseTimeouts::StartToClose(policy.start_to_close),
845 };
846 ActivityOptions::with_close_timeouts(close_timeouts)
847 .retry_policy(
848 temporalio_common::RetryPolicy::builder()
849 .maximum_attempts(policy.max_attempts.unwrap_or(3))
850 .build(),
851 )
852 .maybe_heartbeat_timeout(policy.heartbeat_timeout)
853 .build()
854}
855
856fn trip(
857 guardrails: &[Arc<dyn crate::guardrail::Guardrail>],
858 text: &str,
859) -> Option<(String, String)> {
860 for guardrail in guardrails {
861 if let GuardrailVerdict::Block { reason } = guardrail.check(text) {
862 return Some((guardrail.name().to_owned(), reason));
863 }
864 }
865 None
866}
867
868#[derive(Debug, Clone, Serialize, Deserialize)]
870pub struct TurnActivityInput {
871 pub agent: String,
873 pub turn: u32,
875 pub input: String,
877 pub session: SessionDirective,
879}
880
881#[derive(Debug, Clone, Serialize, Deserialize)]
884pub struct TurnActivityOutput {
885 pub session_id: String,
887 pub text: String,
889 pub usage: TurnUsage,
891 #[serde(default, skip_serializing_if = "Option::is_none")]
894 pub limit: Option<ProviderLimitStatus>,
895}
896
897impl From<TurnOutcome> for TurnActivityOutput {
898 fn from(outcome: TurnOutcome) -> Self {
899 Self {
900 session_id: outcome.session_id,
901 text: outcome.text,
902 usage: outcome.usage,
903 limit: None,
904 }
905 }
906}
907
908#[derive(Debug, Clone, Default, Serialize, Deserialize)]
911#[serde(default)]
912pub struct TurnHeartbeat {
913 pub session_id: Option<String>,
915 pub usage: TurnUsage,
919}
920
921#[derive(Debug, Clone)]
924pub struct Providers {
925 default: Arc<dyn Provider>,
926 by_name: std::collections::HashMap<String, Arc<dyn Provider>>,
927}
928
929impl Providers {
930 pub fn new(default: Arc<dyn Provider>) -> Self {
932 let mut by_name = std::collections::HashMap::new();
933 by_name.insert(default.name().to_owned(), default.clone());
934 Self { default, by_name }
935 }
936
937 pub fn with(mut self, provider: Arc<dyn Provider>) -> Self {
939 self.by_name.insert(provider.name().to_owned(), provider);
940 self
941 }
942
943 fn resolve(&self, name: Option<&str>) -> Result<Arc<dyn Provider>, UnknownProvider> {
944 match name {
945 None => Ok(self.default.clone()),
946 Some(name) => self
947 .by_name
948 .get(name)
949 .cloned()
950 .ok_or_else(|| UnknownProvider {
951 name: name.to_owned(),
952 }),
953 }
954 }
955}
956
957#[derive(Debug, Clone, Error)]
959#[error("no provider registered under the name {name:?}")]
960pub struct UnknownProvider {
961 pub name: String,
963}
964
965#[derive(Debug)]
968pub struct TurnActivities {
969 registry: Arc<AgentRegistry>,
970 providers: Providers,
971 attachments: Option<Arc<dyn crate::provider::AttachmentSource>>,
972}
973
974impl TurnActivities {
975 pub fn new(registry: Arc<AgentRegistry>, providers: Providers) -> Self {
977 Self {
978 registry,
979 providers,
980 attachments: None,
981 }
982 }
983
984 pub fn with_attachments(
987 mut self,
988 attachments: Arc<dyn crate::provider::AttachmentSource>,
989 ) -> Self {
990 self.attachments = Some(attachments);
991 self
992 }
993
994 pub fn registry_handle(&self) -> Arc<AgentRegistry> {
997 self.registry.clone()
998 }
999}
1000
1001#[activities]
1002impl TurnActivities {
1003 #[activity]
1011 pub async fn execute_turn(
1012 self: Arc<Self>,
1013 ctx: ActivityContext,
1014 input: TurnActivityInput,
1015 ) -> Result<TurnActivityOutput, ActivityError> {
1016 let agent = self
1017 .registry
1018 .get(&input.agent)
1019 .map_err(|error| terminal_failure("odori::run::unknown_agent", &error))?;
1020 let provider = self
1021 .providers
1022 .resolve(agent.provider())
1023 .map_err(|error| terminal_failure("odori::run::unknown_provider", &error))?;
1024
1025 let info = ctx.info();
1026 let attempt = info.attempt;
1027 let prior = if attempt > 1 {
1028 ctx.heartbeat_details()
1029 .deserialize::<TurnHeartbeat>()
1030 .ok()
1031 .flatten()
1032 .unwrap_or_default()
1033 } else {
1034 TurnHeartbeat::default()
1035 };
1036 let mut session = input.session;
1037 if attempt > 1
1038 && matches!(session, SessionDirective::Start)
1039 && let Some(session_id) = prior.session_id.clone()
1040 {
1041 session = SessionDirective::ResumeForked { session_id };
1042 }
1043
1044 let identity = TurnIdentity {
1045 run_id: info.workflow_run_id.clone().unwrap_or_default(),
1046 turn: input.turn,
1047 attempt,
1048 };
1049 let workflow_id = info.workflow_id.clone().unwrap_or_default();
1050 let mut request =
1051 TurnRequest::new(identity.clone(), agent.directives(), input.input, session);
1052 request.tooling = agent.tooling();
1053 if let Some(source) = &self.attachments
1054 && let Some(attachment) = source.attachment_for(&workflow_id, &identity, agent.name())
1055 {
1056 request.tooling.mcp_servers.push(attachment.mcp_server);
1057 request.tooling.mcp_timeout = attachment.mcp_timeout;
1058 let allowed = request.tooling.allowed_native_tools.get_or_insert_default();
1059 allowed.extend(attachment.allowed_tools);
1060 }
1061
1062 let span = telemetry::turn_span(
1063 &workflow_id,
1064 agent.name(),
1065 provider.name(),
1066 request.directives.model.as_deref(),
1067 input.turn,
1068 attempt,
1069 );
1070 let _turn_registration = telemetry::register_turn(&workflow_id, &span);
1071
1072 let (sender, mut receiver) = mpsc::channel::<TurnEvent>(64);
1075 let (event_sink, usage_snapshot) = TurnEventSink::tracked(sender);
1076 let heartbeat_ctx = ctx.clone();
1077 let prior_usage = prior.usage.clone();
1078 let pump_prior_usage = prior_usage.clone();
1079 let pump_usage_snapshot = usage_snapshot.clone();
1080 let pump_span = span.clone();
1081 let latest_limit: Arc<Mutex<Option<ProviderLimitStatus>>> = Arc::new(Mutex::new(None));
1084 let pump_limit = latest_limit.clone();
1085 let pump = tokio::spawn(
1086 async move {
1087 let mut state = prior;
1088 while let Some(event) = receiver.recv().await {
1089 match &event {
1090 TurnEvent::SessionStarted { session_id } => {
1091 state.session_id = Some(session_id.clone());
1092 telemetry::record_session(&pump_span, session_id);
1093 }
1094 TurnEvent::ToolUse { name } => {
1095 telemetry::record_harness_tool_use(&pump_span, name);
1096 }
1097 TurnEvent::LimitObserved { limit } if !limit.is_empty() => {
1098 *pump_limit.lock().expect("turn limit lock") = Some(limit.clone());
1099 }
1100 _ => {}
1101 }
1102 state.usage = carry_usage_snapshot(
1103 &pump_prior_usage,
1104 &pump_usage_snapshot
1105 .lock()
1106 .expect("turn usage snapshot lock"),
1107 );
1108 let _ = heartbeat_ctx.record_heartbeat(state.clone()).await;
1109 }
1110 state
1111 }
1112 .instrument(span.clone()),
1113 );
1114
1115 let outcome = provider
1116 .execute_turn(request, event_sink)
1117 .instrument(span.clone())
1118 .await;
1119 let mut heartbeat = pump.await.unwrap_or_default();
1120 heartbeat.usage = carry_usage_snapshot(
1121 &prior_usage,
1122 &usage_snapshot.lock().expect("turn usage snapshot lock"),
1123 );
1124 let _ = ctx.record_heartbeat(heartbeat.clone()).await;
1127
1128 match outcome {
1129 Ok(mut outcome) => {
1130 outcome.usage = aggregate_success_usage(&prior_usage, &outcome.usage);
1131 let mut output: TurnActivityOutput = outcome.into();
1132 output.limit = latest_limit.lock().expect("turn limit lock").take();
1133 telemetry::record_turn_success(&span, &output);
1134 Ok(output)
1135 }
1136 Err(error) => {
1137 telemetry::record_turn_failure(&span, &error);
1138 Err(turn_failure(&error))
1139 }
1140 }
1141 }
1142}
1143
1144fn add_option_u64(prior: Option<u64>, current: Option<u64>) -> Option<u64> {
1145 current.map(|value| prior.unwrap_or(0) + value)
1146}
1147
1148fn carry_usage_snapshot(prior: &TurnUsage, current: &TurnUsage) -> TurnUsage {
1149 TurnUsage {
1150 total_cost_usd: current
1151 .total_cost_usd
1152 .map(|value| prior.total_cost_usd.unwrap_or(0.0) + value)
1153 .or(prior.total_cost_usd),
1154 input_tokens: add_option_u64(prior.input_tokens, current.input_tokens)
1155 .or(prior.input_tokens),
1156 output_tokens: add_option_u64(prior.output_tokens, current.output_tokens)
1157 .or(prior.output_tokens),
1158 cached_input_tokens: add_option_u64(prior.cached_input_tokens, current.cached_input_tokens)
1159 .or(prior.cached_input_tokens),
1160 cache_creation_input_tokens: add_option_u64(
1161 prior.cache_creation_input_tokens,
1162 current.cache_creation_input_tokens,
1163 )
1164 .or(prior.cache_creation_input_tokens),
1165 reasoning_output_tokens: add_option_u64(
1166 prior.reasoning_output_tokens,
1167 current.reasoning_output_tokens,
1168 )
1169 .or(prior.reasoning_output_tokens),
1170 duration: current
1171 .duration
1172 .map(|value| prior.duration.unwrap_or_default() + value)
1173 .or(prior.duration),
1174 }
1175}
1176
1177fn aggregate_success_usage(prior: &TurnUsage, successful: &TurnUsage) -> TurnUsage {
1178 TurnUsage {
1179 total_cost_usd: successful
1180 .total_cost_usd
1181 .map(|value| prior.total_cost_usd.unwrap_or(0.0) + value),
1182 input_tokens: add_option_u64(prior.input_tokens, successful.input_tokens),
1183 output_tokens: add_option_u64(prior.output_tokens, successful.output_tokens),
1184 cached_input_tokens: add_option_u64(
1185 prior.cached_input_tokens,
1186 successful.cached_input_tokens,
1187 ),
1188 cache_creation_input_tokens: add_option_u64(
1189 prior.cache_creation_input_tokens,
1190 successful.cache_creation_input_tokens,
1191 ),
1192 reasoning_output_tokens: add_option_u64(
1193 prior.reasoning_output_tokens,
1194 successful.reasoning_output_tokens,
1195 ),
1196 duration: successful
1197 .duration
1198 .map(|value| prior.duration.unwrap_or_default() + value),
1199 }
1200}
1201
1202fn terminal_failure(
1203 error_type: &str,
1204 error: &(dyn std::error::Error + Send + Sync),
1205) -> ActivityError {
1206 ActivityError::Application(Box::new(
1207 ApplicationFailure::builder(anyhow::anyhow!("{error}"))
1208 .type_name(error_type.to_owned())
1209 .non_retryable(true)
1210 .build(),
1211 ))
1212}
1213
1214fn turn_failure(error: &TurnError) -> ActivityError {
1215 ActivityError::Application(Box::new(
1216 ApplicationFailure::builder(anyhow::anyhow!("{error}"))
1217 .type_name(error.error_type().to_owned())
1218 .non_retryable(!error.is_retryable())
1219 .build(),
1220 ))
1221}
1222
1223#[cfg(test)]
1224mod tests {
1225 use super::*;
1226 use crate::provider::TurnUsage;
1227 use proptest::prelude::*;
1228
1229 #[test]
1230 fn usage_records_unknowns_instead_of_treating_them_as_zero() {
1231 let mut usage = RunUsage::default();
1232 usage.absorb_turn(&TurnUsage::default());
1233 let reported = TurnUsage {
1234 total_cost_usd: Some(0.25),
1235 output_tokens: Some(10),
1236 ..TurnUsage::default()
1237 };
1238 usage.absorb_turn(&reported);
1239 assert!((usage.total_cost_usd - 0.25).abs() < f64::EPSILON);
1240 assert_eq!(usage.output_tokens, 0);
1241 assert_eq!(usage.input_tokens, 0);
1242 assert_eq!(usage.turns_with_unknown_tokens, 2);
1243 assert_eq!(usage.turns_with_unknown_cost, 1);
1244 }
1245
1246 #[test]
1247 fn heartbeat_carryover_aggregates_failed_attempt_spend() {
1248 let failed = TurnUsage {
1249 total_cost_usd: Some(0.20),
1250 input_tokens: Some(100),
1251 output_tokens: Some(25),
1252 cached_input_tokens: Some(60),
1253 duration: Some(Duration::from_secs(2)),
1254 ..TurnUsage::default()
1255 };
1256 let retry_snapshot = TurnUsage {
1257 total_cost_usd: Some(0.30),
1258 input_tokens: Some(150),
1259 output_tokens: Some(40),
1260 cached_input_tokens: Some(90),
1261 reasoning_output_tokens: Some(12),
1262 duration: Some(Duration::from_secs(3)),
1263 ..TurnUsage::default()
1264 };
1265 let carried = carry_usage_snapshot(&failed, &retry_snapshot);
1266 assert_eq!(carried.total_cost_usd, Some(0.50));
1267 assert_eq!(carried.input_tokens, Some(250));
1268 assert_eq!(carried.cached_input_tokens, Some(150));
1269 assert_eq!(
1270 carried.reasoning_output_tokens,
1271 Some(12),
1272 "a figure only the retry reported carries forward alone"
1273 );
1274 assert_eq!(
1275 carried.cache_creation_input_tokens, None,
1276 "a figure neither attempt reported stays unknown"
1277 );
1278 assert_eq!(carried.output_tokens, Some(65));
1279 assert_eq!(carried.duration, Some(Duration::from_secs(5)));
1280 let aggregate = aggregate_success_usage(&failed, &retry_snapshot);
1281 assert_eq!(aggregate.total_cost_usd, carried.total_cost_usd);
1282 assert_eq!(aggregate.input_tokens, carried.input_tokens);
1283 assert_eq!(aggregate.output_tokens, carried.output_tokens);
1284 assert_eq!(aggregate.duration, carried.duration);
1285 }
1286
1287 #[test]
1288 fn failure_before_generation_carries_zero_incremental_spend() {
1289 let prior = TurnUsage {
1290 input_tokens: Some(12),
1291 output_tokens: Some(3),
1292 ..TurnUsage::default()
1293 };
1294 let carried = carry_usage_snapshot(&prior, &TurnUsage::default());
1295 assert_eq!(carried.input_tokens, prior.input_tokens);
1296 assert_eq!(carried.output_tokens, prior.output_tokens);
1297 assert_eq!(carried.total_cost_usd, prior.total_cost_usd);
1298 }
1299
1300 #[test]
1301 fn new_accounting_fields_default_when_replaying_older_history() {
1302 let budget: RunBudget = serde_json::from_value(serde_json::json!({
1303 "max_turns": 2,
1304 "max_cost_usd": 1.5
1305 }))
1306 .expect("old budget shape");
1307 assert_eq!(budget.max_total_tokens, None);
1308
1309 let usage: RunUsage = serde_json::from_value(serde_json::json!({
1310 "total_cost_usd": 0.5,
1311 "input_tokens": 10,
1312 "output_tokens": 5
1313 }))
1314 .expect("old usage shape");
1315 assert_eq!(usage.turns_with_unknown_tokens, 0);
1316 assert_eq!(usage.turns_with_unknown_cost, 0);
1317
1318 let heartbeat: TurnHeartbeat = serde_json::from_value(serde_json::json!({
1319 "session_id": "session-old"
1320 }))
1321 .expect("old heartbeat shape");
1322 assert_eq!(heartbeat.session_id.as_deref(), Some("session-old"));
1323 assert!(heartbeat.usage.input_tokens.is_none());
1324
1325 let record: TurnRecord = serde_json::from_value(serde_json::json!({
1328 "turn": 0,
1329 "input": "hello",
1330 "text": "hi",
1331 "session_id": "session-old",
1332 "usage": {"total_cost_usd": 0.1, "input_tokens": 5, "output_tokens": 2,
1333 "duration": null}
1334 }))
1335 .expect("old transcript record shape");
1336 assert_eq!(record.usage.cached_input_tokens, None);
1337 assert!(record.limit.is_none());
1338 let json = serde_json::to_value(&record).expect("serialize");
1339 assert!(
1340 json.get("limit").is_none(),
1341 "an unobserved limit serializes to nothing"
1342 );
1343 }
1344
1345 #[test]
1346 fn session_rollups_sum_recorded_turns() {
1347 let record = |cost: f64, cached: Option<u64>| TurnRecord {
1348 turn: 0,
1349 input: String::new(),
1350 text: String::new(),
1351 session_id: "session".to_owned(),
1352 usage: TurnUsage {
1353 total_cost_usd: Some(cost),
1354 input_tokens: Some(10),
1355 output_tokens: Some(4),
1356 cached_input_tokens: cached,
1357 ..TurnUsage::default()
1358 },
1359 limit: None,
1360 };
1361 let transcript = [record(0.1, Some(6)), record(0.2, None)];
1362 let usage = RunUsage::from_turns(&transcript);
1363 assert!((usage.total_cost_usd - 0.3).abs() < f64::EPSILON);
1364 assert_eq!(usage.input_tokens, 20);
1365 assert_eq!(
1366 usage.cached_input_tokens, 6,
1367 "absent per-turn figures contribute nothing, not zeroed turns"
1368 );
1369 assert_eq!(usage.turns_with_unknown_tokens, 0);
1370 }
1371
1372 proptest! {
1373 #![proptest_config(ProptestConfig::with_cases(512))]
1374
1375 #[test]
1378 fn budget_termination_is_exact_and_overshoot_is_one_aggregate_turn(
1379 turns in prop::collection::vec((0_u16..500, 0_u16..500, 0_u16..100), 0..40),
1380 max_turns in prop::option::of(0_u8..30),
1381 max_tokens in prop::option::of(0_u16..10_000),
1382 max_cost in prop::option::of(0_u16..2_000),
1383 ) {
1384 let budget = RunBudget {
1385 max_turns: max_turns.map(u32::from),
1386 max_total_tokens: max_tokens.map(u64::from),
1387 max_cost_usd: max_cost.map(f64::from),
1388 };
1389 let expected_termination = budget.max_turns.is_some_and(|cap| turns.len() as u32 > cap)
1390 || budget.max_total_tokens.is_some_and(|cap| {
1391 let mut spent = 0_u64;
1392 turns.iter().enumerate().any(|(index, (input, output, _))| {
1393 spent += u64::from(*input) + u64::from(*output);
1394 spent > cap || (index + 1 < turns.len() && spent >= cap)
1395 })
1396 })
1397 || budget.max_cost_usd.is_some_and(|cap| {
1398 let mut spent = 0_f64;
1399 turns.iter().enumerate().any(|(index, (_, _, cost))| {
1400 spent += f64::from(*cost);
1401 spent > cap || (index + 1 < turns.len() && spent >= cap)
1402 })
1403 });
1404
1405 let mut usage = RunUsage::default();
1406 let mut completed = 0_u32;
1407 let mut last_tokens = 0_u64;
1408 let mut last_cost = 0_f64;
1409 let mut termination = None;
1410 for (input, output, cost) in &turns {
1411 if let Some(cap) = pre_turn_budget_cap(&budget, &usage, completed) {
1412 termination = Some(cap);
1413 break;
1414 }
1415 last_tokens = u64::from(*input) + u64::from(*output);
1416 last_cost = f64::from(*cost);
1417 usage.absorb_turn(&TurnUsage {
1418 total_cost_usd: Some(last_cost),
1419 input_tokens: Some(u64::from(*input)),
1420 output_tokens: Some(u64::from(*output)),
1421 ..TurnUsage::default()
1422 });
1423 completed += 1;
1424 if let Some(cap) = post_turn_budget_cap(&budget, &usage, completed) {
1425 termination = Some(cap);
1426 break;
1427 }
1428 }
1429
1430 prop_assert_eq!(termination.is_some(), expected_termination);
1431 if let Some(cap) = termination {
1432 match cap {
1433 BudgetCap::Turns { spent, cap } => {
1434 prop_assert!(spent <= cap.saturating_add(1));
1435 }
1436 BudgetCap::TotalTokens { spent, cap } => {
1437 prop_assert!(spent <= cap.saturating_add(last_tokens));
1438 }
1439 BudgetCap::CostUsd { spent, cap } => {
1440 prop_assert!(spent <= cap + last_cost);
1441 }
1442 }
1443 }
1444 }
1445 }
1446
1447 #[test]
1448 fn providers_resolve_default_named_and_unknown() {
1449 #[derive(Debug)]
1450 struct Fake(&'static str);
1451 #[async_trait::async_trait]
1452 impl Provider for Fake {
1453 fn name(&self) -> &str {
1454 self.0
1455 }
1456 async fn execute_turn(
1457 &self,
1458 _request: TurnRequest,
1459 _events: TurnEventSink,
1460 ) -> Result<TurnOutcome, TurnError> {
1461 Err(TurnError::Config {
1462 message: "unused".into(),
1463 })
1464 }
1465 }
1466 let providers = Providers::new(Arc::new(Fake("claude"))).with(Arc::new(Fake("codex")));
1467 assert_eq!(providers.resolve(None).expect("default").name(), "claude");
1468 assert_eq!(
1469 providers.resolve(Some("codex")).expect("named").name(),
1470 "codex"
1471 );
1472 assert!(providers.resolve(Some("gemini")).is_err());
1473 }
1474}
1475
1476#[derive(Debug, Clone, Serialize, Deserialize)]
1479pub struct ToolInvocation {
1480 pub identity: InvocationId,
1482 pub tool: String,
1484 pub arguments: serde_json::Value,
1486}
1487
1488#[derive(Debug, Clone, Serialize, Deserialize)]
1490pub enum ToolInvocationReply {
1491 Completed(ToolCallResult),
1494 Rejected(InvocationRejection),
1496}
1497
1498#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1501pub enum InvocationRejection {
1502 Fenced,
1504 UnknownTool,
1506 UnknownTurn,
1508 InvalidCallId,
1510 RunCancelled,
1512}
1513
1514#[derive(Debug, Clone, Serialize, Deserialize)]
1516pub struct ExecuteToolInput {
1517 pub agent: String,
1519 pub tool: String,
1521 pub arguments: serde_json::Value,
1523 pub identity: InvocationId,
1526}
1527
1528#[derive(Debug)]
1531pub struct ToolActivities {
1532 registry: Arc<AgentRegistry>,
1533 max_result_bytes: usize,
1534}
1535
1536pub const DEFAULT_MAX_RESULT_BYTES: usize = 256 * 1024;
1542
1543impl ToolActivities {
1544 pub fn new(registry: Arc<AgentRegistry>) -> Self {
1546 Self {
1547 registry,
1548 max_result_bytes: DEFAULT_MAX_RESULT_BYTES,
1549 }
1550 }
1551
1552 pub fn with_max_result_bytes(mut self, max_result_bytes: usize) -> Self {
1555 self.max_result_bytes = max_result_bytes;
1556 self
1557 }
1558}
1559
1560#[activities]
1561impl ToolActivities {
1562 #[activity]
1564 pub async fn execute_tool(
1565 self: Arc<Self>,
1566 ctx: ActivityContext,
1567 input: ExecuteToolInput,
1568 ) -> Result<ToolCallResult, ActivityError> {
1569 let info = ctx.info();
1570 let span = telemetry::tool_span(
1571 info.workflow_id.as_deref().unwrap_or_default(),
1572 &input.agent,
1573 &input.tool,
1574 &input.identity.call_id,
1575 input.identity.turn,
1576 input.identity.attempt,
1577 );
1578 let agent = match self.registry.get(&input.agent) {
1579 Ok(agent) => agent,
1580 Err(error) => {
1581 telemetry::record_tool_failure(&span, "odori::tool::unknown_agent");
1582 return Err(terminal_failure("odori::tool::unknown_agent", &error));
1583 }
1584 };
1585 let Some(tool) = agent.tools().iter().find(|tool| tool.name() == input.tool) else {
1586 telemetry::record_tool_failure(&span, "odori::tool::unknown_tool");
1587 return Err(terminal_failure(
1588 "odori::tool::unknown_tool",
1589 &crate::agent::UnknownAgent {
1590 name: input.tool.clone(),
1591 },
1592 ));
1593 };
1594 let context = ToolContext {
1595 run_id: info.workflow_run_id.clone().unwrap_or_default(),
1596 turn: input.identity.turn,
1597 attempt: input.identity.attempt,
1598 invocation_id: input.identity.call_id.clone(),
1599 };
1600 match tool
1601 .invoke(context, input.arguments)
1602 .instrument(span.clone())
1603 .await
1604 {
1605 Ok(value) => {
1606 let result = cap_result(tool_output_to_result(value), self.max_result_bytes);
1607 telemetry::record_tool_result(&span, result.is_error);
1608 Ok(result)
1609 }
1610 Err(failure) => {
1611 telemetry::record_tool_failure(&span, "odori::tool::failed");
1612 Err(tool_failure(&failure))
1613 }
1614 }
1615 }
1616}
1617
1618fn cap_result(result: ToolCallResult, max_result_bytes: usize) -> ToolCallResult {
1623 let size = result.content.to_string().len();
1624 if size <= max_result_bytes {
1625 return result;
1626 }
1627 ToolCallResult::error(format!(
1628 "tool result too large: {size} bytes exceeds the {max_result_bytes}-byte limit; write large output to a file and return its path instead"
1629 ))
1630}
1631
1632fn tool_output_to_result(value: serde_json::Value) -> ToolCallResult {
1635 let text = match value {
1636 serde_json::Value::String(text) => text,
1637 other => other.to_string(),
1638 };
1639 ToolCallResult::text(text)
1640}
1641
1642fn tool_failure(failure: &ToolFailure) -> ActivityError {
1643 ActivityError::Application(Box::new(
1644 ApplicationFailure::builder(anyhow::anyhow!("{failure}"))
1645 .type_name("odori::tool::failed".to_owned())
1646 .non_retryable(!failure.retryable)
1647 .build(),
1648 ))
1649}
1650
1651#[cfg(test)]
1652mod q4_tests {
1653 use super::*;
1654
1655 #[test]
1656 fn oversized_results_become_model_visible_failures() {
1657 let big = ToolCallResult::text("x".repeat(1024));
1658 let capped = cap_result(big.clone(), 256);
1659 assert!(capped.is_error);
1660 let text = capped.content.to_string();
1661 assert!(
1662 text.contains("too large") && text.contains("256-byte limit"),
1663 "{text}"
1664 );
1665 assert_eq!(cap_result(big.clone(), 1_000_000), big);
1667 }
1668}