1use super::agent::RunOutput;
29use std::sync::Arc;
30
31use chrono::Utc;
32use futures_util::StreamExt;
33use tokio::time::timeout;
34use tracing::{debug, info, warn};
35use uuid::Uuid;
36
37use behest_provider::{ChatRequest, ChatStreamEvent, FinishReason, Message, TokenUsage, ToolCall};
38
39use super::accumulator::StreamAccumulator;
40use super::agent::AgentRuntime;
41use super::compaction::CompactionCircuitBreaker;
42use super::doom_loop::{DoomLoopDetector, DoomLoopType};
43use super::error::{RuntimeError, RuntimeResult};
44use super::run::{RunId, RunRequest, RunStatus};
45use super::turn::{TurnAction, TurnOutcome, TurnState, TurnTransition};
46use behest_event::{
47 AgentEvent, CompactionCircuitOpened, ContextBuilt, DoomLoopDetected, MessageCommitted,
48 ModelStarted, RunCompleted, TextDelta, ToolCallCompleted, ToolCallDelta,
49 ToolCallStarted as ToolCallStartedEvent, UsageRecorded,
50};
51
52impl AgentRuntime {
53 #[allow(clippy::too_many_arguments)]
84 pub(super) async fn run_loop(
85 &self,
86 run_id: RunId,
87 session_id: Uuid,
88 provider: Arc<dyn behest_provider::ChatProvider>,
89 request: RunRequest,
90 tool_specs: Vec<behest_provider::ToolSpec>,
91 has_tools: bool,
92 iteration: usize,
93 total_usage: TokenUsage,
94 last_finish: Option<FinishReason>,
95 assistant_message: Option<Message>,
96 assistant_msg_id: Option<Uuid>,
97 start_state: TurnState,
98 doom_detector: &mut DoomLoopDetector,
99 breaker: &mut CompactionCircuitBreaker,
100 output_recovery_count: u32,
101 ) -> RuntimeResult<RunOutput> {
102 let result = self
103 .run_loop_inner(
104 run_id,
105 session_id,
106 provider,
107 request,
108 tool_specs,
109 has_tools,
110 iteration,
111 total_usage,
112 last_finish,
113 assistant_message,
114 assistant_msg_id,
115 start_state,
116 doom_detector,
117 breaker,
118 output_recovery_count,
119 )
120 .await;
121
122 let _ = self.delete_snapshot_helper(run_id).await;
123
124 result
125 }
126
127 #[allow(clippy::too_many_lines, clippy::too_many_arguments)]
128 async fn run_loop_inner(
129 &self,
130 run_id: RunId,
131 session_id: Uuid,
132 provider: Arc<dyn behest_provider::ChatProvider>,
133 request: RunRequest,
134 tool_specs: Vec<behest_provider::ToolSpec>,
135 has_tools: bool,
136 mut iteration: usize,
137 mut total_usage: TokenUsage,
138 mut last_finish: Option<FinishReason>,
139 mut assistant_message: Option<Message>,
140 mut assistant_msg_id: Option<Uuid>,
141 start_state: TurnState,
142 doom_detector: &mut DoomLoopDetector,
143 breaker: &mut CompactionCircuitBreaker,
144 mut output_recovery_count: u32,
145 ) -> RuntimeResult<RunOutput> {
146 let mut resume_from = start_state;
147
148 loop {
149 if resume_from == TurnState::CheckingPolicy {
150 iteration += 1;
151 }
152
153 if resume_from == TurnState::CheckingPolicy {
155 self.save_snapshot_helper(
156 run_id,
157 session_id,
158 iteration,
159 TurnState::CheckingPolicy,
160 total_usage,
161 last_finish.as_ref(),
162 assistant_message.as_ref(),
163 assistant_msg_id,
164 &request,
165 output_recovery_count,
166 )
167 .await?;
168
169 let outcome = if iteration > self.policy.max_iterations {
170 TurnOutcome::PolicyExceeded {
171 reason: format!(
172 "iteration {iteration} exceeds limit {}",
173 self.policy.max_iterations
174 ),
175 }
176 } else if let Some(budget) = self.policy.max_tokens {
177 let budget_u64 = budget as u64;
178 if total_usage.total_tokens >= budget_u64 {
179 #[allow(clippy::cast_possible_truncation)]
180 TurnOutcome::PolicyExceeded {
181 reason: format!(
182 "token budget {budget} exceeded: {} used",
183 total_usage.total_tokens
184 ),
185 }
186 } else {
187 TurnOutcome::Success
188 }
189 } else {
190 TurnOutcome::Success
191 };
192
193 match TurnTransition::resolve(TurnState::CheckingPolicy, &outcome) {
194 TurnAction::Fail { reason: _ } => {
195 if iteration > self.policy.max_iterations {
196 let err =
197 RuntimeError::IterationLimitExceeded(self.policy.max_iterations);
198 self.fail_run(run_id, &err).await;
199 return Err(err);
200 }
201 #[allow(clippy::cast_possible_truncation)]
202 let err = RuntimeError::TokenBudgetExceeded {
203 used: total_usage.total_tokens as usize,
204 limit: self.policy.max_tokens.unwrap_or(0),
205 };
206 self.fail_run(run_id, &err).await;
207 return Err(err);
208 }
209 TurnAction::Continue { .. } => {}
210 _ => unreachable!("CheckingPolicy only produces Fail or Continue"),
211 }
212 }
213
214 let mut chat_request = None;
216 if resume_from == TurnState::CheckingPolicy
217 || resume_from == TurnState::BuildingContext
218 || resume_from == TurnState::CallingModel
219 {
220 self.save_snapshot_helper(
221 run_id,
222 session_id,
223 iteration,
224 TurnState::BuildingContext,
225 total_usage,
226 last_finish.as_ref(),
227 assistant_message.as_ref(),
228 assistant_msg_id,
229 &request,
230 output_recovery_count,
231 )
232 .await?;
233
234 self.update_status(
235 run_id,
236 TurnTransition::status_for(TurnState::BuildingContext),
237 )
238 .await?;
239
240 if self.policy.compaction.auto && !breaker.is_open() {
241 let caps = provider.capabilities();
242 if let (Some(model_ctx), Some(max_out)) =
243 (caps.max_input_tokens, caps.max_output_tokens)
244 {
245 let records = self
246 .store
247 .sessions()
248 .list_messages(&session_id)
249 .await
250 .map_err(RuntimeError::from)?;
251
252 let compact_result = self
253 .compaction
254 .compact_if_needed(
255 &records,
256 model_ctx,
257 max_out,
258 self.store.sessions(),
259 session_id,
260 )
261 .await;
262
263 match compact_result {
264 Ok(Some(result)) => {
265 breaker.record_success();
266 debug!(
267 run_id = %run_id,
268 tokens_saved = result.tokens_saved,
269 "proactive compaction completed"
270 );
271 }
272 Ok(None) => {}
273 Err(e) => {
274 if breaker.record_failure() {
275 warn!(
276 run_id = %run_id,
277 failures = breaker.consecutive_failures(),
278 "compaction circuit breaker opened"
279 );
280 self.emit(&AgentEvent::CompactionCircuitOpened(
281 CompactionCircuitOpened {
282 run_id,
283 consecutive_failures: breaker.consecutive_failures(),
284 timestamp: Utc::now(),
285 },
286 ));
287 }
288 return Err(e);
289 }
290 }
291 }
292 }
293
294 let req = self
295 .context
296 .build(
297 &self.store,
298 session_id,
299 request.model.clone(),
300 if iteration == 1 {
301 Some(&request.input)
302 } else {
303 None
304 },
305 if has_tools { Some(&tool_specs) } else { None },
306 )
307 .await?;
308
309 self.emit(&AgentEvent::ContextBuilt(ContextBuilt {
310 run_id,
311 message_count: req.messages.len(),
312 timestamp: Utc::now(),
313 }));
314
315 chat_request = Some(req);
316 }
317
318 let (next_assistant_message, next_finish_reason, _usage) = if resume_from
320 == TurnState::CheckingPolicy
321 || resume_from == TurnState::CallingModel
322 {
323 resume_from = TurnState::CheckingPolicy;
324
325 self.save_snapshot_helper(
326 run_id,
327 session_id,
328 iteration,
329 TurnState::CallingModel,
330 total_usage,
331 last_finish.as_ref(),
332 assistant_message.as_ref(),
333 assistant_msg_id,
334 &request,
335 output_recovery_count,
336 )
337 .await?;
338
339 self.update_status(run_id, TurnTransition::status_for(TurnState::CallingModel))
340 .await?;
341
342 self.emit(&AgentEvent::ModelStarted(ModelStarted {
343 run_id,
344 provider: request.provider.clone(),
345 model: request.model.clone(),
346 iteration,
347 timestamp: Utc::now(),
348 }));
349
350 let req = chat_request.take().ok_or_else(|| {
351 RuntimeError::RecoveryFailed("chat request missing in CallingModel".to_string())
352 })?;
353
354 let (msg, finish, usg) = {
355 let model_result = self.call_model(&provider, req, run_id).await;
356 let outcome = match &model_result {
357 Ok(_) => TurnOutcome::Success,
358 Err(RuntimeError::Provider(e)) if e.is_context_overflow() => {
359 TurnOutcome::ContextOverflow
360 }
361 Err(e) => TurnOutcome::ProviderError {
362 message: e.to_string(),
363 },
364 };
365
366 match TurnTransition::resolve(TurnState::CallingModel, &outcome) {
367 TurnAction::Continue { .. } => model_result?,
368 TurnAction::CompactAndRetry => {
369 if self.policy.compaction.auto {
370 let caps = provider.capabilities();
371 let model_ctx = caps.max_input_tokens.unwrap_or(128_000);
372 let max_out = caps.max_output_tokens.unwrap_or(16_384);
373 let records = self
374 .store
375 .sessions()
376 .list_messages(&session_id)
377 .await
378 .map_err(RuntimeError::from)?;
379
380 let compact_result = self
381 .compaction
382 .compact_after_overflow(
383 &records,
384 model_ctx,
385 max_out,
386 self.store.sessions(),
387 session_id,
388 )
389 .await;
390
391 match compact_result {
392 Ok(result) => {
393 breaker.record_success();
394 debug!(
395 run_id = %run_id,
396 tokens_saved = result.tokens_saved,
397 "reactive compaction after provider overflow"
398 );
399 }
400 Err(e) => {
401 if breaker.record_failure() {
402 warn!(
403 run_id = %run_id,
404 failures = breaker.consecutive_failures(),
405 "compaction circuit breaker opened"
406 );
407 self.emit(&AgentEvent::CompactionCircuitOpened(
408 CompactionCircuitOpened {
409 run_id,
410 consecutive_failures: breaker
411 .consecutive_failures(),
412 timestamp: Utc::now(),
413 },
414 ));
415 }
416 return Err(e);
417 }
418 }
419 }
420 continue;
421 }
422 TurnAction::Fail { .. } => match model_result {
423 Err(e) => return Err(e),
424 Ok(_) => unreachable!("Fail action but model call succeeded"),
425 },
426 TurnAction::BreakLoop => {
427 unreachable!("CallingModel never produces BreakLoop")
428 }
429 }
430 };
431
432 if let Some(u) = &usg {
433 total_usage = total_usage.merge(*u);
434 self.emit(&AgentEvent::UsageRecorded(UsageRecorded {
435 run_id,
436 usage: *u,
437 timestamp: Utc::now(),
438 }));
439 self.emit_cache_metrics(run_id, u).await;
440 }
441
442 last_finish = Some(finish.clone());
443
444 let msg_id = self.store.append_message(session_id, &msg).await?;
445
446 self.emit(&AgentEvent::AssistantMessageCommitted(MessageCommitted {
447 run_id,
448 message_id: msg_id,
449 timestamp: Utc::now(),
450 }));
451
452 assistant_message = Some(msg);
453 assistant_msg_id = Some(msg_id);
454
455 (assistant_message.clone(), last_finish.clone(), usg)
456 } else {
457 (assistant_message.clone(), last_finish.clone(), None)
458 };
459
460 let mut tool_calls = Vec::new();
462 if resume_from == TurnState::CheckingPolicy
463 || resume_from == TurnState::ProcessingResponse
464 {
465 resume_from = TurnState::CheckingPolicy;
466
467 let msg = next_assistant_message.as_ref().ok_or_else(|| {
468 RuntimeError::RecoveryFailed(
469 "assistant message missing in ProcessingResponse".to_string(),
470 )
471 })?;
472 let finish = next_finish_reason.as_ref().ok_or_else(|| {
473 RuntimeError::RecoveryFailed(
474 "last finish missing in ProcessingResponse".to_string(),
475 )
476 })?;
477
478 if matches!(finish, FinishReason::Length)
479 && (output_recovery_count as usize) < self.policy.max_output_recovery_attempts
480 {
481 output_recovery_count += 1;
482 let continue_msg = Message::user_text(
483 "Your previous response was truncated due to output length limit. \
484 Please continue from where you left off.",
485 );
486 self.store.append_message(session_id, &continue_msg).await?;
487 let outcome = TurnOutcome::OutputTruncated;
488 match TurnTransition::resolve(TurnState::ProcessingResponse, &outcome) {
489 TurnAction::Continue { .. } => continue,
490 _ => unreachable!(),
491 }
492 }
493
494 let calls = match msg {
495 Message::Assistant { tool_calls, .. } if !tool_calls.is_empty() => {
496 tool_calls.clone()
497 }
498 _ => Vec::new(),
499 };
500
501 let response_outcome = if calls.is_empty() {
502 TurnOutcome::NoToolCalls
503 } else if !matches!(finish, FinishReason::ToolCalls) {
504 TurnOutcome::NotToolCalls {
505 finish_reason: finish.clone(),
506 }
507 } else {
508 TurnOutcome::Success
509 };
510
511 match TurnTransition::resolve(TurnState::ProcessingResponse, &response_outcome) {
512 TurnAction::BreakLoop => break,
513 TurnAction::Continue { .. } => {
514 tool_calls = calls;
515 }
516 _ => unreachable!(
517 "ProcessingResponse only produces BreakLoop, Continue, or OutputTruncated"
518 ),
519 }
520 } else if let Some(Message::Assistant {
521 tool_calls: calls, ..
522 }) = &assistant_message
523 {
524 tool_calls = calls.clone();
525 }
526
527 if resume_from == TurnState::CheckingPolicy || resume_from == TurnState::ExecutingTools
529 {
530 resume_from = TurnState::CheckingPolicy;
531
532 self.save_snapshot_helper(
533 run_id,
534 session_id,
535 iteration,
536 TurnState::ExecutingTools,
537 total_usage,
538 last_finish.as_ref(),
539 assistant_message.as_ref(),
540 assistant_msg_id,
541 &request,
542 output_recovery_count,
543 )
544 .await?;
545
546 self.update_status(
547 run_id,
548 TurnTransition::status_for(TurnState::ExecutingTools),
549 )
550 .await?;
551
552 let msg_id = assistant_msg_id.ok_or_else(|| {
553 RuntimeError::RecoveryFailed(
554 "assistant message ID missing in ExecutingTools".to_string(),
555 )
556 })?;
557
558 for call in &tool_calls {
559 if let Some(loop_type) =
560 doom_detector.record_and_check(&call.name, &call.arguments)
561 {
562 let description = match &loop_type {
563 DoomLoopType::ConsecutiveDuplicate { tool_name, count } => {
564 format!("consecutive duplicate: {tool_name} called {count} times")
565 }
566 DoomLoopType::Cycle {
567 pattern,
568 repetitions,
569 } => {
570 format!(
571 "cycle detected: [{}] repeated {repetitions} times",
572 pattern.join(", ")
573 )
574 }
575 };
576 self.emit(&AgentEvent::DoomLoopDetected(DoomLoopDetected {
577 run_id,
578 description: description.clone(),
579 timestamp: Utc::now(),
580 }));
581 let err = RuntimeError::DoomLoopDetected { description };
582 self.fail_run(run_id, &err).await;
583 return Err(err);
584 }
585 }
586
587 let outcomes = self
588 .tools
589 .execute_batch(
590 tool_calls,
591 session_id,
592 msg_id,
593 Some(self.store.executions()),
594 )
595 .await?;
596
597 for outcome in &outcomes {
598 let tool_msg_id = self
599 .store
600 .append_message(session_id, &outcome.message)
601 .await?;
602
603 self.emit(&AgentEvent::ToolMessageCommitted(MessageCommitted {
604 run_id,
605 message_id: tool_msg_id,
606 timestamp: Utc::now(),
607 }));
608 }
609 }
610
611 if resume_from == TurnState::CheckingPolicy || resume_from == TurnState::Persisting {
613 resume_from = TurnState::CheckingPolicy;
614
615 self.save_snapshot_helper(
616 run_id,
617 session_id,
618 iteration,
619 TurnState::Persisting,
620 total_usage,
621 last_finish.as_ref(),
622 assistant_message.as_ref(),
623 assistant_msg_id,
624 &request,
625 output_recovery_count,
626 )
627 .await?;
628
629 self.update_status(run_id, TurnTransition::status_for(TurnState::Persisting))
630 .await?;
631
632 let finish = last_finish.as_ref().ok_or_else(|| {
633 RuntimeError::RecoveryFailed("last finish missing in Persisting".to_string())
634 })?;
635
636 let persisting_outcome = if matches!(finish, FinishReason::ToolCalls) {
637 TurnOutcome::Success
638 } else {
639 TurnOutcome::NotToolCalls {
640 finish_reason: finish.clone(),
641 }
642 };
643
644 match TurnTransition::resolve(TurnState::Persisting, &persisting_outcome) {
645 TurnAction::BreakLoop => break,
646 TurnAction::Continue { .. } => {}
647 _ => unreachable!(),
648 }
649 }
650
651 assistant_message = None;
652 assistant_msg_id = None;
653 }
654
655 self.update_status(run_id, RunStatus::Completed).await?;
656
657 let final_finish = last_finish.clone().unwrap_or(FinishReason::Stop);
658 self.emit(&AgentEvent::RunCompleted(RunCompleted {
659 run_id,
660 finish_reason: final_finish.clone(),
661 iterations: iteration,
662 timestamp: Utc::now(),
663 }));
664
665 info!(%run_id, iterations = iteration, "run completed");
666
667 Ok(RunOutput {
668 run_id,
669 session_id,
670 iterations: iteration,
671 finish_reason: final_finish,
672 total_usage,
673 })
674 }
675
676 async fn call_model(
677 &self,
678 provider: &Arc<dyn behest_provider::ChatProvider>,
679 request: ChatRequest,
680 run_id: RunId,
681 ) -> RuntimeResult<(Message, FinishReason, Option<TokenUsage>)> {
682 let caps = provider.capabilities();
683
684 if caps.chat_stream {
685 match self.call_streaming(provider, request.clone(), run_id).await {
686 Ok(result) => return Ok(result),
687 Err(e)
688 if matches!(
689 e,
690 RuntimeError::Provider(behest_core::error::ProviderError::Timeout { .. })
691 ) =>
692 {
693 return Err(e);
694 }
695 Err(e) => {
696 warn!(error = %e, "streaming failed, falling back to complete");
697 }
698 }
699 }
700
701 self.call_complete(provider, request, run_id).await
702 }
703
704 async fn call_streaming(
705 &self,
706 provider: &Arc<dyn behest_provider::ChatProvider>,
707 request: ChatRequest,
708 run_id: RunId,
709 ) -> RuntimeResult<(Message, FinishReason, Option<TokenUsage>)> {
710 let stream = timeout(self.policy.provider_timeout, provider.stream(request))
711 .await
712 .map_err(|_| {
713 RuntimeError::Provider(behest_core::error::ProviderError::Timeout {
714 provider: provider.id(),
715 })
716 })?
717 .map_err(RuntimeError::from)?;
718
719 let mut accumulator = StreamAccumulator::new();
720 let mut finish_reason = FinishReason::Stop;
721 let mut usage: Option<TokenUsage> = None;
722
723 tokio::pin!(stream);
724
725 while let Some(event_result) = timeout(self.policy.provider_timeout, stream.next())
726 .await
727 .map_err(|_| {
728 RuntimeError::Provider(behest_core::error::ProviderError::Timeout {
729 provider: provider.id(),
730 })
731 })?
732 {
733 let event = event_result.map_err(RuntimeError::from)?;
734
735 match &event {
736 ChatStreamEvent::TextDelta { delta } => {
737 accumulator.append_text(delta);
738 self.emit(&AgentEvent::TextDelta(TextDelta {
739 run_id,
740 delta: delta.clone(),
741 timestamp: Utc::now(),
742 }));
743 }
744 ChatStreamEvent::ToolCallStarted { id, name } => {
745 accumulator.start_tool_call(id.clone(), name.clone());
746 self.emit(&AgentEvent::ToolCallStarted(ToolCallStartedEvent {
747 run_id,
748 call_id: id.clone(),
749 tool_name: name.clone(),
750 timestamp: Utc::now(),
751 }));
752 }
753 ChatStreamEvent::ToolCallArgumentsDelta { id, delta } => {
754 accumulator.append_tool_arguments(id, delta);
755 self.emit(&AgentEvent::ToolCallDelta(ToolCallDelta {
756 run_id,
757 call_id: id.clone(),
758 delta: delta.clone(),
759 timestamp: Utc::now(),
760 }));
761 }
762 ChatStreamEvent::ToolCallCompleted { call } => {
763 self.emit(&AgentEvent::ToolCallCompleted(ToolCallCompleted {
764 run_id,
765 call: call.clone(),
766 timestamp: Utc::now(),
767 }));
768 }
769 ChatStreamEvent::Finished {
770 finish_reason: fr,
771 usage: u,
772 } => {
773 finish_reason = fr.clone();
774 usage = *u;
775 }
776 ChatStreamEvent::Started { .. } => {}
777 _ => {}
778 }
779 }
780
781 let message = accumulator.to_message();
782 Ok((message, finish_reason, usage))
783 }
784
785 async fn call_complete(
786 &self,
787 provider: &Arc<dyn behest_provider::ChatProvider>,
788 request: ChatRequest,
789 run_id: RunId,
790 ) -> RuntimeResult<(Message, FinishReason, Option<TokenUsage>)> {
791 let response = timeout(self.policy.provider_timeout, provider.complete(request))
792 .await
793 .map_err(|_| {
794 RuntimeError::Provider(behest_core::error::ProviderError::Timeout {
795 provider: provider.id(),
796 })
797 })?
798 .map_err(RuntimeError::from)?;
799
800 if let Some(text) = extract_assistant_text(&response.message) {
801 self.emit(&AgentEvent::TextDelta(TextDelta {
802 run_id,
803 delta: text,
804 timestamp: Utc::now(),
805 }));
806 }
807
808 for tc in extract_tool_calls(&response.message) {
809 self.emit(&AgentEvent::ToolCallCompleted(ToolCallCompleted {
810 run_id,
811 call: tc.clone(),
812 timestamp: Utc::now(),
813 }));
814 }
815
816 Ok((response.message, response.finish_reason, response.usage))
817 }
818}
819
820fn extract_assistant_text(message: &Message) -> Option<String> {
821 match message {
822 Message::Assistant { content, .. } => {
823 let text: String = content
824 .iter()
825 .filter_map(|p| match p {
826 behest_provider::ContentPart::Text { text, .. } => Some(text.as_str()),
827 _ => None,
828 })
829 .collect();
830 if text.is_empty() { None } else { Some(text) }
831 }
832 _ => None,
833 }
834}
835
836fn extract_tool_calls(message: &Message) -> Vec<ToolCall> {
837 match message {
838 Message::Assistant { tool_calls, .. } => tool_calls.clone(),
839 _ => Vec::new(),
840 }
841}