bamboo_engine/runtime/runner/
image_fallback.rs

1//! Image fallback and attachment resolution helpers for the agent loop runner.
2
3use std::sync::Arc;
4
5use crate::runtime::config::ImageFallbackConfig;
6use bamboo_agent_core::storage::AttachmentReader;
7use bamboo_agent_core::{AgentError, Message};
8use bamboo_llm::{LLMProvider, ResolvedModel};
9
10mod attachment_urls;
11#[cfg(windows)]
12mod ocr;
13mod placeholder;
14mod rewrite;
15mod vision_target;
16pub(crate) use vision_target::configured_vision_resolver;
17
18#[cfg(windows)]
19pub async fn ensure_session_image_ocr_cached(
20    session: &mut bamboo_agent_core::Session,
21    attachment_reader: Option<&dyn AttachmentReader>,
22) -> bool {
23    ocr::ensure_session_image_ocr_cached(session, attachment_reader).await
24}
25
26pub async fn resolve_bamboo_attachments_for_llm(
27    messages: &mut [Message],
28    reader: &dyn AttachmentReader,
29) -> std::result::Result<(), AgentError> {
30    attachment_urls::resolve_bamboo_attachments_for_llm(messages, reader).await
31}
32
33#[cfg(any(test, windows))]
34pub fn persistable_image_urls(parts: &[bamboo_domain::MessagePart]) -> Vec<String> {
35    placeholder::persistable_image_urls(parts)
36}
37
38/// The independent legacy hook omits an image when its description request
39/// fails. It must not inherit the stricter tool-result error contract.
40pub async fn apply_image_fallback_to_llm_messages(
41    messages: &mut [Message],
42    fallback: ImageFallbackConfig,
43    attachment_reader: Option<&dyn AttachmentReader>,
44    llm: Option<&Arc<dyn LLMProvider>>,
45) -> std::result::Result<(), AgentError> {
46    if fallback.mode == crate::runtime::config::ImageFallbackMode::Vision {
47        bamboo_llm::validate_tool_image_budget(messages.iter()).map_err(AgentError::LLM)?;
48    }
49    rewrite::apply_image_fallback_to_llm_messages(
50        messages,
51        fallback,
52        attachment_reader,
53        llm,
54        rewrite::VisionFailurePolicy::OmitImage,
55    )
56    .await
57}
58
59/// Fresh ViewImage results fail atomically; a partial description is never
60/// substituted for a successful tool result.
61async fn apply_strict_image_fallback_to_llm_messages(
62    messages: &mut [Message],
63    fallback: ImageFallbackConfig,
64    attachment_reader: Option<&dyn AttachmentReader>,
65    llm: Option<&Arc<dyn LLMProvider>>,
66) -> std::result::Result<(), AgentError> {
67    bamboo_llm::validate_tool_image_budget(messages.iter()).map_err(AgentError::LLM)?;
68    tokio::time::timeout(
69        VISION_TIMEOUT,
70        rewrite::apply_image_fallback_to_llm_messages(
71            messages,
72            fallback,
73            attachment_reader,
74            llm,
75            rewrite::VisionFailurePolicy::Error,
76        ),
77    )
78    .await
79    .map_err(|_| AgentError::LLM(VISION_TIMEOUT_MESSAGE.into()))?
80}
81
82#[cfg(test)]
83mod tests;
84
85/// A host resolver exclusively for ViewImage tool results, kept separate
86/// from model arguments. Never apply it to conversation uploads or history.
87pub(crate) type VisionModelResolver = Arc<
88    dyn Fn() -> futures::future::BoxFuture<'static, Result<ResolvedModel, AgentError>>
89        + Send
90        + Sync,
91>;
92
93pub(crate) const VISION_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
94const VISION_TIMEOUT_MESSAGE: &str =
95    "Vision description timed out; no image or partial description was sent to the main model.";
96pub(crate) const VISION_CONFIGURATION_ERROR: &str = "ViewImage requires a configured Vision model for a main model without Vision. Configure defaults.vision (provider instance and model), or vision_model on the active provider. No image was sent.";
97
98pub(crate) async fn resolve_vision_target(
99    config: &crate::runtime::config::AgentLoopConfig,
100) -> Result<ResolvedModel, AgentError> {
101    let resolver = config
102        .vision_model_resolver
103        .as_ref()
104        .ok_or_else(|| AgentError::LLM(VISION_CONFIGURATION_ERROR.into()))?;
105    tokio::time::timeout(VISION_TIMEOUT, async {
106        let target = resolver().await?;
107        if target.model_name.trim().is_empty() {
108            return Err(AgentError::LLM(VISION_CONFIGURATION_ERROR.into()));
109        }
110        if !target.provider.supports_vision(&target.model_name).await {
111            return Err(AgentError::LLM("Configured Vision model does not support Vision. Enable supports_vision for that exact provider/model or select another configured Vision model. No image was sent.".into()));
112        }
113        Ok(target)
114    }).await.map_err(|_| AgentError::LLM("Vision model resolution timed out. No image was sent.".into()))?
115}
116
117pub(crate) async fn describe_tool_images(
118    target: &ResolvedModel,
119    result: &mut bamboo_agent_core::ToolResult,
120) -> Result<(), AgentError> {
121    bamboo_llm::validate_tool_result_images(&result.images).map_err(AgentError::LLM)?;
122    let mut messages = [Message::tool_result_with_images(
123        "vision-description",
124        &result.result,
125        true,
126        result.images.clone(),
127    )];
128    apply_strict_image_fallback_to_llm_messages(
129        &mut messages,
130        ImageFallbackConfig {
131            mode: crate::runtime::config::ImageFallbackMode::Vision,
132            vision_model: Some(target.model_name.clone()),
133        },
134        None,
135        Some(&target.provider),
136    )
137    .await?;
138    result.result = std::mem::take(&mut messages[0].content);
139    result.images.clear();
140    Ok(())
141}
bamboo_engine/runtime/runner/image_fallback.rs

bamboo_engine/runtime/runner/
image_fallback.rs
