bamboo_engine/runtime/runner/
image_fallback.rs

1//! Image fallback and attachment resolution helpers for the agent loop runner.
2
3use std::sync::Arc;
4
5use crate::runtime::config::ImageFallbackConfig;
6use bamboo_agent_core::storage::AttachmentReader;
7use bamboo_agent_core::{AgentError, Message};
8use bamboo_llm::{LLMProvider, ResolvedModel};
9
10mod attachment_urls;
11mod input_references;
12#[cfg(test)]
13pub(crate) use input_references::conversation_image_reference;
14pub(crate) use input_references::{
15    image_reference_reader, project_nonvision_images, round_needs_image_references,
16};
17mod legacy_target;
18#[cfg(windows)]
19mod ocr;
20mod placeholder;
21mod rewrite;
22mod vision_target;
23pub(crate) use legacy_target::legacy_vision_resolver;
24pub(crate) use vision_target::{configured_vision_resolver, validate_configured_vision_target};
25
26#[cfg(windows)]
27pub async fn ensure_session_image_ocr_cached(
28    session: &mut bamboo_agent_core::Session,
29    attachment_reader: Option<&dyn AttachmentReader>,
30) -> bool {
31    ocr::ensure_session_image_ocr_cached(session, attachment_reader).await
32}
33
34pub async fn resolve_bamboo_attachments_for_llm(
35    messages: &mut [Message],
36    reader: &dyn AttachmentReader,
37) -> std::result::Result<(), AgentError> {
38    attachment_urls::resolve_bamboo_attachments_for_llm(messages, reader).await
39}
40
41#[cfg(any(test, windows))]
42pub fn persistable_image_urls(parts: &[bamboo_domain::MessagePart]) -> Vec<String> {
43    placeholder::persistable_image_urls(parts)
44}
45
46/// The independent legacy hook omits an image when its description request
47/// fails. It must not inherit the stricter tool-result error contract.
48pub async fn apply_image_fallback_to_llm_messages(
49    messages: &mut [Message],
50    fallback: ImageFallbackConfig,
51    attachment_reader: Option<&dyn AttachmentReader>,
52    llm: Option<&Arc<dyn LLMProvider>>,
53) -> std::result::Result<(), AgentError> {
54    if fallback.mode == crate::runtime::config::ImageFallbackMode::Vision {
55        bamboo_llm::validate_tool_image_budget(messages.iter()).map_err(AgentError::LLM)?;
56    }
57    rewrite::apply_image_fallback_to_llm_messages(
58        messages,
59        fallback,
60        attachment_reader,
61        llm,
62        rewrite::VisionFailurePolicy::OmitImage,
63    )
64    .await
65}
66
67/// Fresh ViewImage results fail atomically; a partial description is never
68/// substituted for a successful tool result.
69async fn apply_strict_image_fallback_to_llm_messages(
70    messages: &mut [Message],
71    fallback: ImageFallbackConfig,
72    attachment_reader: Option<&dyn AttachmentReader>,
73    llm: Option<&Arc<dyn LLMProvider>>,
74) -> std::result::Result<(), AgentError> {
75    bamboo_llm::validate_tool_image_budget(messages.iter()).map_err(AgentError::LLM)?;
76    tokio::time::timeout(
77        VISION_TIMEOUT,
78        rewrite::apply_image_fallback_to_llm_messages(
79            messages,
80            fallback,
81            attachment_reader,
82            llm,
83            rewrite::VisionFailurePolicy::Error,
84        ),
85    )
86    .await
87    .map_err(|_| AgentError::LLM(VISION_TIMEOUT_MESSAGE.into()))?
88}
89
90#[cfg(test)]
91mod tests;
92
93/// A host resolver exclusively for ViewImage tool results, kept separate
94/// from model arguments. Never apply it to conversation uploads or history.
95pub(crate) type VisionModelResolver = Arc<
96    dyn Fn() -> futures::future::BoxFuture<'static, Result<ResolvedModel, AgentError>>
97        + Send
98        + Sync,
99>;
100
101/// Independent port for an explicitly opted-in legacy conversation transform.
102/// It must never consume the strict ViewImage tool-result resolver field.
103pub(crate) type LegacyVisionModelResolver = Arc<
104    dyn Fn() -> futures::future::BoxFuture<'static, Result<ResolvedModel, AgentError>>
105        + Send
106        + Sync,
107>;
108
109pub(crate) const VISION_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
110const VISION_TIMEOUT_MESSAGE: &str =
111    "Vision description timed out; no image or partial description was sent to the main model.";
112pub(crate) const VISION_CONFIGURATION_ERROR: &str = "ViewImage requires a configured Vision model for a main model without Vision. Original images and attachment references were preserved. Configure defaults.vision (provider instance and model), or vision_model on the active provider, or select a genuinely Vision-capable main model. No image was sent.";
113
114pub(crate) async fn resolve_vision_target(
115    config: &crate::runtime::config::AgentLoopConfig,
116) -> Result<ResolvedModel, AgentError> {
117    let resolver = config
118        .vision_model_resolver
119        .as_ref()
120        .ok_or_else(|| AgentError::LLM(VISION_CONFIGURATION_ERROR.into()))?;
121    resolve_checked_vision_target(resolver).await
122}
123
124pub(crate) async fn resolve_legacy_vision_target(
125    config: &crate::runtime::config::AgentLoopConfig,
126) -> Result<ResolvedModel, AgentError> {
127    let resolver = config
128        .legacy_vision_model_resolver
129        .as_ref()
130        .ok_or_else(|| {
131            AgentError::LLM(
132                "Legacy Vision target is unavailable; original images were preserved.".into(),
133            )
134        })?;
135    resolve_checked_vision_target(resolver).await
136}
137
138async fn resolve_checked_vision_target(
139    resolver: &VisionModelResolver,
140) -> Result<ResolvedModel, AgentError> {
141    tokio::time::timeout(VISION_TIMEOUT, async {
142        let target = resolver().await?;
143        if target.model_name.trim().is_empty() {
144            return Err(AgentError::LLM(VISION_CONFIGURATION_ERROR.into()));
145        }
146        if !target.provider.supports_vision(&target.model_name).await {
147            return Err(AgentError::LLM(format!(
148                "Configured Vision model does not support Vision. {VISION_CONFIGURATION_ERROR}"
149            )));
150        }
151        Ok(target)
152    })
153    .await
154    .map_err(|_| AgentError::LLM("Vision model resolution timed out. No image was sent.".into()))?
155}
156
157pub(crate) async fn describe_tool_images(
158    target: &ResolvedModel,
159    result: &mut bamboo_agent_core::ToolResult,
160) -> Result<(), AgentError> {
161    bamboo_llm::validate_tool_result_images(&result.images).map_err(AgentError::LLM)?;
162    let mut messages = [Message::tool_result_with_images(
163        "vision-description",
164        &result.result,
165        true,
166        result.images.clone(),
167    )];
168    apply_strict_image_fallback_to_llm_messages(
169        &mut messages,
170        ImageFallbackConfig {
171            mode: crate::runtime::config::ImageFallbackMode::Vision,
172            vision_model: Some(target.model_name.clone()),
173        },
174        None,
175        Some(&target.provider),
176    )
177    .await?;
178    result.result = std::mem::take(&mut messages[0].content);
179    result.images.clear();
180    Ok(())
181}
bamboo_engine/runtime/runner/image_fallback.rs

bamboo_engine/runtime/runner/
image_fallback.rs
