#![allow(dead_code)]
use super::recall_guardrails::text_relates_to_critical_identity;
use super::sliding_window::summarize_tool_result;
use super::*;
use crate::config::{AudioConfig, SttConfig, VisionConfig};
use crate::events::TerminalState;
use crate::traits::AttachmentProvenance;
pub(crate) const RENDERER_VERSION: u32 = 3;
#[derive(Clone, Debug)]
pub(crate) struct RenderOptions {
pub vision: VisionConfig,
pub audio: AudioConfig,
pub stt: SttConfig,
pub model: String,
}
impl Default for RenderOptions {
fn default() -> Self {
Self {
vision: VisionConfig {
enabled: true,
max_image_bytes: 4 * 1_048_576,
mime_types: vec![
"image/jpeg".to_string(),
"image/png".to_string(),
"image/gif".to_string(),
"image/webp".to_string(),
],
},
audio: AudioConfig::from_files(&crate::config::FilesConfig::default()),
stt: SttConfig::from_files(&crate::config::FilesConfig::default()),
model: String::new(),
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum RenderMode {
Current,
Archived { terminal_state: TerminalState },
}
pub(crate) fn render_turn(
turn_messages: &[Message],
mode: RenderMode,
_version: u32,
options: &RenderOptions,
) -> Vec<Value> {
match mode {
RenderMode::Current => render_current(turn_messages, options),
RenderMode::Archived { terminal_state } => {
render_archived(turn_messages, terminal_state, options)
}
}
}
pub(crate) fn is_failure_boilerplate(content: &str) -> bool {
let t = content.trim_start();
t.starts_with("I wasn't able to process that request.")
|| t.starts_with("I wasn't able to complete this task.")
|| t.starts_with("I made some progress but wasn't able to fully complete")
|| t.starts_with("I seem to be stuck on this task.")
|| t.starts_with("I've reached my processing limit")
|| t.starts_with("This goal hit its daily processing budget")
|| t.starts_with("This scheduled goal hit its daily processing budget")
|| t.starts_with("This scheduled run hit its per-run processing budget")
|| t.starts_with("I sent the requested file(s), but ran into issues")
|| t.starts_with("I completed the main deliverable but wasn't able to finish")
}
fn truncate_old_assistant(content: &str) -> String {
if content.chars().count() > MAX_OLD_ASSISTANT_CONTENT_CHARS {
let truncated: String = content
.chars()
.take(MAX_OLD_ASSISTANT_CONTENT_CHARS)
.collect();
format!("{}…", truncated)
} else {
content.to_string()
}
}
fn tool_result_ids(turn_messages: &[Message]) -> std::collections::HashSet<&str> {
turn_messages
.iter()
.filter(|m| m.role == "tool" && m.tool_name.as_ref().is_some_and(|n| !n.is_empty()))
.filter_map(|m| m.tool_call_id.as_deref())
.collect()
}
fn wire_tool_calls(tc_json: &str, result_ids: &std::collections::HashSet<&str>) -> Vec<Value> {
let Ok(tcs) = serde_json::from_str::<Vec<ToolCall>>(tc_json) else {
return Vec::new();
};
tcs.iter()
.filter(|tc| result_ids.contains(tc.id.as_str()))
.map(|tc| {
let mut val = json!({
"id": tc.id,
"type": "function",
"function": {
"name": tc.name,
"arguments": tc.arguments
}
});
if let Some(ref extra) = tc.extra_content {
val["extra_content"] = extra.clone();
}
val
})
.collect()
}
fn attach_tool_routing(obj: &mut Value, m: &Message) {
if let Some(name) = &m.tool_name {
if !name.is_empty() {
obj["name"] = json!(name);
}
}
if let Some(tcid) = &m.tool_call_id {
obj["tool_call_id"] = json!(tcid);
}
}
fn append_tool_observation_messages(
out: &mut Vec<Value>,
m: &Message,
options: &RenderOptions,
vision_skipped: &mut bool,
) {
if m.role != "tool" {
return;
}
if !m
.attachments
.iter()
.any(|a| a.provenance == AttachmentProvenance::ToolObservation)
{
return;
}
let tool_name = m.tool_name.as_deref().unwrap_or("tool");
let hint = m.content.as_deref().unwrap_or("");
let label = crate::agent::vision::format_tool_observation_label(tool_name, hint);
let built = crate::agent::vision::build_tool_observation_content(
&label,
&m.attachments,
&options.vision,
);
if built.vision_skipped {
*vision_skipped = true;
}
out.push(json!({
"role": "user",
"content": built.content,
}));
}
fn render_current(turn_messages: &[Message], options: &RenderOptions) -> Vec<Value> {
let result_ids = tool_result_ids(turn_messages);
let mut vision_skipped = false;
let mut audio_skipped = false;
let mut stt_failed = false;
let mut rendered: Vec<Value> = Vec::new();
for m in turn_messages
.iter()
.filter(|m| !(m.role == "tool" && m.tool_name.as_ref().is_none_or(|n| n.is_empty())))
{
if m.role == "assistant"
&& m.tool_calls_json.is_none()
&& m.content.as_deref().is_some_and(is_failure_boilerplate)
{
continue;
}
let content = if m.role == "user" && !m.attachments.is_empty() {
let text = m.content.as_deref().unwrap_or("");
let built = crate::agent::attachment_content::build_attachment_content(
text,
&m.attachments,
RenderMode::Current,
&options.vision,
&options.audio,
&options.model,
);
if built.vision_skipped {
vision_skipped = true;
}
if built.audio_skipped && !crate::agent::stt::content_has_transcription(text) {
if options.stt.enabled
&& crate::agent::stt::should_run_stt_fallback(
&options.stt,
&options.audio,
&options.model,
&m.attachments,
)
{
stt_failed = true;
} else {
audio_skipped = true;
}
}
built.content
} else {
json!(m.content)
};
let mut obj = json!({
"role": m.role,
"content": content,
});
if let Some(tc_json) = &m.tool_calls_json {
let filtered = wire_tool_calls(tc_json, &result_ids);
if !filtered.is_empty() {
obj["tool_calls"] = json!(filtered);
if m.content.is_none() {
obj["content"] = Value::Null;
}
} else if m.content.is_none()
|| m.content.as_deref().is_some_and(|c| c.trim().is_empty())
{
obj["content"] = json!("[Action completed]");
}
}
attach_tool_routing(&mut obj, m);
rendered.push(obj);
append_tool_observation_messages(&mut rendered, m, options, &mut vision_skipped);
}
if vision_skipped {
rendered.insert(
0,
json!({
"role": "system",
"content": crate::agent::vision::VISION_SKIPPED_SYSTEM_HINT,
}),
);
}
if stt_failed {
rendered.insert(
0,
json!({
"role": "system",
"content": crate::agent::stt::STT_FAILED_SYSTEM_HINT,
}),
);
} else if audio_skipped {
rendered.insert(
0,
json!({
"role": "system",
"content": crate::agent::audio::AUDIO_SKIPPED_SYSTEM_HINT,
}),
);
}
rendered
}
fn render_archived(
turn_messages: &[Message],
terminal_state: TerminalState,
options: &RenderOptions,
) -> Vec<Value> {
let result_ids = tool_result_ids(turn_messages);
let last_substantive_assistant = turn_messages
.iter()
.enumerate()
.filter(|(_, m)| {
m.role == "assistant"
&& m.content.as_deref().is_some_and(|c| {
!c.trim().is_empty()
&& !is_failure_boilerplate(c)
&& !text_relates_to_critical_identity(c)
})
})
.map(|(i, _)| i)
.next_back();
let tool_step_count = turn_messages
.iter()
.filter(|m| m.role == "tool" && m.tool_name.as_ref().is_some_and(|n| !n.is_empty()))
.count();
let last_assistant_or_tool = turn_messages
.iter()
.rposition(|m| m.role == "assistant" || m.role == "tool");
let need_placeholder = last_substantive_assistant.is_none();
let mut vision_skipped = false;
let mut out: Vec<Value> = Vec::with_capacity(turn_messages.len());
for (idx, m) in turn_messages.iter().enumerate() {
if m.content
.as_deref()
.is_some_and(text_relates_to_critical_identity)
{
let mut obj = json!({
"role": m.role,
"content": m.content,
});
if let Some(tc_json) = &m.tool_calls_json {
let filtered = wire_tool_calls(tc_json, &result_ids);
if !filtered.is_empty() {
obj["tool_calls"] = json!(filtered);
}
}
attach_tool_routing(&mut obj, m);
out.push(obj);
maybe_emit_placeholder(
&mut out,
idx,
need_placeholder,
last_assistant_or_tool,
terminal_state,
tool_step_count,
);
continue;
}
match m.role.as_str() {
"user" => {
out.push(json!({ "role": "user", "content": m.content }));
}
"assistant" => {
if Some(idx) == last_substantive_assistant {
let truncated = m
.content
.as_deref()
.map(truncate_old_assistant)
.unwrap_or_default();
let mut obj = json!({
"role": "assistant",
"content": truncated,
});
if let Some(tc_json) = &m.tool_calls_json {
let filtered = wire_tool_calls(tc_json, &result_ids);
if !filtered.is_empty() {
obj["tool_calls"] = json!(filtered);
}
}
out.push(obj);
} else if m.content.as_deref().is_some_and(is_failure_boilerplate) {
emit_tool_call_only_assistant(&mut out, m, &result_ids);
} else if m.tool_calls_json.is_some() {
emit_tool_call_only_assistant(&mut out, m, &result_ids);
}
}
"tool" if m.tool_name.as_ref().is_some_and(|n| !n.is_empty()) => {
let args_json = m
.tool_call_id
.as_deref()
.and_then(|cid| tool_args_for_call(turn_messages, cid))
.unwrap_or_default();
let tool_name = m.tool_name.as_deref().unwrap_or("unknown");
let result = m.content.as_deref().unwrap_or("");
let summary = summarize_tool_result(tool_name, &args_json, result);
let mut obj = json!({ "role": "tool", "content": summary });
attach_tool_routing(&mut obj, m);
out.push(obj);
append_tool_observation_messages(&mut out, m, options, &mut vision_skipped);
}
_ => {}
}
maybe_emit_placeholder(
&mut out,
idx,
need_placeholder,
last_assistant_or_tool,
terminal_state,
tool_step_count,
);
}
if vision_skipped {
out.insert(
0,
json!({
"role": "system",
"content": crate::agent::vision::VISION_SKIPPED_SYSTEM_HINT,
}),
);
}
out
}
fn emit_tool_call_only_assistant(
out: &mut Vec<Value>,
m: &Message,
result_ids: &std::collections::HashSet<&str>,
) {
if let Some(tc_json) = &m.tool_calls_json {
let filtered = wire_tool_calls(tc_json, result_ids);
if !filtered.is_empty() {
out.push(json!({
"role": "assistant",
"content": Value::Null,
"tool_calls": filtered,
}));
}
}
}
fn maybe_emit_placeholder(
out: &mut Vec<Value>,
idx: usize,
need_placeholder: bool,
last_assistant_or_tool: Option<usize>,
terminal_state: TerminalState,
tool_step_count: usize,
) {
if need_placeholder && last_assistant_or_tool == Some(idx) {
out.push(json!({
"role": "assistant",
"content": terminal_state.placeholder(tool_step_count),
}));
}
}
fn tool_args_for_call(turn_messages: &[Message], call_id: &str) -> Option<String> {
for m in turn_messages.iter() {
if m.role == "assistant" {
if let Some(tc_json) = &m.tool_calls_json {
if let Ok(tcs) = serde_json::from_str::<Vec<ToolCall>>(tc_json) {
for tc in &tcs {
if tc.id == call_id {
return Some(tc.arguments.clone());
}
}
}
}
}
}
None
}
#[cfg(test)]
mod tests {
use super::*;
fn user(c: &str) -> Message {
Message {
role: "user".to_string(),
content: Some(c.to_string()),
..Message::runtime_defaults()
}
}
fn assistant(c: &str) -> Message {
Message {
role: "assistant".to_string(),
content: Some(c.to_string()),
..Message::runtime_defaults()
}
}
fn assistant_empty_with_tool_call() -> Message {
Message {
role: "assistant".to_string(),
content: None,
tool_calls_json: Some(
r#"[{"id":"c1","name":"terminal","arguments":"{}"}]"#.to_string(),
),
..Message::runtime_defaults()
}
}
fn tool(name: &str, call_id: &str, result: &str) -> Message {
let content = if name == "terminal" {
if let Some(code) = result.strip_prefix("exit ") {
format!("exit_code: {code}")
} else {
result.to_string()
}
} else {
result.to_string()
};
Message {
role: "tool".to_string(),
content: Some(content),
tool_call_id: Some(call_id.to_string()),
tool_name: Some(name.to_string()),
..Message::runtime_defaults()
}
}
#[test]
fn archived_keeps_user_full_and_last_nonempty_assistant_truncated() {
let turn = vec![
user("please do the long thing with lots of detail ...full text..."),
assistant_empty_with_tool_call(),
tool("terminal", "c1", "exit 0"),
assistant(&"X".repeat(500)), assistant(""), ];
let out = render_turn(
&turn,
RenderMode::Archived {
terminal_state: TerminalState::Completed,
},
RENDERER_VERSION,
&RenderOptions::default(),
);
let joined = serde_json::to_string(&out).unwrap();
assert!(
joined.contains("...full text..."),
"user text survives in full"
);
assert!(
joined.contains(&"X".repeat(MAX_OLD_ASSISTANT_CONTENT_CHARS)),
"assistant truncated to cap"
);
assert!(!joined.contains(&"X".repeat(MAX_OLD_ASSISTANT_CONTENT_CHARS + 1)));
assert!(
joined.contains("terminal: -> exit 0"),
"tool result summarized deterministically"
);
}
#[test]
fn archived_no_text_reply_uses_terminal_state_placeholder() {
let turn = vec![
user("run it"),
assistant_empty_with_tool_call(),
tool("terminal", "c1", "exit 1"),
];
let out = render_turn(
&turn,
RenderMode::Archived {
terminal_state: TerminalState::Failed,
},
RENDERER_VERSION,
&RenderOptions::default(),
);
let joined = serde_json::to_string(&out).unwrap();
assert!(joined.contains("[failed: 1 tool steps, no text reply]"));
}
#[test]
fn archived_interrupted_turn_renders_interrupted_placeholder() {
let turn = vec![user("hello"), tool("terminal", "c1", "exit 0")];
let out = render_turn(
&turn,
RenderMode::Archived {
terminal_state: TerminalState::Interrupted,
},
RENDERER_VERSION,
&RenderOptions::default(),
);
assert!(serde_json::to_string(&out)
.unwrap()
.contains("[task interrupted]"));
}
#[test]
fn archived_preserves_identity_critical_verbatim() {
let turn = vec![user("my name is David Loor"), assistant("noted")];
let out = render_turn(
&turn,
RenderMode::Archived {
terminal_state: TerminalState::Completed,
},
RENDERER_VERSION,
&RenderOptions::default(),
);
assert!(serde_json::to_string(&out)
.unwrap()
.contains("my name is David Loor"));
}
#[test]
fn render_is_deterministic() {
let turn = vec![
user("hi"),
assistant("there"),
tool("read_file", "c1", "12 lines"),
];
let a = render_turn(
&turn,
RenderMode::Archived {
terminal_state: TerminalState::Completed,
},
RENDERER_VERSION,
&RenderOptions::default(),
);
let b = render_turn(
&turn,
RenderMode::Archived {
terminal_state: TerminalState::Completed,
},
RENDERER_VERSION,
&RenderOptions::default(),
);
assert_eq!(a, b);
}
#[test]
fn current_mode_is_append_only_full() {
let turn = vec![user("hi"), assistant("there")];
let out = render_turn(
&turn,
RenderMode::Current,
RENDERER_VERSION,
&RenderOptions::default(),
);
assert_eq!(out.len(), 2);
assert_eq!(out[0]["content"], "hi");
assert_eq!(out[1]["content"], "there");
}
#[test]
fn archived_output_order_is_chronological() {
let turn = vec![
user("u"),
assistant_empty_with_tool_call(),
tool("terminal", "c1", "exit 0"),
assistant("final"),
];
let out = render_turn(
&turn,
RenderMode::Archived {
terminal_state: TerminalState::Completed,
},
RENDERER_VERSION,
&RenderOptions::default(),
);
let roles: Vec<&str> = out.iter().map(|m| m["role"].as_str().unwrap()).collect();
assert_eq!(roles, vec!["user", "assistant", "tool", "assistant"]);
}
#[test]
fn archived_no_user_message_turn_renders_without_synthesizing_user() {
let turn = vec![
assistant_empty_with_tool_call(),
tool("terminal", "c1", "exit 0"),
];
let out = render_turn(
&turn,
RenderMode::Archived {
terminal_state: TerminalState::Completed,
},
RENDERER_VERSION,
&RenderOptions::default(),
);
assert!(
out.iter().all(|m| m["role"] != "user"),
"no synthetic user message"
);
assert!(
serde_json::to_string(&out)
.unwrap()
.contains("1 tool steps")
|| out.iter().any(|m| m["role"] == "assistant")
);
}
#[test]
fn archived_identity_message_emitted_once_not_duplicated() {
let turn = vec![
user("hi"),
assistant("my name is David Loor"),
assistant("ok done"),
];
let out = render_turn(
&turn,
RenderMode::Archived {
terminal_state: TerminalState::Completed,
},
RENDERER_VERSION,
&RenderOptions::default(),
);
let joined = serde_json::to_string(&out).unwrap();
assert!(
joined.contains("my name is David Loor"),
"identity survives verbatim"
);
assert_eq!(
joined.matches("my name is David Loor").count(),
1,
"emitted once, not duplicated"
);
}
#[test]
fn current_mode_drops_learned_helplessness_but_archived_uses_placeholder() {
let turn = vec![
user("do it"),
assistant_empty_with_tool_call(),
tool("terminal", "c1", "exit 1"),
assistant("I wasn't able to complete this task."),
];
let cur = render_turn(
&turn,
RenderMode::Current,
RENDERER_VERSION,
&RenderOptions::default(),
);
assert!(
!serde_json::to_string(&cur)
.unwrap()
.contains("I wasn't able to complete"),
"learned-helplessness dropped in Current"
);
let arch = render_turn(
&turn,
RenderMode::Archived {
terminal_state: TerminalState::Failed,
},
RENDERER_VERSION,
&RenderOptions::default(),
);
let aj = serde_json::to_string(&arch).unwrap();
assert!(aj.contains("[failed: 1 tool steps, no text reply]"));
assert!(
!aj.contains("I wasn't able to complete"),
"Archived failure boilerplate is replaced by the deterministic terminal placeholder"
);
}
#[test]
fn current_mode_encodes_image_attachments_as_multimodal_array() {
use crate::traits::MessageAttachment;
use std::io::Write;
use tempfile::NamedTempFile;
let mut file = NamedTempFile::new().unwrap();
file.write_all(&[0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A])
.unwrap();
let stub = format!(
"[File received: photo.png (0 KB, image/png)\nSaved to: {}]",
file.path().display()
);
let turn = vec![Message {
role: "user".to_string(),
content: Some(stub),
attachments: vec![MessageAttachment {
local_path: file.path().to_string_lossy().into_owned(),
filename: "photo.png".to_string(),
mime_type: "image/png".to_string(),
size_bytes: 8,
..Default::default()
}],
..Message::runtime_defaults()
}];
let out = render_turn(
&turn,
RenderMode::Current,
RENDERER_VERSION,
&RenderOptions::default(),
);
let user_msg = out
.iter()
.find(|m| m["role"] == "user")
.expect("user message");
let blocks = user_msg["content"].as_array().expect("multimodal array");
assert!(blocks.iter().any(|b| b["type"] == "text"));
assert!(blocks.iter().any(|b| b["type"] == "image_url"));
}
#[test]
fn current_mode_emits_synthetic_user_observation_after_tool_screenshot() {
use crate::traits::{AttachmentProvenance, MessageAttachment};
use std::io::Write;
use tempfile::NamedTempFile;
let mut file = NamedTempFile::new().unwrap();
file.write_all(&[0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A])
.unwrap();
let turn = vec![
assistant_empty_with_tool_call(),
Message {
role: "tool".to_string(),
content: Some("Screenshot captured and delivered to chat.".to_string()),
tool_call_id: Some("c1".to_string()),
tool_name: Some("browser".to_string()),
attachments: vec![MessageAttachment {
local_path: file.path().to_string_lossy().into_owned(),
filename: "screenshot.png".to_string(),
mime_type: "image/png".to_string(),
size_bytes: 8,
provenance: AttachmentProvenance::ToolObservation,
source_tool: Some("browser".to_string()),
}],
..Message::runtime_defaults()
},
];
let out = render_turn(
&turn,
RenderMode::Current,
RENDERER_VERSION,
&RenderOptions::default(),
);
let observation = out
.iter()
.filter(|m| m["role"] == "user")
.find(|m| {
m["content"]
.as_str()
.is_some_and(|text| text.contains("Tool observation image from browser"))
|| m["content"].as_array().is_some_and(|blocks| {
blocks.iter().any(|b| {
b.get("text")
.and_then(|t| t.as_str())
.is_some_and(|text| text.contains("Tool observation image"))
})
})
})
.expect("synthetic tool observation user message");
let blocks = observation["content"]
.as_array()
.expect("multimodal observation");
assert!(blocks.iter().any(|b| b["type"] == "image_url"));
}
#[test]
fn archived_mode_keeps_image_attachments_as_text_stub() {
let turn = vec![Message {
role: "user".to_string(),
content: Some(
"[File received: photo.png (1 KB, image/png)\nSaved to: /tmp/x.png]".to_string(),
),
attachments: vec![crate::traits::MessageAttachment {
local_path: "/tmp/x.png".to_string(),
filename: "photo.png".to_string(),
mime_type: "image/png".to_string(),
size_bytes: 1024,
..Default::default()
}],
..Message::runtime_defaults()
}];
let out = render_turn(
&turn,
RenderMode::Archived {
terminal_state: TerminalState::Completed,
},
RENDERER_VERSION,
&RenderOptions::default(),
);
assert!(out[0]["content"].is_string());
}
}