1use car_engine::{builtin_tool_labels, format_tool_result, tool_output_is_external, Runtime};
14use car_inference::tasks::generate::{ContentBlock, Message, Provenance, ToolCall};
15use car_inference::{GenerateParams, GenerateRequest};
16use car_ir::{ActionProposal, ActionStatus};
17use serde_json::{json, Value};
18use std::collections::HashMap;
19use std::sync::Arc;
20use std::time::Duration;
21
22use super::memory::MemoryTools;
23use super::tool_memory::{approach_from_call, FailureSignature, ToolMemory, RECOVERY_WINDOW_TURNS};
24use crate::coder::native_loop::TurnGenerator;
25
26pub const OBSERVATION_CAP: usize = 16 * 1024;
37
38pub const VALUE_STORE_PREVIEWS_DEFAULT: bool = false;
76
77pub enum AssistantEvent {
81 ModelServed {
85 model_id: String,
86 local_last_resort: bool,
87 },
88 Text(String),
90 ToolCall { name: String, params: Value },
92 ToolResult {
94 name: String,
95 ok: bool,
96 content: String,
97 },
98 Done { text: String },
100 Error(String),
102 GoalEvaluated {
106 iteration: u32,
107 met: bool,
108 grounded: bool,
109 reason: String,
110 },
111}
112
113#[derive(Clone)]
115pub struct AssistantConfig {
116 pub model: Option<String>,
119 pub strict_model: bool,
122 pub max_turns: u32,
124 pub tools: Vec<Value>,
126 pub gated_tools: Vec<String>,
130 pub approval_policy: Option<ApprovalPolicyFn>,
136 pub proactive_memory: Option<Arc<MemoryTools>>,
141 pub tool_memory: Option<Arc<ToolMemory>>,
151 pub tool_labels: Option<HashMap<String, car_verify::infoflow::ToolLabels>>,
162 pub todos: Option<Arc<tokio::sync::Mutex<super::todo::TodoList>>>,
165 pub value_store_previews: bool,
185 pub response_format: Option<car_inference::ResponseFormat>,
202 pub context_window_override: Option<usize>,
211 pub refuse_unadvertised_tools: bool,
221 pub response_format_validator: Option<ResponseFormatValidator>,
229 pub delegate_budget: Option<DelegateBudget>,
234}
235
236pub type ResponseFormatValidator = Arc<dyn Fn(&Value) -> bool + Send + Sync>;
239
240#[derive(Debug, Clone, Copy, PartialEq, Eq)]
244pub struct DelegateBudget {
245 pub max_delegations: u32,
247 pub max_child_turns: u32,
249}
250
251impl Default for DelegateBudget {
252 fn default() -> Self {
253 Self {
254 max_delegations: 20,
255 max_child_turns: 300,
256 }
257 }
258}
259
260pub fn resolve_context_window(
269 override_tokens: Option<usize>,
270 registry_window: usize,
271) -> (usize, Option<String>) {
272 match override_tokens {
273 None => (registry_window, None),
274 Some(requested) if registry_window > 0 && requested > registry_window => (
275 registry_window,
276 Some(format!(
277 "context window override {requested} exceeds the model's known window \
278 {registry_window}; using {registry_window} (a larger value would let the \
279 history overflow the real window and truncate the task provider-side, \
280 which is what compaction exists to prevent)"
281 )),
282 ),
283 Some(requested) => (requested, None),
284 }
285}
286
287pub fn final_text_matches_format(
296 text: &str,
297 format: &car_inference::ResponseFormat,
298 validator: Option<&ResponseFormatValidator>,
299) -> bool {
300 let payload = extract_json_payload(text);
301 match format {
302 car_inference::ResponseFormat::JsonObject => {
303 serde_json::from_str::<Value>(payload).is_ok_and(|v| v.is_object())
304 }
305 car_inference::ResponseFormat::JsonSchema { .. } => {
306 match serde_json::from_str::<Value>(payload) {
307 Ok(v) => validator.is_none_or(|is_valid| is_valid(&v)),
310 Err(_) => false,
311 }
312 }
313 }
314}
315
316pub fn extract_json_payload(text: &str) -> &str {
319 let t = text.trim();
320 let Some(rest) = t.strip_prefix("```") else {
321 return t;
322 };
323 let Some(rest) = rest.strip_suffix("```") else {
324 return t;
325 };
326 let rest = match rest.split_once('\n') {
328 Some((tag, body)) if tag.trim().chars().all(|c| c.is_ascii_alphanumeric()) => body,
329 _ => rest,
330 };
331 rest.trim()
332}
333
334const FORMAT_REPAIR_NUDGE: &str =
339 "Your previous answer was not the requested JSON. Return only the JSON object — \
340 no prose, no code fence, no tool calls.";
341const FORMAT_REPAIR_NUDGE_SCHEMA: &str =
344 "Your previous answer did not match the required JSON Schema. Return only JSON that \
345 conforms to the schema — no prose, no code fence, no tool calls.";
346
347fn format_repair_nudge(format: &car_inference::ResponseFormat) -> &'static str {
349 match format {
350 car_inference::ResponseFormat::JsonObject => FORMAT_REPAIR_NUDGE,
351 car_inference::ResponseFormat::JsonSchema { .. } => FORMAT_REPAIR_NUDGE_SCHEMA,
352 }
353}
354
355pub const FORMAT_REPAIR_NOTICE: &str =
359 "[format repair: final answer was not the requested JSON; re-asked the model once with no tools]";
360pub const FORMAT_REPAIR_STILL_INVALID: &str =
363 "[format repair: the repaired answer still does not match the requested format; returning it as-is]";
364pub const FORMAT_REPAIR_FAILED_PREFIX: &str = "[format repair failed:";
369
370pub const DELEGATE_TOOL: &str = "delegate";
376pub const DELEGATE_DEFAULT_MAX_TURNS: u32 = 25;
378pub const DELEGATE_MAX_TURNS_CAP: u32 = 60;
380
381fn delegable_tool_names(tools: &[Value]) -> Vec<String> {
384 tools
385 .iter()
386 .filter_map(|d| d.get("name").and_then(Value::as_str))
387 .filter(|n| *n != DELEGATE_TOOL)
388 .map(str::to_string)
389 .collect()
390}
391
392pub fn delegate_tool_def(parent_tools: &[Value]) -> Value {
403 let names = delegable_tool_names(parent_tools);
404 json!({
405 "name": DELEGATE_TOOL,
406 "tier": "read_only",
407 "mutating": true,
408 "description": "Hand one self-contained sub-task to a fresh sub-agent that shares \
409 your model, permissions, and working directory but starts with an EMPTY \
410 transcript: it sees only the goal you write, not this conversation. Use it \
411 to keep a long exploration or a noisy batch of tool output out of your own \
412 context. It runs to completion before this call returns and you receive \
413 ONLY its final written answer, so put everything it needs in `goal` and \
414 ask it to report exactly what you need back. It cannot delegate further.",
415 "parameters": {
416 "type": "object",
417 "properties": {
418 "goal": {
419 "type": "string",
420 "description": "The single, self-contained task, with all the context the sub-agent needs and what to report back."
421 },
422 "tools": {
423 "type": "array",
424 "items": { "type": "string", "enum": names },
425 "description": "Tools to grant the sub-agent. Must be a subset of your own; omit for all of them."
426 },
427 "max_turns": {
428 "type": "integer",
429 "minimum": 1,
430 "maximum": DELEGATE_MAX_TURNS_CAP,
431 "description": "Turn budget for the sub-agent (default 25). It reports an error if it runs out."
432 }
433 },
434 "required": ["goal"]
435 }
436 })
437}
438
439#[derive(Debug, Clone, PartialEq)]
441pub struct DelegateRequest {
442 pub goal: String,
443 pub tools: Option<Vec<String>>,
445 pub max_turns: u32,
446}
447
448pub fn parse_delegate_params(params: &Value) -> Result<DelegateRequest, String> {
451 let goal = params
452 .get("goal")
453 .and_then(Value::as_str)
454 .map(str::trim)
455 .filter(|g| !g.is_empty())
456 .ok_or("delegate needs a non-empty `goal` string")?
457 .to_string();
458 let tools = match params.get("tools") {
459 None | Some(Value::Null) => None,
460 Some(Value::Array(items)) => Some(
461 items
462 .iter()
463 .map(|v| {
464 v.as_str().map(str::to_string).ok_or_else(|| {
465 "delegate `tools` must be an array of tool names".to_string()
466 })
467 })
468 .collect::<Result<Vec<_>, _>>()?,
469 ),
470 Some(_) => return Err("delegate `tools` must be an array of tool names".into()),
471 };
472 let max_turns = match params.get("max_turns") {
473 None | Some(Value::Null) => DELEGATE_DEFAULT_MAX_TURNS,
474 Some(v) => {
475 let n = v
476 .as_u64()
477 .filter(|n| *n >= 1)
478 .ok_or("delegate `max_turns` must be a positive integer")?;
479 (n.min(DELEGATE_MAX_TURNS_CAP as u64)) as u32
480 }
481 };
482 Ok(DelegateRequest {
483 goal,
484 tools,
485 max_turns,
486 })
487}
488
489pub fn delegate_child_config(
504 parent: &AssistantConfig,
505 req: &DelegateRequest,
506) -> Result<AssistantConfig, String> {
507 let delegable = delegable_tool_names(&parent.tools);
508 let requested: Vec<String> = match &req.tools {
509 Some(list) => list.clone(),
510 None => delegable.clone(),
511 };
512 let escalations: Vec<&String> = requested
513 .iter()
514 .filter(|t| !delegable.iter().any(|d| d == *t))
515 .collect();
516 if !escalations.is_empty() {
517 let nested = escalations.iter().any(|t| *t == DELEGATE_TOOL);
518 return Err(format!(
519 "privilege escalation rejected: sub-agent tools {escalations:?} are not a subset of \
520 your own tools{}",
521 if nested {
522 " (a sub-agent cannot delegate further)"
523 } else {
524 ""
525 }
526 ));
527 }
528 let tools: Vec<Value> = parent
529 .tools
530 .iter()
531 .filter(|d| {
532 d.get("name")
533 .and_then(Value::as_str)
534 .is_some_and(|n| requested.iter().any(|r| r == n))
535 })
536 .cloned()
537 .collect();
538 Ok(AssistantConfig {
539 tools,
540 refuse_unadvertised_tools: true,
541 response_format_validator: None,
542 delegate_budget: None,
543 max_turns: req.max_turns,
544 todos: None,
545 response_format: None,
546 ..parent.clone()
547 })
548}
549
550fn delegate_child_history(parent_messages: &[Message], goal: &str) -> Vec<Message> {
553 let mut history: Vec<Message> = parent_messages
554 .iter()
555 .take_while(|m| matches!(m, Message::System { .. }))
556 .cloned()
557 .collect();
558 history.push(Message::User {
559 content: goal.to_string(),
560 });
561 history
562}
563
564struct DelegateOutcome {
566 ok: bool,
567 content: String,
570 turns: u32,
571 external: bool,
574 receipts: Vec<AssistantToolReceipt>,
576 spawned: bool,
579}
580
581#[allow(clippy::too_many_arguments)]
589fn run_delegate<'a>(
590 generator: &'a dyn TurnGenerator,
591 runtime: &'a Runtime,
592 parent: &'a AssistantConfig,
593 parent_messages: &'a [Message],
594 params: &'a Value,
595 cancel: &'a std::sync::atomic::AtomicBool,
596 approval: Option<&'a dyn ApprovalGate>,
597 runtime_session_id: Option<&'a str>,
598 redrive_ungrounded_summary: bool,
599 tool_labels: &'a HashMap<String, car_verify::infoflow::ToolLabels>,
600) -> std::pin::Pin<Box<dyn std::future::Future<Output = DelegateOutcome> + Send + 'a>> {
601 Box::pin(async move {
602 let req = match parse_delegate_params(params) {
603 Ok(r) => r,
604 Err(e) => {
605 return DelegateOutcome {
606 ok: false,
607 content: cap(json!({ "error": e }).to_string()),
608 turns: 0,
609 external: false,
610 receipts: Vec::new(),
611 spawned: false,
612 }
613 }
614 };
615 let child_cfg = match delegate_child_config(parent, &req) {
616 Ok(c) => c,
617 Err(e) => {
618 return DelegateOutcome {
619 ok: false,
620 content: cap(json!({ "error": e }).to_string()),
621 turns: 0,
622 external: false,
623 receipts: Vec::new(),
624 spawned: false,
625 }
626 }
627 };
628 let mut child_messages = delegate_child_history(parent_messages, &req.goal);
629 let mut child_emit: &mut (dyn FnMut(AssistantEvent) + Send) = &mut |_| {};
635 let child = run_assistant_loop_cancellable_in_session_durable(
636 generator,
637 runtime,
638 &child_cfg,
639 &mut child_messages,
640 cancel,
641 approval,
642 None,
643 runtime_session_id,
644 None,
645 None,
646 redrive_ungrounded_summary,
647 &mut child_emit,
648 )
649 .await;
650 let external = child
651 .tool_receipts
652 .iter()
653 .any(|r| tool_output_is_external(&r.tool, tool_labels));
654 if child.status == "success" {
655 DelegateOutcome {
656 ok: true,
657 content: cap(child.summary),
658 turns: child.turns,
659 external,
660 receipts: child.tool_receipts,
661 spawned: true,
662 }
663 } else {
664 DelegateOutcome {
668 ok: false,
669 content: cap(json!({
670 "error": format!(
671 "delegate did not finish (status: {}) after {} turns: {}",
672 child.status, child.turns, child.summary
673 )
674 })
675 .to_string()),
676 turns: child.turns,
677 external,
678 receipts: child.tool_receipts,
679 spawned: true,
680 }
681 }
682 })
683}
684
685pub type ApprovalPolicyFn =
689 std::sync::Arc<dyn Fn(&str, &Value) -> ToolApprovalDecision + Send + Sync>;
690
691pub enum ToolApprovalDecision {
693 Allow,
695 RequireApproval,
697 Deny(String),
699}
700
701pub enum ApprovalDecision {
703 Approved,
704 Denied(String),
705}
706
707#[async_trait::async_trait]
712pub trait ApprovalGate: Send + Sync {
713 async fn request(&self, tool: &str, params: &Value) -> ApprovalDecision;
714
715 async fn request_action(&self, _call_id: &str, tool: &str, params: &Value) -> ApprovalDecision {
716 self.request(tool, params).await
717 }
718
719 async fn before_dispatch(
722 &self,
723 _call_id: &str,
724 _tool: &str,
725 _params: &Value,
726 ) -> Result<(), String> {
727 Ok(())
728 }
729
730 async fn after_dispatch(
733 &self,
734 _call_id: &str,
735 _tool: &str,
736 _params: &Value,
737 _ok: bool,
738 _receipt: &Value,
739 ) -> Result<(), String> {
740 Ok(())
741 }
742}
743
744#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
746pub struct AssistantModelAttribution {
747 pub model_id: String,
749 pub local_last_resort: bool,
751}
752
753pub struct AssistantOutcome {
754 pub status: &'static str,
756 pub summary: String,
758 pub turns: u32,
760 pub tools_called: Vec<String>,
762 pub tool_receipts: Vec<AssistantToolReceipt>,
765 pub models_served: Vec<AssistantModelAttribution>,
769 pub model_used: String,
774}
775
776#[derive(Clone, Debug)]
777pub struct AssistantToolReceipt {
778 pub tool: String,
779 pub call_id: Option<String>,
780 pub ok: bool,
781 pub params: Value,
782 pub via: Option<String>,
787}
788
789fn transcript_tool_receipts(messages: &[Message]) -> Vec<AssistantToolReceipt> {
790 let mut calls: std::collections::HashMap<String, (String, Value)> =
791 std::collections::HashMap::new();
792 let mut receipts = Vec::new();
793 for message in messages {
794 match message {
795 Message::Assistant { tool_calls, .. } => {
796 for call in tool_calls {
797 if let Some(id) = call.id.as_deref() {
798 calls.insert(
799 id.to_string(),
800 (
801 call.name.clone(),
802 serde_json::to_value(&call.arguments)
803 .unwrap_or_else(|_| Value::Object(Default::default())),
804 ),
805 );
806 }
807 }
808 }
809 Message::ToolResult {
810 tool_use_id,
811 content,
812 ..
813 } => {
814 let Some((tool, params)) = calls.get(tool_use_id).cloned() else {
815 continue;
816 };
817 let parsed = serde_json::from_str::<Value>(content).ok();
818 let ok = parsed
819 .as_ref()
820 .map(|value| {
821 value.get("error").is_none()
822 && value.get("ok").and_then(Value::as_bool) != Some(false)
823 && value.get("status").and_then(Value::as_str) != Some("Failed")
824 })
825 .unwrap_or_else(|| {
826 let lower = content.to_ascii_lowercase();
827 !lower.contains("declined by user")
828 && !lower.contains("tool call denied")
829 && !lower.starts_with("error:")
830 });
831 receipts.push(AssistantToolReceipt {
832 tool,
833 call_id: Some(tool_use_id.clone()),
834 ok,
835 params,
836 via: None,
837 });
838 }
839 _ => {}
840 }
841 }
842 receipts
843}
844
845fn cap(mut s: String) -> String {
859 let total = s.len();
860 if total <= OBSERVATION_CAP {
861 return s;
862 }
863 let mut end = OBSERVATION_CAP;
864 while !s.is_char_boundary(end) {
865 end -= 1;
866 }
867 let elided = total - end;
868 s.truncate(end);
869 s.push_str(&format!(
872 "\n…[truncated: showing first {end} of {total} bytes; {elided} bytes elided \
873 and NOT retained. To see the rest, re-run this tool with a narrower \
874 query — the elided bytes cannot be recovered by asking for them.]…"
875 ));
876 s
877}
878
879const STATE_BLOCK_OPEN: &str = "\n\n<runtime-state>\n";
884const STATE_BLOCK_CLOSE: &str = "\n</runtime-state>";
885
886fn append_state_block(messages: &mut [Message], block: &str) {
901 let Some(last) = messages.last_mut() else {
902 return;
903 };
904 let fenced = format!("{STATE_BLOCK_OPEN}{block}{STATE_BLOCK_CLOSE}");
905 match last {
906 Message::System { content }
907 | Message::User { content }
908 | Message::Assistant { content, .. }
909 | Message::ToolResult { content, .. } => content.push_str(&fenced),
910 _ => {}
913 }
914}
915
916const STATE_BLOCK_MAX_FACTS: usize = 5;
919
920fn recent_fact_subjects(receipts: &[AssistantToolReceipt]) -> Vec<String> {
928 let mut subjects: Vec<String> = Vec::new();
929 for receipt in receipts.iter().filter(|r| r.ok && r.tool == "remember") {
930 let Some(subject) = receipt.params.get("subject").and_then(Value::as_str) else {
931 continue;
932 };
933 let subject = subject.trim();
934 if subject.is_empty() {
935 continue;
936 }
937 subjects.retain(|s| s != subject);
941 subjects.push(subject.to_string());
942 }
943 subjects
944}
945
946fn render_state_block(todo: Option<String>, facts: &[String]) -> Option<String> {
956 let mut sections: Vec<String> = Vec::new();
957 if let Some(todo) = todo {
958 sections.push(todo);
959 }
960 if !facts.is_empty() {
961 let hidden = facts.len().saturating_sub(STATE_BLOCK_MAX_FACTS);
964 let listed = facts
965 .iter()
966 .skip(hidden)
967 .map(String::as_str)
968 .collect::<Vec<_>>()
969 .join(", ");
970 let mut line = format!("remembered this run: {listed}");
971 if hidden > 0 {
972 line.push_str(&format!(" (+{hidden} earlier)"));
973 }
974 line.push_str("\n subjects only — call `recall` for the content");
975 sections.push(line);
976 }
977 (!sections.is_empty()).then(|| sections.join("\n"))
978}
979
980const HISTORY_MIN_TAIL: usize = 6;
983
984pub(crate) const HISTORY_BUDGET_NUMERATOR: usize = 3;
1000pub(crate) const HISTORY_BUDGET_DENOMINATOR: usize = 4;
1002
1003pub(crate) fn history_budget(context_window: usize) -> usize {
1007 context_window / HISTORY_BUDGET_DENOMINATOR * HISTORY_BUDGET_NUMERATOR
1008}
1009
1010const COMPACTION_NOTICE_PREFIX: &str = "[history compacted:";
1016
1017#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
1028pub(crate) enum CompactionRecovery {
1029 #[default]
1032 EventsQuery,
1033 Unrecoverable,
1035}
1036
1037fn format_compaction_notice(turns: usize, tokens: usize, recovery: CompactionRecovery) -> String {
1046 match recovery {
1047 CompactionRecovery::EventsQuery => format!(
1048 "{COMPACTION_NOTICE_PREFIX} {turns} earlier turns removed to fit the context \
1049 window, ~{tokens} tokens. They are gone from this transcript but the run's \
1050 event log still has them — call `events_query` (e.g. {{\"kinds\": \
1051 [\"action_failed\"], \"limit\": 5}}) to see what was already tried, rather \
1052 than assuming you never tried it.]"
1053 ),
1054 CompactionRecovery::Unrecoverable => format!(
1055 "{COMPACTION_NOTICE_PREFIX} {turns} earlier turns dropped to fit the model's \
1056 context window, ~{tokens} tokens. They are not recoverable in this run — \
1057 work from what is still in this transcript, and do not assume something was \
1058 never tried just because you cannot see it.]"
1059 ),
1060 }
1061}
1062
1063fn parse_compaction_notice(message: &Message) -> Option<(usize, usize)> {
1071 let Message::System { content } = message else {
1072 return None;
1073 };
1074 let rest = content.strip_prefix(COMPACTION_NOTICE_PREFIX)?;
1075 let turns: usize = rest.split_whitespace().next()?.parse().ok()?;
1076 let tokens: usize = rest
1077 .split('~')
1078 .nth(1)?
1079 .split_whitespace()
1080 .next()?
1081 .parse()
1082 .ok()?;
1083 Some((turns, tokens))
1084}
1085
1086fn approx_message_tokens(m: &Message) -> usize {
1092 car_inference::media_tokens::messages_history_tokens(std::slice::from_ref(m))
1093}
1094
1095pub(crate) fn compact_history_to_window(messages: &mut Vec<Message>, context_window: usize) {
1108 compact_history_measured(messages, context_window, PromptMeasure::default())
1109}
1110
1111#[derive(Debug, Clone, Copy, Default)]
1124pub(crate) struct PromptMeasure {
1125 pub fixed_overhead: usize,
1128 pub reported: Option<(usize, usize)>,
1133}
1134
1135pub(crate) fn message_estimates(messages: &[Message]) -> Vec<usize> {
1143 messages.iter().map(approx_message_tokens).collect()
1144}
1145
1146pub(crate) fn measure_scale(estimates: &[usize], measure: PromptMeasure) -> f64 {
1162 let Some((reported, covered)) = measure.reported else {
1163 return 1.0;
1164 };
1165 let covered = covered.min(estimates.len());
1166 let covered_est: usize = estimates[..covered].iter().sum::<usize>() + measure.fixed_overhead;
1167 if covered_est > 0 && reported * 4 > covered_est * 5 {
1168 reported as f64 / covered_est as f64
1169 } else {
1170 1.0
1171 }
1172}
1173
1174pub(crate) fn scaled_prompt_tokens(
1182 messages: &[Message],
1183 fixed_overhead: usize,
1184 scale: f64,
1185) -> usize {
1186 let estimate =
1187 car_inference::media_tokens::request_prompt_tokens("", None, None, None, Some(messages))
1188 + fixed_overhead;
1189 (estimate as f64 * scale).round() as usize
1190}
1191
1192pub(crate) fn compact_history_measured(
1198 messages: &mut Vec<Message>,
1199 context_window: usize,
1200 measure: PromptMeasure,
1201) {
1202 compact_history_measured_with_recovery(
1203 messages,
1204 context_window,
1205 measure,
1206 CompactionRecovery::default(),
1207 )
1208}
1209
1210pub(crate) fn compact_history_measured_with_recovery(
1215 messages: &mut Vec<Message>,
1216 context_window: usize,
1217 measure: PromptMeasure,
1218 recovery: CompactionRecovery,
1219) {
1220 if context_window == 0 {
1221 return;
1222 }
1223 let budget = history_budget(context_window);
1224 let estimates: Vec<usize> = message_estimates(messages);
1225 let estimated: usize =
1231 car_inference::media_tokens::request_prompt_tokens("", None, None, None, Some(messages))
1232 + measure.fixed_overhead;
1233 let (total, scale, reported) = match measure.reported {
1235 Some((reported, covered)) => {
1236 let covered = covered.min(messages.len());
1237 let appended: usize = estimates[covered..].iter().sum();
1238 let scale = measure_scale(&estimates, measure);
1239 let appended_scaled = (appended as f64 * scale).round() as usize;
1240 (reported + appended_scaled, scale, Some(reported))
1241 }
1242 None => (estimated, 1.0, None),
1243 };
1244 let scaled = |tokens: usize| (tokens as f64 * scale).round() as usize;
1245 if total <= budget {
1246 return;
1247 }
1248 tracing::info!(
1249 reported_prompt_tokens = reported,
1250 estimated_prompt_tokens = estimated,
1251 measured_prompt_tokens = total,
1252 scale,
1253 budget,
1254 context_window,
1255 "history exceeds the compaction budget"
1256 );
1257
1258 let mut head_end = 0;
1260 while head_end < messages.len() && matches!(messages[head_end], Message::System { .. }) {
1261 head_end += 1;
1262 }
1263 if head_end < messages.len()
1264 && matches!(
1265 messages[head_end],
1266 Message::User { .. } | Message::UserMultimodal { .. }
1267 )
1268 {
1269 head_end += 1;
1270 }
1271 let existing_notice = messages
1276 .get(head_end)
1277 .and_then(parse_compaction_notice)
1278 .map(|totals| {
1279 let at = head_end;
1280 head_end += 1;
1281 (at, totals)
1282 });
1283
1284 if messages.len().saturating_sub(head_end) <= HISTORY_MIN_TAIL {
1286 return;
1287 }
1288 let max_drop = messages.len() - HISTORY_MIN_TAIL;
1289
1290 let mut drop_end = head_end;
1292 let mut running = total;
1293 while running > budget && drop_end < max_drop {
1294 running = running.saturating_sub(scaled(estimates[drop_end]));
1295 drop_end += 1;
1296 }
1297 if drop_end > head_end
1302 && drop_end < messages.len()
1303 && matches!(messages[drop_end - 1], Message::ProviderOutputItems { .. })
1304 && matches!(messages[drop_end], Message::Assistant { .. })
1305 {
1306 drop_end += 1;
1307 }
1308 while drop_end < messages.len() && matches!(messages[drop_end], Message::ToolResult { .. }) {
1309 drop_end += 1;
1310 }
1311 if drop_end <= head_end {
1312 return;
1313 }
1314 let dropped = drop_end - head_end;
1315 let dropped_tokens: usize = estimates[head_end..drop_end]
1318 .iter()
1319 .map(|t| scaled(*t))
1320 .sum();
1321 messages.drain(head_end..drop_end);
1322 match existing_notice {
1335 Some((at, (prior_turns, prior_tokens))) => {
1336 messages[at] = Message::System {
1337 content: format_compaction_notice(
1338 prior_turns + dropped,
1339 prior_tokens + dropped_tokens,
1340 recovery,
1341 ),
1342 };
1343 }
1344 None => messages.insert(
1345 head_end,
1346 Message::System {
1347 content: format_compaction_notice(dropped, dropped_tokens, recovery),
1348 },
1349 ),
1350 }
1351 tracing::debug!(
1352 dropped_messages = dropped,
1353 kept = messages.len(),
1354 context_window,
1355 budget,
1356 "compacted assistant history to fit the model context window"
1357 );
1358}
1359
1360fn message_memory_text(message: &Message) -> Option<String> {
1361 match message {
1362 Message::System { content }
1363 | Message::User { content }
1364 | Message::Assistant { content, .. }
1365 | Message::ToolResult { content, .. } => {
1366 let trimmed = content.trim();
1367 (!trimmed.is_empty()).then(|| trimmed.to_string())
1368 }
1369 Message::UserMultimodal { content } => {
1370 let text = content
1371 .iter()
1372 .filter_map(|block| match block {
1373 ContentBlock::Text { text } => Some(text.trim()),
1374 _ => None,
1375 })
1376 .filter(|s| !s.is_empty())
1377 .collect::<Vec<_>>()
1378 .join("\n");
1379 (!text.is_empty()).then_some(text)
1380 }
1381 _ => None,
1382 }
1383}
1384
1385fn proactive_query_from_messages(messages: &[Message]) -> String {
1386 messages
1387 .iter()
1388 .rev()
1389 .find_map(|m| match m {
1390 Message::User { content } => {
1391 let trimmed = content.trim();
1392 (!trimmed.is_empty()).then(|| trimmed.to_string())
1393 }
1394 Message::UserMultimodal { .. } => message_memory_text(m),
1395 _ => None,
1396 })
1397 .unwrap_or_default()
1398}
1399
1400fn append_context_block(req: &mut GenerateRequest, title: &str, body: &str) {
1401 let block = format!("## {title}\n{body}");
1402 req.context = Some(match req.context.take() {
1403 Some(existing) if !existing.trim().is_empty() => format!("{existing}\n\n{block}"),
1404 _ => block,
1405 });
1406}
1407
1408fn proactive_maintenance_event_data(
1409 report: &car_memgine::ProactiveMaintenanceReport,
1410) -> std::collections::HashMap<String, Value> {
1411 let mut data = proactive_trigger_event_data(&report.trigger);
1412 data.insert(
1413 "saved_count".to_string(),
1414 Value::from(report.saved.len() as u64),
1415 );
1416 data.insert(
1417 "skipped_existing".to_string(),
1418 Value::from(report.skipped_existing as u64),
1419 );
1420 data.insert(
1421 "status_updated".to_string(),
1422 Value::from(report.status.is_some()),
1423 );
1424 data
1425}
1426
1427fn proactive_intervention_event_data(
1428 decision: &car_memgine::ProactiveMemoryDecision,
1429) -> std::collections::HashMap<String, Value> {
1430 let mut data = std::collections::HashMap::new();
1431 match decision {
1432 car_memgine::ProactiveMemoryDecision::Inject {
1433 selected,
1434 candidates,
1435 bank,
1436 ..
1437 } => {
1438 data.insert("decision".to_string(), Value::from("inject"));
1439 data.insert("selected_id".to_string(), Value::from(selected.id.clone()));
1440 data.insert(
1441 "selected_kind".to_string(),
1442 Value::from(format!("{:?}", selected.kind).to_ascii_lowercase()),
1443 );
1444 data.insert(
1445 "candidate_count".to_string(),
1446 Value::from(candidates.len() as u64),
1447 );
1448 data.insert(
1449 "bank_knowledge".to_string(),
1450 Value::from(bank.knowledge as u64),
1451 );
1452 data.insert(
1453 "bank_procedural".to_string(),
1454 Value::from(bank.procedural as u64),
1455 );
1456 data.insert(
1457 "bank_open_subgoals".to_string(),
1458 Value::from(bank.open_subgoals as u64),
1459 );
1460 }
1461 car_memgine::ProactiveMemoryDecision::Silent {
1462 reason,
1463 candidates,
1464 bank,
1465 } => {
1466 data.insert("decision".to_string(), Value::from("silent"));
1467 data.insert("reason".to_string(), Value::from(reason.clone()));
1468 data.insert(
1469 "candidate_count".to_string(),
1470 Value::from(candidates.len() as u64),
1471 );
1472 data.insert(
1473 "bank_knowledge".to_string(),
1474 Value::from(bank.knowledge as u64),
1475 );
1476 data.insert(
1477 "bank_procedural".to_string(),
1478 Value::from(bank.procedural as u64),
1479 );
1480 data.insert(
1481 "bank_open_subgoals".to_string(),
1482 Value::from(bank.open_subgoals as u64),
1483 );
1484 }
1485 }
1486 data
1487}
1488
1489fn proactive_trigger_event_data(
1490 trigger: &car_memgine::ProactiveMemoryTrigger,
1491) -> std::collections::HashMap<String, Value> {
1492 std::collections::HashMap::from([
1493 (
1494 "repeated_failures".to_string(),
1495 Value::from(trigger.repeated_failures as u64),
1496 ),
1497 ("tool_error".to_string(), Value::from(trigger.tool_error)),
1498 (
1499 "explicit_uncertainty".to_string(),
1500 Value::from(trigger.explicit_uncertainty),
1501 ),
1502 (
1503 "high_risk_action".to_string(),
1504 Value::from(trigger.high_risk_action),
1505 ),
1506 (
1507 "context_shift".to_string(),
1508 Value::from(trigger.context_shift),
1509 ),
1510 ])
1511}
1512
1513async fn record_inference_metered(runtime: &Runtime, result: &car_inference::InferenceResult) {
1541 let mut data: HashMap<String, Value> = HashMap::new();
1542 data.insert(
1543 "model_id".to_string(),
1544 Value::from(result.served_model_id().to_string()),
1545 );
1546 data.insert(
1549 "usage_measured".to_string(),
1550 Value::from(result.usage.is_some()),
1551 );
1552
1553 let metrics = match &result.usage {
1554 Some(u) => car_eventlog::Metrics::inference(u.prompt_tokens, u.completion_tokens, None)
1555 .with_duration(result.latency_ms as f64),
1556 None => car_eventlog::Metrics::latency(result.latency_ms as f64),
1557 };
1558
1559 runtime.log.lock().await.append_metered(
1560 car_eventlog::EventKind::InferenceMetered,
1561 None,
1562 None,
1563 data,
1564 metrics,
1565 );
1566}
1567
1568async fn maybe_apply_assistant_proactive_memory(
1569 cfg: &AssistantConfig,
1570 runtime: &Runtime,
1571 req: &mut GenerateRequest,
1572 messages: &[Message],
1573) {
1574 let Some(memory) = &cfg.proactive_memory else {
1575 return;
1576 };
1577 let query = proactive_query_from_messages(messages);
1578 if query.trim().is_empty() {
1579 return;
1580 }
1581 let mut recent = messages
1582 .iter()
1583 .rev()
1584 .filter_map(message_memory_text)
1585 .take(6)
1586 .collect::<Vec<_>>();
1587 recent.reverse();
1588 let events = {
1589 let log = runtime.log.lock().await;
1590 log.events().to_vec()
1591 };
1592 let (maintenance, decision) = match memory.proactive_intervention(&query, recent, &events).await
1593 {
1594 Ok(out) => out,
1595 Err(e) => {
1596 tracing::debug!(error = %e, "assistant proactive memory pass failed");
1597 return;
1598 }
1599 };
1600 {
1601 let mut log = runtime.log.lock().await;
1602 log.append(
1603 car_eventlog::EventKind::ProactiveMemoryMaintained,
1604 None,
1605 None,
1606 proactive_maintenance_event_data(&maintenance),
1607 );
1608 log.append(
1609 car_eventlog::EventKind::ProactiveMemoryIntervention,
1610 None,
1611 None,
1612 proactive_intervention_event_data(&decision),
1613 );
1614 }
1615 if let car_memgine::ProactiveMemoryDecision::Inject { reminder, .. } = decision {
1616 append_context_block(req, "Proactive Memory", &reminder);
1617 }
1618}
1619
1620#[derive(Default)]
1628struct OpenFailures {
1629 by_tool: HashMap<String, OpenFailure>,
1640 offered: std::collections::HashSet<String>,
1645 penalized: std::collections::HashSet<String>,
1648}
1649
1650struct OpenFailure {
1651 sig: FailureSignature,
1652 turn: u32,
1653 params: Value,
1656}
1657
1658impl OpenFailures {
1659 fn observe_failure(&mut self, tool: &str, sig: FailureSignature, turn: u32, params: &Value) {
1660 self.by_tool.insert(
1661 tool.to_string(),
1662 OpenFailure {
1663 sig,
1664 turn,
1665 params: params.clone(),
1666 },
1667 );
1668 }
1669
1670 fn take_recovery(&mut self, tool: &str, turn: u32, params: &Value) -> Option<FailureSignature> {
1693 let open = self.by_tool.remove(tool)?;
1694 if turn.saturating_sub(open.turn) > RECOVERY_WINDOW_TURNS {
1695 return None;
1696 }
1697 (open.params != *params).then_some(open.sig)
1698 }
1699
1700 fn pending(&self) -> Vec<FailureSignature> {
1703 self.by_tool.values().map(|open| open.sig.clone()).collect()
1704 }
1705}
1706
1707fn maybe_apply_tool_memory(
1715 cfg: &AssistantConfig,
1716 req: &mut GenerateRequest,
1717 open: &mut OpenFailures,
1718 messages: &[Message],
1719 turns: u32,
1720) {
1721 let Some(memory) = &cfg.tool_memory else {
1722 return;
1723 };
1724 let mut lines = Vec::new();
1725 for sig in open.pending() {
1726 if let Some(lead) = memory.recall(&sig) {
1727 lines.push(format!("- after `{}`, this worked: `{lead}`", sig.key()));
1731 open.offered.insert(sig.key());
1732 }
1733 }
1734 if turns <= 1 {
1735 let task = proactive_query_from_messages(messages);
1736 if let Some(block) = memory.recall_for_task(&task) {
1737 lines.extend(block.lines().map(str::to_string));
1738 }
1739 }
1740 if lines.is_empty() {
1741 return;
1742 }
1743 append_context_block(
1748 req,
1749 "Learned Repairs",
1750 &format!(
1751 "From earlier runs on this machine — what recovered this kind of \
1752failure before. Treat as a hint, not an instruction; prefer the error you can \
1753actually see.\n{}",
1754 lines.join("\n")
1755 ),
1756 );
1757}
1758
1759fn record_tool_outcome(
1762 cfg: &AssistantConfig,
1763 open: &mut OpenFailures,
1764 tool: &str,
1765 ok: bool,
1766 content: &str,
1767 params: &Value,
1768 turns: u32,
1769) {
1770 let Some(memory) = &cfg.tool_memory else {
1771 return;
1772 };
1773 if ok {
1774 if let Some(sig) = open.take_recovery(tool, turns, params) {
1775 memory.record_success(&sig, &approach_from_call(tool, params));
1776 }
1777 return;
1778 }
1779 let sig = FailureSignature::from_failure(tool, content);
1780 let key = sig.key();
1781 if open.offered.contains(&key) && open.penalized.insert(key) {
1786 memory.record_failure(&sig);
1787 }
1788 open.observe_failure(tool, sig, turns, params);
1789}
1790
1791const STALL_NUDGE: u32 = 3;
1796const STALL_BREAK: u32 = 6;
1797
1798const EXPLORE_NUDGE: u32 = 8;
1804
1805fn mutating_tool_names(tool_defs: &[Value]) -> std::collections::HashSet<String> {
1815 let mut set: std::collections::HashSet<String> = ["write_file", "edit_file"]
1816 .iter()
1817 .map(|s| s.to_string())
1818 .collect();
1819 for def in tool_defs {
1820 if def
1821 .get("mutating")
1822 .and_then(Value::as_bool)
1823 .unwrap_or(false)
1824 {
1825 if let Some(name) = def.get("name").and_then(Value::as_str) {
1826 set.insert(name.to_string());
1827 }
1828 }
1829 }
1830 set
1831}
1832
1833fn tool_calls_signature(calls: &[ToolCall]) -> String {
1837 let mut parts: Vec<String> = calls
1838 .iter()
1839 .map(|c| {
1840 format!(
1841 "{}({})",
1842 c.name,
1843 serde_json::to_string(&c.arguments).unwrap_or_default()
1844 )
1845 })
1846 .collect();
1847 parts.sort();
1848 parts.join("|")
1849}
1850
1851#[derive(Debug, PartialEq, Eq)]
1854enum GuardStep {
1855 Progress,
1857 Continue,
1859 Nudge,
1861 Break,
1863}
1864
1865#[derive(Default)]
1875struct NoProgressGuard {
1876 seen_sigs: std::collections::HashSet<String>,
1877 stall_repeats: u32,
1878 turns_since_mutation: u32,
1879 nudged: bool,
1880}
1881
1882impl NoProgressGuard {
1883 fn observe(&mut self, sig: &str, mutated_ok: bool) -> GuardStep {
1886 let sig_is_new = self.seen_sigs.insert(sig.to_string());
1887 if mutated_ok && sig_is_new {
1888 self.seen_sigs.clear();
1891 self.seen_sigs.insert(sig.to_string());
1892 self.stall_repeats = 0;
1893 self.turns_since_mutation = 0;
1894 self.nudged = false;
1895 return GuardStep::Progress;
1896 }
1897 self.turns_since_mutation += 1;
1899 if !sig_is_new {
1900 self.stall_repeats += 1;
1901 if self.stall_repeats >= STALL_BREAK {
1902 return GuardStep::Break;
1903 }
1904 if self.stall_repeats >= STALL_NUDGE && !self.nudged {
1905 self.nudged = true;
1906 return GuardStep::Nudge;
1907 }
1908 }
1909 if self.turns_since_mutation >= EXPLORE_NUDGE && !self.nudged {
1910 self.nudged = true;
1911 return GuardStep::Nudge;
1912 }
1913 GuardStep::Continue
1914 }
1915}
1916
1917fn build_proposal(
1927 source: &str,
1928 call: &ToolCall,
1929 parameters: &Value,
1930) -> Result<ActionProposal, String> {
1931 serde_json::from_value(json!({
1932 "source": source,
1933 "actions": [{
1934 "id": call.id,
1935 "type": "tool_call",
1936 "tool": call.name,
1937 "parameters": parameters,
1938 }],
1939 }))
1940 .map_err(|e| format!("malformed proposal: {e}"))
1941}
1942
1943pub async fn run_assistant_loop(
1947 generator: &dyn TurnGenerator,
1948 runtime: &Runtime,
1949 cfg: &AssistantConfig,
1950 messages: &mut Vec<Message>,
1951 emit: impl FnMut(AssistantEvent),
1952) -> AssistantOutcome {
1953 let never = std::sync::atomic::AtomicBool::new(false);
1954 run_assistant_loop_cancellable(generator, runtime, cfg, messages, &never, None, None, emit)
1955 .await
1956}
1957
1958pub async fn run_assistant_loop_cancellable(
1962 generator: &dyn TurnGenerator,
1963 runtime: &Runtime,
1964 cfg: &AssistantConfig,
1965 messages: &mut Vec<Message>,
1966 cancel: &std::sync::atomic::AtomicBool,
1967 approval: Option<&dyn ApprovalGate>,
1968 images: Option<&[ContentBlock]>,
1969 emit: impl FnMut(AssistantEvent),
1970) -> AssistantOutcome {
1971 run_assistant_loop_cancellable_in_session(
1972 generator, runtime, cfg, messages, cancel, approval, images, None, emit,
1973 )
1974 .await
1975}
1976
1977pub async fn run_assistant_loop_cancellable_in_session(
1981 generator: &dyn TurnGenerator,
1982 runtime: &Runtime,
1983 cfg: &AssistantConfig,
1984 messages: &mut Vec<Message>,
1985 cancel: &std::sync::atomic::AtomicBool,
1986 approval: Option<&dyn ApprovalGate>,
1987 images: Option<&[ContentBlock]>,
1988 runtime_session_id: Option<&str>,
1989 emit: impl FnMut(AssistantEvent),
1990) -> AssistantOutcome {
1991 run_assistant_loop_cancellable_in_session_durable(
1992 generator,
1993 runtime,
1994 cfg,
1995 messages,
1996 cancel,
1997 approval,
1998 images,
1999 runtime_session_id,
2000 None,
2001 None,
2002 true,
2003 emit,
2004 )
2005 .await
2006}
2007
2008pub async fn run_assistant_loop_cancellable_in_session_durable(
2012 generator: &dyn TurnGenerator,
2013 runtime: &Runtime,
2014 cfg: &AssistantConfig,
2015 messages: &mut Vec<Message>,
2016 cancel: &std::sync::atomic::AtomicBool,
2017 approval: Option<&dyn ApprovalGate>,
2018 images: Option<&[ContentBlock]>,
2019 runtime_session_id: Option<&str>,
2020 durable_session_id: Option<&str>,
2021 durability: Option<&dyn super::governance::AssistantDurability>,
2022 redrive_ungrounded_summary: bool,
2023 mut emit: impl FnMut(AssistantEvent),
2024) -> AssistantOutcome {
2025 use std::sync::atomic::Ordering;
2026 let tools = if cfg.tools.is_empty() {
2027 None
2028 } else {
2029 Some(cfg.tools.clone())
2030 };
2031 let mut tools_called: Vec<String> = Vec::new();
2032 let mut tool_receipts: Vec<AssistantToolReceipt> = transcript_tool_receipts(messages);
2036 let mut values = super::value_store::SessionValues::new();
2040 let mut last_text = String::new();
2041 let mut last_model = String::new();
2042 let mut models_served = Vec::new();
2043 let mut turns = 0u32;
2044 let mut claim_corrections = 0u8;
2045 let registry_window = cfg
2050 .model
2051 .as_deref()
2052 .map(|m| generator.context_window(m))
2053 .unwrap_or(0);
2054 let (context_window, window_advisory) =
2055 resolve_context_window(cfg.context_window_override, registry_window);
2056 if let Some(advisory) = window_advisory {
2057 tracing::warn!(
2058 requested = cfg.context_window_override,
2059 registry_window,
2060 "{advisory}"
2061 );
2062 emit(AssistantEvent::Text(format!(
2063 "[context window: {advisory}]"
2064 )));
2065 }
2066 let mut prompt_measure = PromptMeasure {
2071 fixed_overhead: car_inference::media_tokens::tool_defs_tokens(&cfg.tools),
2072 reported: None,
2073 };
2074 let mutating_tools = mutating_tool_names(&cfg.tools);
2079 let advertised_names: std::collections::HashSet<String> = cfg
2082 .tools
2083 .iter()
2084 .filter_map(|d| d.get("name").and_then(Value::as_str))
2085 .map(str::to_string)
2086 .collect();
2087 let delegate_advertised = advertised_names.contains(DELEGATE_TOOL);
2088 let delegate_budget = cfg.delegate_budget.unwrap_or_default();
2090 let mut delegations_spawned: u32 = 0;
2091 let mut child_turns_used: u32 = 0;
2092 let builtin_labels;
2097 let tool_labels = match &cfg.tool_labels {
2098 Some(m) => m,
2099 None => {
2100 builtin_labels = builtin_tool_labels();
2101 &builtin_labels
2102 }
2103 };
2104 let mut guard = NoProgressGuard::default();
2109 let mut open_failures = OpenFailures::default();
2112
2113 while turns < cfg.max_turns {
2114 if cancel.load(Ordering::Relaxed) {
2115 return AssistantOutcome {
2116 status: "cancelled",
2117 summary: "cancelled".to_string(),
2118 turns,
2119 tools_called,
2120 tool_receipts,
2121 models_served: models_served.clone(),
2122 model_used: last_model.clone(),
2123 };
2124 }
2125 turns += 1;
2126
2127 let before_compaction = messages.clone();
2131 compact_history_measured(messages, context_window, prompt_measure);
2132 if before_compaction != *messages {
2133 prompt_measure.reported = None;
2136 if let (Some(store), Some(session_id)) = (durability, durable_session_id) {
2137 if let Err(e) = store
2138 .checkpoint(session_id, messages, "history_compacted", None)
2139 .await
2140 {
2141 let msg = format!("durable checkpoint failed after compaction: {e}");
2142 emit(AssistantEvent::Error(msg.clone()));
2143 return AssistantOutcome {
2144 status: "error",
2145 summary: msg,
2146 turns,
2147 tools_called,
2148 tool_receipts,
2149 models_served: models_served.clone(),
2150 model_used: last_model,
2151 };
2152 }
2153 }
2154 }
2155
2156 let mut req = GenerateRequest {
2157 prompt: String::new(),
2158 model: cfg.model.clone(),
2159 params: GenerateParams {
2160 temperature: 0.0,
2161 strict_model: cfg.strict_model,
2162 ..Default::default()
2163 },
2164 context: None,
2165 context_stable_prefix: None,
2166 tools: tools.clone(),
2167 images: if turns == 1 {
2170 images.map(|imgs| imgs.to_vec())
2171 } else {
2172 None
2173 },
2174 messages: Some(messages.clone()),
2175 cache_control: false,
2176 response_format: if tools.is_none() {
2181 cfg.response_format.clone()
2182 } else {
2183 None
2184 },
2185 intent: None,
2186 client_ref: None,
2187 expected_row_digest: None,
2188 expected_catalog_revision: None,
2189 caller: None,
2190 };
2191 maybe_apply_assistant_proactive_memory(cfg, runtime, &mut req, messages).await;
2192 maybe_apply_tool_memory(cfg, &mut req, &mut open_failures, messages, turns);
2193
2194 let todo_render = match &cfg.todos {
2210 Some(todos) => todos.lock().await.render(),
2211 None => None,
2212 };
2213 if let Some(block) = render_state_block(todo_render, &recent_fact_subjects(&tool_receipts))
2214 {
2215 if let Some(msgs) = req.messages.as_mut() {
2216 append_state_block(msgs, &block);
2217 }
2218 }
2219 let request_covers = messages.len();
2227
2228 let mut result = match generator.generate(req).await {
2229 Ok(r) => r,
2230 Err(e) => {
2231 let msg = format!("inference failed: {e}");
2232 emit(AssistantEvent::Error(msg.clone()));
2233 return AssistantOutcome {
2234 status: "error",
2235 summary: msg,
2236 turns,
2237 tools_called,
2238 tool_receipts,
2239 models_served: models_served.clone(),
2240 model_used: last_model.clone(),
2241 };
2242 }
2243 };
2244 record_inference_metered(runtime, &result).await;
2255 let attribution = AssistantModelAttribution {
2256 model_id: result.served_model_id().to_string(),
2257 local_last_resort: result.local_last_resort,
2258 };
2259 emit(AssistantEvent::ModelServed {
2260 model_id: attribution.model_id.clone(),
2261 local_last_resort: attribution.local_last_resort,
2262 });
2263 models_served.push(attribution);
2264 if let Some(u) = &result.usage {
2268 let input = u.prompt_tokens + u.cache_read_input_tokens + u.cache_creation_input_tokens;
2269 if input > 0 {
2270 prompt_measure.reported = Some((input as usize, request_covers));
2271 }
2272 }
2273 result.text = car_inference::tasks::generate::strip_leaked_reasoning(&result.text);
2279 last_model = result.served_model_id().to_string();
2280
2281 if result.tool_calls.is_empty() {
2283 last_text = result.text.clone();
2284 let ungrounded = ungrounded_summary_claims(&last_text, &tool_receipts);
2285 if redrive_ungrounded_summary && !ungrounded.is_empty() {
2286 result.append_assistant_history(messages, vec![]);
2287 if claim_corrections < 2 && turns < cfg.max_turns {
2288 claim_corrections += 1;
2289 messages.push(Message::User {
2290 content: format!(
2291 "Evidence check rejected the draft's unsupported operational claim(s): {}. \
2292 Rewrite the answer using only claims supported by successful transcript \
2293 tool receipts. Preserve useful source findings, explicitly mark missing \
2294 live evidence, and do not rerun completed actions merely to support prose.",
2295 ungrounded.join(", ")
2296 ),
2297 });
2298 if let (Some(store), Some(session_id)) = (durability, durable_session_id) {
2299 if let Err(e) = store
2300 .checkpoint(session_id, messages, "ungrounded_summary_redrive", None)
2301 .await
2302 {
2303 let msg =
2304 format!("durable checkpoint failed before claim correction: {e}");
2305 emit(AssistantEvent::Error(msg.clone()));
2306 return AssistantOutcome {
2307 status: "error",
2308 summary: msg,
2309 turns,
2310 tools_called,
2311 tool_receipts,
2312 models_served: models_served.clone(),
2313 model_used: last_model,
2314 };
2315 }
2316 }
2317 continue;
2318 }
2319 let summary = annotate_summary_with_claim_note(&last_text, &ungrounded);
2320 emit(AssistantEvent::Error(summary.clone()));
2321 return AssistantOutcome {
2322 status: "error",
2323 summary,
2324 turns,
2325 tools_called,
2326 tool_receipts,
2327 models_served: models_served.clone(),
2328 model_used: last_model,
2329 };
2330 }
2331 let mut final_appended = false;
2344 if let Some(format) = cfg.response_format.as_ref().filter(|f| {
2345 !final_text_matches_format(&last_text, f, cfg.response_format_validator.as_ref())
2346 }) {
2347 emit(AssistantEvent::Text(FORMAT_REPAIR_NOTICE.to_string()));
2348 result.append_assistant_history(messages, vec![]);
2349 messages.push(Message::User {
2350 content: format_repair_nudge(format).to_string(),
2351 });
2352 let repair = GenerateRequest {
2353 prompt: String::new(),
2354 model: cfg.model.clone(),
2355 params: GenerateParams {
2356 temperature: 0.0,
2357 strict_model: cfg.strict_model,
2358 ..Default::default()
2359 },
2360 context: None,
2361 context_stable_prefix: None,
2362 tools: None,
2363 images: None,
2364 messages: Some(messages.clone()),
2365 cache_control: false,
2366 response_format: Some(format.clone()),
2367 intent: None,
2368 client_ref: None,
2369 expected_row_digest: None,
2370 expected_catalog_revision: None,
2371 caller: None,
2372 };
2373 match generator.generate(repair).await {
2374 Ok(mut repaired) => {
2375 record_inference_metered(runtime, &repaired).await;
2376 let attribution = AssistantModelAttribution {
2377 model_id: repaired.served_model_id().to_string(),
2378 local_last_resort: repaired.local_last_resort,
2379 };
2380 emit(AssistantEvent::ModelServed {
2381 model_id: attribution.model_id.clone(),
2382 local_last_resort: attribution.local_last_resort,
2383 });
2384 models_served.push(attribution);
2385 repaired.text =
2386 car_inference::tasks::generate::strip_leaked_reasoning(&repaired.text);
2387 if !final_text_matches_format(
2388 &repaired.text,
2389 format,
2390 cfg.response_format_validator.as_ref(),
2391 ) {
2392 emit(AssistantEvent::Text(
2393 FORMAT_REPAIR_STILL_INVALID.to_string(),
2394 ));
2395 }
2396 last_model = repaired.served_model_id().to_string();
2397 last_text = repaired.text.clone();
2398 result = repaired;
2399 }
2400 Err(e) => {
2401 if matches!(messages.last(), Some(Message::User { content }) if content == format_repair_nudge(format))
2407 {
2408 messages.pop();
2409 }
2410 emit(AssistantEvent::Text(format!(
2411 "{FORMAT_REPAIR_FAILED_PREFIX} {e}; returning the draft answer as-is]"
2412 )));
2413 final_appended = true;
2414 }
2415 }
2416 }
2417 if !final_appended {
2418 result.append_assistant_history(messages, vec![]);
2419 }
2420 if let (Some(store), Some(session_id)) = (durability, durable_session_id) {
2421 if let Err(e) = store
2422 .checkpoint(session_id, messages, "assistant_final", None)
2423 .await
2424 {
2425 let msg = format!("durable checkpoint failed after assistant response: {e}");
2426 emit(AssistantEvent::Error(msg.clone()));
2427 return AssistantOutcome {
2428 status: "error",
2429 summary: msg,
2430 turns,
2431 tools_called,
2432 tool_receipts,
2433 models_served: models_served.clone(),
2434 model_used: last_model,
2435 };
2436 }
2437 }
2438 runtime
2443 .record_turn_completed(
2444 "empty_tool_calls",
2445 result.stop_reason.as_deref(),
2446 result.was_truncated(),
2447 turns,
2448 &last_model,
2449 )
2450 .await;
2451 emit(AssistantEvent::Done {
2452 text: last_text.clone(),
2453 });
2454 return AssistantOutcome {
2455 status: "success",
2456 summary: last_text,
2457 turns,
2458 tools_called,
2459 tool_receipts,
2460 models_served: models_served.clone(),
2461 model_used: last_model.clone(),
2462 };
2463 }
2464
2465 if !result.text.trim().is_empty() {
2466 last_text = result.text.clone();
2467 emit(AssistantEvent::Text(result.text.clone()));
2468 }
2469
2470 let mut calls = result.tool_calls.clone();
2472 for (i, call) in calls.iter_mut().enumerate() {
2473 if call.id.is_none() {
2474 call.id = Some(format!("call_{turns}_{i}"));
2475 }
2476 }
2477
2478 result.append_assistant_history(messages, calls.clone());
2479 if let (Some(store), Some(session_id)) = (durability, durable_session_id) {
2480 if let Err(e) = store
2481 .checkpoint(session_id, messages, "assistant_tool_calls", None)
2482 .await
2483 {
2484 let msg = format!("durable checkpoint failed before tool dispatch: {e}");
2485 emit(AssistantEvent::Error(msg.clone()));
2486 return AssistantOutcome {
2487 status: "error",
2488 summary: msg,
2489 turns,
2490 tools_called,
2491 tool_receipts,
2492 models_served: models_served.clone(),
2493 model_used: last_model,
2494 };
2495 }
2496 }
2497
2498 let mut mutated_ok = false;
2503
2504 for call in &calls {
2507 let id = call.id.clone().expect("ids assigned above");
2508 emit(AssistantEvent::ToolCall {
2509 name: call.name.clone(),
2510 params: serde_json::to_value(&call.arguments).unwrap_or_default(),
2511 });
2512
2513 let mut params_val = serde_json::to_value(&call.arguments).unwrap_or_default();
2517 if cfg.value_store_previews {
2526 let resolved = values.resolve_refs(&mut params_val);
2527 if !resolved.is_empty() {
2528 tracing::debug!(
2529 tool = %call.name,
2530 handles = ?resolved,
2531 "resolved retained-value references in tool arguments"
2532 );
2533 }
2534 }
2535 let params_val = params_val;
2536 let posture = match &cfg.approval_policy {
2537 Some(policy) => policy(&call.name, ¶ms_val),
2538 None => {
2539 if cfg.gated_tools.iter().any(|t| t == &call.name) {
2540 ToolApprovalDecision::RequireApproval
2541 } else {
2542 ToolApprovalDecision::Allow
2543 }
2544 }
2545 };
2546 let posture = if cfg.refuse_unadvertised_tools && !advertised_names.contains(&call.name)
2550 {
2551 ToolApprovalDecision::Deny(format!(
2552 "tool '{}' is not granted to this delegate; use only: {}",
2553 call.name,
2554 advertised_names
2555 .iter()
2556 .cloned()
2557 .collect::<Vec<_>>()
2558 .join(", ")
2559 ))
2560 } else {
2561 posture
2562 };
2563 let needs_approval = matches!(&posture, ToolApprovalDecision::RequireApproval);
2564
2565 let refusal: Option<String> = match posture {
2566 ToolApprovalDecision::Allow => None,
2567 ToolApprovalDecision::Deny(reason) => Some(reason),
2568 ToolApprovalDecision::RequireApproval => {
2569 let decision = match approval {
2570 Some(gate) => gate.request_action(&id, &call.name, ¶ms_val).await,
2571 None => ApprovalDecision::Denied(format!(
2572 "'{}' needs approval: re-run with --full-access to allow it on this host, \
2573 or use the default sandbox where edits are isolated",
2574 call.name
2575 )),
2576 };
2577 match decision {
2578 ApprovalDecision::Approved => None,
2579 ApprovalDecision::Denied(reason) => Some(reason),
2580 }
2581 }
2582 };
2583 if let Some(reason) = refusal {
2584 let content = cap(json!({ "error": reason }).to_string());
2585 emit(AssistantEvent::ToolResult {
2586 name: call.name.clone(),
2587 ok: false,
2588 content: content.clone(),
2589 });
2590 messages.push(Message::ToolResult {
2591 tool_use_id: id,
2592 content,
2593 provenance: Provenance::Internal,
2595 });
2596 if let (Some(store), Some(session_id)) = (durability, durable_session_id) {
2597 if let Err(e) = store
2598 .checkpoint(session_id, messages, "tool_refused", None)
2599 .await
2600 {
2601 let msg = format!("durable checkpoint failed after refusal: {e}");
2602 emit(AssistantEvent::Error(msg.clone()));
2603 return AssistantOutcome {
2604 status: "error",
2605 summary: msg,
2606 turns,
2607 tools_called,
2608 tool_receipts,
2609 models_served: models_served.clone(),
2610 model_used: last_model,
2611 };
2612 }
2613 }
2614 continue;
2615 }
2616
2617 if delegate_advertised && call.name == DELEGATE_TOOL {
2621 let goal_brief = params_val
2622 .get("goal")
2623 .and_then(Value::as_str)
2624 .unwrap_or_default()
2625 .replace('\n', " ");
2626 let goal_brief: String = goal_brief.chars().take(80).collect();
2627 let over_budget = delegations_spawned >= delegate_budget.max_delegations
2628 || child_turns_used >= delegate_budget.max_child_turns;
2629 let done = if over_budget {
2630 DelegateOutcome {
2631 ok: false,
2632 content: cap(json!({
2633 "error": format!(
2634 "delegation budget exhausted ({delegations_spawned} delegations / \
2635 {child_turns_used} child turns used; limits {} / {}) — finish with \
2636 what you have",
2637 delegate_budget.max_delegations, delegate_budget.max_child_turns
2638 )
2639 })
2640 .to_string()),
2641 turns: 0,
2642 external: false,
2643 receipts: Vec::new(),
2644 spawned: false,
2645 }
2646 } else {
2647 run_delegate(
2648 generator,
2649 runtime,
2650 cfg,
2651 messages,
2652 ¶ms_val,
2653 cancel,
2654 approval,
2655 runtime_session_id,
2656 redrive_ungrounded_summary,
2657 tool_labels,
2658 )
2659 .await
2660 };
2661 if done.spawned {
2662 delegations_spawned += 1;
2663 child_turns_used = child_turns_used.saturating_add(done.turns);
2664 }
2665 emit(AssistantEvent::Text(format!(
2666 "[delegate: {goal_brief} — {} turns, {}]",
2667 done.turns,
2668 if done.ok { "ok" } else { "error" }
2669 )));
2670 if done.ok {
2671 tools_called.push(call.name.clone());
2672 if mutating_tools.contains(&call.name) {
2673 mutated_ok = true;
2674 }
2675 }
2676 tool_receipts.push(AssistantToolReceipt {
2677 tool: call.name.clone(),
2678 call_id: Some(id.clone()),
2679 ok: done.ok,
2680 params: params_val.clone(),
2681 via: None,
2682 });
2683 let via = format!("{DELEGATE_TOOL}:{id}");
2689 tool_receipts.extend(done.receipts.into_iter().map(|mut r| {
2690 r.via = Some(via.clone());
2691 r
2692 }));
2693 emit(AssistantEvent::ToolResult {
2694 name: call.name.clone(),
2695 ok: done.ok,
2696 content: done.content.clone(),
2697 });
2698 messages.push(Message::ToolResult {
2699 tool_use_id: id,
2700 content: done.content,
2701 provenance: if done.external {
2705 Provenance::External
2706 } else {
2707 Provenance::Internal
2708 },
2709 });
2710 if let (Some(store), Some(session_id)) = (durability, durable_session_id) {
2711 if let Err(e) = store
2712 .checkpoint(session_id, messages, "tool_result", None)
2713 .await
2714 {
2715 let msg = format!("durable checkpoint failed after delegate result: {e}");
2716 emit(AssistantEvent::Error(msg.clone()));
2717 return AssistantOutcome {
2718 status: "error",
2719 summary: msg,
2720 turns,
2721 tools_called,
2722 tool_receipts,
2723 models_served: models_served.clone(),
2724 model_used: last_model,
2725 };
2726 }
2727 }
2728 continue;
2729 }
2730
2731 let proposal = match build_proposal(&result.model_used, call, ¶ms_val) {
2732 Ok(p) => p,
2733 Err(e) => {
2734 let content = cap(json!({ "error": e }).to_string());
2737 emit(AssistantEvent::ToolResult {
2738 name: call.name.clone(),
2739 ok: false,
2740 content: content.clone(),
2741 });
2742 messages.push(Message::ToolResult {
2743 tool_use_id: id,
2744 content,
2745 provenance: Provenance::Internal,
2747 });
2748 if let (Some(store), Some(session_id)) = (durability, durable_session_id) {
2749 if let Err(e) = store
2750 .checkpoint(session_id, messages, "malformed_tool_call", None)
2751 .await
2752 {
2753 let msg = format!("durable checkpoint failed after tool error: {e}");
2754 emit(AssistantEvent::Error(msg.clone()));
2755 return AssistantOutcome {
2756 status: "error",
2757 summary: msg,
2758 turns,
2759 tools_called,
2760 tool_receipts,
2761 models_served: models_served.clone(),
2762 model_used: last_model,
2763 };
2764 }
2765 }
2766 continue;
2767 }
2768 };
2769
2770 if needs_approval {
2771 let dispatch = match approval {
2772 Some(gate) => gate.before_dispatch(&id, &call.name, ¶ms_val).await,
2773 None => Err("approval gate disappeared before dispatch".into()),
2774 };
2775 if let Err(e) = dispatch {
2776 let content =
2777 cap(json!({ "error": format!("dispatch refused: {e}") }).to_string());
2778 emit(AssistantEvent::ToolResult {
2779 name: call.name.clone(),
2780 ok: false,
2781 content: content.clone(),
2782 });
2783 messages.push(Message::ToolResult {
2784 tool_use_id: id,
2785 content,
2786 provenance: Provenance::Internal,
2787 });
2788 if let (Some(store), Some(session_id)) = (durability, durable_session_id) {
2789 let _ = store
2790 .checkpoint(session_id, messages, "dispatch_refused", None)
2791 .await;
2792 }
2793 continue;
2794 }
2795 }
2796
2797 let exec = match runtime_session_id {
2798 Some(session_id) => runtime.execute_with_session(&proposal, session_id).await,
2799 None => runtime.execute(&proposal).await,
2800 };
2801 let action = exec.results.first();
2802 let runtime_succeeded = action
2803 .map(|r| matches!(r.status, ActionStatus::Succeeded))
2804 .unwrap_or(false);
2805 let ok = runtime_succeeded
2811 && (call.name != "shell"
2812 || action
2813 .and_then(|result| result.output.as_ref())
2814 .and_then(|output| output.get("exit_code"))
2815 .and_then(Value::as_i64)
2816 == Some(0));
2817 if needs_approval {
2818 let receipt = json!({
2819 "ok": ok,
2820 "action_id": action.map(|result| result.action_id.clone()),
2821 "status": action.map(|result| format!("{:?}", result.status)),
2822 });
2823 if let Some(gate) = approval {
2824 if let Err(e) = gate
2825 .after_dispatch(&id, &call.name, ¶ms_val, ok, &receipt)
2826 .await
2827 {
2828 let msg = format!(
2829 "action executed but its durable terminal receipt failed: {e}; action is indeterminate"
2830 );
2831 emit(AssistantEvent::Error(msg.clone()));
2832 return AssistantOutcome {
2833 status: "error",
2834 summary: msg,
2835 turns,
2836 tools_called,
2837 tool_receipts,
2838 models_served: models_served.clone(),
2839 model_used: last_model,
2840 };
2841 }
2842 }
2843 }
2844 let content = match action {
2857 Some(r)
2858 if cfg.value_store_previews && matches!(r.status, ActionStatus::Succeeded) =>
2859 {
2860 let rendered = format_tool_result(r);
2861 match (&r.output, rendered.len() > OBSERVATION_CAP) {
2862 (Some(v), true) => {
2863 let handle = values.put(v.clone());
2864 format!(
2865 "{}{}",
2866 super::value_store::render_preview(&handle, v),
2867 super::value_store::reference_hint(&handle)
2868 )
2869 }
2870 _ => cap(rendered),
2873 }
2874 }
2875 Some(r) => cap(format_tool_result(r)),
2876 None => cap(format!("tool '{}' produced no result", call.name)),
2877 };
2878 if ok {
2879 tools_called.push(call.name.clone());
2880 if mutating_tools.contains(&call.name) {
2881 mutated_ok = true;
2882 }
2883 }
2884 tool_receipts.push(AssistantToolReceipt {
2885 tool: call.name.clone(),
2886 call_id: action.map(|r| r.action_id.clone()),
2887 ok,
2888 params: params_val.clone(),
2889 via: None,
2890 });
2891 record_tool_outcome(
2898 cfg,
2899 &mut open_failures,
2900 &call.name,
2901 ok,
2902 &content,
2903 ¶ms_val,
2904 turns,
2905 );
2906 emit(AssistantEvent::ToolResult {
2907 name: call.name.clone(),
2908 ok,
2909 content: content.clone(),
2910 });
2911 messages.push(Message::ToolResult {
2912 tool_use_id: id,
2913 content,
2914 provenance: if tool_output_is_external(&call.name, tool_labels) {
2919 Provenance::External
2920 } else {
2921 Provenance::Internal
2922 },
2923 });
2924 if let (Some(store), Some(session_id)) = (durability, durable_session_id) {
2925 if let Err(e) = store
2926 .checkpoint(session_id, messages, "tool_result", None)
2927 .await
2928 {
2929 let msg = format!("durable checkpoint failed after tool result: {e}");
2930 emit(AssistantEvent::Error(msg.clone()));
2931 return AssistantOutcome {
2932 status: "error",
2933 summary: msg,
2934 turns,
2935 tools_called,
2936 tool_receipts,
2937 models_served: models_served.clone(),
2938 model_used: last_model,
2939 };
2940 }
2941 }
2942 }
2943
2944 let mut inject_nudge = false;
2951 match guard.observe(&tool_calls_signature(&calls), mutated_ok) {
2952 GuardStep::Break => {
2953 let summary = format!(
2954 "Stopped: repeated the same action {} times without changing \
2955 anything — no progress was being made.",
2956 guard.stall_repeats
2957 );
2958 runtime
2959 .record_turn_completed("stalled", None, false, turns, &last_model)
2960 .await;
2961 emit(AssistantEvent::Done {
2962 text: summary.clone(),
2963 });
2964 return AssistantOutcome {
2965 status: "stalled",
2966 summary,
2967 turns,
2968 tools_called,
2969 tool_receipts,
2970 models_served: models_served.clone(),
2971 model_used: last_model.clone(),
2972 };
2973 }
2974 GuardStep::Nudge => inject_nudge = true,
2975 GuardStep::Progress | GuardStep::Continue => {}
2976 }
2977
2978 if inject_nudge {
2981 messages.push(Message::User {
2982 content: "You have repeated the same action several times without \
2983 changing anything or making progress. Stop re-reading and \
2984 either take a concrete action (write or edit a file, run a \
2985 command) or, if the task is genuinely complete, finish now \
2986 with your summary."
2987 .into(),
2988 });
2989 if let (Some(store), Some(session_id)) = (durability, durable_session_id) {
2990 if let Err(e) = store
2991 .checkpoint(session_id, messages, "progress_nudge", None)
2992 .await
2993 {
2994 let msg = format!("durable checkpoint failed after progress nudge: {e}");
2995 emit(AssistantEvent::Error(msg.clone()));
2996 return AssistantOutcome {
2997 status: "error",
2998 summary: msg,
2999 turns,
3000 tools_called,
3001 tool_receipts,
3002 models_served: models_served.clone(),
3003 model_used: last_model,
3004 };
3005 }
3006 }
3007 }
3008 }
3009
3010 runtime
3011 .record_turn_completed("max_turns", None, false, turns, &last_model)
3012 .await;
3013 AssistantOutcome {
3014 status: "max_turns",
3015 summary: if last_text.is_empty() {
3016 format!("stopped after {} turns without finishing", cfg.max_turns)
3017 } else {
3018 last_text
3019 },
3020 turns,
3021 tools_called,
3022 tool_receipts,
3023 models_served: models_served.clone(),
3024 model_used: last_model.clone(),
3025 }
3026}
3027
3028#[derive(Debug, Clone)]
3029struct SummaryClaimRequirement {
3030 label: &'static str,
3031 tools: &'static [&'static str],
3032 require_ok: bool,
3033 shell_terms: &'static [&'static str],
3034 paths: Vec<String>,
3035}
3036
3037const TEST_TERMS: &[&str] = &[
3038 "test",
3039 "pytest",
3040 "cargo test",
3041 "cargo nextest",
3042 "npm test",
3043 "npm run test",
3044 "pnpm test",
3045 "pnpm run test",
3046 "yarn test",
3047 "bun test",
3048 "go test",
3049 "swift test",
3050 "dotnet test",
3051 "ctest",
3052 "cmake --build",
3053 "make test",
3054];
3055const BUILD_TERMS: &[&str] = &[
3056 "build",
3057 "cargo check",
3058 "cargo build",
3059 "npm run build",
3060 "pnpm build",
3061 "yarn build",
3062 "bun run build",
3063 "cmake --build",
3064 "go build",
3065 "swift build",
3066 "dotnet build",
3067 "mvn package",
3068 "gradle build",
3069 "./gradlew build",
3070];
3071const CHECK_TERMS: &[&str] = &[
3072 "cargo check",
3073 "git diff --check",
3074 "npm run lint",
3075 "npm run check",
3076 "pnpm check",
3077 "pnpm lint",
3078 "yarn check",
3079 "yarn lint",
3080 "bun run check",
3081 "eslint",
3082 "clippy",
3083 "swiftlint",
3084 "ruff",
3085 "mypy",
3086 "biome check",
3087];
3088const READ_TERMS: &[&str] = &[
3100 "cat ", "sed ", "rg ", "grep ", "ls ", "find ", "type ", "findstr ", ];
3103const WRITE_TERMS: &[&str] = &[
3104 "touch ",
3105 "cat >",
3106 "tee ",
3107 "python ",
3108 "node ",
3109 "perl ", "type nul >",
3111 "echo >", ];
3113const GIT_STATUS_TERMS: &[&str] = &["git status"];
3114const GIT_REVISION_TERMS: &[&str] = &["git rev-parse", "git log", "git show"];
3115const APP_INSIGHTS_TERMS: &[&str] = &["az monitor app-insights query"];
3116const DEPLOYMENT_EVIDENCE_TERMS: &[&str] = &[
3117 "az pipelines show",
3118 "az pipelines runs show",
3119 "az devops invoke",
3120];
3121const SUMMARY_PATH_EXTENSIONS: &[&str] = &[
3122 ".rs", ".py", ".js", ".ts", ".tsx", ".jsx", ".go", ".swift", ".java", ".kt", ".kts", ".c",
3123 ".h", ".cc", ".hh", ".cpp", ".hpp", ".cxx", ".hxx", ".cs", ".fs", ".vb", ".php", ".rb", ".ex",
3124 ".exs", ".md", ".txt", ".json", ".yaml", ".yml", ".toml", ".html", ".css", ".xml", ".sh",
3125 ".sql",
3126];
3127
3128fn normalize_summary_path_token(raw: &str) -> Option<String> {
3129 let token = raw.trim_matches(|c: char| {
3130 matches!(
3131 c,
3132 '"' | '\'' | '`' | ',' | ';' | ':' | ')' | '(' | '[' | ']' | '{' | '}' | '.'
3133 )
3134 });
3135 if token.is_empty() || token.starts_with('-') || token.contains("://") || token.contains("..") {
3136 return None;
3137 }
3138 let looks_like_path = token.contains('/')
3139 || SUMMARY_PATH_EXTENSIONS
3140 .iter()
3141 .any(|ext| token.to_ascii_lowercase().ends_with(ext));
3142 if !looks_like_path {
3143 return None;
3144 }
3145 Some(
3146 token
3147 .trim_start_matches("./")
3148 .replace('\\', "/")
3149 .to_ascii_lowercase(),
3150 )
3151}
3152
3153fn summary_path_hints(summary: &str) -> Vec<String> {
3154 let mut paths = Vec::new();
3155 for raw in summary.split_whitespace() {
3156 if let Some(path) = normalize_summary_path_token(raw) {
3157 if !paths.contains(&path) {
3158 paths.push(path);
3159 }
3160 }
3161 }
3162 paths
3163}
3164
3165fn summary_claim_requirements(summary: &str) -> Vec<SummaryClaimRequirement> {
3166 let s = summary.to_ascii_lowercase();
3167 let units: Vec<&str> = s
3168 .split(['\n', '.'])
3169 .map(str::trim)
3170 .filter(|unit| !unit.is_empty())
3171 .collect();
3172 let path_hints = summary_path_hints(summary);
3173 let mut claims = Vec::new();
3174 if units.iter().any(|unit| {
3175 unit.contains("ran the test")
3176 || unit.contains("ran tests")
3177 || (unit.contains("verified with") && unit.contains("test"))
3178 || (unit.contains("test")
3179 && (unit.contains("passed")
3180 || unit.contains("green")
3181 || unit.contains("succeeded")
3182 || unit.contains("successful")))
3183 }) {
3184 claims.push(SummaryClaimRequirement {
3185 label: "tests were run/passed",
3186 tools: &["shell"],
3187 require_ok: true,
3188 shell_terms: TEST_TERMS,
3189 paths: Vec::new(),
3190 });
3191 }
3192 if units.iter().any(|unit| {
3193 (unit.contains("build") || unit.contains("cargo check"))
3194 && (unit.contains("passed")
3195 || unit.contains("succeeded")
3196 || unit.contains("successful")
3197 || unit.contains("built")
3198 || unit.contains("green")
3199 || unit.contains("ran the build")
3200 || unit.contains("ran cargo check"))
3201 }) {
3202 claims.push(SummaryClaimRequirement {
3203 label: "build succeeded",
3204 tools: &["shell"],
3205 require_ok: true,
3206 shell_terms: BUILD_TERMS,
3207 paths: Vec::new(),
3208 });
3209 }
3210 if units.iter().any(|unit| {
3211 (unit.contains("check") || unit.contains("lint"))
3212 && (unit.contains("passed")
3213 || unit.contains("green")
3214 || unit.contains("succeeded")
3215 || unit.contains("successful"))
3216 }) {
3217 claims.push(SummaryClaimRequirement {
3218 label: "checks were run/passed",
3219 tools: &["shell"],
3220 require_ok: true,
3221 shell_terms: CHECK_TERMS,
3222 paths: Vec::new(),
3223 });
3224 }
3225 if units.iter().any(|unit| {
3226 (unit.contains("read ") || unit.contains("inspected ") || unit.contains("looked at "))
3227 && (unit.contains("file") || unit.contains("source"))
3228 }) {
3229 claims.push(SummaryClaimRequirement {
3230 label: "files were read/inspected",
3231 tools: &["read_file", "list_dir", "find_files", "grep_files", "shell"],
3232 require_ok: true,
3233 shell_terms: READ_TERMS,
3234 paths: path_hints.clone(),
3235 });
3236 }
3237 if units.iter().any(|unit| {
3238 (unit.contains("created")
3239 || unit.contains("wrote")
3240 || unit.contains("updated")
3241 || unit.contains("edited"))
3242 && unit.contains("file")
3243 }) {
3244 claims.push(SummaryClaimRequirement {
3245 label: "files were created/updated",
3246 tools: &["write_file", "edit_file", "shell"],
3247 require_ok: true,
3248 shell_terms: WRITE_TERMS,
3249 paths: path_hints.clone(),
3250 });
3251 }
3252 if units.iter().any(|unit| {
3253 unit.contains("repository is clean")
3254 || unit.contains("repo is clean")
3255 || unit.contains("working tree is clean")
3256 || unit.contains("status: clean")
3257 }) {
3258 claims.push(SummaryClaimRequirement {
3259 label: "repository cleanliness was verified",
3260 tools: &["shell"],
3261 require_ok: true,
3262 shell_terms: GIT_STATUS_TERMS,
3263 paths: Vec::new(),
3264 });
3265 }
3266 if units.iter().any(|unit| {
3267 unit.contains("head matches origin")
3268 || unit.contains("head is aligned with origin")
3269 || unit.contains("head and origin are identical")
3270 }) {
3271 claims.push(SummaryClaimRequirement {
3272 label: "repository revision/remote relationship was verified",
3273 tools: &["shell"],
3274 require_ok: true,
3275 shell_terms: GIT_REVISION_TERMS,
3276 paths: Vec::new(),
3277 });
3278 }
3279 if units.iter().any(|unit| {
3280 (unit.contains("app insights")
3281 || unit.contains("application insights")
3282 || unit.contains("telemetry"))
3283 && (unit.contains("query showed")
3284 || unit.contains("query confirmed")
3285 || unit.contains("we observed")
3286 || unit.contains("live telemetry showed")
3287 || unit.contains("no recurrence")
3288 || unit.contains("recurred after"))
3289 && !unit.contains("not obtained")
3290 && !unit.contains("unable")
3291 }) {
3292 claims.push(SummaryClaimRequirement {
3293 label: "live Application Insights evidence was observed",
3294 tools: &["shell", "browse_observe"],
3295 require_ok: true,
3296 shell_terms: APP_INSIGHTS_TERMS,
3297 paths: Vec::new(),
3298 });
3299 }
3300 if units.iter().any(|unit| {
3301 (unit.contains("production") || unit.contains("live"))
3302 && (unit.contains("browser") || unit.contains("portal") || unit.contains("page"))
3303 && (unit.contains("inspected")
3304 || unit.contains("observed")
3305 || unit.contains("verified"))
3306 && !unit.contains("not obtained")
3307 && !unit.contains("unable")
3308 }) {
3309 claims.push(SummaryClaimRequirement {
3310 label: "production browser state was observed",
3311 tools: &["browse_observe"],
3312 require_ok: true,
3313 shell_terms: &[],
3314 paths: Vec::new(),
3315 });
3316 }
3317 if units.iter().any(|unit| {
3318 unit.contains("deployment")
3319 && (unit.contains("successfully fixed")
3320 || unit.contains("was deployed")
3321 || unit.contains("after fix")
3322 || unit.contains("post-deployment"))
3323 && !unit.contains("cannot")
3324 && !unit.contains("not obtained")
3325 }) {
3326 claims.push(SummaryClaimRequirement {
3327 label: "deployment state/change was verified",
3328 tools: &["shell"],
3329 require_ok: true,
3330 shell_terms: DEPLOYMENT_EVIDENCE_TERMS,
3331 paths: Vec::new(),
3332 });
3333 }
3334 if units.iter().any(|unit| {
3335 unit.contains("subscription")
3336 && (unit.contains("outside") || unit.contains("not in"))
3337 && unit
3338 .as_bytes()
3339 .windows(2)
3340 .any(|window| window[0] == b'n' && window[1].is_ascii_digit())
3341 && !unit.contains("cannot")
3342 && !unit.contains("not verified")
3343 && !unit.contains("not obtained")
3344 && !unit.contains("insufficient evidence")
3345 }) {
3346 claims.push(SummaryClaimRequirement {
3347 label: "named aircraft subscription status was observed live",
3348 tools: &["shell"],
3349 require_ok: true,
3350 shell_terms: APP_INSIGHTS_TERMS,
3351 paths: Vec::new(),
3352 });
3353 }
3354 claims
3355}
3356
3357fn shell_command(params: &Value) -> Option<String> {
3358 params
3359 .get("command")
3360 .and_then(Value::as_str)
3361 .map(|s| s.to_ascii_lowercase())
3362}
3363
3364fn normalized_receipt_path(params: &Value) -> Option<String> {
3365 params.get("path").and_then(Value::as_str).map(|path| {
3366 path.trim_start_matches("./")
3367 .replace('\\', "/")
3368 .to_ascii_lowercase()
3369 })
3370}
3371
3372fn text_mentions_summary_path(text: &str, path: &str) -> bool {
3373 let text = text.replace('\\', "/").to_ascii_lowercase();
3374 text.contains(path) || text.contains(&format!("./{path}"))
3375}
3376
3377fn receipt_mentions_summary_path(receipt: &AssistantToolReceipt, path: &str) -> bool {
3378 if receipt.tool == "shell" {
3379 return shell_command(&receipt.params)
3380 .map(|cmd| text_mentions_summary_path(&cmd, path))
3381 .unwrap_or(false);
3382 }
3383 normalized_receipt_path(&receipt.params)
3384 .map(|receipt_path| text_mentions_summary_path(&receipt_path, path))
3385 .unwrap_or(false)
3386}
3387
3388fn receipt_satisfies_claim(
3389 receipt: &AssistantToolReceipt,
3390 claim: &SummaryClaimRequirement,
3391) -> bool {
3392 if claim.require_ok && !receipt.ok {
3393 return false;
3394 }
3395 if !claim.tools.iter().any(|t| *t == receipt.tool) {
3396 return false;
3397 }
3398 if !claim.paths.is_empty()
3399 && !claim
3400 .paths
3401 .iter()
3402 .any(|path| receipt_mentions_summary_path(receipt, path))
3403 {
3404 return false;
3405 }
3406 if receipt.tool != "shell" || claim.shell_terms.is_empty() {
3407 return true;
3408 }
3409 let Some(cmd) = shell_command(&receipt.params) else {
3410 return false;
3411 };
3412 claim.shell_terms.iter().any(|term| cmd.contains(term))
3413}
3414
3415pub fn ungrounded_summary_claims(
3429 summary: &str,
3430 receipts: &[AssistantToolReceipt],
3431) -> Vec<&'static str> {
3432 summary_claim_requirements(summary)
3433 .into_iter()
3434 .filter(|claim| {
3435 !receipts
3436 .iter()
3437 .any(|receipt| receipt_satisfies_claim(receipt, claim))
3438 })
3439 .map(|claim| claim.label)
3440 .collect()
3441}
3442
3443fn apply_summary_claim_grounding(
3444 mut verdict: car_verify::goal::GoalVerdict,
3445 outcome: &AssistantOutcome,
3446) -> car_verify::goal::GoalVerdict {
3447 if !verdict.met {
3448 return verdict;
3449 }
3450 let ungrounded = ungrounded_summary_claims(&outcome.summary, &outcome.tool_receipts);
3451 if ungrounded.is_empty() {
3452 return verdict;
3453 }
3454 verdict.grounded = false;
3455 verdict.reason = format!(
3456 "{}; ungrounded assistant summary claim(s): {}",
3457 verdict.reason,
3458 ungrounded.join(", ")
3459 );
3460 verdict
3461}
3462
3463pub fn annotate_summary_with_claim_note(summary: &str, ungrounded: &[&'static str]) -> String {
3473 if ungrounded.is_empty() {
3474 return summary.to_string();
3475 }
3476 format!(
3477 "{summary}\n\n[claim check] unverified summary claim(s) this run \
3478 (no matching tool receipt): {}",
3479 ungrounded.join(", ")
3480 )
3481}
3482
3483pub struct GoalLoopResult {
3487 pub outcome: AssistantOutcome,
3488 pub run: car_verify::goal::GoalRun,
3489}
3490
3491const GOAL_EVALUATION_TIMEOUT: Duration =
3499 Duration::from_secs(crate::coder::shell_tool::DEFAULT_SHELL_TIMEOUT_SECS);
3500
3501pub async fn run_assistant_goal_loop<G, GF>(
3515 generator: &dyn TurnGenerator,
3516 runtime: &Runtime,
3517 cfg: &AssistantConfig,
3518 messages: &mut Vec<Message>,
3519 cancel: &std::sync::atomic::AtomicBool,
3520 approval: Option<&dyn ApprovalGate>,
3521 spec: &car_verify::goal::GoalSpec,
3522 gather: G,
3523 emit: impl FnMut(AssistantEvent),
3524) -> GoalLoopResult
3525where
3526 G: FnMut(&AssistantOutcome) -> GF,
3527 GF: std::future::Future<Output = car_engine::GoalGather>,
3528{
3529 run_assistant_goal_loop_in_session(
3530 generator, runtime, cfg, messages, cancel, approval, spec, None, gather, emit,
3531 )
3532 .await
3533}
3534
3535pub async fn run_assistant_goal_loop_in_session<G, GF>(
3537 generator: &dyn TurnGenerator,
3538 runtime: &Runtime,
3539 cfg: &AssistantConfig,
3540 messages: &mut Vec<Message>,
3541 cancel: &std::sync::atomic::AtomicBool,
3542 approval: Option<&dyn ApprovalGate>,
3543 spec: &car_verify::goal::GoalSpec,
3544 runtime_session_id: Option<&str>,
3545 gather: G,
3546 emit: impl FnMut(AssistantEvent),
3547) -> GoalLoopResult
3548where
3549 G: FnMut(&AssistantOutcome) -> GF,
3550 GF: std::future::Future<Output = car_engine::GoalGather>,
3551{
3552 run_assistant_goal_loop_in_session_durable(
3553 generator,
3554 runtime,
3555 cfg,
3556 messages,
3557 cancel,
3558 approval,
3559 spec,
3560 runtime_session_id,
3561 None,
3562 None,
3563 gather,
3564 emit,
3565 )
3566 .await
3567}
3568
3569pub async fn run_assistant_goal_loop_in_session_durable<G, GF>(
3570 generator: &dyn TurnGenerator,
3571 runtime: &Runtime,
3572 cfg: &AssistantConfig,
3573 messages: &mut Vec<Message>,
3574 cancel: &std::sync::atomic::AtomicBool,
3575 approval: Option<&dyn ApprovalGate>,
3576 spec: &car_verify::goal::GoalSpec,
3577 runtime_session_id: Option<&str>,
3578 durable_session_id: Option<&str>,
3579 durability: Option<&dyn super::governance::AssistantDurability>,
3580 mut gather: G,
3581 mut emit: impl FnMut(AssistantEvent),
3582) -> GoalLoopResult
3583where
3584 G: FnMut(&AssistantOutcome) -> GF,
3585 GF: std::future::Future<Output = car_engine::GoalGather>,
3586{
3587 use car_verify::goal::{
3588 anchor_directive, evaluate_goal, governor_check, GoalHalt, GoalRun, GoalRunState,
3589 GoalStatus, GoalVerdict,
3590 };
3591 use std::sync::atomic::Ordering;
3592
3593 let start = std::time::Instant::now();
3594 let mut run_state = GoalRunState::default();
3595 let mut evidence: Vec<GoalVerdict> = Vec::new();
3596 let mut all_models_served = Vec::new();
3597 let mut last_reason = String::new();
3598 let mut last_outcome = AssistantOutcome {
3599 status: "goal_pending",
3600 summary: String::new(),
3601 turns: 0,
3602 tools_called: Vec::new(),
3603 tool_receipts: Vec::new(),
3604 models_served: Vec::new(),
3605 model_used: String::new(),
3606 };
3607
3608 let finish = |status: GoalStatus,
3609 grounded: bool,
3610 reason: String,
3611 iterations: u32,
3612 evidence: Vec<GoalVerdict>,
3613 outcome: AssistantOutcome|
3614 -> GoalLoopResult {
3615 GoalLoopResult {
3616 run: GoalRun {
3617 status,
3618 iterations,
3619 grounded,
3620 cost_usd: 0.0,
3621 last_reason: reason,
3622 evidence,
3623 },
3624 outcome,
3625 }
3626 };
3627
3628 loop {
3629 run_state.elapsed_secs = start.elapsed().as_secs();
3630 if cancel.load(Ordering::Relaxed) {
3631 return finish(
3632 GoalStatus::Halted {
3633 halt: GoalHalt::Cancelled,
3634 },
3635 evidence.last().map(|v| v.grounded).unwrap_or(true),
3636 "cancelled".into(),
3637 run_state.turns,
3638 evidence,
3639 last_outcome,
3640 );
3641 }
3642 if let Some(halt) = governor_check(&spec.governor, &run_state) {
3643 return finish(
3644 GoalStatus::Halted { halt },
3645 evidence.last().map(|v| v.grounded).unwrap_or(true),
3646 if last_reason.is_empty() {
3647 halt.as_str().to_string()
3648 } else {
3649 format!("{} ({})", halt.as_str(), last_reason)
3650 },
3651 run_state.turns,
3652 evidence,
3653 last_outcome,
3654 );
3655 }
3656
3657 let directive = anchor_directive(&spec.goal, &last_reason);
3660 messages.push(Message::User { content: directive });
3661 if let (Some(store), Some(session_id)) = (durability, durable_session_id) {
3662 if let Err(e) = store
3663 .checkpoint(
3664 session_id,
3665 messages,
3666 "goal_directive",
3667 serde_json::to_value(spec).ok(),
3668 )
3669 .await
3670 {
3671 last_outcome.status = "error";
3672 last_outcome.summary = format!("durable goal checkpoint failed: {e}");
3673 return finish(
3674 GoalStatus::Halted {
3675 halt: GoalHalt::Cancelled,
3676 },
3677 false,
3678 last_outcome.summary.clone(),
3679 run_state.turns,
3680 evidence,
3681 last_outcome,
3682 );
3683 }
3684 }
3685
3686 let mut outcome = run_assistant_loop_cancellable_in_session_durable(
3687 generator,
3688 runtime,
3689 cfg,
3690 messages,
3691 cancel,
3692 approval,
3693 None,
3694 runtime_session_id,
3695 durable_session_id,
3696 durability,
3697 false,
3698 &mut emit,
3699 )
3700 .await;
3701 all_models_served.append(&mut outcome.models_served);
3702 outcome.models_served = all_models_served.clone();
3703 run_state.turns += 1;
3704 if outcome.tools_called.is_empty() {
3707 run_state.turns_since_progress += 1;
3708 } else {
3709 run_state.turns_since_progress = 0;
3710 }
3711
3712 if outcome.status == "cancelled" {
3713 return finish(
3714 GoalStatus::Halted {
3715 halt: GoalHalt::Cancelled,
3716 },
3717 evidence.last().map(|v| v.grounded).unwrap_or(true),
3718 "cancelled".into(),
3719 run_state.turns,
3720 evidence,
3721 outcome,
3722 );
3723 }
3724
3725 let g = match tokio::time::timeout(GOAL_EVALUATION_TIMEOUT, gather(&outcome)).await {
3733 Ok(g) => g,
3734 Err(_) => {
3735 let reason = format!(
3736 "goal check did not complete within {}s — treating this turn's reply as \
3737 unevaluated rather than blocking on it",
3738 GOAL_EVALUATION_TIMEOUT.as_secs()
3739 );
3740 let verdict = GoalVerdict {
3753 met: false,
3754 grounded: false,
3755 reason: reason.clone(),
3756 };
3757 evidence.push(verdict.clone());
3758 runtime
3759 .record_goal_evaluated(
3760 &spec.goal,
3761 &spec.condition,
3762 run_state.turns,
3763 verdict.met,
3764 verdict.grounded,
3765 &verdict.reason,
3766 &outcome.model_used,
3767 )
3768 .await;
3769 tracing::warn!(
3770 target: "car::goal",
3771 iteration = run_state.turns,
3772 timeout_secs = GOAL_EVALUATION_TIMEOUT.as_secs(),
3773 "goal evaluation timed out — halting with the primary reply intact"
3774 );
3775 emit(AssistantEvent::GoalEvaluated {
3776 iteration: run_state.turns,
3777 met: false,
3778 grounded: false,
3779 reason: reason.clone(),
3780 });
3781 return finish(
3782 GoalStatus::Halted {
3783 halt: GoalHalt::EvaluationTimeout,
3784 },
3785 false,
3786 reason,
3787 run_state.turns,
3788 evidence,
3789 outcome,
3790 );
3791 }
3792 };
3793 let inputs = runtime.gather_goal_inputs(&g).await;
3794
3795 let base = evaluate_goal(&spec.condition, &inputs);
3800 let verdict = if base.met && base.grounded {
3801 let ungrounded = ungrounded_summary_claims(&outcome.summary, &outcome.tool_receipts);
3812 if !ungrounded.is_empty() {
3813 tracing::info!(
3814 target: "car::goal",
3815 iteration = run_state.turns,
3816 claims = %ungrounded.join(", "),
3817 "deterministic goal check passed; final-summary claim(s) unmatched \
3818 to a tool receipt — annotating reply, keeping grounded=true"
3819 );
3820 outcome.summary = annotate_summary_with_claim_note(&outcome.summary, &ungrounded);
3821 }
3822 base
3823 } else {
3824 apply_summary_claim_grounding(base, &outcome)
3830 };
3831 evidence.push(verdict.clone());
3832 runtime
3833 .record_goal_evaluated(
3834 &spec.goal,
3835 &spec.condition,
3836 run_state.turns,
3837 verdict.met,
3838 verdict.grounded,
3839 &verdict.reason,
3840 &outcome.model_used,
3841 )
3842 .await;
3843 tracing::info!(
3846 target: "car::goal",
3847 iteration = run_state.turns,
3848 met = verdict.met,
3849 grounded = verdict.grounded,
3850 reason = %verdict.reason,
3851 "goal evaluated"
3852 );
3853 emit(AssistantEvent::GoalEvaluated {
3854 iteration: run_state.turns,
3855 met: verdict.met,
3856 grounded: verdict.grounded,
3857 reason: verdict.reason.clone(),
3858 });
3859
3860 if verdict.met && verdict.grounded {
3861 return finish(
3862 GoalStatus::Achieved,
3863 verdict.grounded,
3864 verdict.reason,
3865 run_state.turns,
3866 evidence,
3867 outcome,
3868 );
3869 }
3870 last_reason = verdict.reason;
3871 last_outcome = outcome;
3872 }
3873}
3874
3875#[cfg(test)]
3876mod tests {
3877 use super::*;
3878 use crate::assistant::executor::GeneralExecutor;
3879 use async_trait::async_trait;
3880 use car_engine::{LocalSubstrate, Runtime, Substrate, ToolExecutor};
3881 use car_inference::{InferenceEngine, InferenceResult};
3882 use std::sync::atomic::{AtomicUsize, Ordering};
3883 use std::sync::{Arc, Mutex as StdMutex};
3884
3885 #[test]
3895 fn truncation_reports_true_size_and_elided_amount() {
3896 let total = OBSERVATION_CAP + 5_000;
3897 let out = cap("x".repeat(total));
3898
3899 assert!(
3900 out.contains(&format!("of {total} bytes")),
3901 "the TRUE size must be reported, not just the fact of truncation: {}",
3902 &out[out.len().saturating_sub(200)..]
3903 );
3904 assert!(
3905 out.contains("5000 bytes elided"),
3906 "the elided amount must be reported so the model can judge the loss: {}",
3907 &out[out.len().saturating_sub(200)..]
3908 );
3909 assert!(
3910 out.contains("NOT retained"),
3911 "the model must be told re-running is the only recovery"
3912 );
3913 assert!(out.starts_with(&"x".repeat(1_000)));
3915 }
3916
3917 #[test]
3920 fn observations_within_the_cap_are_unmodified() {
3921 let small = "y".repeat(OBSERVATION_CAP);
3922 assert_eq!(cap(small.clone()), small);
3923 let tiny = "hello".to_string();
3924 assert_eq!(cap(tiny.clone()), tiny);
3925 }
3926
3927 #[test]
3930 fn truncation_respects_char_boundaries() {
3931 let s = "€".repeat(OBSERVATION_CAP);
3933 let out = cap(s);
3934 assert!(out.contains("bytes elided"));
3935 assert!(out.is_char_boundary(0));
3936 }
3937
3938 #[test]
3941 fn guard_breaks_on_repeated_mutation_not_just_reads() {
3942 let mut g = NoProgressGuard::default();
3947 assert_eq!(
3949 g.observe("remember({\"body\":\"x\"})", true),
3950 GuardStep::Progress
3951 );
3952 let sig = "remember({\"body\":\"x\"})";
3955 let mut steps = vec![];
3956 for _ in 0..STALL_BREAK {
3957 steps.push(g.observe(sig, true));
3958 }
3959 assert!(
3960 steps.contains(&GuardStep::Break),
3961 "repeated identical mutation must eventually Break, got {steps:?}"
3962 );
3963 assert!(
3964 steps.contains(&GuardStep::Nudge),
3965 "should nudge before breaking"
3966 );
3967 }
3968
3969 #[test]
3970 fn guard_treats_distinct_mutations_as_progress() {
3971 let mut g = NoProgressGuard::default();
3973 for i in 0..20 {
3974 let sig = format!("remember({{\"body\":\"fact-{i}\"}})");
3975 assert_eq!(g.observe(&sig, true), GuardStep::Progress);
3976 }
3977 }
3978
3979 #[test]
3980 fn guard_read_only_repeat_still_breaks() {
3981 let mut g = NoProgressGuard::default();
3983 let mut steps = vec![];
3984 for _ in 0..(STALL_BREAK + 1) {
3985 steps.push(g.observe("recall({\"q\":\"x\"})", false));
3986 }
3987 assert!(steps.contains(&GuardStep::Break));
3988 }
3989
3990 #[test]
3991 fn guard_read_only_task_never_hard_stops_without_repeat() {
3992 let mut g = NoProgressGuard::default();
3995 let mut steps = vec![];
3996 for i in 0..(EXPLORE_NUDGE + 5) {
3997 steps.push(g.observe(&format!("read_file({{\"p\":\"f{i}\"}})"), false));
3998 }
3999 assert!(
4000 !steps.contains(&GuardStep::Break),
4001 "distinct reads must not Break"
4002 );
4003 assert!(
4004 steps.contains(&GuardStep::Nudge),
4005 "should soft-nudge after EXPLORE_NUDGE"
4006 );
4007 }
4008
4009 fn sys(t: &str) -> Message {
4012 Message::System { content: t.into() }
4013 }
4014 fn usr(t: &str) -> Message {
4015 Message::User { content: t.into() }
4016 }
4017 fn asst_call(id: &str) -> Message {
4018 Message::Assistant {
4019 content: String::new(),
4020 tool_calls: vec![serde_json::from_value(json!({
4021 "name": "write_file",
4022 "arguments": {"path": "a.js"},
4023 "id": id
4024 }))
4025 .unwrap()],
4026 thinking: vec![],
4027 model_id: None,
4028 local_last_resort: false,
4029 }
4030 }
4031 fn tool_res(id: &str, body: &str) -> Message {
4032 Message::ToolResult {
4033 tool_use_id: id.into(),
4034 content: body.into(),
4035 provenance: Default::default(),
4036 }
4037 }
4038 fn provider_item(id: &str, body: &str) -> Message {
4039 Message::ProviderOutputItems {
4040 protocol: car_inference::protocol::OPENAI_RESPONSES_PROTOCOL.into(),
4041 items: vec![json!({
4042 "type": "reasoning",
4043 "id": id,
4044 "status": "completed",
4045 "encrypted_content": body,
4046 })],
4047 }
4048 }
4049
4050 fn no_orphan_tool_results(msgs: &[Message]) -> bool {
4053 let mut seen_call_ids: std::collections::HashSet<String> = Default::default();
4054 for m in msgs {
4055 match m {
4056 Message::Assistant { tool_calls, .. } => {
4057 for c in tool_calls {
4058 if let Some(id) = &c.id {
4059 seen_call_ids.insert(id.clone());
4060 }
4061 }
4062 }
4063 Message::ToolResult { tool_use_id, .. } if !seen_call_ids.contains(tool_use_id) => {
4064 return false;
4065 }
4066 _ => {}
4067 }
4068 }
4069 true
4070 }
4071
4072 #[test]
4073 fn mutating_tools_are_derived_from_metadata_plus_builtin_file_writers() {
4074 let tools = vec![
4075 json!({"name": "remember", "mutating": true}),
4076 json!({"name": "recall"}),
4077 json!({"name": "generate_image", "mutating": true}),
4078 ];
4079 let names = mutating_tool_names(&tools);
4080
4081 assert!(names.contains("write_file"));
4082 assert!(names.contains("edit_file"));
4083 assert!(names.contains("remember"));
4084 assert!(names.contains("generate_image"));
4085 assert!(!names.contains("recall"));
4086 }
4087
4088 #[test]
4089 fn compaction_is_noop_under_budget_and_when_window_unknown() {
4090 let mut m = vec![
4091 sys("s"),
4092 usr("task"),
4093 asst_call("c1"),
4094 tool_res("c1", "small"),
4095 ];
4096 let before = m.clone();
4097 compact_history_to_window(&mut m, 128_000); assert_eq!(m, before, "under-budget history must be untouched");
4099 compact_history_to_window(&mut m, 0); assert_eq!(m, before, "unknown window must be a no-op");
4101 }
4102
4103 #[test]
4104 fn compaction_pins_system_and_task_keeps_tail_no_orphans() {
4105 let big = "x".repeat(20_000); let mut m = vec![sys("system"), usr("THE ORIGINAL TASK")];
4107 for i in 0..12 {
4108 m.push(asst_call(&format!("c{i}")));
4109 m.push(tool_res(&format!("c{i}"), &big));
4110 }
4111 let window = 20_000; compact_history_to_window(&mut m, window);
4113
4114 assert!(matches!(&m[0], Message::System { .. }), "system pinned");
4116 assert!(
4117 matches!(&m[1], Message::User { content } if content == "THE ORIGINAL TASK"),
4118 "original task pinned"
4119 );
4120 assert!(
4122 matches!(m.last(), Some(Message::ToolResult { tool_use_id, .. }) if tool_use_id == "c11"),
4123 "most-recent tool result kept"
4124 );
4125 assert!(
4127 no_orphan_tool_results(&m),
4128 "no orphaned tool results after trim"
4129 );
4130 assert!(m.len() < 26, "history was compacted (was 26 msgs)");
4132 }
4133
4134 #[test]
4143 fn state_block_lands_at_the_tail_inside_the_last_message() {
4144 let mut messages = vec![
4145 sys("system prompt"),
4146 usr("do the thing"),
4147 tool_res("c1", "tool output here"),
4148 ];
4149 let before_prefix = format!("{:?}{:?}", messages[0], messages[1]);
4150
4151 append_state_block(&mut messages, "todo: 1/3 done\n [ ] 2 wire the CLI");
4152
4153 let Message::ToolResult { content, .. } = &messages[2] else {
4155 panic!("last message should still be the tool result");
4156 };
4157 assert!(content.starts_with("tool output here"), "{content}");
4158 assert!(
4159 content.contains("wire the CLI"),
4160 "state must be present: {content}"
4161 );
4162 assert!(content.contains("<runtime-state>"), "{content}");
4164 assert!(content.contains("</runtime-state>"), "{content}");
4165 assert_eq!(messages.len(), 3);
4167 assert_eq!(
4169 before_prefix,
4170 format!("{:?}{:?}", messages[0], messages[1]),
4171 "appending state must not perturb the cached prefix"
4172 );
4173 }
4174
4175 #[tokio::test]
4178 async fn state_block_never_enters_the_durable_history() {
4179 let dir = tempfile::tempdir().unwrap();
4180 let rt = runtime_for(dir.path()).await;
4181 let todos = Arc::new(tokio::sync::Mutex::new(super::super::todo::TodoList::new()));
4182 todos
4183 .lock()
4184 .await
4185 .write(&[json!({"text": "wire the CLI"})])
4186 .unwrap();
4187
4188 let seen = Arc::new(StdMutex::new(Vec::new()));
4189 let script = CapturingScript {
4190 turns: vec![turn("done", json!([]))],
4191 cursor: AtomicUsize::new(0),
4192 seen: Arc::clone(&seen),
4193 };
4194 let mut messages = vec![sys("sys"), usr("do it")];
4195 let mut cfg = cfg();
4196 cfg.todos = Some(Arc::clone(&todos));
4197 run_assistant_loop(&script, &rt, &cfg, &mut messages, |_| {}).await;
4198
4199 let sent = seen.lock().unwrap();
4201 let sent_msgs = sent[0].messages.as_ref().expect("messages sent");
4202 let tail = format!("{:?}", sent_msgs.last().unwrap());
4203 assert!(
4204 tail.contains("wire the CLI"),
4205 "the model must see live state: {tail}"
4206 );
4207
4208 assert!(
4210 !messages
4211 .iter()
4212 .any(|m| format!("{m:?}").contains("<runtime-state>")),
4213 "the block must not persist into history, or it stacks one copy per turn"
4214 );
4215 }
4216
4217 #[tokio::test]
4220 async fn no_state_block_when_there_is_nothing_to_say() {
4221 let dir = tempfile::tempdir().unwrap();
4222 let rt = runtime_for(dir.path()).await;
4223 let seen = Arc::new(StdMutex::new(Vec::new()));
4224 let script = CapturingScript {
4225 turns: vec![turn("done", json!([]))],
4226 cursor: AtomicUsize::new(0),
4227 seen: Arc::clone(&seen),
4228 };
4229 let mut messages = vec![sys("sys"), usr("do it")];
4230 let mut cfg = cfg();
4231 cfg.todos = Some(Arc::new(tokio::sync::Mutex::new(
4232 super::super::todo::TodoList::new(),
4233 )));
4234 run_assistant_loop(&script, &rt, &cfg, &mut messages, |_| {}).await;
4235
4236 let sent = seen.lock().unwrap();
4237 let all = format!("{:?}", sent[0].messages);
4238 assert!(
4239 !all.contains("<runtime-state>"),
4240 "empty plan must render nothing: {all}"
4241 );
4242 }
4243
4244 fn remember_receipt(subject: &str, ok: bool) -> AssistantToolReceipt {
4245 AssistantToolReceipt {
4246 tool: "remember".to_string(),
4247 call_id: None,
4248 ok,
4249 params: json!({"subject": subject, "body": "…"}),
4250 via: None,
4251 }
4252 }
4253
4254 #[test]
4261 fn written_facts_reach_the_state_block_without_being_asked_for() {
4262 let receipts = [
4263 remember_receipt("deploy target", true),
4264 AssistantToolReceipt {
4265 tool: "read_file".to_string(),
4266 call_id: None,
4267 ok: true,
4268 params: json!({"path": "x"}),
4269 via: None,
4270 },
4271 remember_receipt("user timezone", true),
4272 ];
4273
4274 let subjects = recent_fact_subjects(&receipts);
4275 assert_eq!(subjects, vec!["deploy target", "user timezone"]);
4276
4277 let block = render_state_block(None, &subjects).expect("facts alone must render a block");
4278 assert!(block.contains("deploy target"), "{block}");
4279 assert!(block.contains("user timezone"), "{block}");
4280 assert!(
4282 block.contains("recall"),
4283 "must point at the content: {block}"
4284 );
4285 assert!(!block.contains('…'), "bodies must not be inlined: {block}");
4286 }
4287
4288 #[test]
4292 fn a_failed_remember_is_not_reported_as_known() {
4293 let receipts = [
4294 remember_receipt("landed fact", true),
4295 remember_receipt("rejected fact", false),
4296 ];
4297 assert_eq!(recent_fact_subjects(&receipts), vec!["landed fact"]);
4298 }
4299
4300 #[test]
4304 fn re_remembering_a_subject_moves_it_instead_of_duplicating() {
4305 let receipts = [
4306 remember_receipt("api base url", true),
4307 remember_receipt("deploy target", true),
4308 remember_receipt("api base url", true),
4309 ];
4310 assert_eq!(
4311 recent_fact_subjects(&receipts),
4312 vec!["deploy target", "api base url"]
4313 );
4314 }
4315
4316 #[test]
4319 fn the_fact_list_is_bounded_and_says_what_it_dropped() {
4320 let subjects: Vec<String> = (0..12).map(|i| format!("fact {i}")).collect();
4321 let block = render_state_block(None, &subjects).expect("must render");
4322
4323 assert!(block.contains("fact 11"), "newest must survive: {block}");
4324 assert!(!block.contains("fact 6"), "oldest must be cut: {block}");
4325 assert!(
4326 block.contains("+7 earlier"),
4327 "a silent cut reads as 'that's all there is': {block}"
4328 );
4329 }
4330
4331 #[test]
4334 fn sections_render_independently_and_nothing_renders_nothing() {
4335 assert!(render_state_block(None, &[]).is_none());
4336 assert!(render_state_block(Some("todo: 0/1 done".into()), &[]).is_some());
4337 assert!(render_state_block(None, &["a fact".to_string()]).is_some());
4338
4339 let both = render_state_block(Some("todo: 0/1 done".into()), &["a fact".to_string()])
4340 .expect("must render");
4341 assert!(both.contains("todo:"), "{both}");
4342 assert!(both.contains("a fact"), "{both}");
4343 }
4344
4345 #[test]
4352 fn compaction_leaves_a_marker_the_model_can_see() {
4353 let big = "x".repeat(20_000);
4354 let mut m = vec![sys("system"), usr("THE ORIGINAL TASK")];
4355 for i in 0..12 {
4356 m.push(asst_call(&format!("c{i}")));
4357 m.push(tool_res(&format!("c{i}"), &big));
4358 }
4359 compact_history_to_window(&mut m, 20_000);
4360
4361 let notice = m
4362 .iter()
4363 .find_map(|msg| match msg {
4364 Message::System { content } if content.starts_with(COMPACTION_NOTICE_PREFIX) => {
4365 Some(content.clone())
4366 }
4367 _ => None,
4368 })
4369 .expect("a compaction notice must be left in place of the removed turns");
4370
4371 assert!(
4372 notice.contains("earlier turns removed"),
4373 "the notice must say turns were removed: {notice}"
4374 );
4375 assert!(
4376 notice.contains("events_query"),
4377 "a notice that says something is missing without saying how to look \
4378 only turns a silent failure into a visible dead end: {notice}"
4379 );
4380 assert!(
4383 matches!(&m[2], Message::System { content } if content.starts_with(COMPACTION_NOTICE_PREFIX)),
4384 "notice belongs where the turns were, after system + task"
4385 );
4386 }
4387
4388 #[test]
4393 fn repeated_compaction_accumulates_into_one_notice() {
4394 let big = "x".repeat(20_000);
4395 let mut m = vec![sys("system"), usr("THE ORIGINAL TASK")];
4396 for i in 0..12 {
4397 m.push(asst_call(&format!("c{i}")));
4398 m.push(tool_res(&format!("c{i}"), &big));
4399 }
4400 compact_history_to_window(&mut m, 20_000);
4401 let (first_turns, first_tokens) =
4402 parse_compaction_notice(&m[2]).expect("first notice parses");
4403
4404 for i in 12..24 {
4406 m.push(asst_call(&format!("c{i}")));
4407 m.push(tool_res(&format!("c{i}"), &big));
4408 }
4409 compact_history_to_window(&mut m, 20_000);
4410
4411 let notices: Vec<&String> = m
4412 .iter()
4413 .filter_map(|msg| match msg {
4414 Message::System { content } if content.starts_with(COMPACTION_NOTICE_PREFIX) => {
4415 Some(content)
4416 }
4417 _ => None,
4418 })
4419 .collect();
4420 assert_eq!(
4421 notices.len(),
4422 1,
4423 "exactly one notice, not a stack: {notices:?}"
4424 );
4425
4426 let (turns, tokens) = parse_compaction_notice(&m[2]).expect("notice still parses");
4427 assert!(
4428 turns > first_turns && tokens > first_tokens,
4429 "totals must accumulate across compactions ({first_turns}/{first_tokens} \
4430 -> {turns}/{tokens})"
4431 );
4432 }
4433
4434 #[test]
4437 fn compaction_notice_round_trips() {
4438 let rendered = format_compaction_notice(12, 34_000, CompactionRecovery::default());
4442 let parsed = parse_compaction_notice(&Message::System { content: rendered });
4443 assert_eq!(parsed, Some((12, 34_000)));
4444 assert_eq!(
4446 parse_compaction_notice(&sys("ordinary system prompt")),
4447 None
4448 );
4449 assert_eq!(parse_compaction_notice(&usr("a user turn")), None);
4450 }
4451
4452 #[test]
4453 fn compaction_keeps_responses_item_with_its_assistant_turn() {
4454 let big = "x".repeat(20_000);
4455 let mut messages = vec![sys("system"), usr("THE ORIGINAL TASK")];
4456 for i in 0..12 {
4457 messages.push(provider_item(&format!("rs_{i}"), &big));
4458 messages.push(asst_call(&format!("c{i}")));
4459 messages.push(tool_res(&format!("c{i}"), "ok"));
4460 }
4461
4462 compact_history_to_window(&mut messages, 20_000);
4463
4464 for (index, message) in messages.iter().enumerate() {
4465 if matches!(message, Message::ProviderOutputItems { .. }) {
4466 assert!(
4467 matches!(messages.get(index + 1), Some(Message::Assistant { .. })),
4468 "provider continuity item was orphaned from its assistant"
4469 );
4470 }
4471 }
4472 assert!(
4473 no_orphan_tool_results(&messages),
4474 "compacted history contains an orphan tool result"
4475 );
4476 }
4477
4478 #[tokio::test]
4484 async fn loop_compacts_history_to_window() {
4485 let dir = tempfile::tempdir().unwrap();
4486 let rt = runtime_for(dir.path()).await;
4487
4488 struct WindowedBig {
4492 cursor: AtomicUsize,
4493 }
4494 #[async_trait]
4495 impl TurnGenerator for WindowedBig {
4496 async fn generate(&self, _req: GenerateRequest) -> Result<InferenceResult, String> {
4497 let i = self.cursor.fetch_add(1, Ordering::SeqCst);
4498 if i < 6 {
4499 Ok(turn(
4502 &"x".repeat(8000),
4503 json!([{ "id": format!("c{i}"), "name": "calculate",
4504 "arguments": { "expression": format!("1+{i}") } }]),
4505 ))
4506 } else {
4507 Ok(turn("done", json!([])))
4508 }
4509 }
4510 fn context_window(&self, _model: &str) -> usize {
4511 4000
4512 }
4513 }
4514
4515 let generator = WindowedBig {
4516 cursor: AtomicUsize::new(0),
4517 };
4518 let mut messages = vec![
4519 Message::System {
4520 content: "system".into(),
4521 },
4522 Message::User {
4523 content: "THE TASK".into(),
4524 },
4525 ];
4526 let mut c = cfg();
4527 c.max_turns = 8;
4528
4529 let out = run_assistant_loop(&generator, &rt, &c, &mut messages, |_e| {}).await;
4530
4531 assert_eq!(out.status, "success");
4532 assert!(
4535 messages.len() <= 11,
4536 "history bounded by compaction, got {} messages",
4537 messages.len()
4538 );
4539 assert!(
4540 matches!(&messages[0], Message::System { .. }),
4541 "system stays pinned"
4542 );
4543 assert!(
4544 matches!(&messages[1], Message::User { content } if content == "THE TASK"),
4545 "original task stays pinned"
4546 );
4547 assert!(
4548 no_orphan_tool_results(&messages),
4549 "no orphaned tool results in the live loop"
4550 );
4551 }
4552
4553 #[tokio::test]
4557 async fn loop_halts_a_no_progress_repeat_loop() {
4558 let dir = tempfile::tempdir().unwrap();
4559 let rt = runtime_for(dir.path()).await;
4560
4561 struct Stuck;
4562 #[async_trait]
4563 impl TurnGenerator for Stuck {
4564 async fn generate(&self, _req: GenerateRequest) -> Result<InferenceResult, String> {
4565 Ok(turn(
4567 "re-reading",
4568 json!([{ "name": "read_file", "arguments": { "path": "app.js" } }]),
4569 ))
4570 }
4571 }
4572
4573 let mut messages = vec![
4574 Message::System {
4575 content: "sys".into(),
4576 },
4577 Message::User {
4578 content: "task".into(),
4579 },
4580 ];
4581 let mut c = cfg();
4582 c.max_turns = 40; let out = run_assistant_loop(&Stuck, &rt, &c, &mut messages, |_e| {}).await;
4585
4586 assert_eq!(
4587 out.status, "stalled",
4588 "a no-progress loop must halt as `stalled`, not run to max_turns"
4589 );
4590 assert!(
4591 out.turns < 40,
4592 "must stop well before the turn cap, got {} turns",
4593 out.turns
4594 );
4595 }
4596
4597 #[tokio::test]
4602 async fn loop_halts_a_read_plus_readonly_shell_cycle() {
4603 let dir = tempfile::tempdir().unwrap();
4604 let rt = runtime_for(dir.path()).await;
4605
4606 struct Cycle {
4607 cursor: AtomicUsize,
4608 }
4609 #[async_trait]
4610 impl TurnGenerator for Cycle {
4611 async fn generate(&self, _req: GenerateRequest) -> Result<InferenceResult, String> {
4612 let i = self.cursor.fetch_add(1, Ordering::SeqCst);
4613 if i.is_multiple_of(2) {
4614 Ok(turn(
4615 "read",
4616 json!([{ "name": "read_file", "arguments": { "path": "app.js" } }]),
4617 ))
4618 } else {
4619 Ok(turn(
4620 "probe",
4621 json!([{ "name": "shell", "arguments": { "command": "wc -l app.js" } }]),
4622 ))
4623 }
4624 }
4625 }
4626
4627 let mut messages = vec![
4628 Message::System {
4629 content: "sys".into(),
4630 },
4631 Message::User {
4632 content: "task".into(),
4633 },
4634 ];
4635 let mut c = cfg();
4636 c.max_turns = 40;
4637
4638 let out = run_assistant_loop(
4639 &Cycle {
4640 cursor: AtomicUsize::new(0),
4641 },
4642 &rt,
4643 &c,
4644 &mut messages,
4645 |_e| {},
4646 )
4647 .await;
4648
4649 assert_eq!(
4650 out.status, "stalled",
4651 "a read/read-only-shell cycle with no file change must halt"
4652 );
4653 assert!(out.turns < 40, "stopped before the cap, got {}", out.turns);
4654 }
4655
4656 #[tokio::test]
4661 async fn loop_halts_a_repeatedly_failing_mutation() {
4662 let dir = tempfile::tempdir().unwrap();
4663 let rt = runtime_for(dir.path()).await;
4664
4665 struct FailWrite;
4666 #[async_trait]
4667 impl TurnGenerator for FailWrite {
4668 async fn generate(&self, _req: GenerateRequest) -> Result<InferenceResult, String> {
4669 Ok(turn(
4672 "writing",
4673 json!([{ "name": "write_file",
4674 "arguments": { "path": "../../etc/evil", "content": "x" } }]),
4675 ))
4676 }
4677 }
4678
4679 let mut messages = vec![
4680 Message::System {
4681 content: "sys".into(),
4682 },
4683 Message::User {
4684 content: "task".into(),
4685 },
4686 ];
4687 let mut c = cfg();
4688 c.max_turns = 40;
4689
4690 let out = run_assistant_loop(&FailWrite, &rt, &c, &mut messages, |_e| {}).await;
4691
4692 assert_eq!(
4693 out.status, "stalled",
4694 "a repeatedly-failing mutation makes no progress and must halt (not reset the guard)"
4695 );
4696 assert!(out.turns < 40, "stopped before the cap, got {}", out.turns);
4697 }
4698
4699 fn turn_with_usage(
4703 text: &str,
4704 tool_calls: Value,
4705 prompt_tokens: u64,
4706 completion_tokens: u64,
4707 ) -> InferenceResult {
4708 serde_json::from_value(json!({
4709 "text": text,
4710 "tool_calls": tool_calls,
4711 "trace_id": "t",
4712 "model_used": "scripted",
4713 "latency_ms": 25,
4714 "usage": {
4715 "prompt_tokens": prompt_tokens,
4716 "completion_tokens": completion_tokens,
4717 "total_tokens": prompt_tokens + completion_tokens,
4718 "context_window": 8192,
4719 },
4720 }))
4721 .expect("scripted InferenceResult shape with usage")
4722 }
4723
4724 #[tokio::test]
4735 async fn assistant_loop_meters_every_model_call_with_real_tokens() {
4736 let dir = tempfile::tempdir().unwrap();
4737 let rt = runtime_for(dir.path()).await;
4738 let mut fallback_turn = turn_with_usage("The answer is 42.", json!([]), 200, 15);
4741 fallback_turn.model_identity.resolved_model_id = "mlx/qwen3-4b:4bit".into();
4742 fallback_turn.local_last_resort = true;
4743 let script = Script {
4744 turns: vec![
4745 turn_with_usage(
4746 "computing",
4747 json!([{ "id": "c1", "name": "calculate", "arguments": { "expression": "6*7" } }]),
4748 120,
4749 30,
4750 ),
4751 fallback_turn,
4752 ],
4753 cursor: AtomicUsize::new(0),
4754 };
4755 let mut messages = vec![
4756 Message::System {
4757 content: "sys".into(),
4758 },
4759 Message::User {
4760 content: "what is 6*7?".into(),
4761 },
4762 ];
4763 let mut assistant_events = Vec::new();
4764 let outcome = run_assistant_loop(&script, &rt, &cfg(), &mut messages, |event| {
4765 assistant_events.push(event)
4766 })
4767 .await;
4768 assert_eq!(outcome.status, "success");
4769 assert_eq!(
4770 outcome.models_served,
4771 vec![
4772 AssistantModelAttribution {
4773 model_id: "scripted".into(),
4774 local_last_resort: false,
4775 },
4776 AssistantModelAttribution {
4777 model_id: "mlx/qwen3-4b:4bit".into(),
4778 local_last_resort: true,
4779 },
4780 ],
4781 "the terminal run receipt must retain every turn, including the local fallback"
4782 );
4783 assert_eq!(
4784 outcome.model_used, "mlx/qwen3-4b:4bit",
4785 "final attribution must use the canonical resolved model id"
4786 );
4787
4788 let transcript_attributions: Vec<_> = messages
4789 .iter()
4790 .filter_map(|message| match message {
4791 Message::Assistant {
4792 model_id,
4793 local_last_resort,
4794 ..
4795 } => Some((model_id.as_deref(), *local_last_resort)),
4796 _ => None,
4797 })
4798 .collect();
4799 assert_eq!(
4800 transcript_attributions,
4801 vec![(Some("scripted"), false), (Some("mlx/qwen3-4b:4bit"), true)],
4802 "the exact replayable transcript must carry each serving attribution"
4803 );
4804
4805 let attributions: Vec<_> = assistant_events
4806 .iter()
4807 .filter_map(|event| match event {
4808 AssistantEvent::ModelServed {
4809 model_id,
4810 local_last_resort,
4811 } => Some((model_id.as_str(), *local_last_resort)),
4812 _ => None,
4813 })
4814 .collect();
4815 assert_eq!(
4816 attributions,
4817 vec![("scripted", false), ("mlx/qwen3-4b:4bit", true)],
4818 "every completed assistant turn must emit its canonical serving model and fallback marker"
4819 );
4820
4821 let events = rt.log.lock().await.events().to_vec();
4822 let metered: Vec<_> = events
4823 .iter()
4824 .filter(|e| e.kind == car_eventlog::EventKind::InferenceMetered)
4825 .collect();
4826 assert_eq!(
4827 metered.len(),
4828 2,
4829 "one InferenceMetered per model call; the loop made 2 generate() calls"
4830 );
4831 let metered_models: Vec<_> = metered
4832 .iter()
4833 .map(|event| event.data.get("model_id").and_then(Value::as_str))
4834 .collect();
4835 assert_eq!(
4836 metered_models,
4837 vec![Some("scripted"), Some("mlx/qwen3-4b:4bit")],
4838 "metered events must use the same canonical serving ids"
4839 );
4840 for ev in &metered {
4841 assert_eq!(
4842 ev.data.get("usage_measured").and_then(|v| v.as_bool()),
4843 Some(true)
4844 );
4845 }
4846
4847 let m = car_eventlog::harness_metrics::compute_harness_metrics(&events);
4848 assert_eq!(
4849 m.trajectory_efficiency.model_calls, 2,
4850 "harness metrics must see the model calls"
4851 );
4852 assert_eq!(
4853 m.trajectory_efficiency.total_tokens,
4854 120 + 30 + 200 + 15,
4855 "tokens must be the sum of the scripted usage, not an estimate"
4856 );
4857 assert!(m.trajectory_efficiency.wall_clock_ms > 0.0);
4858
4859 assert!(
4866 m.trajectory_efficiency.actions_succeeded > 0,
4867 "the executed `calculate` call must be recorded as a succeeded action; \
4868 got {m:?}"
4869 );
4870 assert!(
4871 m.trajectory_efficiency.success_rate.is_some(),
4872 "success_rate is the evolution gate's only regression guard and must be measured"
4873 );
4874 }
4875
4876 #[tokio::test]
4881 async fn unmeasured_usage_still_counts_the_call_but_fabricates_no_tokens() {
4882 let dir = tempfile::tempdir().unwrap();
4883 let rt = runtime_for(dir.path()).await;
4884 let script = Script {
4886 turns: vec![turn("done, no usage reported", json!([]))],
4887 cursor: AtomicUsize::new(0),
4888 };
4889 let mut messages = vec![
4890 Message::System {
4891 content: "sys".into(),
4892 },
4893 Message::User {
4894 content: "hi".into(),
4895 },
4896 ];
4897 let outcome = run_assistant_loop(&script, &rt, &cfg(), &mut messages, |_| {}).await;
4898 assert_eq!(outcome.status, "success");
4899
4900 let events = rt.log.lock().await.events().to_vec();
4901 let metered: Vec<_> = events
4902 .iter()
4903 .filter(|e| e.kind == car_eventlog::EventKind::InferenceMetered)
4904 .collect();
4905 assert_eq!(metered.len(), 1, "the call happened, so it is counted");
4906 assert_eq!(
4907 metered[0]
4908 .data
4909 .get("usage_measured")
4910 .and_then(|v| v.as_bool()),
4911 Some(false),
4912 "the journal must say the count was unavailable, not imply a zero"
4913 );
4914
4915 let m = car_eventlog::harness_metrics::compute_harness_metrics(&events);
4916 assert_eq!(m.trajectory_efficiency.model_calls, 1);
4917 assert_eq!(
4918 m.trajectory_efficiency.total_tokens, 0,
4919 "no usage reported means no tokens attributed — absent, not invented"
4920 );
4921 }
4922
4923 fn turn(text: &str, tool_calls: Value) -> InferenceResult {
4924 serde_json::from_value(json!({
4925 "text": text,
4926 "tool_calls": tool_calls,
4927 "trace_id": "t",
4928 "model_used": "scripted",
4929 "latency_ms": 0,
4930 }))
4931 .expect("scripted InferenceResult shape")
4932 }
4933
4934 struct Script {
4935 turns: Vec<InferenceResult>,
4936 cursor: AtomicUsize,
4937 }
4938
4939 #[async_trait]
4940 impl TurnGenerator for Script {
4941 async fn generate(&self, _req: GenerateRequest) -> Result<InferenceResult, String> {
4942 let i = self.cursor.fetch_add(1, Ordering::SeqCst);
4943 self.turns.get(i).cloned().ok_or("script exhausted".into())
4944 }
4945 }
4946
4947 struct CapturingGenerator {
4948 seen: Arc<StdMutex<Vec<GenerateRequest>>>,
4949 }
4950
4951 #[async_trait]
4952 impl TurnGenerator for CapturingGenerator {
4953 async fn generate(&self, req: GenerateRequest) -> Result<InferenceResult, String> {
4954 self.seen.lock().unwrap().push(req);
4955 Ok(turn("done", json!([])))
4956 }
4957 }
4958
4959 struct CapturingScript {
4960 turns: Vec<InferenceResult>,
4961 cursor: AtomicUsize,
4962 seen: Arc<StdMutex<Vec<GenerateRequest>>>,
4963 }
4964
4965 #[async_trait]
4966 impl TurnGenerator for CapturingScript {
4967 async fn generate(&self, req: GenerateRequest) -> Result<InferenceResult, String> {
4968 self.seen.lock().unwrap().push(req);
4969 let i = self.cursor.fetch_add(1, Ordering::SeqCst);
4970 self.turns.get(i).cloned().ok_or("script exhausted".into())
4971 }
4972 }
4973
4974 async fn runtime_for(dir: &std::path::Path) -> Runtime {
4978 let substrate: Arc<dyn Substrate> = Arc::new(LocalSubstrate::new());
4979 let exec: Arc<dyn ToolExecutor> =
4980 Arc::new(GeneralExecutor::new(substrate.clone(), dir, true));
4981 let engine = Arc::new(InferenceEngine::new(Default::default()));
4982 let rt = Runtime::new()
4983 .with_inference(engine)
4984 .with_executor(exec)
4985 .with_substrate(substrate);
4986 rt.register_agent_basics().await;
4987 rt.register_tool_entry(
4988 car_engine::ToolEntry::builtin(car_ir::builtins::shell()).with_side_effects(true),
4989 )
4990 .await;
4991 rt
4992 }
4993
4994 fn cfg() -> AssistantConfig {
4995 AssistantConfig {
4996 model: Some("scripted".into()),
4997 strict_model: false,
4998 max_turns: 6,
4999 tools: GeneralExecutor::tool_defs(),
5000 gated_tools: Vec::new(),
5001 approval_policy: None,
5002 proactive_memory: None,
5003 tool_memory: None,
5004 tool_labels: None,
5008 todos: None,
5009 value_store_previews: false,
5010 response_format: None,
5011 context_window_override: None,
5012 refuse_unadvertised_tools: false,
5013 response_format_validator: None,
5014 delegate_budget: None,
5015 }
5016 }
5017
5018 #[test]
5028 fn the_shipped_default_is_the_measured_one() {
5029 assert!(
5030 !VALUE_STORE_PREVIEWS_DEFAULT,
5031 "retained previews stay OFF because the 3-replicate car-bench-harness \
5032 A/B did not meet #813's fewer-calls criterion. Changing this needs a \
5033 new measurement, not an edit."
5034 );
5035 }
5036
5037 #[test]
5050 fn no_production_call_site_hard_codes_the_preview_default() {
5051 let crate_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
5052 for rel in [
5053 "src/assistant/agent_loop.rs",
5054 "src/assistant/chat.rs",
5055 "src/coder/discuss.rs",
5056 "src/mcp_assistant.rs",
5057 ] {
5058 let src = std::fs::read_to_string(crate_dir.join(rel))
5059 .unwrap_or_else(|e| panic!("reading {rel}: {e}"));
5060 let production = match src.find("\n#[cfg(test)]") {
5061 Some(cut) => &src[..cut],
5062 None => src.as_str(),
5063 };
5064 for literal in ["value_store_previews: false", "value_store_previews: true"] {
5065 assert!(
5066 !production.contains(literal),
5067 "{rel} hard-codes the preview arm in production code. The \
5068 shipped default is VALUE_STORE_PREVIEWS_DEFAULT, chosen from a \
5069 measured A/B; a literal here forks it silently."
5070 );
5071 }
5072 }
5073 }
5074
5075 #[tokio::test]
5085 async fn the_off_arm_still_truncates_exactly_as_before() {
5086 assert!(
5087 !cfg().value_store_previews,
5088 "this fixture is the OFF arm — it pins the pre-#813 observation path, \
5089 not the shipped default (see VALUE_STORE_PREVIEWS_DEFAULT)"
5090 );
5091
5092 let dir = tempfile::tempdir().unwrap();
5093 let big = "x".repeat(OBSERVATION_CAP + 40_000);
5094 std::fs::write(dir.path().join("big.txt"), &big).unwrap();
5095 let rt = runtime_for(dir.path()).await;
5096
5097 let script = Script {
5098 turns: vec![
5099 turn(
5100 "reading",
5101 json!([{ "id": "c1", "name": "read_file", "arguments": { "path": "./big.txt" } }]),
5102 ),
5103 turn("done", json!([])),
5104 ],
5105 cursor: AtomicUsize::new(0),
5106 };
5107 let mut messages = vec![
5108 Message::System {
5109 content: "sys".into(),
5110 },
5111 Message::User {
5112 content: "read it".into(),
5113 },
5114 ];
5115 run_assistant_loop(&script, &rt, &cfg(), &mut messages, |_| {}).await;
5116
5117 let observation = messages
5118 .iter()
5119 .find_map(|m| match m {
5120 Message::ToolResult { content, .. } => Some(content.clone()),
5121 _ => None,
5122 })
5123 .expect("a tool observation");
5124 assert!(
5125 observation.contains("…[truncated:"),
5126 "off path must still truncate destructively: {}",
5127 &observation[observation.len().saturating_sub(200)..]
5128 );
5129 assert!(
5130 !observation.contains("[full value retained"),
5131 "no handle may leak into the default transcript"
5132 );
5133 }
5134
5135 #[tokio::test]
5144 async fn a_retained_value_survives_the_transcript_and_can_be_used_by_a_later_tool() {
5145 let dir = tempfile::tempdir().unwrap();
5146 let big = format!(
5148 "HEAD-MARKER\n{}\nTAIL-MARKER",
5149 "z".repeat(OBSERVATION_CAP + 40_000)
5150 );
5151 std::fs::write(dir.path().join("big.txt"), &big).unwrap();
5152 let rt = runtime_for(dir.path()).await;
5153
5154 let script = Script {
5155 turns: vec![
5156 turn(
5157 "reading",
5158 json!([{ "id": "c1", "name": "read_file", "arguments": { "path": "./big.txt" } }]),
5159 ),
5160 turn(
5161 "copying",
5162 json!([{ "id": "c2", "name": "write_file",
5163 "arguments": { "path": "./copy.txt", "content": "$r1.content" } }]),
5164 ),
5165 turn("done", json!([])),
5166 ],
5167 cursor: AtomicUsize::new(0),
5168 };
5169 let mut messages = vec![
5170 Message::System {
5171 content: "sys".into(),
5172 },
5173 Message::User {
5174 content: "copy it".into(),
5175 },
5176 ];
5177 let mut cfg = cfg();
5178 cfg.value_store_previews = true;
5179 cfg.max_turns = 8;
5180 run_assistant_loop(&script, &rt, &cfg, &mut messages, |_| {}).await;
5181
5182 let observation = messages
5183 .iter()
5184 .find_map(|m| match m {
5185 Message::ToolResult { content, .. } => Some(content.clone()),
5186 _ => None,
5187 })
5188 .expect("a tool observation");
5189
5190 assert!(
5192 observation.contains("content: text(len="),
5193 "the large field must announce its size and shape: {observation}"
5194 );
5195 assert!(
5196 observation.contains("[full value retained"),
5197 "the model must be told the value is reachable: {observation}"
5198 );
5199 assert!(
5200 observation.len() < 2_000,
5201 "preview must be bounded, got {} bytes",
5202 observation.len()
5203 );
5204 assert!(
5205 !observation.contains(&"z".repeat(1_000)),
5206 "the payload itself must not be in the transcript"
5207 );
5208
5209 let copied = std::fs::read_to_string(dir.path().join("copy.txt"))
5216 .expect("the second tool must have run with the resolved value");
5217 assert!(
5218 copied.len() > OBSERVATION_CAP,
5219 "only {} bytes came back; the value was not retained in full",
5220 copied.len()
5221 );
5222 assert!(
5223 copied.contains("HEAD-MARKER"),
5224 "the head — the only part destructive truncation ever kept — is missing"
5225 );
5226 assert!(
5227 copied.contains("TAIL-MARKER"),
5228 "the TAIL is the part cap() always destroyed; recovering it is the \
5229 whole point of #813"
5230 );
5231 assert!(
5233 !observation.contains("TAIL-MARKER"),
5234 "the tail must have come from the store, not the context: {observation}"
5235 );
5236 }
5237
5238 #[tokio::test]
5239 async fn loop_runs_a_tool_then_finishes() {
5240 let dir = tempfile::tempdir().unwrap();
5241 let rt = runtime_for(dir.path()).await;
5242 let script = Script {
5244 turns: vec![
5245 turn(
5246 "computing",
5247 json!([{ "id": "c1", "name": "calculate", "arguments": { "expression": "6*7" } }]),
5248 ),
5249 turn("The answer is 42.", json!([])),
5250 ],
5251 cursor: AtomicUsize::new(0),
5252 };
5253 let mut messages = vec![
5254 Message::System {
5255 content: "sys".into(),
5256 },
5257 Message::User {
5258 content: "what is 6*7?".into(),
5259 },
5260 ];
5261 let mut events = Vec::new();
5262 let outcome =
5263 run_assistant_loop(&script, &rt, &cfg(), &mut messages, |e| events.push(e)).await;
5264
5265 assert_eq!(outcome.status, "success");
5266 assert_eq!(outcome.summary, "The answer is 42.");
5267 assert!(outcome.tools_called.contains(&"calculate".to_string()));
5268 assert!(events
5269 .iter()
5270 .any(|e| matches!(e, AssistantEvent::ToolResult { name, ok: true, .. } if name == "calculate")));
5271 }
5272
5273 #[tokio::test]
5274 async fn loop_replays_managed_responses_continuity_on_second_turn() {
5275 let dir = tempfile::tempdir().unwrap();
5276 let rt = runtime_for(dir.path()).await;
5277 let reasoning = json!({
5278 "type": "reasoning",
5279 "id": "rs_agent",
5280 "status": "completed",
5281 "summary": [{"type": "summary_text", "text": "safe"}],
5282 "encrypted_content": "opaque-agent",
5283 });
5284 let mut first = turn(
5285 "checking",
5286 json!([{ "id": "c1", "name": "calculate", "arguments": { "expression": "6*7" } }]),
5287 );
5288 first.provider_output_items = vec![reasoning.clone()];
5289 let seen = Arc::new(StdMutex::new(Vec::new()));
5290 let script = CapturingScript {
5291 turns: vec![first, turn("done", json!([]))],
5292 cursor: AtomicUsize::new(0),
5293 seen: seen.clone(),
5294 };
5295 let mut messages = vec![
5296 Message::System {
5297 content: "sys".into(),
5298 },
5299 Message::User {
5300 content: "calculate".into(),
5301 },
5302 ];
5303
5304 let outcome = run_assistant_loop(&script, &rt, &cfg(), &mut messages, |_e| {}).await;
5305
5306 assert_eq!(outcome.status, "success");
5307 assert!(
5308 !outcome.summary.contains("opaque-agent"),
5309 "opaque continuity must never become user-visible text"
5310 );
5311 let seen = seen.lock().unwrap();
5312 let second = seen[1].messages.as_ref().expect("second-turn history");
5313 assert!(matches!(
5314 &second[2],
5315 Message::ProviderOutputItems { protocol, items }
5316 if protocol == car_inference::protocol::OPENAI_RESPONSES_PROTOCOL
5317 && items == &vec![reasoning]
5318 ));
5319 assert!(matches!(
5320 &second[3],
5321 Message::Assistant { content, .. } if content == "checking"
5322 ));
5323 assert!(matches!(&second[4], Message::ToolResult { .. }));
5324 }
5325
5326 #[tokio::test]
5327 async fn loop_injects_proactive_memory_before_generation() {
5328 let dir = tempfile::tempdir().unwrap();
5329 let rt = runtime_for(dir.path()).await;
5330 let memory = Arc::new(crate::assistant::memory::MemoryTools::open(
5331 dir.path().join("assistant-memory.json"),
5332 ));
5333 memory
5334 .execute(
5335 "remember",
5336 &json!({
5337 "subject": "phoenix task requirement",
5338 "body": "Requirement: for phoenix task work, run pytest before finishing."
5339 }),
5340 )
5341 .await
5342 .unwrap();
5343 let seen = Arc::new(StdMutex::new(Vec::new()));
5344 let generator = CapturingGenerator { seen: seen.clone() };
5345 let mut cfg = cfg();
5346 cfg.proactive_memory = Some(memory);
5347 let mut messages = vec![
5348 Message::System {
5349 content: "sys".into(),
5350 },
5351 Message::User {
5352 content: "finish the phoenix task".into(),
5353 },
5354 ];
5355
5356 let outcome = run_assistant_loop(&generator, &rt, &cfg, &mut messages, |_| {}).await;
5357
5358 assert_eq!(outcome.status, "success");
5359 {
5362 let captured = seen.lock().unwrap();
5363 let context = captured[0].context.as_deref().unwrap_or("");
5364 assert!(
5365 context.contains("## Proactive Memory"),
5366 "request context should carry proactive memory: {context}"
5367 );
5368 assert!(
5369 context.contains("run pytest before finishing"),
5370 "selected memory should be injected: {context}"
5371 );
5372 }
5373 let log = rt.log.lock().await;
5374 assert!(log
5375 .events()
5376 .iter()
5377 .any(|e| e.kind == car_eventlog::EventKind::ProactiveMemoryMaintained));
5378 assert!(log.events().iter().any(|e| {
5379 e.kind == car_eventlog::EventKind::ProactiveMemoryIntervention
5380 && e.data.get("decision") == Some(&json!("inject"))
5381 }));
5382 }
5383
5384 #[tokio::test]
5385 async fn loop_learns_the_call_that_recovered_a_failed_tool() {
5386 let dir = tempfile::tempdir().unwrap();
5390 let rt = runtime_for(dir.path()).await;
5391 let memory = Arc::new(crate::assistant::tool_memory::ToolMemory::open(
5392 dir.path().join("repairs.json"),
5393 ));
5394 let script = Script {
5395 turns: vec![
5396 turn(
5397 "trying",
5398 json!([{ "id": "c1", "name": "calculate", "arguments": { "expression": "6 ** ** 7" } }]),
5399 ),
5400 turn(
5401 "retrying",
5402 json!([{ "id": "c2", "name": "calculate", "arguments": { "expression": "6*7" } }]),
5403 ),
5404 turn("done", json!([])),
5405 ],
5406 cursor: AtomicUsize::new(0),
5407 };
5408 let mut cfg = cfg();
5409 cfg.tool_memory = Some(memory.clone());
5410 let mut messages = vec![
5411 Message::System {
5412 content: "sys".into(),
5413 },
5414 Message::User {
5415 content: "compute six times seven".into(),
5416 },
5417 ];
5418
5419 let outcome = run_assistant_loop(&script, &rt, &cfg, &mut messages, |_| {}).await;
5420
5421 assert_eq!(outcome.status, "success");
5422 assert_eq!(
5423 memory.learned_count(),
5424 1,
5425 "the recovering call should have been learned"
5426 );
5427 }
5428
5429 #[tokio::test]
5430 async fn a_learned_repair_reaches_the_next_run_that_hits_the_same_failure() {
5431 let dir = tempfile::tempdir().unwrap();
5435 let rt = runtime_for(dir.path()).await;
5436 let memory = Arc::new(crate::assistant::tool_memory::ToolMemory::open(
5437 dir.path().join("repairs.json"),
5438 ));
5439 let mut cfg = cfg();
5440 cfg.tool_memory = Some(memory.clone());
5441
5442 let learning = Script {
5443 turns: vec![
5444 turn(
5445 "trying",
5446 json!([{ "id": "c1", "name": "calculate", "arguments": { "expression": "6 ** ** 7" } }]),
5447 ),
5448 turn(
5449 "retrying",
5450 json!([{ "id": "c2", "name": "calculate", "arguments": { "expression": "6*7" } }]),
5451 ),
5452 turn("done", json!([])),
5453 ],
5454 cursor: AtomicUsize::new(0),
5455 };
5456 let mut messages = vec![
5457 Message::System {
5458 content: "sys".into(),
5459 },
5460 Message::User {
5461 content: "compute six times seven".into(),
5462 },
5463 ];
5464 run_assistant_loop(&learning, &rt, &cfg, &mut messages, |_| {}).await;
5465 assert_eq!(memory.learned_count(), 1, "run one must learn something");
5466
5467 let seen = Arc::new(StdMutex::new(Vec::new()));
5469 let second = CapturingScript {
5470 turns: vec![
5471 turn(
5472 "trying",
5473 json!([{ "id": "d1", "name": "calculate", "arguments": { "expression": "9 ** ** 9" } }]),
5474 ),
5475 turn("done", json!([])),
5476 ],
5477 cursor: AtomicUsize::new(0),
5478 seen: seen.clone(),
5479 };
5480 let mut messages = vec![
5481 Message::System {
5482 content: "sys".into(),
5483 },
5484 Message::User {
5485 content: "compute nine times nine".into(),
5486 },
5487 ];
5488 run_assistant_loop(&second, &rt, &cfg, &mut messages, |_| {}).await;
5489
5490 let captured = seen.lock().unwrap();
5491 let first_context = captured[0].context.as_deref().unwrap_or("");
5492 assert!(
5493 !first_context.contains("## Learned Repairs"),
5494 "nothing has failed yet on this run: {first_context}"
5495 );
5496 let after_failure = captured[1].context.as_deref().unwrap_or("");
5497 assert!(
5498 after_failure.contains("## Learned Repairs"),
5499 "the turn after the failure should carry the lead: {after_failure}"
5500 );
5501 assert!(
5502 after_failure.contains("6*7"),
5503 "the lead should be the call that actually recovered: {after_failure}"
5504 );
5505 }
5506
5507 #[tokio::test]
5508 async fn an_unrelated_later_success_is_not_credited_as_a_repair() {
5509 let dir = tempfile::tempdir().unwrap();
5513 let rt = runtime_for(dir.path()).await;
5514 let memory = Arc::new(crate::assistant::tool_memory::ToolMemory::open(
5515 dir.path().join("repairs.json"),
5516 ));
5517 let script = Script {
5518 turns: vec![
5519 turn(
5520 "trying",
5521 json!([{ "id": "c1", "name": "calculate", "arguments": { "expression": "6 ** ** 7" } }]),
5522 ),
5523 turn(
5524 "moving on",
5525 json!([{ "id": "c2", "name": "write_file", "arguments": { "path": "note.txt", "content": "hi" } }]),
5526 ),
5527 turn("done", json!([])),
5528 ],
5529 cursor: AtomicUsize::new(0),
5530 };
5531 let mut cfg = cfg();
5532 cfg.tool_memory = Some(memory.clone());
5533 let mut messages = vec![
5534 Message::System {
5535 content: "sys".into(),
5536 },
5537 Message::User {
5538 content: "do two things".into(),
5539 },
5540 ];
5541
5542 run_assistant_loop(&script, &rt, &cfg, &mut messages, |_| {}).await;
5543
5544 assert_eq!(
5545 memory.learned_count(),
5546 0,
5547 "a different tool succeeding is not a repair for the one that failed"
5548 );
5549 }
5550
5551 async fn learn_over_gap(gap: usize, recover_with: &str) -> usize {
5555 let dir = tempfile::tempdir().unwrap();
5556 let rt = runtime_for(dir.path()).await;
5557 let memory = Arc::new(crate::assistant::tool_memory::ToolMemory::open(
5558 dir.path().join("repairs.json"),
5559 ));
5560 let mut turns = vec![turn(
5561 "trying",
5562 json!([{ "id": "c0", "name": "calculate", "arguments": { "expression": "6 ** ** 7" } }]),
5563 )];
5564 for i in 0..gap {
5566 turns.push(turn(
5567 "thinking",
5568 json!([{ "id": format!("g{i}"), "name": "todo_write",
5569 "arguments": { "items": [{"task": format!("step {i}"), "status": "pending"}] } }]),
5570 ));
5571 }
5572 turns.push(turn(
5573 "retrying",
5574 json!([{ "id": "cN", "name": "calculate", "arguments": { "expression": recover_with } }]),
5575 ));
5576 turns.push(turn("done", json!([])));
5577 let script = Script {
5578 turns,
5579 cursor: AtomicUsize::new(0),
5580 };
5581 let mut cfg = cfg();
5582 cfg.max_turns = 12;
5583 cfg.tool_memory = Some(memory.clone());
5584 let mut messages = vec![
5585 Message::System {
5586 content: "sys".into(),
5587 },
5588 Message::User {
5589 content: "compute six times seven".into(),
5590 },
5591 ];
5592 run_assistant_loop(&script, &rt, &cfg, &mut messages, |_| {}).await;
5593 memory.learned_count()
5594 }
5595
5596 #[tokio::test]
5597 async fn a_recovery_inside_the_window_is_learned_and_one_outside_it_is_not() {
5598 assert_eq!(learn_over_gap(0, "6*7").await, 1, "next turn is a recovery");
5602 assert_eq!(
5603 learn_over_gap(RECOVERY_WINDOW_TURNS as usize - 1, "6*7").await,
5604 1,
5605 "the last turn inside the window still counts"
5606 );
5607 assert_eq!(
5608 learn_over_gap(RECOVERY_WINDOW_TURNS as usize + 2, "6*7").await,
5609 0,
5610 "well past the window is not a repair"
5611 );
5612 }
5613
5614 #[tokio::test]
5615 async fn an_identical_retry_that_happens_to_work_is_not_a_repair() {
5616 assert_eq!(
5621 learn_over_gap(0, "6 ** ** 7").await,
5622 0,
5623 "same arguments succeeding is a transient, not a repair"
5624 );
5625 }
5626
5627 #[tokio::test]
5628 async fn a_success_on_a_different_tool_never_closes_another_tools_failure() {
5629 let dir = tempfile::tempdir().unwrap();
5634 let rt = runtime_for(dir.path()).await;
5635 let memory = Arc::new(crate::assistant::tool_memory::ToolMemory::open(
5636 dir.path().join("repairs.json"),
5637 ));
5638 let script = Script {
5639 turns: vec![
5640 turn(
5641 "trying",
5642 json!([{ "id": "c1", "name": "calculate", "arguments": { "expression": "6 ** ** 7" } }]),
5643 ),
5644 turn(
5645 "different tool",
5646 json!([{ "id": "c2", "name": "todo_write",
5647 "arguments": { "items": [{"task": "unrelated", "status": "pending"}] } }]),
5648 ),
5649 turn("done", json!([])),
5650 ],
5651 cursor: AtomicUsize::new(0),
5652 };
5653 let mut cfg = cfg();
5654 cfg.tool_memory = Some(memory.clone());
5655 let mut messages = vec![
5656 Message::System {
5657 content: "sys".into(),
5658 },
5659 Message::User {
5660 content: "do things".into(),
5661 },
5662 ];
5663 run_assistant_loop(&script, &rt, &cfg, &mut messages, |_| {}).await;
5664 assert_eq!(
5665 memory.learned_count(),
5666 0,
5667 "a different tool's success is not a repair for this one"
5668 );
5669 }
5670
5671 #[tokio::test]
5672 async fn one_stale_lead_costs_exactly_one_failure_however_many_retries() {
5673 let dir = tempfile::tempdir().unwrap();
5677 let rt = runtime_for(dir.path()).await;
5678 let store = dir.path().join("repairs.json");
5679 let memory = Arc::new(crate::assistant::tool_memory::ToolMemory::open(
5680 store.clone(),
5681 ));
5682 let mut cfg = cfg();
5683 cfg.max_turns = 12;
5684 cfg.tool_memory = Some(memory.clone());
5685
5686 let learn = Script {
5688 turns: vec![
5689 turn(
5690 "trying",
5691 json!([{ "id": "a1", "name": "calculate", "arguments": { "expression": "6 ** ** 7" } }]),
5692 ),
5693 turn(
5694 "retrying",
5695 json!([{ "id": "a2", "name": "calculate", "arguments": { "expression": "6*7" } }]),
5696 ),
5697 turn("done", json!([])),
5698 ],
5699 cursor: AtomicUsize::new(0),
5700 };
5701 let mut messages = vec![
5702 Message::System {
5703 content: "sys".into(),
5704 },
5705 Message::User {
5706 content: "compute".into(),
5707 },
5708 ];
5709 run_assistant_loop(&learn, &rt, &cfg, &mut messages, |_| {}).await;
5710 assert_eq!(memory.learned_count(), 1);
5711
5712 let mut turns = Vec::new();
5715 for i in 0..5 {
5716 turns.push(turn(
5717 "failing",
5718 json!([{ "id": format!("b{i}"), "name": "calculate",
5719 "arguments": { "expression": format!("{i} ** ** {i}") } }]),
5720 ));
5721 }
5722 turns.push(turn("giving up", json!([])));
5723 let retry_storm = Script {
5724 turns,
5725 cursor: AtomicUsize::new(0),
5726 };
5727 let mut messages = vec![
5728 Message::System {
5729 content: "sys".into(),
5730 },
5731 Message::User {
5732 content: "compute".into(),
5733 },
5734 ];
5735 run_assistant_loop(&retry_storm, &rt, &cfg, &mut messages, |_| {}).await;
5736
5737 let sig = crate::assistant::tool_memory::FailureSignature::from_failure(
5738 "calculate",
5739 "[FAILED] bad expression",
5740 );
5741 assert!(
5742 memory.recall(&sig).is_some(),
5743 "one offered lead must cost one failure, not one per retry — \
5744 five penalties would have degraded it"
5745 );
5746 }
5747
5748 #[tokio::test]
5749 async fn penalty_markers_do_not_leak_between_runs() {
5750 let dir = tempfile::tempdir().unwrap();
5754 let rt = runtime_for(dir.path()).await;
5755 let memory = Arc::new(crate::assistant::tool_memory::ToolMemory::open(
5756 dir.path().join("repairs.json"),
5757 ));
5758 let mut cfg = cfg();
5759 cfg.tool_memory = Some(memory.clone());
5760
5761 let learn = Script {
5762 turns: vec![
5763 turn(
5764 "trying",
5765 json!([{ "id": "a1", "name": "calculate", "arguments": { "expression": "6 ** ** 7" } }]),
5766 ),
5767 turn(
5768 "retrying",
5769 json!([{ "id": "a2", "name": "calculate", "arguments": { "expression": "6*7" } }]),
5770 ),
5771 turn("done", json!([])),
5772 ],
5773 cursor: AtomicUsize::new(0),
5774 };
5775 let mut messages = vec![
5776 Message::System {
5777 content: "sys".into(),
5778 },
5779 Message::User {
5780 content: "compute".into(),
5781 },
5782 ];
5783 run_assistant_loop(&learn, &rt, &cfg, &mut messages, |_| {}).await;
5784
5785 for round in 0..3 {
5788 let single = Script {
5789 turns: vec![
5790 turn(
5791 "trying",
5792 json!([{ "id": format!("r{round}"), "name": "calculate",
5793 "arguments": { "expression": format!("{round} ** ** {round}") } }]),
5794 ),
5795 turn("done", json!([])),
5796 ],
5797 cursor: AtomicUsize::new(0),
5798 };
5799 let mut messages = vec![
5800 Message::System {
5801 content: "sys".into(),
5802 },
5803 Message::User {
5804 content: "compute".into(),
5805 },
5806 ];
5807 run_assistant_loop(&single, &rt, &cfg, &mut messages, |_| {}).await;
5808 }
5809
5810 let sig = crate::assistant::tool_memory::FailureSignature::from_failure(
5811 "calculate",
5812 "[FAILED] bad expression",
5813 );
5814 assert!(
5815 memory.recall(&sig).is_some(),
5816 "a first failure in a fresh run is not evidence against a lead"
5817 );
5818 }
5819
5820 #[tokio::test]
5821 async fn the_none_path_writes_nothing_to_disk() {
5822 let dir = tempfile::tempdir().unwrap();
5825 let rt = runtime_for(dir.path()).await;
5826 let store = dir.path().join("repairs.json");
5827 let script = Script {
5828 turns: vec![
5829 turn(
5830 "trying",
5831 json!([{ "id": "c1", "name": "calculate", "arguments": { "expression": "6 ** ** 7" } }]),
5832 ),
5833 turn(
5834 "retrying",
5835 json!([{ "id": "c2", "name": "calculate", "arguments": { "expression": "6*7" } }]),
5836 ),
5837 turn("done", json!([])),
5838 ],
5839 cursor: AtomicUsize::new(0),
5840 };
5841 let mut messages = vec![
5842 Message::System {
5843 content: "sys".into(),
5844 },
5845 Message::User {
5846 content: "compute".into(),
5847 },
5848 ];
5849 run_assistant_loop(&script, &rt, &cfg(), &mut messages, |_| {}).await;
5850 assert!(
5851 !store.exists(),
5852 "a surface that did not opt in must leave no store behind"
5853 );
5854 }
5855
5856 #[test]
5857 fn a_delegate_child_inherits_the_learning_store() {
5858 let dir = tempfile::tempdir().unwrap();
5862 let memory = Arc::new(crate::assistant::tool_memory::ToolMemory::open(
5863 dir.path().join("repairs.json"),
5864 ));
5865 let mut parent = cfg();
5866 parent.tools = vec![json!({
5867 "name": "calculate",
5868 "description": "d",
5869 "input_schema": {"type": "object"}
5870 })];
5871 parent.tool_memory = Some(memory.clone());
5872 let child = delegate_child_config(
5873 &parent,
5874 &DelegateRequest {
5875 goal: "sub".into(),
5876 tools: None,
5877 max_turns: 2,
5878 },
5879 )
5880 .expect("child config");
5881 let inherited = child.tool_memory.expect("child inherits the store");
5882 assert!(
5883 Arc::ptr_eq(&inherited, &memory),
5884 "the child must learn into the SAME store, not a fresh one"
5885 );
5886 }
5887
5888 #[tokio::test]
5889 async fn a_secret_in_a_recovering_call_never_reaches_the_store_through_the_loop() {
5890 let dir = tempfile::tempdir().unwrap();
5893 let rt = runtime_for(dir.path()).await;
5894 let store = dir.path().join("repairs.json");
5895 let memory = Arc::new(crate::assistant::tool_memory::ToolMemory::open(
5896 store.clone(),
5897 ));
5898 let script = Script {
5899 turns: vec![
5900 turn(
5901 "trying",
5902 json!([{ "id": "c1", "name": "write_file",
5903 "arguments": { "path": "x.txt", "content": "nope", "bogus": true } }]),
5904 ),
5905 turn(
5906 "retrying",
5907 json!([{ "id": "c2", "name": "write_file",
5908 "arguments": { "path": "x.txt",
5909 "content": "token ghp_ABCDEFGHIJKLMNOPQRST" } }]),
5910 ),
5911 turn("done", json!([])),
5912 ],
5913 cursor: AtomicUsize::new(0),
5914 };
5915 let mut cfg = cfg();
5916 cfg.tool_memory = Some(memory.clone());
5917 let mut messages = vec![
5918 Message::System {
5919 content: "sys".into(),
5920 },
5921 Message::User {
5922 content: "write the file".into(),
5923 },
5924 ];
5925 run_assistant_loop(&script, &rt, &cfg, &mut messages, |_| {}).await;
5926 if store.exists() {
5927 let on_disk = std::fs::read_to_string(&store).unwrap();
5928 assert!(
5929 !on_disk.contains("ghp_ABCDEFGHIJKLMNOPQRST"),
5930 "a credential must not survive into the durable store: {on_disk}"
5931 );
5932 }
5933 }
5934
5935 #[tokio::test]
5936 async fn learning_is_off_unless_the_surface_opted_in() {
5937 let dir = tempfile::tempdir().unwrap();
5940 let rt = runtime_for(dir.path()).await;
5941 let seen = Arc::new(StdMutex::new(Vec::new()));
5942 let script = CapturingScript {
5943 turns: vec![
5944 turn(
5945 "trying",
5946 json!([{ "id": "c1", "name": "calculate", "arguments": { "expression": "6 ** ** 7" } }]),
5947 ),
5948 turn("done", json!([])),
5949 ],
5950 cursor: AtomicUsize::new(0),
5951 seen: seen.clone(),
5952 };
5953 let mut messages = vec![
5954 Message::System {
5955 content: "sys".into(),
5956 },
5957 Message::User {
5958 content: "compute".into(),
5959 },
5960 ];
5961
5962 run_assistant_loop(&script, &rt, &cfg(), &mut messages, |_| {}).await;
5963
5964 let captured = seen.lock().unwrap();
5965 assert!(
5966 captured.iter().all(|req| !req
5967 .context
5968 .as_deref()
5969 .unwrap_or("")
5970 .contains("Learned Repairs")),
5971 "no surface opted in, so nothing should be recalled"
5972 );
5973 }
5974
5975 #[tokio::test]
5976 async fn loop_journals_turn_completed_at_empty_tool_calls_terminal() {
5977 let dir = tempfile::tempdir().unwrap();
5983 let rt = runtime_for(dir.path()).await;
5984 let script = Script {
5985 turns: vec![
5986 turn(
5987 "computing",
5988 json!([{ "id": "c1", "name": "calculate", "arguments": { "expression": "6*7" } }]),
5989 ),
5990 turn("The answer is 42.", json!([])),
5991 ],
5992 cursor: AtomicUsize::new(0),
5993 };
5994 let mut messages = vec![
5995 Message::System {
5996 content: "sys".into(),
5997 },
5998 Message::User {
5999 content: "what is 6*7?".into(),
6000 },
6001 ];
6002 let mut events = Vec::new();
6003 let outcome =
6004 run_assistant_loop(&script, &rt, &cfg(), &mut messages, |e| events.push(e)).await;
6005 assert_eq!(outcome.status, "success");
6006 assert_eq!(outcome.model_used, "scripted");
6008
6009 let log = rt.log.lock().await;
6010 let tc = log
6011 .events()
6012 .iter()
6013 .find(|e| e.kind == car_eventlog::EventKind::TurnCompleted)
6014 .expect("empty-tool-calls terminal must journal a TurnCompleted");
6015 assert_eq!(
6016 tc.data.get("decision"),
6017 Some(&serde_json::json!("empty_tool_calls"))
6018 );
6019 assert_eq!(tc.data.get("turns"), Some(&serde_json::json!(2)));
6020 assert_eq!(
6021 tc.data.get("model_id"),
6022 Some(&serde_json::json!("scripted"))
6023 );
6024 assert_eq!(
6026 tc.data.get("model_tier"),
6027 Some(&serde_json::json!("unknown"))
6028 );
6029 }
6030
6031 #[tokio::test]
6032 async fn loop_journals_turn_completed_at_max_turns_terminal() {
6033 let dir = tempfile::tempdir().unwrap();
6038 let rt = runtime_for(dir.path()).await;
6039 let tool_turn = || {
6040 turn(
6041 "still going",
6042 json!([{ "id": "c", "name": "calculate", "arguments": { "expression": "1+1" } }]),
6043 )
6044 };
6045 let script = Script {
6046 turns: (0..10).map(|_| tool_turn()).collect(),
6047 cursor: AtomicUsize::new(0),
6048 };
6049 let mut messages = vec![
6050 Message::System {
6051 content: "sys".into(),
6052 },
6053 Message::User {
6054 content: "loop".into(),
6055 },
6056 ];
6057 let cfg = AssistantConfig {
6059 max_turns: 3,
6060 ..cfg()
6061 };
6062 let mut events = Vec::new();
6063 let outcome =
6064 run_assistant_loop(&script, &rt, &cfg, &mut messages, |e| events.push(e)).await;
6065 assert_eq!(outcome.status, "max_turns");
6066
6067 let log = rt.log.lock().await;
6068 let tc = log
6069 .events()
6070 .iter()
6071 .find(|e| {
6072 e.kind == car_eventlog::EventKind::TurnCompleted
6073 && e.data.get("decision") == Some(&serde_json::json!("max_turns"))
6074 })
6075 .expect("max_turns terminal must journal a TurnCompleted");
6076 assert_eq!(tc.data.get("turns"), Some(&serde_json::json!(3)));
6077 }
6078
6079 #[test]
6080 fn summary_claim_grounding_requires_matching_receipts() {
6081 let ungrounded = ungrounded_summary_claims("I ran the tests and they passed.", &[]);
6082 assert_eq!(ungrounded, vec!["tests were run/passed"]);
6083
6084 let grounded = ungrounded_summary_claims(
6085 "I ran the tests and they passed.",
6086 &[AssistantToolReceipt {
6087 tool: "shell".into(),
6088 call_id: Some("s1".into()),
6089 ok: true,
6090 params: json!({ "command": "cargo test -q" }),
6091 via: None,
6092 }],
6093 );
6094 assert!(grounded.is_empty(), "{grounded:?}");
6095
6096 let failed = ungrounded_summary_claims(
6097 "I ran the tests and they passed.",
6098 &[AssistantToolReceipt {
6099 tool: "shell".into(),
6100 call_id: Some("s1".into()),
6101 ok: false,
6102 params: json!({ "command": "cargo test -q" }),
6103 via: None,
6104 }],
6105 );
6106 assert_eq!(failed, vec!["tests were run/passed"]);
6107 }
6108
6109 #[test]
6110 fn summary_claim_grounding_catches_verification_and_check_claims() {
6111 assert_eq!(
6112 ungrounded_summary_claims("Verified with cargo test.", &[]),
6113 vec!["tests were run/passed"]
6114 );
6115 assert_eq!(
6116 ungrounded_summary_claims("cargo check passed.", &[]),
6117 vec!["build succeeded", "checks were run/passed"]
6118 );
6119 assert_eq!(
6120 ungrounded_summary_claims("All checks are green.", &[]),
6121 vec!["checks were run/passed"]
6122 );
6123
6124 let cargo_check = [AssistantToolReceipt {
6125 tool: "shell".into(),
6126 call_id: Some("s1".into()),
6127 ok: true,
6128 params: json!({ "command": "cargo check -p car-server-core" }),
6129 via: None,
6130 }];
6131 assert!(
6132 ungrounded_summary_claims("cargo check passed.", &cargo_check).is_empty(),
6133 "cargo check receipt should ground both build and check claims"
6134 );
6135
6136 let diff_check = [AssistantToolReceipt {
6137 tool: "shell".into(),
6138 call_id: Some("s2".into()),
6139 ok: true,
6140 params: json!({ "command": "git diff --check" }),
6141 via: None,
6142 }];
6143 assert!(
6144 ungrounded_summary_claims("All checks are green.", &diff_check).is_empty(),
6145 "diff-check receipt should ground generic check claims"
6146 );
6147
6148 let tests = [AssistantToolReceipt {
6149 tool: "shell".into(),
6150 call_id: Some("s3".into()),
6151 ok: true,
6152 params: json!({ "command": "npm run test -- --watch=false" }),
6153 via: None,
6154 }];
6155 assert!(
6156 ungrounded_summary_claims("Verified with npm run test.", &tests).is_empty(),
6157 "npm run test receipt should ground verification test claims"
6158 );
6159
6160 assert_eq!(
6161 ungrounded_summary_claims("ctest passed.", &[]),
6162 vec!["tests were run/passed"]
6163 );
6164
6165 let ctest = [AssistantToolReceipt {
6166 tool: "shell".into(),
6167 call_id: Some("s4".into()),
6168 ok: true,
6169 params: json!({ "command": "ctest --test-dir build --output-on-failure" }),
6170 via: None,
6171 }];
6172 assert!(
6173 ungrounded_summary_claims("ctest passed.", &ctest).is_empty(),
6174 "ctest receipt should ground CMake test claims"
6175 );
6176
6177 let cmake_build = [AssistantToolReceipt {
6178 tool: "shell".into(),
6179 call_id: Some("s5".into()),
6180 ok: true,
6181 params: json!({ "command": "cmake -S . -B build && cmake --build build" }),
6182 via: None,
6183 }];
6184 assert!(
6185 ungrounded_summary_claims("CMake build succeeded.", &cmake_build).is_empty(),
6186 "cmake --build receipt should ground CMake build claims"
6187 );
6188
6189 let pnpm_check = [AssistantToolReceipt {
6190 tool: "shell".into(),
6191 call_id: Some("s6".into()),
6192 ok: true,
6193 params: json!({ "command": "pnpm check" }),
6194 via: None,
6195 }];
6196 assert!(
6197 ungrounded_summary_claims("Checks passed.", &pnpm_check).is_empty(),
6198 "package check receipts should ground generic check claims"
6199 );
6200 }
6201
6202 #[test]
6203 fn production_investigation_claims_require_matching_live_receipts() {
6204 let summary = "Repository is clean and HEAD matches origin. Application Insights telemetry showed no recurrence. The production portal page was inspected.";
6205 assert_eq!(
6206 ungrounded_summary_claims(summary, &[]),
6207 vec![
6208 "repository cleanliness was verified",
6209 "repository revision/remote relationship was verified",
6210 "live Application Insights evidence was observed",
6211 "production browser state was observed",
6212 ]
6213 );
6214
6215 let receipts = vec![
6216 AssistantToolReceipt {
6217 tool: "shell".into(),
6218 call_id: Some("git".into()),
6219 ok: true,
6220 params: json!({"command": "git status && git rev-parse HEAD && git rev-parse origin/main"}),
6221 via: None,
6222 },
6223 AssistantToolReceipt {
6224 tool: "shell".into(),
6225 call_id: Some("ai".into()),
6226 ok: true,
6227 params: json!({"command": "az monitor app-insights query --analytics-query 'exceptions | summarize count()'"}),
6228 via: None,
6229 },
6230 AssistantToolReceipt {
6231 tool: "browse_observe".into(),
6232 call_id: Some("browser".into()),
6233 ok: true,
6234 params: json!({}),
6235 via: None,
6236 },
6237 ];
6238 assert!(ungrounded_summary_claims(summary, &receipts).is_empty());
6239
6240 let cautious = "Source tests assert Information-level logging without an exception object.\nApplication Insights query: not obtained.\nProduction browser state: not obtained.\nCannot determine whether N744JS is outside the subscription or whether deployment 23708 fixed the issue.";
6241 assert!(
6242 ungrounded_summary_claims(cautious, &[]).is_empty(),
6243 "explicitly source-scoped and negated claims must not be rejected"
6244 );
6245 }
6246
6247 #[test]
6248 fn summary_file_claim_grounding_requires_matching_named_path() {
6249 let other_edit = [AssistantToolReceipt {
6250 tool: "edit_file".into(),
6251 call_id: Some("e1".into()),
6252 ok: true,
6253 params: json!({ "path": "src/other.rs" }),
6254 via: None,
6255 }];
6256 assert_eq!(
6257 ungrounded_summary_claims("Updated file src/lib.rs.", &other_edit),
6258 vec!["files were created/updated"]
6259 );
6260
6261 let matching_edit = [AssistantToolReceipt {
6262 tool: "edit_file".into(),
6263 call_id: Some("e2".into()),
6264 ok: true,
6265 params: json!({ "path": "./src/lib.rs" }),
6266 via: None,
6267 }];
6268 assert!(
6269 ungrounded_summary_claims("Updated file src/lib.rs.", &matching_edit).is_empty(),
6270 "matching edit_file path should ground the specific update claim"
6271 );
6272
6273 let shell_touch = [AssistantToolReceipt {
6274 tool: "shell".into(),
6275 call_id: Some("s1".into()),
6276 ok: true,
6277 params: json!({ "command": "touch src/lib.rs" }),
6278 via: None,
6279 }];
6280 assert!(
6281 ungrounded_summary_claims("Created file src/lib.rs.", &shell_touch).is_empty(),
6282 "matching shell command path should ground the specific creation claim"
6283 );
6284 }
6285
6286 #[test]
6287 fn summary_read_claim_grounding_requires_matching_named_path() {
6288 let other_read = [AssistantToolReceipt {
6289 tool: "read_file".into(),
6290 call_id: Some("r1".into()),
6291 ok: true,
6292 params: json!({ "path": "src/other.rs" }),
6293 via: None,
6294 }];
6295 assert_eq!(
6296 ungrounded_summary_claims("Inspected file src/lib.rs.", &other_read),
6297 vec!["files were read/inspected"]
6298 );
6299
6300 let matching_read = [AssistantToolReceipt {
6301 tool: "read_file".into(),
6302 call_id: Some("r2".into()),
6303 ok: true,
6304 params: json!({ "path": "src/lib.rs" }),
6305 via: None,
6306 }];
6307 assert!(
6308 ungrounded_summary_claims("Inspected file src/lib.rs.", &matching_read).is_empty(),
6309 "matching read_file path should ground the specific inspection claim"
6310 );
6311
6312 let generic_update = [AssistantToolReceipt {
6313 tool: "edit_file".into(),
6314 call_id: Some("e1".into()),
6315 ok: true,
6316 params: json!({ "path": "src/lib.rs" }),
6317 via: None,
6318 }];
6319 assert!(
6320 ungrounded_summary_claims("Updated files.", &generic_update).is_empty(),
6321 "generic file claims should keep the existing tool-class grounding"
6322 );
6323 }
6324
6325 #[tokio::test]
6331 async fn goal_loop_converges_when_the_command_check_passes() {
6332 use car_verify::goal::{GoalCondition, GoalGovernor, GoalSpec, GoalStatus};
6333
6334 let dir = tempfile::tempdir().unwrap();
6335 let rt = runtime_for(dir.path()).await;
6336
6337 let create = crate::coder::test_cmds::touch("donefile");
6340 let script = Script {
6341 turns: vec![
6342 turn("Let me start.", json!([])),
6343 turn(
6344 "creating it",
6345 json!([{ "id": "s1", "name": "shell", "arguments": { "command": create } }]),
6346 ),
6347 turn("Done — created donefile.", json!([])),
6348 ],
6349 cursor: AtomicUsize::new(0),
6350 };
6351
6352 let spec = GoalSpec {
6353 goal: "create a file named donefile".into(),
6354 condition: GoalCondition::Command {
6355 id: "donefile".into(),
6356 expect_exit: 0,
6357 },
6358 governor: GoalGovernor {
6359 max_turns: Some(5),
6360 ..Default::default()
6361 },
6362 };
6363
6364 let mut messages = vec![Message::System {
6365 content: "sys".into(),
6366 }];
6367 let never = std::sync::atomic::AtomicBool::new(false);
6368 let donefile = dir.path().join("donefile");
6369 let mut events = Vec::new();
6370
6371 let result = run_assistant_goal_loop(
6372 &script,
6373 &rt,
6374 &cfg(),
6375 &mut messages,
6376 &never,
6377 None,
6378 &spec,
6379 |_outcome| {
6380 let exists = donefile.exists();
6382 async move {
6383 let mut g = car_engine::GoalGather::default();
6384 g.command_exits
6385 .insert("donefile".into(), if exists { 0 } else { 1 });
6386 g
6387 }
6388 },
6389 |e| events.push(e),
6390 )
6391 .await;
6392
6393 assert_eq!(
6394 result.run.status,
6395 GoalStatus::Achieved,
6396 "{:?}",
6397 result.run.last_reason
6398 );
6399 assert_eq!(
6400 result.run.iterations, 2,
6401 "should converge on the 2nd iteration"
6402 );
6403 assert!(
6404 result.run.grounded,
6405 "a Command-check completion is grounded"
6406 );
6407 assert!(donefile.exists(), "the real file must have been created");
6408 assert_eq!(
6409 result.outcome.models_served.len(),
6410 3,
6411 "the terminal goal-run receipt must retain model calls from every iteration"
6412 );
6413 assert!(result
6414 .outcome
6415 .models_served
6416 .iter()
6417 .all(|attribution| attribution.model_id == "scripted"));
6418 let checks: Vec<_> = events
6419 .iter()
6420 .filter_map(|e| match e {
6421 AssistantEvent::GoalEvaluated {
6422 iteration,
6423 met,
6424 grounded,
6425 reason,
6426 } => Some((*iteration, *met, *grounded, reason.as_str())),
6427 _ => None,
6428 })
6429 .collect();
6430 assert_eq!(checks.len(), 2, "one verifier event per goal iteration");
6431 assert_eq!(checks[0].0, 1);
6432 assert!(
6433 !checks[0].1,
6434 "first iteration should not meet the command condition"
6435 );
6436 assert_eq!(checks[1].0, 2);
6437 assert!(
6438 checks[1].1,
6439 "second iteration should meet the command condition"
6440 );
6441 assert!(checks[1].2, "command-backed completion is grounded");
6442
6443 let log = rt.log.lock().await;
6444 let goal_events: Vec<_> = log
6445 .events()
6446 .iter()
6447 .filter(|e| e.kind == car_eventlog::EventKind::GoalEvaluated)
6448 .collect();
6449 assert_eq!(
6450 goal_events.len(),
6451 2,
6452 "event log should audit each verifier pass"
6453 );
6454 assert_eq!(goal_events[0].data.get("iteration"), Some(&json!(1)));
6455 assert_eq!(goal_events[0].data.get("met"), Some(&json!(false)));
6456 assert_eq!(
6457 goal_events[1].data.get("goal"),
6458 Some(&json!("create a file named donefile"))
6459 );
6460 assert_eq!(
6461 goal_events[1].data.get("condition"),
6462 Some(&json!({"kind": "command", "id": "donefile", "expect_exit": 0}))
6463 );
6464 assert_eq!(goal_events[1].data.get("iteration"), Some(&json!(2)));
6465 assert_eq!(goal_events[1].data.get("met"), Some(&json!(true)));
6466 assert_eq!(goal_events[1].data.get("grounded"), Some(&json!(true)));
6467 }
6468
6469 #[tokio::test]
6475 async fn deterministic_pass_not_reopened_by_ungrounded_prose() {
6476 use car_verify::goal::{GoalCondition, GoalGovernor, GoalSpec, GoalStatus};
6477
6478 let dir = tempfile::tempdir().unwrap();
6479 let rt = runtime_for(dir.path()).await;
6480 let script = Script {
6483 turns: vec![turn("I ran the tests and they passed.", json!([]))],
6484 cursor: AtomicUsize::new(0),
6485 };
6486 let spec = GoalSpec {
6487 goal: "make tests pass".into(),
6488 condition: GoalCondition::Command {
6489 id: "tests".into(),
6490 expect_exit: 0,
6491 },
6492 governor: GoalGovernor {
6493 max_turns: Some(3),
6494 ..Default::default()
6495 },
6496 };
6497 let mut messages = vec![Message::System {
6498 content: "sys".into(),
6499 }];
6500 let never = std::sync::atomic::AtomicBool::new(false);
6501 let mut events = Vec::new();
6502
6503 let result = run_assistant_goal_loop(
6504 &script,
6505 &rt,
6506 &cfg(),
6507 &mut messages,
6508 &never,
6509 None,
6510 &spec,
6511 |_outcome| async move {
6512 let mut g = car_engine::GoalGather::default();
6513 g.command_exits.insert("tests".into(), 0);
6514 g
6515 },
6516 |e| events.push(e),
6517 )
6518 .await;
6519
6520 assert_eq!(
6523 result.run.status,
6524 GoalStatus::Achieved,
6525 "{:?}",
6526 result.run.last_reason
6527 );
6528 assert_eq!(result.run.iterations, 1);
6529 assert!(result.run.grounded, "command-backed completion is grounded");
6530 assert_eq!(result.run.evidence.len(), 1);
6531 assert!(result.run.evidence[0].met && result.run.evidence[0].grounded);
6532 assert!(
6534 result.outcome.summary.contains("[claim check]")
6535 && result.outcome.summary.contains("tests were run/passed"),
6536 "summary should carry the claim-check note: {}",
6537 result.outcome.summary
6538 );
6539 assert!(
6542 !serde_json::to_string(&messages)
6543 .unwrap_or_default()
6544 .contains("[claim check]"),
6545 "the claim-check note must not leak into the thread messages"
6546 );
6547 let streamed: Vec<_> = events
6549 .iter()
6550 .filter_map(|e| match e {
6551 AssistantEvent::GoalEvaluated { grounded, .. } => Some(*grounded),
6552 _ => None,
6553 })
6554 .collect();
6555 assert_eq!(streamed, vec![true], "streamed verdict stays grounded=true");
6556 let log = rt.log.lock().await;
6559 let goal_events: Vec<_> = log
6560 .events()
6561 .iter()
6562 .filter(|e| e.kind == car_eventlog::EventKind::GoalEvaluated)
6563 .collect();
6564 assert_eq!(goal_events.len(), 1);
6565 assert_eq!(goal_events[0].data.get("met"), Some(&json!(true)));
6566 assert_eq!(goal_events[0].data.get("grounded"), Some(&json!(true)));
6567 assert!(
6568 !goal_events[0]
6569 .data
6570 .get("reason")
6571 .and_then(|r| r.as_str())
6572 .unwrap_or("")
6573 .contains("ungrounded assistant summary claim"),
6574 "durable reason must not record the prose mismatch as a failure"
6575 );
6576 }
6577
6578 #[tokio::test]
6584 async fn ungrounded_claim_without_deterministic_pass_still_fails_closed() {
6585 use car_verify::goal::{GoalCondition, GoalGovernor, GoalHalt, GoalSpec, GoalStatus};
6586
6587 let dir = tempfile::tempdir().unwrap();
6588 let rt = runtime_for(dir.path()).await;
6589 let script = Script {
6590 turns: vec![turn("I ran the tests and they passed.", json!([]))],
6591 cursor: AtomicUsize::new(0),
6592 };
6593 let spec = GoalSpec {
6596 goal: "make tests pass".into(),
6597 condition: GoalCondition::ModelJudge { id: "judge".into() },
6598 governor: GoalGovernor {
6599 max_turns: Some(1),
6600 ..Default::default()
6601 },
6602 };
6603 let mut messages = vec![Message::System {
6604 content: "sys".into(),
6605 }];
6606 let never = std::sync::atomic::AtomicBool::new(false);
6607
6608 let result = run_assistant_goal_loop(
6609 &script,
6610 &rt,
6611 &cfg(),
6612 &mut messages,
6613 &never,
6614 None,
6615 &spec,
6616 |_outcome| async move {
6617 let mut g = car_engine::GoalGather::default();
6618 g.model_verdicts.insert("judge".into(), true);
6619 g
6620 },
6621 |_| {},
6622 )
6623 .await;
6624
6625 assert_eq!(
6626 result.run.status,
6627 GoalStatus::Halted {
6628 halt: GoalHalt::TurnBudget
6629 }
6630 );
6631 assert_eq!(result.run.evidence.len(), 1);
6632 assert!(result.run.evidence[0].met);
6633 assert!(
6634 !result.run.evidence[0].grounded,
6635 "a model-judge completion with an ungrounded claim stays ungrounded"
6636 );
6637 assert!(result
6638 .run
6639 .last_reason
6640 .contains("ungrounded assistant summary claim"));
6641 assert!(!result.outcome.summary.contains("[claim check]"));
6644 }
6645
6646 #[tokio::test]
6650 async fn grounded_prose_on_deterministic_pass_unannotated() {
6651 use car_verify::goal::{GoalCondition, GoalGovernor, GoalSpec, GoalStatus};
6652
6653 let dir = tempfile::tempdir().unwrap();
6654 let rt = runtime_for(dir.path()).await;
6655 let create = crate::coder::test_cmds::touch("donefile");
6660 let script = Script {
6661 turns: vec![
6662 turn(
6663 "creating it",
6664 json!([{ "id": "s1", "name": "shell", "arguments": { "command": create } }]),
6665 ),
6666 turn("Done — created donefile.", json!([])),
6667 ],
6668 cursor: AtomicUsize::new(0),
6669 };
6670 let spec = GoalSpec {
6671 goal: "create a file named donefile".into(),
6672 condition: GoalCondition::Command {
6673 id: "donefile".into(),
6674 expect_exit: 0,
6675 },
6676 governor: GoalGovernor {
6677 max_turns: Some(3),
6678 ..Default::default()
6679 },
6680 };
6681 let mut messages = vec![Message::System {
6682 content: "sys".into(),
6683 }];
6684 let never = std::sync::atomic::AtomicBool::new(false);
6685 let donefile = dir.path().join("donefile");
6686
6687 let result = run_assistant_goal_loop(
6688 &script,
6689 &rt,
6690 &cfg(),
6691 &mut messages,
6692 &never,
6693 None,
6694 &spec,
6695 |_outcome| {
6696 let exists = donefile.exists();
6697 async move {
6698 let mut g = car_engine::GoalGather::default();
6699 g.command_exits
6700 .insert("donefile".into(), if exists { 0 } else { 1 });
6701 g
6702 }
6703 },
6704 |_| {},
6705 )
6706 .await;
6707
6708 assert_eq!(
6709 result.run.status,
6710 GoalStatus::Achieved,
6711 "{:?}",
6712 result.run.last_reason
6713 );
6714 assert_eq!(result.run.iterations, 1);
6715 assert!(result.run.grounded);
6716 assert_eq!(result.outcome.summary, "Done — created donefile.");
6718 assert!(!result.outcome.summary.contains("[claim check]"));
6719 }
6720
6721 #[tokio::test]
6724 async fn goal_loop_halts_on_turn_budget() {
6725 use car_verify::goal::{GoalCondition, GoalGovernor, GoalHalt, GoalSpec, GoalStatus};
6726
6727 let dir = tempfile::tempdir().unwrap();
6728 let rt = runtime_for(dir.path()).await;
6729
6730 struct Idle;
6732 #[async_trait]
6733 impl TurnGenerator for Idle {
6734 async fn generate(&self, _req: GenerateRequest) -> Result<InferenceResult, String> {
6735 Ok(turn("thinking...", json!([])))
6736 }
6737 }
6738
6739 let spec = GoalSpec {
6740 goal: "impossible".into(),
6741 condition: GoalCondition::Command {
6742 id: "never".into(),
6743 expect_exit: 0,
6744 },
6745 governor: GoalGovernor {
6746 max_turns: Some(3),
6747 ..Default::default()
6748 },
6749 };
6750 let mut messages = vec![Message::System {
6751 content: "sys".into(),
6752 }];
6753 let never = std::sync::atomic::AtomicBool::new(false);
6754
6755 let result = run_assistant_goal_loop(
6756 &Idle,
6757 &rt,
6758 &cfg(),
6759 &mut messages,
6760 &never,
6761 None,
6762 &spec,
6763 |_o| async {
6764 let mut g = car_engine::GoalGather::default();
6765 g.command_exits.insert("never".into(), 1);
6766 g
6767 },
6768 |_e| {},
6769 )
6770 .await;
6771
6772 assert_eq!(
6773 result.run.status,
6774 GoalStatus::Halted {
6775 halt: GoalHalt::TurnBudget
6776 }
6777 );
6778 assert_eq!(result.run.iterations, 3);
6779 }
6780
6781 #[tokio::test(start_paused = true)]
6790 async fn goal_loop_fails_open_when_the_check_never_resolves() {
6791 use car_verify::goal::{GoalCondition, GoalGovernor, GoalHalt, GoalSpec, GoalStatus};
6792
6793 let dir = tempfile::tempdir().unwrap();
6794 let rt = runtime_for(dir.path()).await;
6795
6796 struct Answers;
6800 #[async_trait]
6801 impl TurnGenerator for Answers {
6802 async fn generate(&self, _req: GenerateRequest) -> Result<InferenceResult, String> {
6803 Ok(turn("Here is your answer.", json!([])))
6804 }
6805 }
6806
6807 let spec = GoalSpec {
6808 goal: "answer the question".into(),
6809 condition: GoalCondition::Command {
6810 id: "verify".into(),
6811 expect_exit: 0,
6812 },
6813 governor: GoalGovernor {
6814 max_turns: Some(8),
6819 ..Default::default()
6820 },
6821 };
6822 let mut messages = vec![Message::System {
6823 content: "sys".into(),
6824 }];
6825 let never = std::sync::atomic::AtomicBool::new(false);
6826 let mut events = Vec::new();
6827
6828 let result = run_assistant_goal_loop(
6829 &Answers,
6830 &rt,
6831 &cfg(),
6832 &mut messages,
6833 &never,
6834 None,
6835 &spec,
6836 |_outcome| std::future::pending::<car_engine::GoalGather>(),
6837 |e| events.push(e),
6838 )
6839 .await;
6840
6841 assert_eq!(
6842 result.run.status,
6843 GoalStatus::Halted {
6844 halt: GoalHalt::EvaluationTimeout
6845 },
6846 "{:?}",
6847 result.run.last_reason
6848 );
6849 assert_eq!(
6850 result.run.iterations, 1,
6851 "must halt on the FIRST stuck evaluation, not burn the rest of the turn budget \
6852 re-running the model against a check that can never be graded"
6853 );
6854 assert!(
6855 result.run.last_reason.contains("did not complete within"),
6856 "{}",
6857 result.run.last_reason
6858 );
6859 assert_eq!(
6860 result.outcome.summary, "Here is your answer.",
6861 "the primary reply must survive an evaluation pass that never resolves"
6862 );
6863 assert!(
6864 events.iter().any(|e| matches!(
6865 e,
6866 AssistantEvent::GoalEvaluated {
6867 met: false,
6868 grounded: false,
6869 ..
6870 }
6871 )),
6872 "the unevaluated outcome must still be streamed as a goal_evaluated event — \
6873 grounded: false, not true: there is no verdict to be grounded, the check \
6874 never ran (car#1113 review)"
6875 );
6876 assert!(
6877 !result.run.grounded,
6878 "GoalRun.grounded must not claim a deterministic verdict exists when the \
6879 check never got the chance to run"
6880 );
6881 }
6882
6883 struct FixedGate(bool);
6884 #[async_trait]
6885 impl ApprovalGate for FixedGate {
6886 async fn request(&self, _tool: &str, _params: &Value) -> ApprovalDecision {
6887 if self.0 {
6888 ApprovalDecision::Approved
6889 } else {
6890 ApprovalDecision::Denied("user declined".into())
6891 }
6892 }
6893 }
6894
6895 struct CapturingGen {
6896 images_seen: std::sync::Arc<std::sync::Mutex<Option<usize>>>,
6897 }
6898 #[async_trait]
6899 impl TurnGenerator for CapturingGen {
6900 async fn generate(&self, req: GenerateRequest) -> Result<InferenceResult, String> {
6901 *self.images_seen.lock().unwrap() = req.images.as_ref().map(|v| v.len());
6902 Ok(turn("done", json!([]))) }
6904 }
6905
6906 #[tokio::test]
6907 async fn images_are_attached_to_the_first_request() {
6908 let dir = tempfile::tempdir().unwrap();
6909 let rt = runtime_for(dir.path()).await;
6910 let seen = std::sync::Arc::new(std::sync::Mutex::new(None));
6911 let generator = CapturingGen {
6912 images_seen: seen.clone(),
6913 };
6914 let img = ContentBlock::ImageUrl {
6915 url: "https://example.com/x.png".into(),
6916 detail: "auto".into(),
6917 };
6918 let mut messages = vec![
6919 Message::System {
6920 content: "s".into(),
6921 },
6922 Message::User {
6923 content: "describe".into(),
6924 },
6925 ];
6926 let never = std::sync::atomic::AtomicBool::new(false);
6927 let imgs = [img];
6928 run_assistant_loop_cancellable(
6929 &generator,
6930 &rt,
6931 &cfg(),
6932 &mut messages,
6933 &never,
6934 None,
6935 Some(&imgs),
6936 |_| {},
6937 )
6938 .await;
6939 assert_eq!(
6940 *seen.lock().unwrap(),
6941 Some(1),
6942 "the image should reach the first request"
6943 );
6944 }
6945
6946 #[tokio::test]
6947 async fn gated_tool_is_denied_without_a_gate() {
6948 let dir = tempfile::tempdir().unwrap();
6949 let rt = runtime_for(dir.path()).await;
6950 let script = Script {
6951 turns: vec![
6952 turn(
6953 "",
6954 json!([{ "id": "w1", "name": "write_file", "arguments": { "path": "x.txt", "content": "no" } }]),
6955 ),
6956 turn("could not write", json!([])),
6957 ],
6958 cursor: AtomicUsize::new(0),
6959 };
6960 let mut cfg = cfg();
6961 cfg.gated_tools = vec!["write_file".into()];
6962 let mut messages = vec![
6963 Message::System {
6964 content: "s".into(),
6965 },
6966 Message::User {
6967 content: "write x".into(),
6968 },
6969 ];
6970 let never = std::sync::atomic::AtomicBool::new(false);
6971 let outcome = run_assistant_loop_cancellable(
6972 &script,
6973 &rt,
6974 &cfg,
6975 &mut messages,
6976 &never,
6977 None,
6978 None,
6979 |_| {},
6980 )
6981 .await;
6982 assert_eq!(outcome.status, "success");
6983 assert!(
6984 !dir.path().join("x.txt").exists(),
6985 "gated write must not run"
6986 );
6987 assert!(!outcome.tools_called.contains(&"write_file".to_string()));
6988 }
6989
6990 #[tokio::test]
6991 async fn gated_tool_runs_when_approved() {
6992 let dir = tempfile::tempdir().unwrap();
6993 let rt = runtime_for(dir.path()).await;
6994 let script = Script {
6995 turns: vec![
6996 turn(
6997 "",
6998 json!([{ "id": "w1", "name": "write_file", "arguments": { "path": "ok.txt", "content": "yes" } }]),
6999 ),
7000 turn("wrote it", json!([])),
7001 ],
7002 cursor: AtomicUsize::new(0),
7003 };
7004 let mut cfg = cfg();
7005 cfg.gated_tools = vec!["write_file".into()];
7006 let gate = FixedGate(true);
7007 let mut messages = vec![
7008 Message::System {
7009 content: "s".into(),
7010 },
7011 Message::User {
7012 content: "write ok".into(),
7013 },
7014 ];
7015 let never = std::sync::atomic::AtomicBool::new(false);
7016 let outcome = run_assistant_loop_cancellable(
7017 &script,
7018 &rt,
7019 &cfg,
7020 &mut messages,
7021 &never,
7022 Some(&gate),
7023 None,
7024 |_| {},
7025 )
7026 .await;
7027 assert_eq!(outcome.status, "success");
7028 assert_eq!(
7029 std::fs::read_to_string(dir.path().join("ok.txt")).unwrap(),
7030 "yes"
7031 );
7032 }
7033
7034 #[tokio::test]
7035 async fn loop_writes_a_file_through_the_runtime() {
7036 let dir = tempfile::tempdir().unwrap();
7037 let rt = runtime_for(dir.path()).await;
7038 let script = Script {
7039 turns: vec![
7040 turn(
7041 "",
7042 json!([{ "id": "w1", "name": "write_file", "arguments": { "path": "hi.txt", "content": "hello" } }]),
7043 ),
7044 turn("Wrote hi.txt.", json!([])),
7045 ],
7046 cursor: AtomicUsize::new(0),
7047 };
7048 let mut messages = vec![
7049 Message::System {
7050 content: "sys".into(),
7051 },
7052 Message::User {
7053 content: "write hi.txt".into(),
7054 },
7055 ];
7056 let outcome = run_assistant_loop(&script, &rt, &cfg(), &mut messages, |_| {}).await;
7057 assert_eq!(outcome.status, "success");
7058 assert_eq!(
7059 std::fs::read_to_string(dir.path().join("hi.txt")).unwrap(),
7060 "hello"
7061 );
7062 }
7063
7064 fn json_object_cfg() -> AssistantConfig {
7067 AssistantConfig {
7068 response_format: Some(car_inference::ResponseFormat::JsonObject),
7069 ..cfg()
7070 }
7071 }
7072
7073 fn repair_notices(events: &[AssistantEvent]) -> (usize, usize) {
7074 let mut fired = 0;
7075 let mut still_invalid = 0;
7076 for e in events {
7077 if let AssistantEvent::Text(t) = e {
7078 if t == FORMAT_REPAIR_NOTICE {
7079 fired += 1;
7080 }
7081 if t == FORMAT_REPAIR_STILL_INVALID {
7082 still_invalid += 1;
7083 }
7084 }
7085 }
7086 (fired, still_invalid)
7087 }
7088
7089 #[tokio::test]
7093 async fn response_format_is_never_on_tool_turns_only_on_the_repair_turn() {
7094 let dir = tempfile::tempdir().unwrap();
7095 let rt = runtime_for(dir.path()).await;
7096 let tool_turn = || {
7097 turn(
7098 "computing",
7099 json!([{ "id": "c", "name": "calculate", "arguments": { "expression": "1+1" } }]),
7100 )
7101 };
7102
7103 let seen = Arc::new(StdMutex::new(Vec::new()));
7105 let script = CapturingScript {
7106 turns: vec![
7107 tool_turn(),
7108 turn("The sum is 2.", json!([])),
7109 turn(r#"{"sum": 2}"#, json!([])),
7110 ],
7111 cursor: AtomicUsize::new(0),
7112 seen: Arc::clone(&seen),
7113 };
7114 let mut messages = vec![sys("sys"), usr("add 1 and 1, answer as JSON")];
7115 let outcome =
7116 run_assistant_loop(&script, &rt, &json_object_cfg(), &mut messages, |_| {}).await;
7117 assert_eq!(outcome.status, "success");
7118 assert_eq!(outcome.summary, r#"{"sum": 2}"#);
7119 {
7120 let reqs = seen.lock().unwrap();
7121 assert_eq!(reqs.len(), 3);
7122 for (i, r) in reqs[..2].iter().enumerate() {
7123 assert!(r.tools.is_some(), "request {i} offers tools");
7124 assert!(
7125 r.response_format.is_none(),
7126 "request {i} offers tools, so it must not be JSON-constrained"
7127 );
7128 }
7129 assert!(reqs[2].tools.is_none(), "the repair turn offers no tools");
7130 assert_eq!(
7131 reqs[2].response_format,
7132 Some(car_inference::ResponseFormat::JsonObject),
7133 "and is the one request that carries the format"
7134 );
7135 }
7136
7137 let seen = Arc::new(StdMutex::new(Vec::new()));
7139 let script = CapturingScript {
7140 turns: vec![turn(r#"{"sum": 2}"#, json!([]))],
7141 cursor: AtomicUsize::new(0),
7142 seen: Arc::clone(&seen),
7143 };
7144 let mut messages = vec![sys("sys"), usr("add 1 and 1, answer as JSON")];
7145 let no_tools_cfg = AssistantConfig {
7146 tools: Vec::new(),
7147 ..json_object_cfg()
7148 };
7149 let outcome = run_assistant_loop(&script, &rt, &no_tools_cfg, &mut messages, |_| {}).await;
7150 assert_eq!(outcome.status, "success");
7151 {
7152 let reqs = seen.lock().unwrap();
7153 assert_eq!(reqs.len(), 1, "a valid answer costs no extra call");
7154 assert!(reqs[0].tools.is_none());
7155 assert_eq!(
7156 reqs[0].response_format,
7157 Some(car_inference::ResponseFormat::JsonObject)
7158 );
7159 }
7160
7161 let seen = Arc::new(StdMutex::new(Vec::new()));
7163 let script = CapturingScript {
7164 turns: vec![tool_turn(), turn("two", json!([]))],
7165 cursor: AtomicUsize::new(0),
7166 seen: Arc::clone(&seen),
7167 };
7168 let mut messages = vec![sys("sys"), usr("add 1 and 1")];
7169 let outcome = run_assistant_loop(&script, &rt, &cfg(), &mut messages, |_| {}).await;
7170 assert_eq!(outcome.status, "success");
7171 let reqs = seen.lock().unwrap();
7172 assert_eq!(reqs.len(), 2);
7173 assert!(reqs.iter().all(|r| r.response_format.is_none()));
7174 }
7175
7176 #[tokio::test]
7181 async fn invalid_final_answer_triggers_exactly_one_toolless_repair() {
7182 let dir = tempfile::tempdir().unwrap();
7183 let rt = runtime_for(dir.path()).await;
7184 let seen = Arc::new(StdMutex::new(Vec::new()));
7185 let script = CapturingScript {
7186 turns: vec![
7187 turn("Sure! The answer is: sum = 2.", json!([])),
7188 turn(r#"{"sum": 2}"#, json!([])),
7189 ],
7190 cursor: AtomicUsize::new(0),
7191 seen: Arc::clone(&seen),
7192 };
7193 let mut messages = vec![sys("sys"), usr("add 1 and 1, answer as JSON")];
7194 let mut events = Vec::new();
7195 let repair_cfg = AssistantConfig {
7196 model: Some("newsroom-editor".into()),
7197 strict_model: true,
7198 ..json_object_cfg()
7199 };
7200 let outcome =
7201 run_assistant_loop(&script, &rt, &repair_cfg, &mut messages, |e| events.push(e)).await;
7202 assert_eq!(outcome.status, "success");
7203 assert_eq!(
7204 outcome.summary, r#"{"sum": 2}"#,
7205 "the repaired text is the answer"
7206 );
7207 assert_eq!(
7208 outcome.turns, 1,
7209 "a repair is a model call, not a loop turn"
7210 );
7211
7212 let reqs = seen.lock().unwrap();
7213 assert_eq!(reqs.len(), 2, "draft + exactly one repair");
7214 for (index, request) in reqs.iter().enumerate() {
7215 assert_eq!(
7216 request.model.as_deref(),
7217 Some("newsroom-editor"),
7218 "request {index} must retain the configured editor model"
7219 );
7220 assert!(
7221 request.params.strict_model,
7222 "request {index} must retain strict model selection"
7223 );
7224 assert_eq!(
7225 request.expected_row_digest, None,
7226 "assistant config exposes no immutable row precondition"
7227 );
7228 assert_eq!(
7229 request.expected_catalog_revision, None,
7230 "assistant config exposes no catalog revision precondition"
7231 );
7232 }
7233 let repair = &reqs[1];
7234 assert!(
7235 repair.tools.is_none(),
7236 "the repair turn advertises no tools"
7237 );
7238 assert_eq!(
7239 repair.response_format,
7240 Some(car_inference::ResponseFormat::JsonObject)
7241 );
7242 let history = repair.messages.as_ref().unwrap();
7243 assert!(
7244 matches!(history.last(), Some(Message::User { content }) if content == FORMAT_REPAIR_NUDGE),
7245 "the nudge is the last message the repair sees"
7246 );
7247 assert!(
7248 matches!(&history[history.len() - 2], Message::Assistant { content, .. } if content.contains("sum = 2")),
7249 "the draft is in the transcript so the model can see what it got wrong"
7250 );
7251
7252 let (fired, still_invalid) = repair_notices(&events);
7253 assert_eq!(fired, 1, "the repair must be visible in the event stream");
7254 assert_eq!(still_invalid, 0);
7255 assert!(
7256 matches!(events.last(), Some(AssistantEvent::Done { text }) if text == r#"{"sum": 2}"#)
7257 );
7258 assert!(
7260 matches!(messages.last(), Some(Message::Assistant { content, .. }) if content == r#"{"sum": 2}"#)
7261 );
7262 assert!(
7263 matches!(&messages[messages.len() - 2], Message::User { content } if content == FORMAT_REPAIR_NUDGE)
7264 );
7265 }
7266
7267 #[tokio::test]
7269 async fn valid_final_answer_triggers_no_repair() {
7270 let dir = tempfile::tempdir().unwrap();
7271 let rt = runtime_for(dir.path()).await;
7272 let seen = Arc::new(StdMutex::new(Vec::new()));
7273 let script = CapturingScript {
7274 turns: vec![turn("```json\n{\"sum\": 2}\n```", json!([]))],
7277 cursor: AtomicUsize::new(0),
7278 seen: Arc::clone(&seen),
7279 };
7280 let mut messages = vec![sys("sys"), usr("add 1 and 1, answer as JSON")];
7281 let mut events = Vec::new();
7282 let outcome = run_assistant_loop(&script, &rt, &json_object_cfg(), &mut messages, |e| {
7283 events.push(e)
7284 })
7285 .await;
7286 assert_eq!(outcome.status, "success");
7287 assert_eq!(seen.lock().unwrap().len(), 1);
7288 assert_eq!(repair_notices(&events), (0, 0));
7289 }
7290
7291 #[tokio::test]
7294 async fn a_repair_that_still_misses_is_reported_not_retried() {
7295 let dir = tempfile::tempdir().unwrap();
7296 let rt = runtime_for(dir.path()).await;
7297 let seen = Arc::new(StdMutex::new(Vec::new()));
7298 let script = CapturingScript {
7299 turns: vec![
7300 turn("not json", json!([])),
7301 turn("still not json", json!([])),
7302 turn(r#"{"never": "reached"}"#, json!([])),
7303 ],
7304 cursor: AtomicUsize::new(0),
7305 seen: Arc::clone(&seen),
7306 };
7307 let mut messages = vec![sys("sys"), usr("answer as JSON")];
7308 let mut events = Vec::new();
7309 let outcome = run_assistant_loop(&script, &rt, &json_object_cfg(), &mut messages, |e| {
7310 events.push(e)
7311 })
7312 .await;
7313 assert_eq!(outcome.status, "success");
7314 assert_eq!(outcome.summary, "still not json");
7315 assert_eq!(seen.lock().unwrap().len(), 2, "one repair, never a second");
7316 assert_eq!(repair_notices(&events), (1, 1));
7317 }
7318
7319 #[test]
7323 fn final_text_format_check_semantics() {
7324 use car_inference::ResponseFormat::{JsonObject, JsonSchema};
7325 let schema = JsonSchema {
7326 schema: json!({"type": "array"}),
7327 strict: false,
7328 name: None,
7329 };
7330 assert!(final_text_matches_format(r#"{"a": 1}"#, &JsonObject, None));
7331 assert!(final_text_matches_format(
7332 "```json\n{\"a\": 1}\n```",
7333 &JsonObject,
7334 None
7335 ));
7336 assert!(
7337 !final_text_matches_format("[1, 2]", &JsonObject, None),
7338 "an array is not an object"
7339 );
7340 assert!(!final_text_matches_format(
7341 "Here: {\"a\": 1}",
7342 &JsonObject,
7343 None
7344 ));
7345 assert!(
7346 final_text_matches_format("[1, 2]", &schema, None),
7347 "schema mode is parse-only"
7348 );
7349 assert!(!final_text_matches_format("nope", &schema, None));
7350 let requires_legs: ResponseFormatValidator = Arc::new(|v| v.get("legs").is_some());
7351 assert!(
7352 final_text_matches_format(r#"{"legs": []}"#, &schema, Some(&requires_legs)),
7353 "conforming JSON passes the caller's schema check"
7354 );
7355 assert!(
7356 !final_text_matches_format(r#"{"nope": 1}"#, &schema, Some(&requires_legs)),
7357 "valid JSON of the wrong shape must fail once a validator exists"
7358 );
7359 assert_eq!(extract_json_payload("```\n[1]\n```"), "[1]");
7360 assert_eq!(extract_json_payload(" [1] "), "[1]");
7361 assert_eq!(
7362 extract_json_payload("```json\n{}"),
7363 "```json\n{}",
7364 "an unclosed fence is left alone"
7365 );
7366 }
7367
7368 struct WindowedScript {
7373 inner: CapturingScript,
7374 window: usize,
7375 }
7376
7377 #[async_trait]
7378 impl TurnGenerator for WindowedScript {
7379 async fn generate(&self, req: GenerateRequest) -> Result<InferenceResult, String> {
7380 self.inner.generate(req).await
7381 }
7382 fn context_window(&self, _model: &str) -> usize {
7383 self.window
7384 }
7385 }
7386
7387 fn windowed(window: usize) -> (WindowedScript, Arc<StdMutex<Vec<GenerateRequest>>>) {
7388 let seen = Arc::new(StdMutex::new(Vec::new()));
7389 let script = WindowedScript {
7390 inner: CapturingScript {
7391 turns: vec![turn("done", json!([]))],
7392 cursor: AtomicUsize::new(0),
7393 seen: Arc::clone(&seen),
7394 },
7395 window,
7396 };
7397 (script, seen)
7398 }
7399
7400 fn long_history() -> Vec<Message> {
7403 let big = "x".repeat(20_000);
7404 let mut m = vec![sys("system"), usr("THE ORIGINAL TASK")];
7405 for i in 0..12 {
7406 m.push(asst_call(&format!("c{i}")));
7407 m.push(tool_res(&format!("c{i}"), &big));
7408 }
7409 m
7410 }
7411
7412 fn has_compaction_notice(messages: &[Message]) -> bool {
7413 messages.iter().any(|m| {
7414 matches!(m, Message::System { content } if content.starts_with(COMPACTION_NOTICE_PREFIX))
7415 })
7416 }
7417
7418 fn window_advisories(events: &[AssistantEvent]) -> usize {
7419 events
7420 .iter()
7421 .filter(|e| matches!(e, AssistantEvent::Text(t) if t.starts_with("[context window:")))
7422 .count()
7423 }
7424
7425 #[tokio::test]
7428 async fn context_window_override_below_the_registry_window_tightens_compaction() {
7429 let dir = tempfile::tempdir().unwrap();
7430 let rt = runtime_for(dir.path()).await;
7431 let (script, seen) = windowed(200_000);
7432 let mut messages = long_history();
7433 let cfg = AssistantConfig {
7434 context_window_override: Some(20_000),
7435 refuse_unadvertised_tools: false,
7436 response_format_validator: None,
7437 delegate_budget: None,
7438 ..cfg()
7439 };
7440 let mut events = Vec::new();
7441 let outcome =
7442 run_assistant_loop(&script, &rt, &cfg, &mut messages, |e| events.push(e)).await;
7443 assert_eq!(outcome.status, "success");
7444 assert!(
7445 has_compaction_notice(&messages),
7446 "the 20k override must compact"
7447 );
7448 let reqs = seen.lock().unwrap();
7450 assert!(has_compaction_notice(reqs[0].messages.as_ref().unwrap()));
7451 assert_eq!(window_advisories(&events), 0, "tightening is not clamped");
7452 }
7453
7454 #[tokio::test]
7456 async fn no_context_window_override_leaves_the_registry_window_in_charge() {
7457 let dir = tempfile::tempdir().unwrap();
7458 let rt = runtime_for(dir.path()).await;
7459 let (script, _seen) = windowed(200_000);
7460 let mut messages = long_history();
7461 let mut events = Vec::new();
7462 let outcome =
7463 run_assistant_loop(&script, &rt, &cfg(), &mut messages, |e| events.push(e)).await;
7464 assert_eq!(outcome.status, "success");
7465 assert!(!has_compaction_notice(&messages), "60k fits a 200k window");
7466 assert_eq!(window_advisories(&events), 0);
7467 }
7468
7469 #[tokio::test]
7474 async fn context_window_override_above_the_registry_window_is_clamped_and_announced() {
7475 let dir = tempfile::tempdir().unwrap();
7476 let rt = runtime_for(dir.path()).await;
7477 let (script, _seen) = windowed(200_000);
7478 let mut messages = long_history();
7479 let cfg = AssistantConfig {
7480 context_window_override: Some(400_000),
7481 refuse_unadvertised_tools: false,
7482 response_format_validator: None,
7483 delegate_budget: None,
7484 ..cfg()
7485 };
7486 let mut events = Vec::new();
7487 let outcome =
7488 run_assistant_loop(&script, &rt, &cfg, &mut messages, |e| events.push(e)).await;
7489 assert_eq!(outcome.status, "success");
7490 assert!(!has_compaction_notice(&messages));
7491 assert_eq!(window_advisories(&events), 1, "the clamp must be visible");
7492 }
7493
7494 #[test]
7495 fn resolve_context_window_clamps_only_upward_against_a_known_window() {
7496 assert_eq!(resolve_context_window(None, 200_000), (200_000, None));
7497 assert_eq!(resolve_context_window(None, 0), (0, None));
7498 assert_eq!(
7499 resolve_context_window(Some(20_000), 200_000),
7500 (20_000, None)
7501 );
7502 let (w, advisory) = resolve_context_window(Some(400_000), 200_000);
7503 assert_eq!(w, 200_000);
7504 assert!(advisory
7505 .unwrap()
7506 .contains("exceeds the model's known window"));
7507 assert_eq!(resolve_context_window(Some(400_000), 0), (400_000, None));
7510 }
7511
7512 #[test]
7513 fn the_assistant_loops_compaction_notice_text_is_unchanged() {
7514 assert_eq!(
7519 format_compaction_notice(3, 1234, CompactionRecovery::default()),
7520 "[history compacted: 3 earlier turns removed to fit the context window, \
7521 ~1234 tokens. They are gone from this transcript but the run's event log \
7522 still has them — call `events_query` (e.g. {\"kinds\": [\"action_failed\"], \
7523 \"limit\": 5}) to see what was already tried, rather than assuming you never \
7524 tried it.]"
7525 );
7526 assert_eq!(
7527 format_compaction_notice(3, 1234, CompactionRecovery::EventsQuery),
7528 format_compaction_notice(3, 1234, CompactionRecovery::default()),
7529 "EventsQuery is the default; no caller changes behavior by omitting it"
7530 );
7531
7532 let unrecoverable = format_compaction_notice(3, 1234, CompactionRecovery::Unrecoverable);
7534 assert!(unrecoverable.starts_with(COMPACTION_NOTICE_PREFIX));
7535 assert!(!unrecoverable.contains("events_query"));
7536 assert!(!unrecoverable.contains("event log"));
7537 assert_eq!(
7538 parse_compaction_notice(&Message::System {
7539 content: unrecoverable,
7540 }),
7541 Some((3, 1234)),
7542 "both arms must round-trip through parse_compaction_notice"
7543 );
7544 }
7545
7546 #[test]
7547 fn history_budget_is_the_same_number_the_inline_expression_produced() {
7548 for window in [0usize, 1, 3, 5, 4_096, 8_192, 131_072, 200_000, 1_048_576] {
7553 assert_eq!(
7554 history_budget(window),
7555 window / 4 * 3,
7556 "budget changed for window {window}"
7557 );
7558 }
7559 assert_eq!(history_budget(200_000), 150_000);
7560 assert_eq!(history_budget(0), 0);
7561 }
7562
7563 fn modest_history() -> Vec<Message> {
7568 let body = "x".repeat(2_000);
7569 let mut m = vec![sys("system"), usr("THE ORIGINAL TASK")];
7570 for i in 0..12 {
7571 m.push(asst_call(&format!("c{i}")));
7572 m.push(tool_res(&format!("c{i}"), &body));
7573 }
7574 m
7575 }
7576
7577 #[tokio::test]
7582 async fn reported_prompt_tokens_far_above_the_estimate_trigger_compaction_next_turn() {
7583 let dir = tempfile::tempdir().unwrap();
7584 let rt = runtime_for(dir.path()).await;
7585 let seen = Arc::new(StdMutex::new(Vec::new()));
7586 let script = WindowedScript {
7587 inner: CapturingScript {
7588 turns: vec![
7589 turn_with_usage(
7590 "computing",
7591 json!([{ "id": "k", "name": "calculate", "arguments": { "expression": "1+1" } }]),
7592 190_000,
7593 10,
7594 ),
7595 turn("done", json!([])),
7596 ],
7597 cursor: AtomicUsize::new(0),
7598 seen: Arc::clone(&seen),
7599 },
7600 window: 200_000,
7601 };
7602 let mut messages = modest_history();
7603 let estimate = messages.iter().map(approx_message_tokens).sum::<usize>();
7604 assert!(
7605 estimate < 20_000,
7606 "fixture estimate must sit far under the 150k budget: {estimate}"
7607 );
7608
7609 let outcome = run_assistant_loop(&script, &rt, &cfg(), &mut messages, |_| {}).await;
7610 assert_eq!(outcome.status, "success");
7611
7612 let reqs = seen.lock().unwrap();
7613 assert_eq!(reqs.len(), 2);
7614 assert!(
7615 !has_compaction_notice(reqs[0].messages.as_ref().unwrap()),
7616 "turn 1 has no report yet and the estimate fits"
7617 );
7618 assert!(
7619 has_compaction_notice(reqs[1].messages.as_ref().unwrap()),
7620 "turn 2 must compact on the 190k the provider reported for turn 1"
7621 );
7622 assert!(has_compaction_notice(&messages));
7623 }
7624
7625 #[tokio::test]
7627 async fn no_usage_report_falls_back_to_the_estimate() {
7628 let dir = tempfile::tempdir().unwrap();
7629 let rt = runtime_for(dir.path()).await;
7630 let seen = Arc::new(StdMutex::new(Vec::new()));
7631 let script = WindowedScript {
7632 inner: CapturingScript {
7633 turns: vec![
7634 turn(
7635 "computing",
7636 json!([{ "id": "k", "name": "calculate", "arguments": { "expression": "1+1" } }]),
7637 ),
7638 turn("done", json!([])),
7639 ],
7640 cursor: AtomicUsize::new(0),
7641 seen: Arc::clone(&seen),
7642 },
7643 window: 200_000,
7644 };
7645 let mut messages = modest_history();
7646 let outcome = run_assistant_loop(&script, &rt, &cfg(), &mut messages, |_| {}).await;
7647 assert_eq!(outcome.status, "success");
7648 assert_eq!(seen.lock().unwrap().len(), 2);
7649 assert!(!has_compaction_notice(&messages));
7650 }
7651
7652 #[test]
7656 fn measured_compaction_scales_the_drop_to_the_reported_size_and_round_trips() {
7657 let mut m = modest_history();
7658 let len = m.len();
7659 let estimate: usize = m.iter().map(approx_message_tokens).sum();
7660 let measure = PromptMeasure {
7662 fixed_overhead: 0,
7663 reported: Some((estimate * 40, len)),
7664 };
7665 compact_history_measured(&mut m, 200_000, measure);
7666 let notice = m
7667 .iter()
7668 .find(|msg| parse_compaction_notice(msg).is_some())
7669 .expect("must compact on the reported size");
7670 let (turns, tokens) = parse_compaction_notice(notice).unwrap();
7671 assert!(turns > 0);
7672 assert!(
7673 tokens > estimate,
7674 "dropped tokens are accounted in the scaled (provider) measure: {tokens} vs raw estimate {estimate}"
7675 );
7676 let remaining: usize = m.iter().map(approx_message_tokens).sum();
7679 assert!(remaining < estimate);
7680 }
7681
7682 #[test]
7685 fn fixed_overhead_counts_toward_the_budget() {
7686 let mut m = modest_history();
7687 let estimate: usize = m.iter().map(approx_message_tokens).sum();
7688 let window = estimate * 4 / 3 + 40;
7690 compact_history_measured(&mut m, window, PromptMeasure::default());
7691 assert!(!has_compaction_notice(&m), "history alone fits");
7692 compact_history_measured(
7693 &mut m,
7694 window,
7695 PromptMeasure {
7696 fixed_overhead: 5_000,
7697 reported: None,
7698 },
7699 );
7700 assert!(has_compaction_notice(&m), "history + tool defs does not");
7701 }
7702
7703 #[test]
7706 fn reported_count_only_rescales_beyond_a_quarter_off() {
7707 let m = modest_history();
7708 let len = m.len();
7709 let estimate: usize = m.iter().map(approx_message_tokens).sum();
7710 let mut close = m.clone();
7712 compact_history_measured(
7713 &mut close,
7714 estimate * 4 / 3,
7715 PromptMeasure {
7716 fixed_overhead: 0,
7717 reported: Some((estimate * 11 / 10, len)),
7718 },
7719 );
7720 let (_, close_tokens) = close
7721 .iter()
7722 .find_map(parse_compaction_notice)
7723 .expect("110% of a budget-sized estimate must compact");
7724 let mut far = m.clone();
7726 compact_history_measured(
7727 &mut far,
7728 estimate * 4 / 3,
7729 PromptMeasure {
7730 fixed_overhead: 0,
7731 reported: Some((estimate * 20, len)),
7732 },
7733 );
7734 let (_, far_tokens) = far.iter().find_map(parse_compaction_notice).unwrap();
7735 assert!(
7736 far_tokens > close_tokens * 5,
7737 "{far_tokens} vs {close_tokens}"
7738 );
7739 }
7740
7741 fn delegate_cfg() -> AssistantConfig {
7745 let mut tools = GeneralExecutor::tool_defs();
7746 tools.push(delegate_tool_def(&tools));
7747 AssistantConfig { tools, ..cfg() }
7748 }
7749
7750 fn delegate_call(params: Value) -> InferenceResult {
7751 turn(
7752 "delegating",
7753 json!([{ "id": "d1", "name": DELEGATE_TOOL, "arguments": params }]),
7754 )
7755 }
7756
7757 fn calc_call() -> InferenceResult {
7758 turn(
7759 "computing",
7760 json!([{ "id": "k", "name": "calculate", "arguments": { "expression": "1+1" } }]),
7761 )
7762 }
7763
7764 fn tool_names(req: &GenerateRequest) -> Vec<String> {
7765 req.tools
7766 .as_deref()
7767 .unwrap_or_default()
7768 .iter()
7769 .filter_map(|d| d.get("name").and_then(Value::as_str))
7770 .map(str::to_string)
7771 .collect()
7772 }
7773
7774 fn delegate_result(messages: &[Message]) -> (String, bool) {
7776 messages
7777 .iter()
7778 .find_map(|m| match m {
7779 Message::ToolResult {
7780 tool_use_id,
7781 content,
7782 provenance,
7783 } if tool_use_id == "d1" => {
7784 Some((content.clone(), *provenance == Provenance::External))
7785 }
7786 _ => None,
7787 })
7788 .expect("the delegate call must have a tool result")
7789 }
7790
7791 #[test]
7792 fn delegate_tool_def_enumerates_parent_tools_and_excludes_itself() {
7793 let mut tools = GeneralExecutor::tool_defs();
7794 let def = delegate_tool_def(&tools);
7795 assert_eq!(def["name"], DELEGATE_TOOL);
7796 assert_eq!(def["tier"], "read_only");
7797 assert_eq!(def["mutating"], true, "a finished delegation is progress");
7798 assert_eq!(def["parameters"]["required"], json!(["goal"]));
7799 let en = def["parameters"]["properties"]["tools"]["items"]["enum"]
7800 .as_array()
7801 .unwrap()
7802 .clone();
7803 assert!(en.iter().any(|v| v == "calculate"));
7804 assert!(!en.iter().any(|v| v == DELEGATE_TOOL));
7805 tools.push(def);
7807 let again = delegate_tool_def(&tools);
7808 assert!(!again["parameters"]["properties"]["tools"]["items"]["enum"]
7809 .as_array()
7810 .unwrap()
7811 .iter()
7812 .any(|v| v == DELEGATE_TOOL));
7813 assert!(mutating_tool_names(&tools).contains(DELEGATE_TOOL));
7814 }
7815
7816 #[test]
7817 fn delegate_params_parse_with_defaults_and_cap() {
7818 let r = parse_delegate_params(&json!({"goal": " count "})).unwrap();
7819 assert_eq!(r.goal, "count");
7820 assert_eq!(r.tools, None);
7821 assert_eq!(r.max_turns, DELEGATE_DEFAULT_MAX_TURNS);
7822 let r =
7823 parse_delegate_params(&json!({"goal": "x", "tools": ["calculate"], "max_turns": 500}))
7824 .unwrap();
7825 assert_eq!(r.tools.as_deref(), Some(&["calculate".to_string()][..]));
7826 assert_eq!(r.max_turns, DELEGATE_MAX_TURNS_CAP);
7827 assert!(parse_delegate_params(&json!({"goal": ""})).is_err());
7828 assert!(parse_delegate_params(&json!({"goal": "x", "max_turns": 0})).is_err());
7829 assert!(parse_delegate_params(&json!({"goal": "x", "tools": "calculate"})).is_err());
7830 }
7831
7832 #[test]
7834 fn delegate_child_config_derives_from_the_parent() {
7835 let mut parent = delegate_cfg();
7836 parent.gated_tools = vec!["shell".into()];
7837 parent.context_window_override = Some(20_000);
7838 parent.response_format = Some(car_inference::ResponseFormat::JsonObject);
7839 parent.todos = Some(Arc::new(tokio::sync::Mutex::new(
7840 super::super::todo::TodoList::new(),
7841 )));
7842 let req = parse_delegate_params(&json!({"goal": "g", "tools": ["calculate"]})).unwrap();
7843 let child = delegate_child_config(&parent, &req).unwrap();
7844 assert_eq!(
7845 child
7846 .tools
7847 .iter()
7848 .map(|d| d["name"].as_str().unwrap())
7849 .collect::<Vec<_>>(),
7850 vec!["calculate"]
7851 );
7852 assert!(child.refuse_unadvertised_tools);
7853 assert_eq!(child.max_turns, DELEGATE_DEFAULT_MAX_TURNS);
7854 assert!(child.todos.is_none());
7855 assert!(child.response_format.is_none(), "children answer in prose");
7856 assert_eq!(
7857 child.gated_tools, parent.gated_tools,
7858 "gates inherited whole"
7859 );
7860 assert_eq!(child.context_window_override, Some(20_000));
7861 assert_eq!(child.model, parent.model);
7862 let all = delegate_child_config(
7864 &parent,
7865 &parse_delegate_params(&json!({"goal": "g"})).unwrap(),
7866 )
7867 .unwrap();
7868 let names: Vec<&str> = all
7869 .tools
7870 .iter()
7871 .map(|d| d["name"].as_str().unwrap())
7872 .collect();
7873 assert!(names.contains(&"calculate"));
7874 assert!(!names.contains(&DELEGATE_TOOL));
7875 assert_eq!(names.len(), parent.tools.len() - 1);
7876 }
7877
7878 #[tokio::test]
7882 async fn delegate_child_runs_with_a_fresh_history_and_returns_only_its_final_text() {
7883 let dir = tempfile::tempdir().unwrap();
7884 let rt = runtime_for(dir.path()).await;
7885 let seen = Arc::new(StdMutex::new(Vec::new()));
7886 let script = CapturingScript {
7887 turns: vec![
7888 delegate_call(json!({"goal": "what is 1+1? reply with the number only"})),
7889 calc_call(), turn("2", json!([])), turn("The answer is 2.", json!([])), ],
7893 cursor: AtomicUsize::new(0),
7894 seen: Arc::clone(&seen),
7895 };
7896 let mut messages = vec![
7897 sys("PARENT SYSTEM PROMPT"),
7898 usr("PARENT TASK: add one and one"),
7899 ];
7900 let mut events = Vec::new();
7901 let outcome = run_assistant_loop(&script, &rt, &delegate_cfg(), &mut messages, |e| {
7902 events.push(e)
7903 })
7904 .await;
7905 assert_eq!(outcome.status, "success");
7906 assert_eq!(outcome.summary, "The answer is 2.");
7907 assert_eq!(outcome.turns, 2, "child turns are not the parent's");
7908
7909 let reqs = seen.lock().unwrap();
7910 assert_eq!(reqs.len(), 4);
7911 let child_first = reqs[1].messages.as_ref().unwrap();
7912 assert_eq!(
7913 child_first.len(),
7914 2,
7915 "exactly system + goal: {child_first:?}"
7916 );
7917 assert!(
7918 matches!(&child_first[0], Message::System { content } if content == "PARENT SYSTEM PROMPT")
7919 );
7920 assert!(
7921 matches!(&child_first[1], Message::User { content } if content.starts_with("what is 1+1?"))
7922 );
7923 assert!(
7924 !serde_json::to_string(child_first)
7925 .unwrap()
7926 .contains("PARENT TASK"),
7927 "nothing from the parent's transcript reaches the child"
7928 );
7929 let child_tools = tool_names(&reqs[1]);
7930 assert!(child_tools.contains(&"calculate".to_string()));
7931 assert!(
7932 !child_tools.contains(&DELEGATE_TOOL.to_string()),
7933 "no nesting"
7934 );
7935 assert!(tool_names(&reqs[0]).contains(&DELEGATE_TOOL.to_string()));
7936 let parent_second = reqs[3].messages.as_ref().unwrap();
7939 assert!(!serde_json::to_string(parent_second)
7940 .unwrap()
7941 .contains("computing"));
7942 drop(reqs);
7943
7944 let (content, external) = delegate_result(&messages);
7945 assert_eq!(content, "2", "only the child's final text comes back");
7946 assert!(!external, "calculate is internal");
7947
7948 assert!(events
7950 .iter()
7951 .any(|e| matches!(e, AssistantEvent::ToolCall { name, .. } if name == DELEGATE_TOOL)));
7952 assert!(events.iter().any(|e| matches!(e, AssistantEvent::ToolResult { name, ok: true, content } if name == DELEGATE_TOOL && content == "2")));
7953 assert!(events.iter().any(|e| matches!(e, AssistantEvent::Text(t) if t.starts_with("[delegate: what is 1+1?") && t.ends_with("2 turns, ok]"))));
7954 assert!(
7955 !events
7956 .iter()
7957 .any(|e| matches!(e, AssistantEvent::ToolCall { name, .. } if name == "calculate")),
7958 "the child's own tool calls are not forwarded"
7959 );
7960 let receipt = outcome
7961 .tool_receipts
7962 .iter()
7963 .find(|r| r.tool == DELEGATE_TOOL)
7964 .unwrap();
7965 assert!(receipt.ok);
7966 assert_eq!(receipt.call_id.as_deref(), Some("d1"));
7967 assert_eq!(outcome.tools_called, vec![DELEGATE_TOOL.to_string()]);
7968 }
7969
7970 #[tokio::test]
7973 async fn delegate_child_tool_subset_is_enforced_at_execution() {
7974 let dir = tempfile::tempdir().unwrap();
7975 let rt = runtime_for(dir.path()).await;
7976 let seen = Arc::new(StdMutex::new(Vec::new()));
7977 let script = CapturingScript {
7978 turns: vec![
7979 delegate_call(json!({"goal": "write hi.txt", "tools": ["calculate"]})),
7980 turn(
7981 "writing",
7982 json!([{ "id": "w", "name": "write_file", "arguments": { "path": "hi.txt", "content": "hello" } }]),
7983 ),
7984 turn("could not write", json!([])),
7985 turn("done", json!([])),
7986 ],
7987 cursor: AtomicUsize::new(0),
7988 seen: Arc::clone(&seen),
7989 };
7990 let mut messages = vec![sys("sys"), usr("task")];
7991 let outcome =
7992 run_assistant_loop(&script, &rt, &delegate_cfg(), &mut messages, |_| {}).await;
7993 assert_eq!(outcome.status, "success");
7994 assert!(
7995 !dir.path().join("hi.txt").exists(),
7996 "the ungranted write must not run"
7997 );
7998 let reqs = seen.lock().unwrap();
7999 assert_eq!(tool_names(&reqs[1]), vec!["calculate".to_string()]);
8000 let child_second = reqs[2].messages.as_ref().unwrap();
8001 let refusal = child_second
8002 .iter()
8003 .find_map(|m| match m {
8004 Message::ToolResult {
8005 tool_use_id,
8006 content,
8007 ..
8008 } if tool_use_id == "w" => Some(content.clone()),
8009 _ => None,
8010 })
8011 .unwrap();
8012 assert!(
8013 refusal.contains("not granted to this delegate"),
8014 "{refusal}"
8015 );
8016 assert!(
8017 refusal.contains("calculate"),
8018 "says what IS allowed: {refusal}"
8019 );
8020 }
8021
8022 #[tokio::test]
8025 async fn delegate_cannot_nest() {
8026 let dir = tempfile::tempdir().unwrap();
8027 let rt = runtime_for(dir.path()).await;
8028 let seen = Arc::new(StdMutex::new(Vec::new()));
8029 let script = CapturingScript {
8030 turns: vec![
8031 delegate_call(json!({"goal": "go deeper"})),
8032 turn(
8033 "nesting",
8034 json!([{ "id": "n", "name": DELEGATE_TOOL, "arguments": { "goal": "deeper still" } }]),
8035 ),
8036 turn("could not nest", json!([])),
8037 turn("done", json!([])),
8038 ],
8039 cursor: AtomicUsize::new(0),
8040 seen: Arc::clone(&seen),
8041 };
8042 let mut messages = vec![sys("sys"), usr("task")];
8043 let outcome =
8044 run_assistant_loop(&script, &rt, &delegate_cfg(), &mut messages, |_| {}).await;
8045 assert_eq!(outcome.status, "success");
8046 {
8047 let reqs = seen.lock().unwrap();
8048 assert_eq!(reqs.len(), 4, "the nested call must not spawn a grandchild");
8049 let child_second = reqs[2].messages.as_ref().unwrap();
8050 let refusal = serde_json::to_string(child_second).unwrap();
8051 assert!(
8052 refusal.contains("not granted to this delegate"),
8053 "{refusal}"
8054 );
8055 }
8056
8057 let seen = Arc::new(StdMutex::new(Vec::new()));
8059 let script = CapturingScript {
8060 turns: vec![
8061 delegate_call(json!({"goal": "go deeper", "tools": [DELEGATE_TOOL]})),
8062 turn("done", json!([])),
8063 ],
8064 cursor: AtomicUsize::new(0),
8065 seen: Arc::clone(&seen),
8066 };
8067 let mut messages = vec![sys("sys"), usr("task")];
8068 let mut events = Vec::new();
8069 run_assistant_loop(&script, &rt, &delegate_cfg(), &mut messages, |e| {
8070 events.push(e)
8071 })
8072 .await;
8073 assert_eq!(seen.lock().unwrap().len(), 2, "no child ran");
8074 let (content, _) = delegate_result(&messages);
8075 assert!(
8076 content.contains("privilege escalation rejected"),
8077 "{content}"
8078 );
8079 assert!(content.contains("cannot delegate further"), "{content}");
8080 assert!(events.iter().any(|e| matches!(e, AssistantEvent::ToolResult { name, ok: false, .. } if name == DELEGATE_TOOL)));
8081 }
8082
8083 #[tokio::test]
8085 async fn delegate_escalation_is_refused_without_spawning() {
8086 let dir = tempfile::tempdir().unwrap();
8087 let rt = runtime_for(dir.path()).await;
8088 let seen = Arc::new(StdMutex::new(Vec::new()));
8089 let script = CapturingScript {
8090 turns: vec![
8091 delegate_call(json!({"goal": "x", "tools": ["calculate", "launch_missiles"]})),
8092 turn("done", json!([])),
8093 ],
8094 cursor: AtomicUsize::new(0),
8095 seen: Arc::clone(&seen),
8096 };
8097 let mut messages = vec![sys("sys"), usr("task")];
8098 let outcome =
8099 run_assistant_loop(&script, &rt, &delegate_cfg(), &mut messages, |_| {}).await;
8100 assert_eq!(outcome.status, "success");
8101 assert_eq!(seen.lock().unwrap().len(), 2);
8102 let (content, _) = delegate_result(&messages);
8103 assert!(content.contains("launch_missiles"), "{content}");
8104 let receipt = outcome
8105 .tool_receipts
8106 .iter()
8107 .find(|r| r.tool == DELEGATE_TOOL)
8108 .unwrap();
8109 assert!(!receipt.ok);
8110 assert!(
8111 outcome.tools_called.is_empty(),
8112 "a refused delegation is not progress"
8113 );
8114 }
8115
8116 #[tokio::test]
8120 async fn delegate_turn_cap_is_an_error_result() {
8121 let dir = tempfile::tempdir().unwrap();
8122 let rt = runtime_for(dir.path()).await;
8123 let seen = Arc::new(StdMutex::new(Vec::new()));
8124 let script = CapturingScript {
8125 turns: vec![
8126 delegate_call(json!({"goal": "keep computing", "max_turns": 1})),
8127 calc_call(), turn("done", json!([])),
8129 ],
8130 cursor: AtomicUsize::new(0),
8131 seen: Arc::clone(&seen),
8132 };
8133 let mut messages = vec![sys("sys"), usr("task")];
8134 let mut events = Vec::new();
8135 let outcome = run_assistant_loop(&script, &rt, &delegate_cfg(), &mut messages, |e| {
8136 events.push(e)
8137 })
8138 .await;
8139 assert_eq!(outcome.status, "success");
8140 assert_eq!(seen.lock().unwrap().len(), 3);
8141 let (content, _) = delegate_result(&messages);
8142 assert!(content.contains("did not finish"), "{content}");
8143 assert!(content.contains("status: max_turns"), "{content}");
8144 assert!(events
8145 .iter()
8146 .any(|e| matches!(e, AssistantEvent::Text(t) if t.ends_with("1 turns, error]"))));
8147 assert!(events.iter().any(|e| matches!(e, AssistantEvent::ToolResult { name, ok: false, .. } if name == DELEGATE_TOOL)));
8148 assert!(outcome.tools_called.is_empty());
8149 }
8150
8151 #[tokio::test]
8153 async fn delegate_summary_is_capped() {
8154 let dir = tempfile::tempdir().unwrap();
8155 let rt = runtime_for(dir.path()).await;
8156 let seen = Arc::new(StdMutex::new(Vec::new()));
8157 let long = "y".repeat(OBSERVATION_CAP * 3);
8158 let script = CapturingScript {
8159 turns: vec![
8160 delegate_call(json!({"goal": "dump"})),
8161 turn(&long, json!([])),
8162 turn("done", json!([])),
8163 ],
8164 cursor: AtomicUsize::new(0),
8165 seen: Arc::clone(&seen),
8166 };
8167 let mut messages = vec![sys("sys"), usr("task")];
8168 run_assistant_loop(&script, &rt, &delegate_cfg(), &mut messages, |_| {}).await;
8169 let (content, _) = delegate_result(&messages);
8170 assert!(content.len() < long.len());
8171 assert!(
8172 content.contains("bytes elided"),
8173 "the cap must say what it dropped"
8174 );
8175 }
8176
8177 #[tokio::test]
8181 async fn delegate_child_inherits_the_parents_approval_gate() {
8182 let dir = tempfile::tempdir().unwrap();
8183 let rt = runtime_for(dir.path()).await;
8184 let seen = Arc::new(StdMutex::new(Vec::new()));
8185 let script = CapturingScript {
8186 turns: vec![
8187 delegate_call(
8188 json!({"goal": "write hi.txt", "tools": ["write_file", "calculate"]}),
8189 ),
8190 turn(
8191 "writing",
8192 json!([{ "id": "w", "name": "write_file", "arguments": { "path": "hi.txt", "content": "hello" } }]),
8193 ),
8194 turn("denied", json!([])),
8195 turn("done", json!([])),
8196 ],
8197 cursor: AtomicUsize::new(0),
8198 seen: Arc::clone(&seen),
8199 };
8200 let mut cfg = delegate_cfg();
8201 cfg.gated_tools = vec!["write_file".into(), "edit_file".into(), "shell".into()];
8202 let mut messages = vec![sys("sys"), usr("task")];
8203 let outcome = run_assistant_loop(&script, &rt, &cfg, &mut messages, |_| {}).await;
8204 assert_eq!(outcome.status, "success");
8205 assert!(!dir.path().join("hi.txt").exists());
8206 let reqs = seen.lock().unwrap();
8207 assert!(tool_names(&reqs[1]).contains(&"write_file".to_string()));
8209 let child_second = serde_json::to_string(reqs[2].messages.as_ref().unwrap()).unwrap();
8210 assert!(child_second.contains("needs approval"), "{child_second}");
8211 assert!(!child_second.contains("not granted"), "{child_second}");
8212 }
8213
8214 #[tokio::test]
8217 async fn delegate_is_not_intercepted_unless_advertised() {
8218 let dir = tempfile::tempdir().unwrap();
8219 let rt = runtime_for(dir.path()).await;
8220 let seen = Arc::new(StdMutex::new(Vec::new()));
8221 let script = CapturingScript {
8222 turns: vec![delegate_call(json!({"goal": "x"})), turn("done", json!([]))],
8223 cursor: AtomicUsize::new(0),
8224 seen: Arc::clone(&seen),
8225 };
8226 let mut messages = vec![sys("sys"), usr("task")];
8227 run_assistant_loop(&script, &rt, &cfg(), &mut messages, |_| {}).await;
8228 assert_eq!(seen.lock().unwrap().len(), 2, "no child ran");
8229 let (content, _) = delegate_result(&messages);
8230 assert!(!content.is_empty());
8231 assert!(
8232 !content.contains("did not finish"),
8233 "not a delegation at all: {content}"
8234 );
8235 }
8236
8237 #[tokio::test]
8243 async fn delegate_child_receipts_ground_the_parents_claims() {
8244 let dir = tempfile::tempdir().unwrap();
8245 let rt = runtime_for(dir.path()).await;
8246 let seen = Arc::new(StdMutex::new(Vec::new()));
8247 let script = CapturingScript {
8248 turns: vec![
8249 delegate_call(json!({"goal": "run the test suite and report", "tools": ["shell"]})),
8250 turn(
8251 "running",
8252 json!([{ "id": "s", "name": "shell", "arguments": { "command": "echo cargo test ok" } }]),
8253 ),
8254 turn("The suite ran.", json!([])),
8255 turn("I ran the tests and they passed.", json!([])),
8256 ],
8257 cursor: AtomicUsize::new(0),
8258 seen: Arc::clone(&seen),
8259 };
8260 let mut messages = vec![sys("sys"), usr("run the tests")];
8261 let outcome =
8262 run_assistant_loop(&script, &rt, &delegate_cfg(), &mut messages, |_| {}).await;
8263 assert_eq!(outcome.status, "success", "{}", outcome.summary);
8266 assert_eq!(outcome.summary, "I ran the tests and they passed.");
8267 let child_shell = outcome
8268 .tool_receipts
8269 .iter()
8270 .find(|r| r.tool == "shell")
8271 .expect("the child's shell receipt must be in the parent's list");
8272 assert!(child_shell.ok);
8273 assert_eq!(child_shell.via.as_deref(), Some("delegate:d1"));
8274 let del = outcome
8276 .tool_receipts
8277 .iter()
8278 .find(|r| r.tool == DELEGATE_TOOL)
8279 .unwrap();
8280 assert!(del.via.is_none());
8281 assert!(
8282 ungrounded_summary_claims(&outcome.summary, &outcome.tool_receipts).is_empty(),
8283 "the merged shell receipt grounds the tests-passed claim"
8284 );
8285 }
8286
8287 #[tokio::test]
8290 async fn delegate_budget_caps_delegations() {
8291 let dir = tempfile::tempdir().unwrap();
8292 let rt = runtime_for(dir.path()).await;
8293 let seen = Arc::new(StdMutex::new(Vec::new()));
8294 let script = CapturingScript {
8295 turns: vec![
8296 delegate_call(json!({"goal": "first"})),
8297 turn("one", json!([])), delegate_call(json!({"goal": "second"})),
8299 turn("done", json!([])),
8301 ],
8302 cursor: AtomicUsize::new(0),
8303 seen: Arc::clone(&seen),
8304 };
8305 let mut cfg = delegate_cfg();
8306 cfg.delegate_budget = Some(DelegateBudget {
8307 max_delegations: 1,
8308 max_child_turns: 300,
8309 });
8310 let mut messages = vec![sys("sys"), usr("task")];
8311 let outcome = run_assistant_loop(&script, &rt, &cfg, &mut messages, |_| {}).await;
8312 assert_eq!(outcome.status, "success");
8313 assert_eq!(seen.lock().unwrap().len(), 4);
8314 let refusals: Vec<&AssistantToolReceipt> = outcome
8315 .tool_receipts
8316 .iter()
8317 .filter(|r| r.tool == DELEGATE_TOOL && !r.ok)
8318 .collect();
8319 assert_eq!(refusals.len(), 1);
8320 let refusal_text = messages
8321 .iter()
8322 .find_map(|m| match m {
8323 Message::ToolResult { content, .. } if content.contains("budget exhausted") => {
8324 Some(content.clone())
8325 }
8326 _ => None,
8327 })
8328 .expect("the refusal must reach the model");
8329 assert!(refusal_text.contains("1 delegations"), "{refusal_text}");
8330 }
8331
8332 #[tokio::test]
8334 async fn delegate_budget_caps_cumulative_child_turns() {
8335 let dir = tempfile::tempdir().unwrap();
8336 let rt = runtime_for(dir.path()).await;
8337 let seen = Arc::new(StdMutex::new(Vec::new()));
8338 let script = CapturingScript {
8339 turns: vec![
8340 delegate_call(json!({"goal": "first"})),
8341 calc_call(), turn("one", json!([])), delegate_call(json!({"goal": "second"})),
8344 turn("done", json!([])),
8345 ],
8346 cursor: AtomicUsize::new(0),
8347 seen: Arc::clone(&seen),
8348 };
8349 let mut cfg = delegate_cfg();
8350 cfg.delegate_budget = Some(DelegateBudget {
8351 max_delegations: 20,
8352 max_child_turns: 2,
8353 });
8354 let mut messages = vec![sys("sys"), usr("task")];
8355 let outcome = run_assistant_loop(&script, &rt, &cfg, &mut messages, |_| {}).await;
8356 assert_eq!(outcome.status, "success");
8357 assert_eq!(seen.lock().unwrap().len(), 5, "no second child spawned");
8358 assert!(
8359 messages.iter().any(|m| matches!(m, Message::ToolResult { content, .. } if content.contains("2 child turns"))),
8360 "the refusal names the spent turn budget"
8361 );
8362 }
8363
8364 #[tokio::test]
8368 async fn json_schema_shape_mismatch_triggers_the_repair() {
8369 let dir = tempfile::tempdir().unwrap();
8370 let rt = runtime_for(dir.path()).await;
8371 let seen = Arc::new(StdMutex::new(Vec::new()));
8372 let script = CapturingScript {
8373 turns: vec![
8374 turn(r#"{"nope": 1}"#, json!([])),
8375 turn(r#"{"legs": []}"#, json!([])),
8376 ],
8377 cursor: AtomicUsize::new(0),
8378 seen: Arc::clone(&seen),
8379 };
8380 let mut cfg = cfg();
8381 cfg.response_format = Some(car_inference::ResponseFormat::JsonSchema {
8382 schema: json!({"type": "object", "required": ["legs"]}),
8383 strict: false,
8384 name: None,
8385 });
8386 cfg.response_format_validator = Some(Arc::new(|v| v.get("legs").is_some()));
8387 let mut messages = vec![sys("sys"), usr("plan the flight, answer as JSON")];
8388 let mut events = Vec::new();
8389 let outcome =
8390 run_assistant_loop(&script, &rt, &cfg, &mut messages, |e| events.push(e)).await;
8391 assert_eq!(outcome.status, "success");
8392 assert_eq!(outcome.summary, r#"{"legs": []}"#);
8393 let reqs = seen.lock().unwrap();
8394 assert_eq!(reqs.len(), 2, "valid-but-wrong-shape JSON must be repaired");
8395 assert!(reqs[1].tools.is_none());
8396 let nudge = reqs[1]
8397 .messages
8398 .as_ref()
8399 .unwrap()
8400 .last()
8401 .and_then(|m| match m {
8402 Message::User { content } => Some(content.clone()),
8403 _ => None,
8404 })
8405 .unwrap();
8406 assert!(
8407 nudge.contains("JSON Schema"),
8408 "schema-worded, not 'object': {nudge}"
8409 );
8410 assert_eq!(repair_notices(&events), (1, 0));
8411 }
8412
8413 #[tokio::test]
8417 async fn a_failed_repair_call_keeps_the_draft_answer() {
8418 let dir = tempfile::tempdir().unwrap();
8419 let rt = runtime_for(dir.path()).await;
8420 let seen = Arc::new(StdMutex::new(Vec::new()));
8421 let script = CapturingScript {
8422 turns: vec![turn("The answer is 2, not JSON.", json!([]))],
8424 cursor: AtomicUsize::new(0),
8425 seen: Arc::clone(&seen),
8426 };
8427 let mut messages = vec![sys("sys"), usr("answer as JSON")];
8428 let mut events = Vec::new();
8429 let outcome = run_assistant_loop(&script, &rt, &json_object_cfg(), &mut messages, |e| {
8430 events.push(e)
8431 })
8432 .await;
8433 assert_eq!(outcome.status, "success", "the draft is still an answer");
8434 assert_eq!(outcome.summary, "The answer is 2, not JSON.");
8435 assert!(
8436 events.iter().any(|e| matches!(e, AssistantEvent::Text(t) if t.starts_with(FORMAT_REPAIR_FAILED_PREFIX))),
8437 "the failure must be visible"
8438 );
8439 assert!(
8440 matches!(messages.last(), Some(Message::Assistant { content, .. }) if content == "The answer is 2, not JSON."),
8441 "the transcript ends on the draft answer, not a dangling nudge: {:?}",
8442 messages.last()
8443 );
8444 }
8445
8446 #[tokio::test]
8450 async fn delegate_child_cannot_touch_the_parents_todo_list() {
8451 let dir = tempfile::tempdir().unwrap();
8452 let rt = runtime_for(dir.path()).await;
8453 let todos = Arc::new(tokio::sync::Mutex::new(super::super::todo::TodoList::new()));
8454 todos
8455 .lock()
8456 .await
8457 .write(&[json!({"text": "the parent's plan"})])
8458 .unwrap();
8459 let before = todos.lock().await.render();
8460
8461 let seen = Arc::new(StdMutex::new(Vec::new()));
8462 let script = CapturingScript {
8463 turns: vec![
8464 delegate_call(json!({"goal": "reorganize", "tools": ["calculate"]})),
8465 turn(
8466 "writing todos",
8467 json!([{ "id": "t", "name": "todo_write", "arguments": { "todos": [{"text": "hijacked"}] } }]),
8468 ),
8469 turn("could not", json!([])),
8470 turn("done", json!([])),
8471 ],
8472 cursor: AtomicUsize::new(0),
8473 seen: Arc::clone(&seen),
8474 };
8475 let mut cfg = delegate_cfg();
8476 cfg.todos = Some(Arc::clone(&todos));
8477 let mut messages = vec![sys("sys"), usr("task")];
8478 let outcome = run_assistant_loop(&script, &rt, &cfg, &mut messages, |_| {}).await;
8479 assert_eq!(outcome.status, "success");
8480 assert_eq!(
8481 todos.lock().await.render(),
8482 before,
8483 "the parent's list is untouched"
8484 );
8485 let child_second =
8486 serde_json::to_string(seen.lock().unwrap()[2].messages.as_ref().unwrap()).unwrap();
8487 assert!(
8488 child_second.contains("not granted to this delegate"),
8489 "{child_second}"
8490 );
8491 }
8492
8493 #[tokio::test]
8496 async fn delegate_marks_the_parent_result_external_when_a_child_receipt_is() {
8497 let dir = tempfile::tempdir().unwrap();
8498 let rt = runtime_for(dir.path()).await;
8499 let mut tools = GeneralExecutor::tool_defs();
8500 tools.push(json!({
8502 "name": "http_request",
8503 "description": "Fetch a URL.",
8504 "parameters": {"type": "object", "properties": {"url": {"type": "string"}}}
8505 }));
8506 tools.push(delegate_tool_def(&tools));
8507 let cfg = AssistantConfig { tools, ..cfg() };
8508
8509 let seen = Arc::new(StdMutex::new(Vec::new()));
8510 let script = CapturingScript {
8511 turns: vec![
8512 delegate_call(json!({"goal": "fetch the page", "tools": ["http_request"]})),
8513 turn(
8514 "fetching",
8515 json!([{ "id": "h", "name": "http_request", "arguments": { "url": "https://example.invalid/" } }]),
8516 ),
8517 turn("could not fetch", json!([])),
8518 turn("done", json!([])),
8519 ],
8520 cursor: AtomicUsize::new(0),
8521 seen: Arc::clone(&seen),
8522 };
8523 let mut messages = vec![sys("sys"), usr("task")];
8524 let outcome = run_assistant_loop(&script, &rt, &cfg, &mut messages, |_| {}).await;
8525 assert_eq!(outcome.status, "success");
8526 let (_, external) = delegate_result(&messages);
8527 assert!(
8528 external,
8529 "a child receipt from an external-labelled tool must mark the parent's result External"
8530 );
8531 }
8532
8533 #[tokio::test]
8538 async fn reported_count_is_reset_after_compaction_not_reused_stale() {
8539 let dir = tempfile::tempdir().unwrap();
8540 let rt = runtime_for(dir.path()).await;
8541 let seen = Arc::new(StdMutex::new(Vec::new()));
8542 let script = WindowedScript {
8543 inner: CapturingScript {
8544 turns: vec![
8545 turn_with_usage(
8546 "computing",
8547 json!([{ "id": "k", "name": "calculate", "arguments": { "expression": "1+1" } }]),
8548 190_000,
8549 10,
8550 ),
8551 turn(
8553 "still computing",
8554 json!([{ "id": "k2", "name": "calculate", "arguments": { "expression": "2+2" } }]),
8555 ),
8556 turn("done", json!([])),
8557 ],
8558 cursor: AtomicUsize::new(0),
8559 seen: Arc::clone(&seen),
8560 },
8561 window: 200_000,
8562 };
8563 let mut messages = modest_history();
8564 let outcome = run_assistant_loop(&script, &rt, &cfg(), &mut messages, |_| {}).await;
8565 assert_eq!(outcome.status, "success");
8566 let reqs = seen.lock().unwrap();
8567 assert_eq!(reqs.len(), 3);
8568 let notice_of = |req: &GenerateRequest| {
8569 req.messages
8570 .as_ref()
8571 .unwrap()
8572 .iter()
8573 .find_map(parse_compaction_notice)
8574 };
8575 let after_first = notice_of(&reqs[1]).expect("turn 2 compacts on the 190k report");
8576 let after_second = notice_of(&reqs[2]).expect("the notice persists");
8577 assert_eq!(
8578 after_first, after_second,
8579 "no second compaction: the stale 190k report must not survive the first one"
8580 );
8581 }
8582}