use super::*;
#[test]
fn code_workflow_rejects_unsafe_raw_source_rewrites() {
let workspace_root = std::env::temp_dir().join(format!(
"tandem-automation-unsafe-write-{}",
uuid::Uuid::new_v4()
));
std::fs::create_dir_all(workspace_root.join("src")).expect("create workspace");
std::fs::write(workspace_root.join("src/lib.rs"), "pub fn before() {}\n").expect("seed source");
let _snapshot = automation_workspace_root_file_snapshot(
workspace_root.to_str().expect("workspace root string"),
);
let _long_handoff = format!(
"# Handoff\n\n{}\n",
"Detailed implementation summary. ".repeat(20)
);
let node = AutomationFlowNode {
knowledge: tandem_orchestrator::KnowledgeBinding::default(),
node_id: "implement".to_string(),
agent_id: "agent-a".to_string(),
objective: "Research".to_string(),
depends_on: Vec::new(),
input_refs: Vec::new(),
output_contract: Some(AutomationFlowOutputContract {
kind: "brief".to_string(),
validator: None,
enforcement: None,
schema: None,
summary_guidance: None,
}),
retry_policy: None,
timeout_ms: None,
max_tool_calls: None,
stage_kind: None,
gate: None,
metadata: None,
};
let mut session = Session::new(Some("synthetic summary".to_string()), None);
session.messages.push(tandem_types::Message::new(
MessageRole::Assistant,
vec![MessagePart::Text {
text: "I completed project analysis steps using tools, but the model returned no final narrative text.\n\nTool result summary:\nTool `glob` result:\n/home/user123/marketing-tandem/marketing-brief.md\nTool `websearch` result:\nAuthorization required for `websearch`.\nThis integration requires authorization before this action can run.\n\nAuthorize here: https://dashboard.exa.ai/api-keys".to_string(),
}],
));
let telemetry = summarize_automation_tool_activity(
&node,
&session,
&[
"glob".to_string(),
"read".to_string(),
"websearch".to_string(),
"write".to_string(),
],
);
assert_eq!(
telemetry
.get("executed_tools")
.and_then(Value::as_array)
.map(|values| values.iter().filter_map(Value::as_str).collect::<Vec<_>>()),
Some(vec!["glob", "websearch"])
);
assert_eq!(
telemetry
.get("workspace_inspection_used")
.and_then(Value::as_bool),
Some(true)
);
assert_eq!(
telemetry.get("web_research_used").and_then(Value::as_bool),
Some(true)
);
assert_eq!(
telemetry
.get("latest_web_research_failure")
.and_then(Value::as_str),
Some("web research authorization required")
);
}
#[test]
fn summarize_automation_tool_activity_counts_auth_failed_websearch_as_attempted() {
let node = AutomationFlowNode {
knowledge: tandem_orchestrator::KnowledgeBinding::default(),
node_id: "research".to_string(),
agent_id: "agent-a".to_string(),
objective: "Research".to_string(),
depends_on: Vec::new(),
input_refs: Vec::new(),
output_contract: Some(AutomationFlowOutputContract {
kind: "brief".to_string(),
validator: None,
enforcement: None,
schema: None,
summary_guidance: None,
}),
retry_policy: None,
timeout_ms: None,
max_tool_calls: None,
stage_kind: None,
gate: None,
metadata: None,
};
let mut session = Session::new(Some("auth failed websearch".to_string()), None);
session.messages.push(tandem_types::Message::new(
MessageRole::Assistant,
vec![MessagePart::ToolInvocation {
tool: "websearch".to_string(),
args: json!({"query":"tandem competitors"}),
result: None,
error: Some("Authorization required for `websearch`.".to_string()),
}],
));
let telemetry = summarize_automation_tool_activity(
&node,
&session,
&[
"glob".to_string(),
"read".to_string(),
"websearch".to_string(),
"write".to_string(),
],
);
assert_eq!(
telemetry
.get("executed_tools")
.and_then(Value::as_array)
.map(|values| values.iter().filter_map(Value::as_str).collect::<Vec<_>>()),
Some(vec!["websearch"])
);
assert_eq!(
telemetry.get("web_research_used").and_then(Value::as_bool),
Some(true)
);
assert_eq!(
telemetry
.get("latest_web_research_failure")
.and_then(Value::as_str),
Some("web research authorization required")
);
}
#[test]
fn summarize_automation_tool_activity_treats_backend_unavailable_websearch_as_unavailable() {
let node = AutomationFlowNode {
knowledge: tandem_orchestrator::KnowledgeBinding::default(),
node_id: "research".to_string(),
agent_id: "agent-a".to_string(),
objective: "Research".to_string(),
depends_on: Vec::new(),
input_refs: Vec::new(),
output_contract: Some(AutomationFlowOutputContract {
kind: "brief".to_string(),
validator: None,
enforcement: None,
schema: None,
summary_guidance: None,
}),
retry_policy: None,
timeout_ms: None,
max_tool_calls: None,
stage_kind: None,
gate: None,
metadata: None,
};
let mut session = Session::new(Some("backend unavailable websearch".to_string()), None);
session.messages.push(tandem_types::Message::new(
MessageRole::Assistant,
vec![MessagePart::ToolInvocation {
tool: "websearch".to_string(),
args: json!({"query":"tandem competitors"}),
result: Some(json!({
"output": "Web search is currently unavailable for `websearch`.",
"metadata": { "error": "backend_unavailable" }
})),
error: None,
}],
));
let telemetry = summarize_automation_tool_activity(
&node,
&session,
&[
"glob".to_string(),
"read".to_string(),
"websearch".to_string(),
"write".to_string(),
],
);
assert_eq!(
telemetry
.get("latest_web_research_failure")
.and_then(Value::as_str),
Some("web research unavailable")
);
assert_eq!(
telemetry
.get("web_research_succeeded")
.and_then(Value::as_bool),
Some(false)
);
}
#[test]
fn summarize_automation_tool_activity_treats_partial_websearch_with_results_as_success() {
let node = AutomationFlowNode {
knowledge: tandem_orchestrator::KnowledgeBinding::default(),
node_id: "research".to_string(),
agent_id: "agent-a".to_string(),
objective: "Research".to_string(),
depends_on: Vec::new(),
input_refs: Vec::new(),
output_contract: Some(AutomationFlowOutputContract {
kind: "brief".to_string(),
validator: None,
enforcement: None,
schema: None,
summary_guidance: None,
}),
retry_policy: None,
timeout_ms: None,
max_tool_calls: None,
stage_kind: None,
gate: None,
metadata: None,
};
let mut session = Session::new(Some("partial websearch".to_string()), None);
session.messages.push(tandem_types::Message::new(
MessageRole::Assistant,
vec![MessagePart::ToolInvocation {
tool: "websearch".to_string(),
args: json!({"query":"autonomous AI agentic workflows 2025"}),
result: Some(json!({
"output": serde_json::to_string(&json!({
"query": "autonomous AI agentic workflows 2025",
"result_count": 2,
"partial": true,
"results": [
{"title": "One", "url": "https://example.com/1"},
{"title": "Two", "url": "https://example.com/2"}
]
})).expect("json output"),
"metadata": {
"count": 2,
"error": "rate_limited",
"partial": true
}
})),
error: None,
}],
));
let telemetry = summarize_automation_tool_activity(
&node,
&session,
&[
"glob".to_string(),
"read".to_string(),
"websearch".to_string(),
"write".to_string(),
],
);
assert_eq!(
telemetry
.get("web_research_succeeded")
.and_then(Value::as_bool),
Some(true)
);
assert_eq!(
telemetry
.get("latest_web_research_failure")
.and_then(Value::as_str),
None
);
}
#[test]
fn summarize_automation_tool_activity_treats_runtime_websearch_string_result_as_success() {
let node = AutomationFlowNode {
knowledge: tandem_orchestrator::KnowledgeBinding::default(),
node_id: "research".to_string(),
agent_id: "agent-a".to_string(),
objective: "Research".to_string(),
depends_on: Vec::new(),
input_refs: Vec::new(),
output_contract: Some(AutomationFlowOutputContract {
kind: "brief".to_string(),
validator: None,
enforcement: None,
schema: None,
summary_guidance: None,
}),
retry_policy: None,
timeout_ms: None,
max_tool_calls: None,
stage_kind: None,
gate: None,
metadata: None,
};
let mut session = Session::new(Some("runtime websearch".to_string()), None);
session.messages.push(tandem_types::Message::new(
MessageRole::Assistant,
vec![MessagePart::ToolInvocation {
tool: "websearch".to_string(),
args: json!({"query":"autonomous AI agentic workflows 2024 2025"}),
result: Some(json!(serde_json::to_string_pretty(&json!({
"attempted_backends": ["brave"],
"backend": "brave",
"configured_backend": "brave",
"partial": false,
"query": "autonomous AI agentic workflows 2024 2025",
"result_count": 2,
"results": [
{
"title": "AI Agents in 2025: Expectations vs. Reality | IBM",
"url": "https://www.ibm.com/think/insights/ai-agents-2025-expectations-vs-reality"
},
{
"title": "Agentic AI strategy | Deloitte Insights",
"url": "https://www.deloitte.com/us/en/insights/topics/technology-management/tech-trends/2026/agentic-ai-strategy.html"
}
]
}))
.expect("json output"))),
error: None,
}],
));
let telemetry = summarize_automation_tool_activity(
&node,
&session,
&[
"glob".to_string(),
"read".to_string(),
"websearch".to_string(),
"write".to_string(),
],
);
assert_eq!(
telemetry.get("web_research_used").and_then(Value::as_bool),
Some(true)
);
assert_eq!(
telemetry
.get("web_research_succeeded")
.and_then(Value::as_bool),
Some(true)
);
assert_eq!(
telemetry
.get("latest_web_research_failure")
.and_then(Value::as_str),
None
);
}
#[test]
fn automation_prompt_preflight_marks_warning_for_large_prompt() {
let prompt = "x".repeat(20_000);
let preflight = crate::app::state::automation::build_automation_prompt_preflight(
&prompt,
&["glob".to_string(), "read".to_string(), "write".to_string()],
&[json!({
"name": "write",
"description": "write a file",
"input_schema": {"type": "object"}
})],
"artifact_write",
&json!({
"required_capabilities": ["artifact_write"],
"resolved": {
"artifact_write": {
"status": "resolved",
"offered_tools": ["write"],
"available_tools": ["write"]
}
},
"missing_capabilities": []
}),
"standard",
false,
);
assert_eq!(
preflight.get("budget_status").and_then(Value::as_str),
Some("high")
);
assert_eq!(
preflight.get("degraded_prompt").and_then(Value::as_bool),
Some(false)
);
}
#[test]
fn build_automation_attempt_evidence_captures_runtime_websearch_success() {
let node = AutomationFlowNode {
knowledge: tandem_orchestrator::KnowledgeBinding::default(),
node_id: "research_sources".to_string(),
agent_id: "agent-a".to_string(),
objective: "Research".to_string(),
depends_on: Vec::new(),
input_refs: vec![AutomationFlowInputRef {
from_step_id: "collect_inputs".to_string(),
alias: "research_brief".to_string(),
}],
output_contract: Some(AutomationFlowOutputContract {
kind: "citations".to_string(),
validator: Some(crate::AutomationOutputValidatorKind::ResearchBrief),
enforcement: None,
schema: None,
summary_guidance: None,
}),
retry_policy: None,
timeout_ms: None,
max_tool_calls: None,
stage_kind: None,
gate: None,
metadata: Some(json!({
"builder": {
"output_path": ".tandem/artifacts/research-sources.json"
}
})),
};
let mut session = Session::new(Some("research".to_string()), None);
session.messages.push(tandem_types::Message::new(
MessageRole::Assistant,
vec![
MessagePart::ToolInvocation {
tool: "read".to_string(),
args: json!({"file_path": ".tandem/artifacts/collect-inputs.json"}),
result: Some(json!("{\"topic\":\"autonomous AI agentic workflows\"}")),
error: None,
},
MessagePart::ToolInvocation {
tool: "websearch".to_string(),
args: json!({"query":"autonomous AI agentic workflows 2024 2025"}),
result: Some(json!(serde_json::to_string_pretty(&json!({
"backend": "brave",
"result_count": 2,
"partial": false,
"results": [
{"title": "IBM", "url": "https://example.com/ibm"},
{"title": "Deloitte", "url": "https://example.com/deloitte"}
]
}))
.expect("json output"))),
error: None,
},
],
));
let tool_telemetry = summarize_automation_tool_activity(
&node,
&session,
&[
"glob".to_string(),
"read".to_string(),
"websearch".to_string(),
"write".to_string(),
],
);
let preflight = crate::app::state::automation::build_automation_prompt_preflight(
"Research prompt",
&[
"glob".to_string(),
"read".to_string(),
"websearch".to_string(),
"write".to_string(),
],
&[
json!({"name":"websearch"}),
json!({"name":"read"}),
json!({"name":"write"}),
],
"artifact_write",
&json!({
"required_capabilities": ["workspace_read", "workspace_discover", "artifact_write", "web_research"],
"resolved": {},
"missing_capabilities": []
}),
"standard",
false,
);
let attempt_evidence = build_automation_attempt_evidence(
&node,
1,
&session,
&session.id,
".",
&tool_telemetry,
&preflight,
&json!({
"required_capabilities": ["web_research"],
"resolved": {},
"missing_capabilities": []
}),
Some(".tandem/artifacts/research-sources.json"),
None,
None,
);
let web_status = attempt_evidence
.get("evidence")
.and_then(Value::as_object)
.and_then(|value| value.get("web_research"))
.and_then(Value::as_object)
.and_then(|value| value.get("status"))
.and_then(Value::as_str);
assert_eq!(web_status, Some("succeeded"));
let succeeded_tools = attempt_evidence
.get("tool_execution")
.and_then(Value::as_object)
.and_then(|value| value.get("succeeded_tools"))
.and_then(Value::as_array);
assert_eq!(
succeeded_tools
.is_some_and(|rows| rows.iter().any(|value| value.as_str() == Some("websearch"))),
true
);
}
#[test]
fn detect_automation_blocker_category_prefers_delivery_category_from_canonical_evidence() {
let node = AutomationFlowNode {
knowledge: tandem_orchestrator::KnowledgeBinding::default(),
node_id: "notify_user".to_string(),
agent_id: "agent-committer".to_string(),
objective: "Send the report by email.".to_string(),
depends_on: Vec::new(),
input_refs: Vec::new(),
output_contract: Some(AutomationFlowOutputContract {
kind: "approval_gate".to_string(),
validator: Some(crate::AutomationOutputValidatorKind::ReviewDecision),
enforcement: None,
schema: None,
summary_guidance: None,
}),
retry_policy: None,
timeout_ms: None,
max_tool_calls: None,
stage_kind: None,
gate: None,
metadata: Some(json!({
"delivery": {
"method": "email",
"to": "test@example.com",
"content_type": "text/html",
"inline_body_only": true,
"attachments": false
}
})),
};
let tool_telemetry = json!({
"executed_tools": ["read"],
"preflight": {"budget_status": "ok"},
"attempt_evidence": {
"capability_resolution": {
"missing_capabilities": []
},
"delivery": {
"status": "not_attempted"
}
}
});
assert_eq!(
detect_automation_blocker_category(
&node,
"blocked",
Some("email delivery to `test@example.com` was requested but no email draft/send tool executed"),
&tool_telemetry,
None,
)
.as_deref(),
Some("delivery_not_executed")
);
}
#[test]
fn report_generation_objective_does_not_imply_email_delivery_execution() {
let node = AutomationFlowNode {
knowledge: tandem_orchestrator::KnowledgeBinding::default(),
node_id: "generate_report".to_string(),
agent_id: "writer".to_string(),
objective: "Draft the report in simple HTML suitable for email body delivery.".to_string(),
depends_on: Vec::new(),
input_refs: Vec::new(),
output_contract: Some(AutomationFlowOutputContract {
kind: "report_markdown".to_string(),
validator: Some(crate::AutomationOutputValidatorKind::GenericArtifact),
enforcement: None,
schema: None,
summary_guidance: None,
}),
retry_policy: None,
timeout_ms: None,
max_tool_calls: None,
stage_kind: None,
gate: None,
metadata: Some(json!({
"builder": {
"output_path": ".tandem/artifacts/generate-report.html"
}
})),
};
assert!(!crate::app::state::automation::automation_node_requires_email_delivery(&node));
}
#[test]
fn execute_goal_objective_with_gmail_draft_or_send_requires_email_delivery() {
let node = AutomationFlowNode {
knowledge: tandem_orchestrator::KnowledgeBinding::default(),
node_id: "execute_goal".to_string(),
agent_id: "operator".to_string(),
objective: "Create a Gmail draft or send the final HTML summary email to test@example.com if mail tools are available.".to_string(),
depends_on: Vec::new(),
input_refs: Vec::new(),
output_contract: Some(AutomationFlowOutputContract {
kind: "approval_gate".to_string(),
validator: Some(crate::AutomationOutputValidatorKind::ReviewDecision),
enforcement: None,
schema: None,
summary_guidance: None,
}),
retry_policy: None,
timeout_ms: None,
max_tool_calls: None,
stage_kind: None,
gate: None,
metadata: None,
};
assert!(crate::app::state::automation::automation_node_requires_email_delivery(&node));
}
#[test]
fn email_delivery_status_uses_recipient_from_objective_when_metadata_missing() {
let node = AutomationFlowNode {
knowledge: tandem_orchestrator::KnowledgeBinding::default(),
node_id: "execute_goal".to_string(),
agent_id: "operator".to_string(),
objective: "Create a Gmail draft or send the final HTML summary email to test@example.com if mail tools are available.".to_string(),
depends_on: Vec::new(),
input_refs: Vec::new(),
output_contract: Some(AutomationFlowOutputContract {
kind: "approval_gate".to_string(),
validator: Some(crate::AutomationOutputValidatorKind::ReviewDecision),
enforcement: None,
schema: None,
summary_guidance: None,
}),
retry_policy: None,
timeout_ms: None,
max_tool_calls: None,
stage_kind: None,
gate: None,
metadata: None,
};
let (status, reason, approved): (String, Option<String>, Option<bool>) =
detect_automation_node_status(
&node,
"A Gmail draft has been created.\n\n{\"status\":\"completed\",\"approved\":true}",
None,
&json!({
"requested_tools": ["glob", "read", "mcp_list"],
"executed_tools": ["read", "glob", "mcp_list"],
"tool_call_counts": {"read": 1, "glob": 1, "mcp_list": 1},
"workspace_inspection_used": true,
"email_delivery_attempted": false,
"email_delivery_succeeded": false,
"latest_email_delivery_failure": null,
"capability_resolution": {
"email_tool_diagnostics": {
"available_tools": ["mcp.composio_1.gmail_send_email", "mcp.composio_1.gmail_create_email_draft"],
"offered_tools": ["mcp.composio_1.gmail_send_email", "mcp.composio_1.gmail_create_email_draft"],
"available_send_tools": ["mcp.composio_1.gmail_send_email"],
"offered_send_tools": ["mcp.composio_1.gmail_send_email"],
"available_draft_tools": ["mcp.composio_1.gmail_create_email_draft"],
"offered_draft_tools": ["mcp.composio_1.gmail_create_email_draft"]
}
}
}),
None,
);
assert_eq!(status, "blocked");
assert_eq!(
reason.as_deref(),
Some(
"email delivery to `test@example.com` was requested but no email draft/send tool executed"
)
);
assert_eq!(approved, Some(true));
}
#[test]
fn research_workflow_failure_kind_detects_missing_citations() {
let node = AutomationFlowNode {
knowledge: tandem_orchestrator::KnowledgeBinding::default(),
node_id: "research".to_string(),
agent_id: "agent-a".to_string(),
objective: "Research".to_string(),
depends_on: Vec::new(),
input_refs: Vec::new(),
output_contract: Some(AutomationFlowOutputContract {
kind: "brief".to_string(),
validator: None,
enforcement: None,
schema: None,
summary_guidance: None,
}),
retry_policy: None,
timeout_ms: None,
max_tool_calls: None,
stage_kind: None,
gate: None,
metadata: Some(json!({
"builder": {
"output_path": "marketing-brief.md",
"web_research_expected": true,
"source_coverage_required": true
}
})),
};
let artifact_validation = json!({
"semantic_block_reason": "research completed without citation-backed claims",
"unmet_requirements": ["citations_missing", "web_sources_reviewed_missing"],
"verification": {
"verification_failed": false
}
});
assert_eq!(
detect_automation_node_failure_kind(
&node,
"blocked",
None,
Some("research completed without citation-backed claims"),
Some(&artifact_validation),
)
.as_deref(),
Some("research_citations_missing")
);
assert_eq!(
detect_automation_node_phase(&node, "blocked", Some(&artifact_validation)),
"research_validation"
);
}
#[test]
fn research_workflow_defaults_to_warning_without_strict_source_coverage() {
let node = AutomationFlowNode {
knowledge: tandem_orchestrator::KnowledgeBinding::default(),
node_id: "research".to_string(),
agent_id: "agent-a".to_string(),
objective: "Research".to_string(),
depends_on: Vec::new(),
input_refs: Vec::new(),
output_contract: Some(AutomationFlowOutputContract {
kind: "brief".to_string(),
validator: None,
enforcement: None,
schema: None,
summary_guidance: None,
}),
retry_policy: None,
timeout_ms: None,
max_tool_calls: None,
stage_kind: None,
gate: None,
metadata: Some(json!({
"builder": {
"output_path": "marketing-brief.md",
"web_research_expected": true,
"allow_preexisting_output_reuse": true
}
})),
};
let artifact_validation = json!({
"unmet_requirements": ["no_concrete_reads", "citations_missing", "missing_successful_web_research"],
"verification": {
"verification_failed": false
}
});
assert_eq!(
detect_automation_node_failure_kind(
&node,
"completed",
None,
None,
Some(&artifact_validation)
),
None
);
assert_eq!(
detect_automation_node_phase(&node, "completed", Some(&artifact_validation)),
"completed"
);
}
#[test]
fn validator_summary_reports_repair_attempt_state() {
let artifact_validation = json!({
"semantic_block_reason": "research completed without citation-backed claims",
"unmet_requirements": ["citations_missing"],
"repair_attempted": true,
"repair_attempt": 2,
"repair_attempts_remaining": 0,
"repair_succeeded": false,
"repair_exhausted": true,
});
let summary = build_automation_validator_summary(
crate::AutomationOutputValidatorKind::ResearchBrief,
"blocked",
Some("research completed without citation-backed claims"),
Some(&artifact_validation),
);
assert!(summary.repair_attempted);
assert_eq!(summary.repair_attempt, 2);
assert_eq!(summary.repair_attempts_remaining, 0);
assert!(!summary.repair_succeeded);
assert!(summary.repair_exhausted);
}
#[test]
fn artifact_validation_uses_structured_repair_exhaustion_state_from_session_text() {
let workspace_root =
std::env::temp_dir().join(format!("tandem-repair-state-test-{}", now_ms()));
std::fs::create_dir_all(workspace_root.join("inputs")).expect("create workspace");
std::fs::write(workspace_root.join("inputs/questions.md"), "Question")
.expect("seed input file");
let node = AutomationFlowNode {
knowledge: tandem_orchestrator::KnowledgeBinding::default(),
node_id: "research".to_string(),
agent_id: "agent-a".to_string(),
objective: "Research".to_string(),
depends_on: Vec::new(),
input_refs: Vec::new(),
output_contract: Some(AutomationFlowOutputContract {
kind: "brief".to_string(),
validator: None,
enforcement: None,
schema: None,
summary_guidance: None,
}),
retry_policy: None,
timeout_ms: None,
max_tool_calls: None,
stage_kind: None,
gate: None,
metadata: Some(json!({
"builder": {
"output_path": "marketing-brief.md",
"web_research_expected": true,
"source_coverage_required": true
}
})),
};
let mut session = Session::new(Some("research repair exhausted".to_string()), None);
session.messages.push(tandem_types::Message::new(
MessageRole::Assistant,
vec![MessagePart::ToolInvocation {
tool: "write".to_string(),
args: json!({
"path":"marketing-brief.md",
"content":"# Marketing Brief\n\n## Findings\nBlocked draft without citations.\n"
}),
result: Some(json!({"output":"written"})),
error: None,
}],
));
let tool_telemetry = summarize_automation_tool_activity(
&node,
&session,
&[
"glob".to_string(),
"read".to_string(),
"websearch".to_string(),
"write".to_string(),
],
);
let session_text = r#"TOOL_MODE_REQUIRED_NOT_SATISFIED: PREWRITE_REQUIREMENTS_EXHAUSTED
{"status":"blocked","reason":"prewrite requirements exhausted before final artifact validation","failureCode":"PREWRITE_REQUIREMENTS_EXHAUSTED","repairAttempt":2,"repairAttemptsRemaining":0,"repairExhausted":true,"unmetRequirements":["concrete_read_required","successful_web_research_required"]}"#;
let (_accepted_output, metadata, rejected) = validate_automation_artifact_output(
&node,
&session,
workspace_root.to_str().expect("workspace root"),
session_text,
&tool_telemetry,
None,
Some((
"marketing-brief.md".to_string(),
"# Marketing Brief\n\n## Findings\nBlocked draft without citations.\n".to_string(),
)),
&std::collections::BTreeSet::new(),
);
assert!(rejected.is_some());
assert_eq!(
metadata.get("repair_attempt").and_then(Value::as_u64),
Some(2)
);
assert_eq!(
metadata
.get("repair_attempts_remaining")
.and_then(Value::as_u64),
Some(0)
);
assert_eq!(
metadata.get("repair_exhausted").and_then(Value::as_bool),
Some(true)
);
let _ = std::fs::remove_dir_all(workspace_root);
}
#[test]
fn research_artifact_validation_requires_citations_and_web_sources_reviewed() {
let workspace_root =
std::env::temp_dir().join(format!("tandem-research-citation-test-{}", now_ms()));
std::fs::create_dir_all(workspace_root.join("inputs")).expect("create workspace");
std::fs::write(workspace_root.join("inputs/questions.md"), "Question")
.expect("seed input file");
let node = AutomationFlowNode {
knowledge: tandem_orchestrator::KnowledgeBinding::default(),
node_id: "research".to_string(),
agent_id: "agent-a".to_string(),
objective: "Research".to_string(),
depends_on: Vec::new(),
input_refs: Vec::new(),
output_contract: Some(AutomationFlowOutputContract {
kind: "brief".to_string(),
validator: None,
enforcement: None,
schema: None,
summary_guidance: None,
}),
retry_policy: None,
timeout_ms: None,
max_tool_calls: None,
stage_kind: None,
gate: None,
metadata: Some(json!({
"builder": {
"output_path": "marketing-brief.md",
"web_research_expected": true,
"source_coverage_required": true
}
})),
};
let mut session = Session::new(Some("research citations".to_string()), None);
session.messages.push(tandem_types::Message::new(
MessageRole::Assistant,
vec![
MessagePart::ToolInvocation {
tool: "read".to_string(),
args: json!({"path":"inputs/questions.md"}),
result: Some(json!({"output":"Question"})),
error: None,
},
MessagePart::ToolInvocation {
tool: "websearch".to_string(),
args: json!({"query":"market trends"}),
result: Some(json!({"output":"Search results found"})),
error: None,
},
MessagePart::ToolInvocation {
tool: "write".to_string(),
args: json!({
"path":"marketing-brief.md",
"content":"# Marketing Brief\n\n## Files reviewed\n- inputs/questions.md\n\n## Files not reviewed\n- inputs/references.md: not available in this run.\n\n## Findings\nClaims are summarized here without explicit citations.\n"
}),
result: Some(json!({"output":"written"})),
error: None,
},
],
));
let tool_telemetry = summarize_automation_tool_activity(
&node,
&session,
&[
"read".to_string(),
"write".to_string(),
"websearch".to_string(),
],
);
let (_, artifact_validation, rejected) = validate_automation_artifact_output(
&node,
&session,
workspace_root.to_str().expect("workspace root"),
"",
&tool_telemetry,
None,
Some((
"marketing-brief.md".to_string(),
"# Marketing Brief\n\n## Files reviewed\n- inputs/questions.md\n\n## Findings\nClaims are summarized here without explicit citations.\n".to_string(),
)),
&std::collections::BTreeSet::new(),
);
assert_eq!(
rejected.as_deref(),
Some("research completed without citation-backed claims")
);
assert_eq!(
artifact_validation
.get("unmet_requirements")
.and_then(Value::as_array)
.cloned()
.unwrap_or_default(),
vec![
json!("citations_missing"),
json!("web_sources_reviewed_missing")
]
);
assert_eq!(
artifact_validation
.get("artifact_candidates")
.and_then(Value::as_array)
.and_then(|rows| rows.first())
.and_then(|value| value.get("citation_count"))
.and_then(Value::as_u64),
Some(0)
);
assert_eq!(
artifact_validation
.get("citation_count")
.and_then(Value::as_u64),
Some(0)
);
assert_eq!(
artifact_validation
.get("web_sources_reviewed_present")
.and_then(Value::as_bool),
Some(false)
);
let _ = std::fs::remove_dir_all(&workspace_root);
}
#[test]
fn research_citations_validation_accepts_external_research_without_files_reviewed_section() {
let workspace_root =
std::env::temp_dir().join(format!("tandem-research-sources-test-{}", now_ms()));
std::fs::create_dir_all(workspace_root.join("inputs")).expect("create workspace");
std::fs::write(workspace_root.join("inputs/questions.md"), "Question")
.expect("seed input file");
let node = AutomationFlowNode {
knowledge: tandem_orchestrator::KnowledgeBinding::default(),
node_id: "research_sources".to_string(),
agent_id: "researcher".to_string(),
objective: "Research current web sources".to_string(),
depends_on: Vec::new(),
input_refs: Vec::new(),
output_contract: Some(AutomationFlowOutputContract {
kind: "citations".to_string(),
validator: Some(crate::AutomationOutputValidatorKind::ResearchBrief),
enforcement: None,
schema: None,
summary_guidance: Some("Return a citation handoff.".to_string()),
}),
retry_policy: None,
timeout_ms: None,
max_tool_calls: None,
stage_kind: None,
gate: None,
metadata: Some(json!({
"builder": {
"output_path": ".tandem/artifacts/research-sources.json",
"web_research_expected": true,
"source_coverage_required": true
}
})),
};
let mut session = Session::new(Some("research sources".to_string()), None);
session.messages.push(tandem_types::Message::new(
MessageRole::Assistant,
vec![
MessagePart::ToolInvocation {
tool: "read".to_string(),
args: json!({"path":"inputs/questions.md"}),
result: Some(json!({"output":"Question"})),
error: None,
},
MessagePart::ToolInvocation {
tool: "websearch".to_string(),
args: json!({"query":"autonomous AI agentic workflows 2024 2025"}),
result: Some(json!({"output":"Search results found"})),
error: None,
},
MessagePart::ToolInvocation {
tool: "write".to_string(),
args: json!({
"path":".tandem/artifacts/research-sources.json",
"content":"# Research Sources\n\n## Summary\nCurrent external research was gathered successfully.\n\n## Citations\n1. AI Agents in 2025: Expectations vs. Reality | IBM. Source note: https://www.ibm.com/think/insights/ai-agents-2025-expectations-vs-reality\n2. Agentic AI, explained | MIT Sloan. Source note: https://mitsloan.mit.edu/ideas-made-to-matter/agentic-ai-explained\n\n## Web sources reviewed\n- https://www.ibm.com/think/insights/ai-agents-2025-expectations-vs-reality\n- https://mitsloan.mit.edu/ideas-made-to-matter/agentic-ai-explained\n"
}),
result: Some(json!({"output":"written"})),
error: None,
},
],
));
let tool_telemetry = summarize_automation_tool_activity(
&node,
&session,
&[
"read".to_string(),
"write".to_string(),
"websearch".to_string(),
],
);
let (_accepted_output, artifact_validation, rejected) = validate_automation_artifact_output(
&node,
&session,
workspace_root.to_str().expect("workspace root"),
"",
&tool_telemetry,
None,
Some((
".tandem/artifacts/research-sources.json".to_string(),
"# Research Sources\n\n## Summary\nCurrent external research was gathered successfully.\n\n## Citations\n1. AI Agents in 2025: Expectations vs. Reality | IBM. Source note: https://www.ibm.com/think/insights/ai-agents-2025-expectations-vs-reality\n2. Agentic AI, explained | MIT Sloan. Source note: https://mitsloan.mit.edu/ideas-made-to-matter/agentic-ai-explained\n\n## Web sources reviewed\n- https://www.ibm.com/think/insights/ai-agents-2025-expectations-vs-reality\n- https://mitsloan.mit.edu/ideas-made-to-matter/agentic-ai-explained\n".to_string(),
)),
&std::collections::BTreeSet::new(),
);
assert!(rejected.is_none());
assert_eq!(
artifact_validation
.get("validation_outcome")
.and_then(Value::as_str),
Some("passed")
);
assert!(!artifact_validation
.get("unmet_requirements")
.and_then(Value::as_array)
.is_some_and(|values| values
.iter()
.any(|value| value.as_str() == Some("files_reviewed_missing"))));
assert!(!artifact_validation
.get("unmet_requirements")
.and_then(Value::as_array)
.is_some_and(|values| values
.iter()
.any(|value| value.as_str() == Some("files_reviewed_not_backed_by_read"))));
let _ = std::fs::remove_dir_all(&workspace_root);
}
#[test]
fn marketing_template_automation_migrates_to_split_research_flow() {
let mut automation = AutomationV2Spec {
automation_id: "automation-v2-test".to_string(),
name: "Marketing Content Pipeline".to_string(),
description: None,
status: AutomationV2Status::Active,
schedule: AutomationV2Schedule {
schedule_type: AutomationV2ScheduleType::Manual,
cron_expression: None,
interval_seconds: None,
timezone: "UTC".to_string(),
misfire_policy: RoutineMisfirePolicy::RunOnce,
},
knowledge: tandem_orchestrator::KnowledgeBinding::default(),
agents: vec![
AutomationAgentProfile {
agent_id: "research".to_string(),
template_id: None,
display_name: "Research".to_string(),
avatar_url: None,
model_policy: None,
skills: vec!["analysis".to_string()],
tool_policy: AutomationAgentToolPolicy {
allowlist: vec![
"read".to_string(),
"write".to_string(),
"websearch".to_string(),
"glob".to_string(),
],
denylist: Vec::new(),
},
mcp_policy: AutomationAgentMcpPolicy {
allowed_servers: Vec::new(),
allowed_tools: None,
},
approval_policy: None,
},
AutomationAgentProfile {
agent_id: "copywriter".to_string(),
template_id: None,
display_name: "Copywriter".to_string(),
avatar_url: None,
model_policy: None,
skills: Vec::new(),
tool_policy: AutomationAgentToolPolicy {
allowlist: vec!["read".to_string(), "write".to_string()],
denylist: Vec::new(),
},
mcp_policy: AutomationAgentMcpPolicy {
allowed_servers: Vec::new(),
allowed_tools: None,
},
approval_policy: None,
},
],
flow: AutomationFlowSpec {
nodes: vec![
AutomationFlowNode {
knowledge: tandem_orchestrator::KnowledgeBinding::default(),
node_id: "research-brief".to_string(),
agent_id: "research".to_string(),
objective: "Legacy research".to_string(),
depends_on: Vec::new(),
input_refs: Vec::new(),
output_contract: Some(AutomationFlowOutputContract {
kind: "brief".to_string(),
validator: None,
enforcement: None,
schema: None,
summary_guidance: Some("Write `marketing-brief.md`.".to_string()),
}),
retry_policy: None,
timeout_ms: None,
max_tool_calls: None,
stage_kind: None,
gate: None,
metadata: Some(json!({
"builder": {
"title": "Research Brief",
"role": "watcher",
"output_path": "marketing-brief.md",
"prompt": "Legacy one-shot research prompt"
},
"studio": {
"output_path": "marketing-brief.md"
}
})),
},
AutomationFlowNode {
knowledge: tandem_orchestrator::KnowledgeBinding::default(),
node_id: "draft-copy".to_string(),
agent_id: "copywriter".to_string(),
objective: "Draft copy".to_string(),
depends_on: vec!["research-brief".to_string()],
input_refs: vec![AutomationFlowInputRef {
from_step_id: "research-brief".to_string(),
alias: "marketing_brief".to_string(),
}],
output_contract: Some(AutomationFlowOutputContract {
kind: "draft".to_string(),
validator: None,
enforcement: None,
schema: None,
summary_guidance: None,
}),
retry_policy: None,
timeout_ms: None,
max_tool_calls: None,
stage_kind: None,
gate: None,
metadata: None,
},
],
},
execution: AutomationExecutionPolicy {
max_parallel_agents: Some(1),
max_total_runtime_ms: None,
max_total_tool_calls: None,
max_total_tokens: None,
max_total_cost_usd: None,
},
output_targets: Vec::new(),
created_at_ms: 1,
updated_at_ms: 1,
creator_id: "test".to_string(),
workspace_root: Some("/tmp/workspace".to_string()),
metadata: Some(json!({
"studio": {
"template_id": "marketing-content-pipeline",
"version": 1,
"agent_drafts": [{"agentId":"research"}],
"node_drafts": [{"nodeId":"research-brief"}]
}
})),
next_fire_at_ms: None,
last_fired_at_ms: None,
scope_policy: None,
watch_conditions: Vec::new(),
handoff_config: None,
};
assert!(migrate_bundled_studio_research_split_automation(
&mut automation
));
assert!(automation
.flow
.nodes
.iter()
.any(|node| node.node_id == "research-discover-sources"));
assert!(automation
.flow
.nodes
.iter()
.any(|node| node.node_id == "research-local-sources"));
assert!(automation
.flow
.nodes
.iter()
.any(|node| node.node_id == "research-external-research"));
let discover_node = automation
.flow
.nodes
.iter()
.find(|node| node.node_id == "research-discover-sources")
.expect("discover node present");
let discover_enforcement = discover_node
.output_contract
.as_ref()
.and_then(|contract| contract.enforcement.as_ref())
.expect("discover enforcement");
assert!(discover_enforcement
.required_tools
.iter()
.any(|tool| tool == "read"));
assert!(discover_enforcement
.prewrite_gates
.iter()
.any(|gate| gate == "workspace_inspection"));
assert!(discover_enforcement
.prewrite_gates
.iter()
.any(|gate| gate == "concrete_reads"));
let final_node = automation
.flow
.nodes
.iter()
.find(|node| node.node_id == "research-brief")
.expect("final node preserved");
assert_eq!(
automation_node_research_stage(final_node).as_deref(),
Some("research_finalize")
);
assert_eq!(final_node.depends_on.len(), 3);
assert!(automation
.agents
.iter()
.any(|agent| agent.agent_id == "research-discover"));
assert!(automation
.agents
.iter()
.any(|agent| agent.agent_id == "research-local-sources"));
assert!(automation
.agents
.iter()
.any(|agent| agent.agent_id == "research-external"));
let studio = automation
.metadata
.as_ref()
.and_then(|value| value.get("studio"))
.and_then(Value::as_object)
.expect("studio metadata");
assert_eq!(studio.get("version").and_then(Value::as_u64), Some(2));
assert_eq!(
studio
.get("workflow_structure_version")
.and_then(Value::as_u64),
Some(2)
);
assert!(!studio.contains_key("agent_drafts"));
assert!(!studio.contains_key("node_drafts"));
}