#![cfg(feature = "postgres")]
use std::path::PathBuf;
use axum::body::{to_bytes, Body, Bytes};
use axum::http::{Request, StatusCode};
use axum::Router;
use rusty_agent_server::{router, GraphRegistry, ServerConfig};
use serde_json::{json, Value};
use tower::ServiceExt;
fn database_url() -> String {
std::env::var("DATABASE_URL").expect(
"DATABASE_URL must point at a scratch Postgres database \
(e.g. postgres://user:pass@localhost/rusty_test)",
)
}
fn postgres_app() -> Router {
let store_path: PathBuf = std::env::temp_dir().join(format!(
"rusty-server-pg-supervision-{}",
uuid::Uuid::new_v4()
));
let config = ServerConfig::new("127.0.0.1:0".parse().unwrap(), store_path)
.with_postgres(database_url())
.with_tenant_key("pg-supervision", "pg-supervision-secret");
router(GraphRegistry::new(), config)
}
async fn call(app: &Router, method: &str, uri: &str, body: Option<Value>) -> (StatusCode, Value) {
let mut builder = Request::builder()
.method(method)
.uri(uri)
.header("x-api-key", "pg-supervision-secret");
let body = match body {
Some(v) => {
builder = builder.header("content-type", "application/json");
Body::from(v.to_string())
}
None => Body::empty(),
};
let response = app
.clone()
.oneshot(builder.body(body).unwrap())
.await
.unwrap();
let status = response.status();
let bytes: Bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap();
let value = if bytes.is_empty() {
Value::Null
} else {
serde_json::from_slice(&bytes).unwrap_or(Value::Null)
};
(status, value)
}
fn uniq() -> String {
uuid::Uuid::new_v4().simple().to_string()
}
fn manifest_with(extra: Value) -> Value {
let mut manifest = json!({
"agent_kind": "worker",
"manifest_version": "worker/1.0.0",
"accepts": {"work": {"kind": "application/json"}}
});
manifest
.as_object_mut()
.unwrap()
.extend(extra.as_object().unwrap().clone());
manifest
}
fn policy(restart: &str, intensity: u32, supervisor: Option<&str>) -> Value {
let mut p = json!({"restart": restart, "intensity": intensity, "period_ms": 60_000});
if let Some(s) = supervisor {
p["supervisor"] = json!(s);
}
p
}
async fn register_with(app: &Router, agent_id: &str, manifest: Value, team_id: Option<&str>) {
let mut body = json!({"agent_id": agent_id, "manifest": manifest});
if let Some(team) = team_id {
body["team_id"] = json!(team);
}
let (status, v) = call(app, "POST", "/agents", Some(body)).await;
assert_eq!(status, StatusCode::CREATED, "register failed: {v}");
}
async fn activate_one(app: &Router, agent_id: &str, worker: &str, lease_ms: u64) -> u64 {
let (status, v) = call(
app,
"POST",
&format!("/agents/{agent_id}/activate"),
Some(json!({"worker_id": worker, "lease_ms": lease_ms})),
)
.await;
assert_eq!(status, StatusCode::OK, "activate failed: {v}");
v["fencing"].as_u64().unwrap()
}
async fn send_one(app: &Router, agent_id: &str, extra: Value) -> String {
let mut body = json!({"kind": "work", "payload": {"n": 1}});
body.as_object_mut()
.unwrap()
.extend(extra.as_object().unwrap().clone());
let (status, v) = call(
app,
"POST",
&format!("/agents/{agent_id}/mailbox"),
Some(body),
)
.await;
assert_eq!(status, StatusCode::CREATED, "send failed: {v}");
v["task_id"].as_str().unwrap().to_string()
}
async fn next(app: &Router, agent_id: &str, worker: &str, fencing: u64) -> (StatusCode, Value) {
call(
app,
"POST",
&format!("/agents/{agent_id}/mailbox/next"),
Some(json!({"worker_id": worker, "fencing": fencing, "lease_ms": 30_000})),
)
.await
}
async fn next_one(app: &Router, agent_id: &str, worker: &str, fencing: u64) -> Value {
let (status, v) = next(app, agent_id, worker, fencing).await;
assert_eq!(status, StatusCode::OK, "claim failed: {v}");
v["task"].clone()
}
async fn fail_turn(app: &Router, task_id: &str, worker: &str, error_class: &str) -> Value {
let (status, v) = call(
app,
"POST",
&format!("/tasks/{task_id}/fail"),
Some(json!({"worker_id": worker, "error_class": error_class,
"message": format!("turn crashed ({error_class})"), "retryable": true})),
)
.await;
assert_eq!(status, StatusCode::OK, "fail failed: {v}");
v
}
async fn supervision(app: &Router, agent_id: &str) -> Value {
let (status, v) = call(app, "GET", &format!("/agents/{agent_id}/supervision"), None).await;
assert_eq!(status, StatusCode::OK, "supervision failed: {v}");
v
}
#[tokio::test]
#[ignore = "requires DATABASE_URL"]
async fn pg_crash_loop_escalates_to_the_supervisor_with_history() {
let app = postgres_app();
let run = uniq();
let boss = format!("boss-{run}");
let looper = format!("looper-{run}");
register_with(
&app,
&boss,
manifest_with(json!({
"accepts": {"escalated": {"kind": "application/json"}}
})),
None,
)
.await;
register_with(
&app,
&looper,
manifest_with(json!({
"supervision": policy("permanent", 2, Some(&boss))
})),
None,
)
.await;
let looper_fence = activate_one(&app, &looper, "host-1", 60_000).await;
let boss_fence = activate_one(&app, &boss, "host-boss", 60_000).await;
let task_id = send_one(&app, &looper, json!({"max_attempts": 10})).await;
let mut escalation_report = Value::Null;
for (round, backoff_ms) in [(1u64, 1_100u64), (2, 2_100), (3, 4_100)] {
let task = next_one(&app, &looper, "host-1", looper_fence).await;
assert_eq!(task["task_id"], json!(task_id));
assert_eq!(task["attempt"], json!(round));
let v = fail_turn(&app, &task_id, "host-1", "transient").await;
if round == 3 {
escalation_report = v["escalation"].clone();
} else {
assert_eq!(v["escalation"], Value::Null);
}
tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await;
}
assert_eq!(escalation_report["kind"], json!("mailbox"));
let escalation = next_one(&app, &boss, "host-boss", boss_fence).await;
assert_eq!(escalation["kind"], json!("escalated"));
assert_eq!(
escalation["idempotency_key"],
json!(format!("escalation:{looper}:3"))
);
let notice = &escalation["payload"];
assert_eq!(notice["agent_id"], json!(looper));
assert_eq!(notice["policy"]["restart"], json!("permanent"));
let attempts = notice["attempts"].as_array().unwrap();
assert_eq!(attempts.len(), 3);
for (i, attempt) in attempts.iter().enumerate() {
assert_eq!(attempt["ordinal"], json!(i as u64 + 1));
assert_eq!(attempt["trigger"], json!("turn_failed"));
assert_eq!(attempt["task_id"], json!(task_id));
}
let v = supervision(&app, &looper).await;
assert_eq!(v["escalated"], json!(true));
assert_eq!(
v["journal_run_id"],
json!(format!("agent-supervision:pg-supervision:{looper}"))
);
let events = v["events"].as_array().unwrap();
assert_eq!(events.len(), 3);
for (event, decision) in events.iter().zip(["restart", "restart", "escalate"]) {
assert_eq!(event["kind"], json!("supervision_event"));
assert_eq!(event["output"]["value"]["decision"], json!(decision));
}
}
#[tokio::test]
#[ignore = "requires DATABASE_URL"]
async fn pg_team_cancel_leaves_zero_orphan_tasks() {
let app = postgres_app();
let run = uniq();
let team = format!("squad-{}", &run[..8]);
let members: Vec<String> = (1..=3).map(|i| format!("m{i}-{run}")).collect();
for member in &members {
register_with(&app, member, manifest_with(json!({})), Some(&team)).await;
}
let m1_leased = send_one(&app, &members[0], json!({})).await;
send_one(&app, &members[0], json!({})).await;
send_one(&app, &members[1], json!({})).await;
send_one(&app, &members[2], json!({})).await;
let fence = activate_one(&app, &members[0], "host-1", 30_000).await;
let turn = next_one(&app, &members[0], "host-1", fence).await;
assert_eq!(turn["task_id"], json!(m1_leased));
let (status, v) = call(&app, "POST", &format!("/teams/{team}/cancel"), None).await;
assert_eq!(status, StatusCode::OK, "team cancel failed: {v}");
let cancelled_members = v["members"].as_array().unwrap();
assert_eq!(cancelled_members.len(), 3);
for member in cancelled_members {
assert!(member["exit_event"].is_string(), "member: {member}");
}
let v = fail_turn(&app, &m1_leased, "host-1", "cancelled").await;
assert_eq!(v["requeued"], json!(false));
let (status, v) = call(&app, "GET", "/tasks", None).await;
assert_eq!(status, StatusCode::OK);
let tasks = v.as_array().unwrap();
let recipients: Vec<String> = members.iter().map(|m| format!("agent:{m}")).collect();
let member_tasks: Vec<&Value> = tasks
.iter()
.filter(|t| {
t["recipient"]
.as_str()
.is_some_and(|r| recipients.iter().any(|m| m == r))
})
.collect();
assert_eq!(member_tasks.len(), 4);
for t in &member_tasks {
assert_eq!(t["status"], json!("cancelled"), "orphan survived: {t}");
assert_eq!(t["error_class"], json!("cancelled"), "orphan survived: {t}");
}
}
#[tokio::test]
#[ignore = "requires DATABASE_URL"]
async fn pg_supervision_state_survives_a_fresh_store_instance() {
let run = uniq();
let tempy = format!("tempy-{run}");
let app = postgres_app();
register_with(
&app,
&tempy,
manifest_with(json!({
"supervision": policy("temporary", 5, None)
})),
None,
)
.await;
let fence = activate_one(&app, &tempy, "host-1", 30_000).await;
let task_id = send_one(&app, &tempy, json!({})).await;
next_one(&app, &tempy, "host-1", fence).await;
let v = fail_turn(&app, &task_id, "host-1", "transient").await;
assert_eq!(v["escalation"]["kind"], json!("dead_letter"));
drop(app);
let app = postgres_app();
let v = supervision(&app, &tempy).await;
assert_eq!(v["escalated"], json!(true));
assert_eq!(v["attempts"].as_array().unwrap().len(), 1);
assert_eq!(v["attempts"][0]["trigger"], json!("turn_failed"));
let events = v["events"].as_array().unwrap();
assert_eq!(events.len(), 1);
assert_eq!(events[0]["output"]["value"]["decision"], json!("escalate"));
let (status, v) = call(&app, "GET", "/tasks?status=dead", None).await;
assert_eq!(status, StatusCode::OK);
let dead: Vec<&Value> = v
.as_array()
.unwrap()
.iter()
.filter(|t| t["payload"]["agent_id"] == json!(tempy))
.collect();
assert_eq!(dead.len(), 1);
assert_eq!(dead[0]["kind"], json!("escalated"));
assert_eq!(
dead[0]["idempotency_key"],
json!(format!("escalation:{tempy}:1"))
);
}