use crate::error::{ApiError, ApiErrorCode};
use crate::ServerState;
use axum::body::Bytes;
use axum::extract::{Path as UrlPath, State};
use axum::http::StatusCode;
use axum::response::IntoResponse;
use axum::Json;
use kranz_engine::backend::{AgentBackend, AgentEvent, PromptMode, SessionExit, SessionSpec};
use kranz_engine::backend_claude::ClaudeBackend;
use kranz_engine::config;
use kranz_engine::cost::{self, CostEstimate};
use kranz_engine::deps;
use kranz_engine::draft::{drive_draft, DraftOutcome};
use kranz_engine::error::EngineError;
use kranz_engine::event_log::{EventLog, LockForce};
use kranz_engine::git_ops::GitRepo;
use kranz_engine::git_ops::KranzCommitMetadata;
use kranz_engine::merge::{
merge_mission_with_external_evidence, MergeReport, StandardsMergeEvidence,
};
use kranz_engine::orchestrator::{MissionEngine, PlanRequest};
use kranz_engine::paths::MissionPaths;
use kranz_engine::planning::plan_identity;
use kranz_engine::queue;
use kranz_engine::ticket::Ticket;
use kranz_engine::types::{MissionConfig, MissionStatus, Plan, TokenUsage};
use serde_json::{json, Value};
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
type EngineCell = Arc<tokio::sync::Mutex<Box<MissionEngine>>>;
enum HostedMission {
Planning {
cell: EngineCell,
last_use: Arc<Mutex<Instant>>,
pending_plan: Arc<Mutex<Option<Plan>>>,
},
Running {
handle: tokio::task::JoinHandle<()>,
_repo_busy: kranz_engine::queue::RepoBusyHold,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PendingApproval {
Approved(String),
NothingParked,
Mismatch { parked: String },
}
pub struct MissionHost {
repo_root: PathBuf,
backend: tokio::sync::OnceCell<Arc<dyn AgentBackend>>,
missions: Arc<Mutex<HashMap<String, HostedMission>>>,
sweeper: Mutex<Option<tokio::task::JoinHandle<()>>>,
drain: Mutex<DrainSlot>,
global_run_permits: Option<Arc<Semaphore>>,
gate_executor: GateExecutor,
readiness_front_cache: Mutex<Option<FrontReadinessCache>>,
readiness_probe: ReadinessProbe,
}
struct FrontReadinessCache {
mission_id: String,
report: Value,
at: Instant,
}
const READINESS_FRONT_CACHE_TTL: Duration = Duration::from_secs(5);
#[derive(Debug, Clone, Default)]
struct DrainState {
live: bool,
current_mission_id: Option<String>,
ran: Vec<String>,
parked: Vec<String>,
}
type GateExecutor = Arc<dyn Fn(&str, &Path) -> (bool, String) + Send + Sync>;
type ReadinessProbe =
fn(
&Path,
&str,
) -> kranz_engine::error::Result<kranz_engine::backend_readiness::ReadinessReport>;
fn injected_backend_readiness(
_repo_root: &Path,
mission_id: &str,
) -> kranz_engine::error::Result<kranz_engine::backend_readiness::ReadinessReport> {
Ok(kranz_engine::backend_readiness::ReadinessReport {
mission_id: mission_id.to_string(),
roles: Vec::new(),
overall: kranz_engine::backend_readiness::ReadinessStatus::Ok,
warnings: Vec::new(),
})
}
fn real_gate_executor() -> GateExecutor {
Arc::new(|command, cwd| kranz_engine::command_exec::run_bounded_gate_command(cwd, command))
}
fn should_auto_drain(auto_work: bool, queue_non_empty: bool, drain_live: bool) -> bool {
auto_work && queue_non_empty && !drain_live
}
fn drain_state_json(state: &DrainState) -> Value {
json!({
"live": state.live,
"currentMissionId": state.current_mission_id,
"ran": state.ran,
"parked": state.parked,
})
}
struct DrainHandle {
join: tokio::task::JoinHandle<()>,
state: Arc<Mutex<DrainState>>,
}
enum DrainSlot {
Idle,
Starting(Arc<Mutex<DrainState>>),
Running(DrainHandle),
}
impl MissionHost {
pub fn new(repo_root: PathBuf) -> Self {
MissionHost {
repo_root,
backend: tokio::sync::OnceCell::new(),
missions: Arc::new(Mutex::new(HashMap::new())),
sweeper: Mutex::new(None),
drain: Mutex::new(DrainSlot::Idle),
global_run_permits: None,
gate_executor: real_gate_executor(),
readiness_front_cache: Mutex::new(None),
readiness_probe: kranz_engine::backend_readiness::probe_mission,
}
}
pub fn with_backend(repo_root: PathBuf, backend: Arc<dyn AgentBackend>) -> Self {
MissionHost {
repo_root,
backend: tokio::sync::OnceCell::new_with(Some(backend)),
missions: Arc::new(Mutex::new(HashMap::new())),
sweeper: Mutex::new(None),
drain: Mutex::new(DrainSlot::Idle),
global_run_permits: None,
gate_executor: real_gate_executor(),
readiness_front_cache: Mutex::new(None),
readiness_probe: injected_backend_readiness,
}
}
pub fn with_gate_executor<F>(repo_root: PathBuf, gate_executor: F) -> Self
where
F: Fn(&str, &Path) -> (bool, String) + Send + Sync + 'static,
{
MissionHost {
repo_root,
backend: tokio::sync::OnceCell::new(),
missions: Arc::new(Mutex::new(HashMap::new())),
sweeper: Mutex::new(None),
drain: Mutex::new(DrainSlot::Idle),
global_run_permits: None,
gate_executor: Arc::new(gate_executor),
readiness_front_cache: Mutex::new(None),
readiness_probe: kranz_engine::backend_readiness::probe_mission,
}
}
pub(crate) fn new_with_global_run_permits(
repo_root: PathBuf,
global_run_permits: Arc<Semaphore>,
) -> Self {
MissionHost {
repo_root,
backend: tokio::sync::OnceCell::new(),
missions: Arc::new(Mutex::new(HashMap::new())),
sweeper: Mutex::new(None),
drain: Mutex::new(DrainSlot::Idle),
global_run_permits: Some(global_run_permits),
gate_executor: real_gate_executor(),
readiness_front_cache: Mutex::new(None),
readiness_probe: kranz_engine::backend_readiness::probe_mission,
}
}
#[cfg(test)]
pub(crate) fn with_backend_and_global_run_permits(
repo_root: PathBuf,
backend: Arc<dyn AgentBackend>,
global_run_permits: Arc<Semaphore>,
) -> Self {
MissionHost {
repo_root,
backend: tokio::sync::OnceCell::new_with(Some(backend)),
missions: Arc::new(Mutex::new(HashMap::new())),
sweeper: Mutex::new(None),
drain: Mutex::new(DrainSlot::Idle),
global_run_permits: Some(global_run_permits),
gate_executor: real_gate_executor(),
readiness_front_cache: Mutex::new(None),
readiness_probe: injected_backend_readiness,
}
}
pub fn repo_root(&self) -> &PathBuf {
&self.repo_root
}
pub(crate) fn try_global_run_permit(&self) -> Result<Option<OwnedSemaphorePermit>, ApiError> {
self.global_run_permits
.as_ref()
.map(|permits| {
Arc::clone(permits).try_acquire_owned().map_err(|_| {
ApiError::conflict(
"host.maxConcurrentRepos is saturated; retry when another repository finishes",
)
.with_code(ApiErrorCode::RepositoryBusy)
})
})
.transpose()
}
async fn backend(
&self,
claude_binary: Option<&str>,
) -> Result<Arc<dyn AgentBackend>, ApiError> {
let configured = claude_binary.map(str::to_string);
self.backend
.get_or_try_init(|| async move {
let backend = ClaudeBackend::discover(configured.as_deref())?;
Ok::<Arc<dyn AgentBackend>, EngineError>(Arc::new(backend))
})
.await
.map(Arc::clone)
.map_err(ApiError::from)
}
pub async fn create(
&self,
goal: &str,
config_patch: Option<&Value>,
) -> Result<String, ApiError> {
let mut cfg = config::load(&self.repo_root)?;
if let Some(patch) = config_patch {
if !patch.is_object() {
return Err(ApiError::bad_request("'config' must be a JSON object"));
}
let mut merged = serde_json::to_value(&cfg)
.map_err(|e| ApiError::internal(format!("config does not serialize: {e}")))?;
config::deep_merge(&mut merged, patch);
cfg = serde_json::from_value(merged).map_err(|e| {
ApiError::bad_request(format!("'config' patch does not deserialize: {e}"))
})?;
}
config::validate(&cfg)?;
let backend = self.backend(cfg.claude_binary.as_deref()).await?;
let engine = MissionEngine::create(backend, self.repo_root.clone(), goal, cfg)?;
let id = engine.mission_id().to_string();
self.missions
.lock()
.expect("missions registry lock")
.insert(id.clone(), new_planning(new_cell(Box::new(engine))));
self.ensure_sweeper_started();
Ok(id)
}
pub async fn draft(&self, slug: &str, then_enqueue: bool) -> Result<DraftOutcome, ApiError> {
Ticket::ensure_valid_slug(slug)?;
let ticket_path = Ticket::tickets_dir(&self.repo_root).join(format!("{slug}.md"));
if !ticket_path.is_file() {
return Err(ApiError::not_found(format!("ticket '{slug}' not found")));
}
let ticket = Ticket::load(&ticket_path)?;
let cfg = config_for_ticket(config::load(&self.repo_root)?, &ticket);
let backend = self.backend(cfg.claude_binary.as_deref()).await?;
let engine =
MissionEngine::create(backend, self.repo_root.clone(), &ticket.mission_goal(), cfg)?;
let id = engine.mission_id().to_string();
let cell = new_cell(Box::new(engine));
self.missions
.lock()
.expect("missions registry lock")
.insert(id.clone(), new_planning(Arc::clone(&cell)));
self.ensure_sweeper_started();
let drive_result = {
let mut engine = cell.lock().await;
drive_draft(&mut engine, &self.repo_root, &ticket, then_enqueue).await
};
self.missions
.lock()
.expect("missions registry lock")
.remove(&id);
drop(cell);
Ok(drive_result?.outcome)
}
pub async fn draft_async(&self, slug: &str, then_enqueue: bool) -> Result<String, ApiError> {
Ticket::ensure_valid_slug(slug)?;
let ticket_path = Ticket::tickets_dir(&self.repo_root).join(format!("{slug}.md"));
if !ticket_path.is_file() {
return Err(ApiError::not_found(format!("ticket '{slug}' not found")));
}
let ticket = Ticket::load(&ticket_path)?;
let cfg = config_for_ticket(config::load(&self.repo_root)?, &ticket);
let backend = self.backend(cfg.claude_binary.as_deref()).await?;
let engine =
MissionEngine::create(backend, self.repo_root.clone(), &ticket.mission_goal(), cfg)?;
let id = engine.mission_id().to_string();
let cell = new_cell(Box::new(engine));
self.missions
.lock()
.expect("missions registry lock")
.insert(id.clone(), new_planning(Arc::clone(&cell)));
self.ensure_sweeper_started();
let repo_root = self.repo_root.clone();
let missions = Arc::clone(&self.missions);
let mission_id = id.clone();
tokio::spawn(async move {
let drive_result = {
let mut engine = cell.lock().await;
drive_draft(&mut engine, &repo_root, &ticket, then_enqueue).await
};
missions
.lock()
.expect("missions registry lock")
.remove(&mission_id);
drop(cell);
if let Err(e) = drive_result {
tracing::error!(mission = %mission_id, error = %e, "hosted ticket draft errored");
}
});
Ok(id)
}
pub fn approve_ticket(
&self,
slug: &str,
force: bool,
) -> Result<deps::ApprovedTicket, ApiError> {
deps::approve_ticket(&self.repo_root, slug, None, force).map_err(ApiError::from)
}
pub async fn planning_turn(&self, id: &str, text: &str) -> Result<String, ApiError> {
let cell = self.planning_cell_or_attach(id).await?;
let mut engine = try_lock(&cell)?;
let reply = engine.planning_turn(text).await?;
Ok(prepend_seed(engine.take_seed_reply(), reply))
}
pub async fn request_plan(&self, id: &str) -> Result<Value, ApiError> {
let cell = self.planning_cell_or_attach(id).await?;
let mut engine = try_lock(&cell)?;
let request = engine.request_plan().await?;
let seed = engine.take_seed_reply();
match request {
PlanRequest::Ready(plan) => {
let calibration = cost::calibrate(&self.repo_root);
let estimate = cost::estimate(&plan, &engine.state().config, &calibration.params);
let estimate = cost::apply_shape(estimate, &plan, &calibration);
self.set_pending_plan(id, Some(plan.clone()));
Ok(json!({
"ready": true,
"planIdentity": plan_identity(&plan),
"plan": plan,
"estimate": estimate_json(&estimate),
"calibration": { "missionsUsed": calibration.missions_used },
}))
}
PlanRequest::NotReady(reply) | PlanRequest::WrongPlan { reason: reply } => {
Ok(json!({ "ready": false, "reply": prepend_seed(seed, reply) }))
}
}
}
pub async fn approve(&self, id: &str, plan: Plan) -> Result<String, ApiError> {
let cell = self.planning_cell_or_attach(id).await?;
let mut engine = try_lock(&cell)?;
engine.approve_plan_as(
plan,
kranz_engine::live_permission::Actor::LocalMutationCapability,
)?;
self.set_pending_plan(id, None);
Ok(engine.state().mission.mission_branch.clone())
}
pub async fn start(&self, id: &str) -> Result<(), ApiError> {
let taken: Option<Box<MissionEngine>> = {
let mut map = self.missions.lock().expect("missions registry lock");
match map.remove(id) {
None => None,
Some(HostedMission::Running { handle, _repo_busy }) => {
if handle.is_finished() {
drop(_repo_busy);
None
} else {
map.insert(
id.to_string(),
HostedMission::Running { handle, _repo_busy },
);
return Err(ApiError::conflict(format!(
"mission '{id}' is already running — observe it via GET \
/api/missions/{id}/state or steer it via POST \
/api/missions/{id}/control"
)));
}
}
Some(HostedMission::Planning {
cell,
last_use,
pending_plan,
}) => match Arc::try_unwrap(cell) {
Err(cell) => {
map.insert(
id.to_string(),
HostedMission::Planning {
cell,
last_use,
pending_plan,
},
);
return Err(turn_in_flight());
}
Ok(mutex) => {
let engine = mutex.into_inner();
if engine.state().mission.status == MissionStatus::Planning {
map.insert(id.to_string(), new_planning(new_cell(engine)));
return Err(ApiError::conflict(format!(
"mission '{id}' has no approved plan yet — approve one via \
POST /api/missions/{id}/approve first"
)));
}
Some(engine)
}
},
}
};
let (engine, from_registry) = match taken {
Some(engine) => (engine, true),
None => {
if !MissionPaths::new(&self.repo_root, id)
.events_file()
.is_file()
{
return Err(ApiError::not_found(format!("unknown mission '{id}'")));
}
let cfg = config::load(&self.repo_root)?;
let backend = self.backend(cfg.claude_binary.as_deref()).await?;
let engine = Box::new(MissionEngine::resume(
backend,
self.repo_root.clone(),
id,
LockForce::No,
)?);
match engine.state().mission.status {
MissionStatus::Planning => {
return Err(ApiError::conflict(format!(
"mission '{id}' is still in planning — approve a plan first \
(POST /api/missions/{id}/approve, or `kranz plan`)"
)))
}
MissionStatus::Complete => {
return Err(ApiError::conflict(format!(
"mission '{id}' is already complete — nothing to run"
)))
}
MissionStatus::Failed => {
return Err(ApiError::conflict(format!(
"mission '{id}' has failed — inspect its log; there is nothing \
the engine can resume"
)))
}
_ => {}
}
(engine, false)
}
};
let global_run_permit = match self.try_global_run_permit() {
Ok(permit) => permit,
Err(error) => {
if from_registry {
self.missions
.lock()
.expect("missions registry lock")
.insert(id.to_string(), new_planning(new_cell(engine)));
}
return Err(error);
}
};
let repo_busy = match kranz_engine::queue::acquire_repo_busy(&self.repo_root, id) {
Ok(hold) => hold,
Err(e) => {
if from_registry {
self.missions
.lock()
.expect("missions registry lock")
.insert(id.to_string(), new_planning(new_cell(engine)));
}
return Err(match e {
e @ EngineError::LockHeld(_) => {
ApiError::from(e).with_code(ApiErrorCode::RepositoryBusy)
}
other => other.into(),
});
}
};
{
let mut map = self.missions.lock().expect("missions registry lock");
let missions = Arc::clone(&self.missions);
let mission_id = id.to_string();
let handle = spawn_with_global_run_permit(
global_run_permit,
run_to_end(engine, mission_id, missions),
);
map.insert(
id.to_string(),
HostedMission::Running {
handle,
_repo_busy: repo_busy,
},
);
}
Ok(())
}
pub async fn merge(&self, id: &str) -> Result<Value, ApiError> {
if !MissionPaths::is_safe_id(id) {
return Err(ApiError::not_found(format!("unknown mission '{id}'")));
}
let paths = MissionPaths::new(&self.repo_root, id);
if !paths.events_file().is_file() {
return Err(ApiError::not_found(format!("unknown mission '{id}'")));
}
let repo_busy = kranz_engine::queue::acquire_repo_busy(&self.repo_root, id).map_err(
|error| match error {
error @ EngineError::LockHeld(_) => {
ApiError::from(error).with_code(ApiErrorCode::RepositoryBusy)
}
other => other.into(),
},
)?;
let events = EventLog::read_events(&paths.events_file())?;
let state = kranz_engine::reducer::fold(&events).map_err(ApiError::from)?;
if state.mission.status != MissionStatus::Complete {
return Err(ApiError::conflict(format!(
"mission '{id}' is {:?}; only a complete mission can be merged",
state.mission.status
)));
}
let base_branch = state.mission.base_branch.clone();
let base_sha = state.mission.base_sha.clone().ok_or_else(|| {
ApiError::conflict(format!(
"mission '{id}' has no pinned base sha — approve a plan first"
))
})?;
let mission_branch = state.mission.mission_branch.clone();
let standards_pin = state.mission.standards_manifest.clone();
let standards_coverage = kranz_engine::standards_coverage::standards_coverage(id, &events);
let standards_evidence = StandardsMergeEvidence::from_mission_events(
id,
standards_pin.as_ref(),
standards_coverage.as_ref(),
&events,
chrono::Utc::now(),
);
let metadata = KranzCommitMetadata {
mission_id: state.mission.id.clone(),
cost_usd: state.total_cost_usd,
tokens: state.totals.clone(),
};
let repo_root = self.repo_root.clone();
let gate_executor = Arc::clone(&self.gate_executor);
let gate_policy = kranz_engine::command_exec::MergeGatePolicy {
sandbox: kranz_engine::command_exec::worker_gate_sandbox(&state.config)?,
mission_dir: paths.mission_dir(),
};
if let Some(note) = gate_policy.degradation_note() {
tracing::warn!(mission = %id, note = %note, "merge gate sandbox cannot wrap; gates fail closed");
}
let merge_paths = paths.clone();
let report = tokio::task::spawn_blocking(move || {
let repo = GitRepo::open(&repo_root)?;
let report = merge_mission_with_external_evidence(
&repo,
&base_branch,
&base_sha,
&mission_branch,
Some(metadata),
standards_pin.as_ref(),
&standards_evidence,
|cmd, cwd| {
if gate_policy.enforces_on_this_host() {
kranz_engine::command_exec::run_bounded_gate_command_sandboxed(
cwd,
cmd,
&gate_policy,
)
} else {
gate_executor(cmd, cwd)
}
},
&merge_paths,
kranz_engine::live_permission::Actor::LocalMutationCapability,
);
drop(repo_busy);
report
})
.await
.map_err(|e| ApiError::internal(format!("merge task panicked: {e}")))?
.map_err(ApiError::from)?;
match report {
MergeReport::Merged { commit, stale_base } => Ok(json!({
"merged": true,
"commit": commit,
"staleBase": stale_base.map(|warning| json!({
"baseSha": warning.base_sha,
"liveBase": warning.live_base,
"mergeCommitsSinceBase": warning.merge_commits_since_base,
"message": format!(
"stale base: {} merge commit(s) landed on {} since the mission base; cross-branch semantic conflicts are more likely, and full gates have run",
warning.merge_commits_since_base,
warning.live_base,
),
})),
})),
MergeReport::RefusedDirtyTree => Err(ApiError::conflict(
"refusing to merge: tracked working tree is dirty",
)),
MergeReport::GateFailed { gate, output } => Err(ApiError::unprocessable(
kranz_engine::scrub::scrub(&format!("{gate} failed:\n{output}")),
)),
MergeReport::GateConfigInvalid { detail } => Err(ApiError::unprocessable(format!(
"refusing to merge without a valid repo gate suite: {detail}"
))),
MergeReport::SecretScanFailed { findings } => Err(ApiError::unprocessable(format!(
"secret scan failed; add a fingerprint to {} only for a reviewed false positive:\n{}",
kranz_engine::scrub::SECRET_ALLOWLIST_PATH,
kranz_engine::scrub::format_findings(&findings)
))),
MergeReport::Conflict { files } => Err(ApiError::conflict(format!(
"merge conflicted in: {}",
files.join(", ")
))),
MergeReport::RefusedPreMerge { detail } => Err(ApiError::conflict(format!(
"merge refused before it started: {detail}"
))),
MergeReport::StandardsDrifted {
approved_digest,
current_digest,
changed_rules,
} => {
if let Err(error) = EventLog::acquire(
&paths,
id,
std::time::Duration::ZERO,
LockForce::No,
)
.and_then(|mut log| {
log.append(kranz_engine::events::EventKind::StandardsDrifted {
approved_digest: approved_digest.clone(),
current_digest: current_digest.clone(),
surface: "merge".to_string(),
changed_rules: changed_rules.clone(),
})
.map(|_| ())
}) {
tracing::warn!(mission = %id, %error, "standards.drifted event could not be appended; the merge refusal stands");
}
Err(ApiError::unprocessable(format!(
"refusing to merge: the live base Flight Rules policy drifted from the \
approved pin (approved sha256:{approved_digest}, current {}) — the \
applicable enforced set changed; revalidate and re-approve the mission:\n{}",
current_digest
.as_deref()
.map(|d| format!("sha256:{d}"))
.unwrap_or_else(|| "<unreadable>".to_string()),
changed_rules.join("\n")
)))
}
MergeReport::StandardsFailed {
rule_id,
checker,
output,
} => Err(ApiError::unprocessable(kranz_engine::scrub::scrub(
&format!(
"Flight Rules merge checker refused {rule_id} ({checker}):\n{output}"
),
))),
}
}
pub async fn ask(&self, question: &str) -> Result<Value, ApiError> {
let question = question.trim();
if question.is_empty() {
return Err(ApiError::bad_request("ask requires a question"));
}
let cfg = config::load(&self.repo_root)?;
config::validate(&cfg)?;
let role = cfg.validator_scrutiny.clone();
let backend = self.backend(cfg.claude_binary.as_deref()).await?;
let prompt = ask_prompt(question, &ask_context(&self.repo_root));
let spec = SessionSpec {
cwd: self.repo_root.clone(),
prompt: PromptMode::SingleShot(prompt),
append_system_prompt: Some(
"You answer read-only questions about this Kranz repository. \
Use only the supplied context; if it is insufficient, say what is missing. \
Do not modify files, run commands, create missions, enqueue work, approve, \
start, or merge anything."
.to_string(),
),
model: role.model,
effort: role.reasoning_effort,
session_id: format!("ask-{}", uuid::Uuid::new_v4()),
resume: None,
permission_mode: Some("plan".to_string()),
allowed_tools: vec![],
disallowed_tools: vec![
"Bash(*)".to_string(),
"Edit(*)".to_string(),
"Write(*)".to_string(),
],
tools: vec![
"Read".to_string(),
"Grep".to_string(),
"Glob".to_string(),
"LS".to_string(),
],
writable: false,
settings_json: None,
json_schema: None,
max_budget_usd: role.max_budget_usd,
max_turns: role.max_turns,
env: HashMap::new(),
sandbox: None,
hook_status: None,
};
let outcome = run_ask_session(backend, spec).await?;
Ok(json!({
"answer": outcome.answer,
"costUsd": outcome.cost_usd,
"tokens": outcome.tokens,
}))
}
pub fn release(&self, id: &str) -> Result<bool, ApiError> {
release_from(&self.missions, id)
}
pub fn sweep_idle(&self, threshold: Duration) -> Vec<String> {
sweep_idle_from(&self.missions, threshold)
}
fn ensure_sweeper_started(&self) {
let mut guard = self.sweeper.lock().expect("sweeper lock");
if guard.is_some() {
return;
}
let repo_root = self.repo_root.clone();
let missions = Arc::clone(&self.missions);
*guard = Some(tokio::spawn(async move {
const SWEEP_INTERVAL: Duration = Duration::from_secs(60);
loop {
tokio::time::sleep(SWEEP_INTERVAL).await;
let minutes = match config::load(&repo_root) {
Ok(cfg) => cfg.planning_idle_release_minutes,
Err(_) => continue,
};
if minutes == 0 {
continue;
}
let threshold = Duration::from_secs(minutes * 60);
let released = sweep_idle_from(&missions, threshold);
for id in released {
tracing::info!(mission = %id, "released idle planning engine");
}
}
}));
}
pub(crate) fn drain_is_live(&self) -> bool {
match &*self.drain.lock().expect("drain tracker lock") {
DrainSlot::Idle => false,
DrainSlot::Starting(_) => true,
DrainSlot::Running(handle) => !handle.join.is_finished(),
}
}
pub(crate) async fn auto_work_tick(&self) -> bool {
let cfg = match config::load(&self.repo_root) {
Ok(cfg) => cfg,
Err(_) => return false,
};
let queue_non_empty = kranz_engine::queue::peek(&self.repo_root).is_some();
if should_auto_drain(cfg.auto_work, queue_non_empty, self.drain_is_live()) {
if kranz_engine::queue::is_repo_busy(&self.repo_root).is_some() {
return false;
}
match self.drain_once().await {
Ok(_) => return true,
Err(e) if e.code == Some(ApiErrorCode::RepositoryBusy) => {}
Err(e) => tracing::error!(error = %e.message, "autoWork drain failed"),
}
}
false
}
pub async fn abandon(&self, id: &str, reason: &str) -> Result<(), ApiError> {
let taken = self
.missions
.lock()
.expect("missions registry lock")
.remove(id);
match taken {
None => {}
Some(HostedMission::Planning {
cell,
last_use,
pending_plan,
}) => match Arc::try_unwrap(cell) {
Ok(mutex) => drop(mutex.into_inner()),
Err(cell) => {
self.missions
.lock()
.expect("missions registry lock")
.insert(
id.to_string(),
HostedMission::Planning {
cell,
last_use,
pending_plan,
},
);
return Err(turn_in_flight());
}
},
Some(HostedMission::Running { handle, _repo_busy }) => {
if !handle.is_finished() {
handle.abort();
}
let _ = handle.await;
drop(_repo_busy);
}
}
kranz_engine::mission_catalog::abandon_mission(
self.repo_root.clone(),
id,
reason,
LockForce::No,
)
.map_err(ApiError::from)?;
Ok(())
}
pub fn clean(&self, id: &str, all: bool) -> Result<(), ApiError> {
use kranz_engine::mission_catalog::{
cleanable_class, mission_lock_is_live, prune_mission_index_file, CleanClass,
};
if self
.missions
.lock()
.expect("missions registry lock")
.contains_key(id)
{
return Err(ApiError::conflict(format!(
"mission '{id}' is hosted by this server (attached or running) — abandon it \
first, or let its run finish"
)));
}
let paths = MissionPaths::new(&self.repo_root, id);
if !paths.events_file().is_file() {
return Err(ApiError::not_found(format!("unknown mission '{id}'")));
}
let events = EventLog::read_events(&paths.events_file())?;
let state = kranz_engine::reducer::fold(&events).map_err(ApiError::from)?;
let has_plan = paths.plan_file().is_file();
match cleanable_class(state.mission.status, has_plan) {
CleanClass::Keep => {
return Err(ApiError::conflict(format!(
"mission '{id}' is live ({:?}) — abandon it before deleting",
state.mission.status
)))
}
CleanClass::CompleteKeepByDefault if !all => {
return Err(ApiError::conflict(format!(
"mission '{id}' is Complete; completed missions feed the cost-calibration \
corpus — pass \"all\": true to delete it anyway"
)))
}
CleanClass::Stale | CleanClass::CompleteKeepByDefault => {}
}
if mission_lock_is_live(&paths) {
return Err(ApiError::conflict(format!(
"mission '{id}' became live — nothing was deleted"
)));
}
kranz_engine::queue::remove(&self.repo_root, id);
std::fs::remove_dir_all(paths.mission_dir())
.map_err(|e| ApiError::internal(format!("removing mission '{id}': {e}")))?;
prune_mission_index_file(&self.repo_root, id);
Ok(())
}
pub fn pending_plan(&self, id: &str) -> Option<Plan> {
let pending = {
let map = self.missions.lock().expect("missions registry lock");
match map.get(id) {
Some(HostedMission::Planning { pending_plan, .. }) => Arc::clone(pending_plan),
_ => return None,
}
};
let plan = pending.lock().expect("pending plan lock").clone();
plan
}
fn set_pending_plan(&self, id: &str, plan: Option<Plan>) {
let map = self.missions.lock().expect("missions registry lock");
if let Some(HostedMission::Planning { pending_plan, .. }) = map.get(id) {
*pending_plan.lock().expect("pending plan lock") = plan;
}
}
pub async fn try_approve_pending(&self, id: &str) -> Result<Option<String>, ApiError> {
match self.approve_parked(id, |_| true)? {
PendingApproval::Approved(branch) => Ok(Some(branch)),
PendingApproval::NothingParked => Ok(None),
PendingApproval::Mismatch { .. } => unreachable!("unconditional approval"),
}
}
pub async fn try_approve_pending_matching(
&self,
id: &str,
expected_identity: Option<&str>,
) -> Result<PendingApproval, ApiError> {
self.approve_parked(id, |plan| {
expected_identity == Some(plan_identity(plan).as_str())
})
}
fn approve_parked(
&self,
id: &str,
matches: impl FnOnce(&Plan) -> bool,
) -> Result<PendingApproval, ApiError> {
let (cell, pending) = {
let map = self.missions.lock().expect("missions registry lock");
let Some(HostedMission::Planning {
cell,
pending_plan,
last_use,
}) = map.get(id)
else {
return Ok(PendingApproval::NothingParked);
};
*last_use.lock().expect("last-use lock") = Instant::now();
(Arc::clone(cell), Arc::clone(pending_plan))
};
let mut engine = try_lock(&cell)?;
let mut parked = pending.lock().expect("pending plan lock");
let Some(plan) = parked.as_ref() else {
return Ok(PendingApproval::NothingParked);
};
if !matches(plan) {
return Ok(PendingApproval::Mismatch {
parked: plan_identity(plan),
});
}
engine.approve_plan_as(
plan.clone(),
kranz_engine::live_permission::Actor::LocalMutationCapability,
)?;
parked.take();
Ok(PendingApproval::Approved(
engine.state().mission.mission_branch.clone(),
))
}
pub async fn approve_pending(
&self,
id: &str,
expected_identity: Option<&str>,
) -> Result<String, ApiError> {
match self.try_approve_pending_matching(id, expected_identity).await? {
PendingApproval::Approved(branch) => Ok(branch),
PendingApproval::NothingParked => Err(ApiError::conflict(format!(
"mission '{id}' has no reviewed plan pending — refresh the plan preview before approving"
)).with_code(ApiErrorCode::StalePlan)),
PendingApproval::Mismatch { .. } => Err(ApiError::conflict(
"reviewed plan identity is missing or stale — refresh the plan preview before approving",
).with_code(ApiErrorCode::StalePlan)),
}
}
pub async fn drain(&self) -> Result<Value, ApiError> {
self.drain_with_mode(false).await
}
async fn drain_once(&self) -> Result<Value, ApiError> {
self.drain_with_mode(true).await
}
async fn drain_with_mode(&self, once: bool) -> Result<Value, ApiError> {
{
let guard = self.drain.lock().expect("drain tracker lock");
match &*guard {
DrainSlot::Starting(state) => {
return Ok(drain_state_json(&state.lock().expect("drain state lock")));
}
DrainSlot::Running(handle) if !handle.join.is_finished() => {
return Ok(drain_state_json(
&handle.state.lock().expect("drain state lock"),
));
}
DrainSlot::Idle | DrainSlot::Running(_) => {}
}
}
let cfg = config::load(&self.repo_root)?;
let backend = self.backend(cfg.claude_binary.as_deref()).await?;
let repo_root = self.repo_root.clone();
let (state, global_run_permit) = {
let mut guard = self.drain.lock().expect("drain tracker lock");
match &*guard {
DrainSlot::Starting(state) => {
return Ok(drain_state_json(&state.lock().expect("drain state lock")));
}
DrainSlot::Running(handle) if !handle.join.is_finished() => {
return Ok(drain_state_json(
&handle.state.lock().expect("drain state lock"),
));
}
DrainSlot::Idle | DrainSlot::Running(_) => {}
}
let global_run_permit = self.try_global_run_permit()?;
let state = Arc::new(Mutex::new(DrainState {
live: true,
current_mission_id: None,
ran: Vec::new(),
parked: Vec::new(),
}));
*guard = DrainSlot::Starting(Arc::clone(&state));
(state, global_run_permit)
};
let task_state = Arc::clone(&state);
let readiness_probe = self.readiness_probe;
let join = tokio::spawn(async move {
let _global_run_permit = global_run_permit;
drain_task(
repo_root.clone(),
task_state,
once,
move |mission_id| {
let backend = Arc::clone(&backend);
let repo_root = repo_root.clone();
async move { run_mission_headless(backend, repo_root, mission_id).await }
},
readiness_probe,
)
.await;
});
let initial = drain_state_json(&state.lock().expect("drain state lock"));
*self.drain.lock().expect("drain tracker lock") =
DrainSlot::Running(DrainHandle { join, state });
Ok(initial)
}
pub fn queue_state(&self) -> Value {
let entries = kranz_engine::queue::list(&self.repo_root);
let busy_with = kranz_engine::queue::is_repo_busy(&self.repo_root);
let drain = match &*self.drain.lock().expect("drain tracker lock") {
DrainSlot::Running(handle) => {
drain_state_json(&handle.state.lock().expect("drain state lock"))
}
DrainSlot::Starting(state) => {
drain_state_json(&state.lock().expect("drain state lock"))
}
DrainSlot::Idle => drain_state_json(&DrainState::default()),
};
let front_readiness = entries.first().map(|e| {
let mid = e.mission_id.as_str();
{
let cache = self
.readiness_front_cache
.lock()
.expect("readiness front cache lock");
if let Some(cached) = cache.as_ref() {
if cached.mission_id == mid && cached.at.elapsed() < READINESS_FRONT_CACHE_TTL {
return (mid.to_string(), cached.report.clone());
}
}
}
let report = (self.readiness_probe)(&self.repo_root, mid)
.ok()
.and_then(|r| serde_json::to_value(r).ok())
.unwrap_or(Value::Null);
*self
.readiness_front_cache
.lock()
.expect("readiness front cache lock") = Some(FrontReadinessCache {
mission_id: mid.to_string(),
report: report.clone(),
at: Instant::now(),
});
(mid.to_string(), report)
});
let entries_json: Vec<Value> = entries
.into_iter()
.map(|e| {
let readiness = front_readiness.as_ref().and_then(|(id, report)| {
if id == &e.mission_id && !report.is_null() {
Some(report.clone())
} else {
None
}
});
json!({
"missionId": e.mission_id,
"ticketSlug": e.ticket_slug,
"priority": e.priority,
"seq": e.seq,
"readiness": readiness,
})
})
.collect();
let mut state = json!({
"entries": entries_json,
"busyWith": busy_with,
"drain": drain,
});
if let Some(permits) = &self.global_run_permits {
let available = permits.available_permits();
state["maxConcurrentReposAvailable"] = json!(available);
state["maxConcurrentReposSaturated"] = json!(available == 0);
}
state
}
fn planning_cell(&self, id: &str) -> Result<EngineCell, ApiError> {
let map = self.missions.lock().expect("missions registry lock");
match map.get(id) {
Some(HostedMission::Planning { cell, last_use, .. }) => {
*last_use.lock().expect("last-use lock") = Instant::now();
Ok(Arc::clone(cell))
}
Some(HostedMission::Running { .. }) => Err(ApiError::conflict(format!(
"mission '{id}' is running — steer it via POST /api/missions/{id}/control"
))),
None => Err(self.not_hosted(id)),
}
}
async fn planning_cell_or_attach(&self, id: &str) -> Result<EngineCell, ApiError> {
let miss = match self.planning_cell(id) {
Ok(cell) => return Ok(cell),
Err(miss) => miss,
};
if !MissionPaths::new(&self.repo_root, id)
.events_file()
.is_file()
|| self
.missions
.lock()
.expect("missions registry lock")
.contains_key(id)
{
return Err(miss);
}
let cfg = config::load(&self.repo_root)?;
let backend = self.backend(cfg.claude_binary.as_deref()).await?;
let engine = match MissionEngine::resume(backend, self.repo_root.clone(), id, LockForce::No)
{
Ok(engine) => Box::new(engine),
Err(EngineError::LockHeld(holder)) => {
return self
.planning_cell(id)
.map_err(|_| ApiError::from(EngineError::LockHeld(holder)))
}
Err(e) => return Err(e.into()),
};
if engine.state().mission.status != MissionStatus::Planning {
return Err(ApiError::conflict(format!(
"mission '{id}' is not in planning (status {:?}) — planning turns only \
apply before a plan is approved",
engine.state().mission.status
)));
}
let cell = new_cell(engine);
let mut map = self.missions.lock().expect("missions registry lock");
map.insert(id.to_string(), new_planning(Arc::clone(&cell)));
drop(map);
self.ensure_sweeper_started();
Ok(cell)
}
fn not_hosted(&self, id: &str) -> ApiError {
let paths = MissionPaths::new(&self.repo_root, id);
if paths.events_file().is_file() {
ApiError::conflict(format!(
"mission '{id}' is not hosted by this server — resume planning with \
`kranz plan --mission {id}`, or start execution via POST \
/api/missions/{id}/start"
))
.with_code(ApiErrorCode::MissionNotHosted)
} else {
ApiError::not_found(format!("unknown mission '{id}'"))
}
}
}
fn spawn_with_global_run_permit<F>(
global_run_permit: Option<OwnedSemaphorePermit>,
task: F,
) -> tokio::task::JoinHandle<()>
where
F: std::future::Future<Output = ()> + Send + 'static,
{
tokio::spawn(async move {
let _global_run_permit = global_run_permit;
task.await;
})
}
async fn run_to_end(
mut engine: Box<MissionEngine>,
mission_id: String,
missions: Arc<Mutex<HashMap<String, HostedMission>>>,
) {
let repo_root = engine.paths().repo_root.clone();
let result = engine.run().await;
match &result {
Ok(status) => {
tracing::info!(mission = %mission_id, status = ?status, "hosted mission run ended")
}
Err(e) => {
tracing::error!(mission = %mission_id, error = %e, "hosted mission run errored")
}
}
drop(engine);
if let Err(e) = kranz_engine::work::reconcile_ticket_for_mission(&repo_root, &mission_id) {
tracing::warn!(mission = %mission_id, error = %e, "failed to reconcile linked ticket");
}
missions
.lock()
.expect("missions registry lock")
.remove(&mission_id);
}
struct AskRunOutcome {
answer: String,
cost_usd: f64,
tokens: TokenUsage,
}
async fn run_ask_session(
backend: Arc<dyn AgentBackend>,
spec: SessionSpec,
) -> Result<AskRunOutcome, ApiError> {
let mut session = backend.start(spec).await.map_err(ApiError::from)?;
let mut streamed_text = String::new();
let mut result_text = None;
let mut tokens = TokenUsage::default();
let mut cost_usd = 0.0;
let mut result_error = false;
while let Some(event) = session.next_event().await.map_err(ApiError::from)? {
match event {
AgentEvent::Text { text, .. } => streamed_text.push_str(&text),
AgentEvent::Result {
text,
is_error,
usage,
cost_usd: cost,
..
} => {
result_error |= is_error;
tokens.add(&usage);
cost_usd += cost.unwrap_or(0.0);
if !text.trim().is_empty() {
result_text = Some(text);
}
}
_ => {}
}
}
match session.exit_status() {
Some(SessionExit::Completed) if !result_error => {
let answer = result_text.unwrap_or(streamed_text).trim().to_string();
if answer.is_empty() {
return Err(ApiError::internal("ask turn produced an empty answer"));
}
Ok(AskRunOutcome {
answer,
cost_usd,
tokens,
})
}
Some(SessionExit::Completed) => Err(ApiError::internal("ask turn failed")),
Some(SessionExit::Failed(reason)) => {
Err(ApiError::internal(format!("ask turn failed: {reason}")))
}
Some(SessionExit::Aborted) => Err(ApiError::internal("ask turn aborted")),
None => Err(ApiError::internal("ask turn ended without an exit status")),
}
}
fn ask_prompt(question: &str, context: &str) -> String {
format!(
"Answer this operator question about the Kranz repository.\n\n\
Rules:\n\
- Ground the answer only in the context below.\n\
- If the context is insufficient, say what is missing.\n\
- Keep the answer concise but specific, citing mission ids or ticket slugs when relevant.\n\
- This is read-only: do not propose that you have changed state.\n\n\
Question:\n{question}\n\nContext:\n{context}"
)
}
fn ask_context(repo_root: &Path) -> String {
let mut out = String::new();
out.push_str("## Missions\n");
let mut ids = MissionPaths::list_missions(repo_root);
ids.sort();
ids.reverse();
if ids.is_empty() {
out.push_str("(none)\n");
}
for id in ids.into_iter().take(20) {
let paths = MissionPaths::new(repo_root, &id);
let Ok(events) = EventLog::read_events(&paths.events_file()) else {
continue;
};
let Ok(state) = kranz_engine::reducer::fold(&events) else {
continue;
};
out.push_str(&format!(
"- {}: {:?}; goal: {}; branch: {}; cost: ${:.4}; tokens in/out/cacheRead/cacheWrite: {}/{}/{}/{}\n",
state.mission.id,
state.mission.status,
one_line(&state.mission.goal),
state.mission.mission_branch,
state.total_cost_usd,
state.totals.input,
state.totals.output,
state.totals.cache_read,
state.totals.cache_write,
));
for decision in state.recent_decisions.iter().rev().take(3) {
out.push_str(&format!(" decision: {}\n", one_line(decision)));
}
let report = paths.report_file();
let mut text = String::new();
let read = kranz_engine::paths::open_read_nofollow(&report)
.and_then(|mut file| {
use std::io::Read as _;
file.read_to_string(&mut text)?;
Ok(())
})
.is_ok();
if read {
out.push_str(&format!(
" report excerpt: {}\n",
truncate(&one_line(&text), 500)
));
}
}
out.push_str("\n## Tickets\n");
let tickets = Ticket::list(repo_root);
if tickets.is_empty() {
out.push_str("(none)\n");
}
for ticket in tickets.iter().take(40) {
let state = Ticket::read_state(repo_root, &ticket.slug);
out.push_str(&format!(
"- {} [{:?}, p{}]: {}; blocked-by: {}\n",
ticket.slug,
state,
ticket.priority,
one_line(&ticket.title),
if ticket.blocked_by.is_empty() {
"none".to_string()
} else {
ticket.blocked_by.join(", ")
}
));
}
out.push_str("\n## Queue\n");
let entries = queue::list(repo_root);
if entries.is_empty() {
out.push_str("(empty)\n");
}
for entry in entries.iter().take(20) {
out.push_str(&format!(
"- {} priority={} ticket={}\n",
entry.mission_id,
entry.priority,
entry.ticket_slug.as_deref().unwrap_or("-")
));
}
out
}
fn one_line(text: &str) -> String {
text.split_whitespace().collect::<Vec<_>>().join(" ")
}
fn truncate(text: &str, max: usize) -> String {
if text.chars().count() <= max {
return text.to_string();
}
let mut out: String = text.chars().take(max.saturating_sub(1)).collect();
out.push('…');
out
}
async fn drain_task<R, Fut>(
repo_root: PathBuf,
state: Arc<Mutex<DrainState>>,
once: bool,
run_mission: R,
readiness_probe: ReadinessProbe,
) where
R: Fn(String) -> Fut,
Fut: std::future::Future<Output = anyhow::Result<i32>>,
{
drain_task_with_probe(repo_root, state, once, run_mission, readiness_probe).await;
}
async fn drain_task_with_probe<R, Fut, P>(
repo_root: PathBuf,
state: Arc<Mutex<DrainState>>,
once: bool,
run_mission: R,
readiness_probe: P,
) where
R: Fn(String) -> Fut,
Fut: std::future::Future<Output = anyhow::Result<i32>>,
P: Fn(
&Path,
&str,
) -> kranz_engine::error::Result<kranz_engine::backend_readiness::ReadinessReport>,
{
let dispatch_branch = GitRepo::open(&repo_root)
.ok()
.and_then(|g| g.current_branch().ok());
let result = kranz_engine::work::drain_queue_with_probe(
&repo_root,
once,
|mission_id| {
let state = Arc::clone(&state);
let fut = run_mission(mission_id.clone());
async move {
state.lock().expect("drain state lock").current_mission_id =
Some(mission_id.clone());
let outcome = fut.await;
let mut guard = state.lock().expect("drain state lock");
guard.current_mission_id = None;
if outcome.is_ok() {
guard.ran.push(mission_id);
}
outcome
}
},
readiness_probe,
)
.await;
match &result {
Ok(report) if !report.stopped_busy => {
{
let mut guard = state.lock().expect("drain state lock");
for id in &report.parked {
if !guard.parked.contains(id) {
guard.parked.push(id.clone());
}
}
}
restore_drain_checkout(&repo_root, dispatch_branch.as_deref());
}
Ok(report) => {
let mut guard = state.lock().expect("drain state lock");
for id in &report.parked {
if !guard.parked.contains(id) {
guard.parked.push(id.clone());
}
}
}
Err(e) => {
tracing::error!(error = %e, "hosted queue drain errored");
restore_drain_checkout(&repo_root, dispatch_branch.as_deref());
}
}
state.lock().expect("drain state lock").live = false;
}
fn restore_drain_checkout(repo_root: &Path, original: Option<&str>) {
let Some(original) = original else { return };
if original.starts_with("kranz/mission-") {
return;
}
let Ok(git) = GitRepo::open(repo_root) else {
return;
};
if git.current_branch().ok().as_deref() == Some(original) {
return;
}
match git.is_clean_tracked() {
Ok(true) => match git.checkout(original) {
Ok(()) => tracing::info!(branch = %original, "hosted drain restored operator checkout"),
Err(e) => {
tracing::warn!(branch = %original, error = %e, "hosted drain could not restore checkout")
}
},
Ok(false) => tracing::warn!(
"hosted drain leaving checkout in place: tracked files have uncommitted changes"
),
Err(e) => {
tracing::warn!(error = %e, "hosted drain could not probe the working tree; checkout left in place")
}
}
}
async fn run_mission_headless(
backend: Arc<dyn AgentBackend>,
repo_root: PathBuf,
mission_id: String,
) -> anyhow::Result<i32> {
let mut engine = MissionEngine::resume(backend, repo_root, &mission_id, LockForce::No)?;
let status = engine.run().await?;
Ok(exit_code_for(status))
}
fn exit_code_for(status: MissionStatus) -> i32 {
match status {
MissionStatus::Complete => 0,
MissionStatus::Blocked => 2,
_ => 1,
}
}
fn config_for_ticket(mut cfg: MissionConfig, ticket: &Ticket) -> MissionConfig {
if let Some(budget) = ticket.max_budget_usd {
cfg.orchestrator.max_budget_usd = Some(budget);
}
cfg
}
#[allow(dead_code)]
fn draft_outcome_json(outcome: &DraftOutcome) -> Value {
match outcome {
DraftOutcome::ParkedForReview {
mission_id,
mission_branch,
} => json!({
"outcome": "parkedForReview",
"missionId": mission_id,
"missionBranch": mission_branch,
}),
DraftOutcome::Enqueued { mission_id } => json!({
"outcome": "enqueued",
"missionId": mission_id,
}),
DraftOutcome::PlanAsProse { mission_id } => json!({
"outcome": "planAsProse",
"missionId": mission_id,
"message": "the orchestrator produced a plan but emitted it as prose instead of \
through the plan channel, so nothing was queued; re-run draft for \
this ticket",
}),
DraftOutcome::NeedsContext {
mission_id,
questions,
} => json!({
"outcome": "needsContext",
"missionId": mission_id,
"questions": questions,
}),
DraftOutcome::WrongPlan { mission_id, reason } => json!({
"outcome": "wrongPlan",
"missionId": mission_id,
"reason": reason,
}),
}
}
fn new_cell(engine: Box<MissionEngine>) -> EngineCell {
Arc::new(tokio::sync::Mutex::new(engine))
}
fn new_planning(cell: EngineCell) -> HostedMission {
HostedMission::Planning {
cell,
last_use: Arc::new(Mutex::new(Instant::now())),
pending_plan: Arc::new(Mutex::new(None)),
}
}
fn release_from(
missions: &Mutex<HashMap<String, HostedMission>>,
id: &str,
) -> Result<bool, ApiError> {
let mut map = missions.lock().expect("missions registry lock");
match map.remove(id) {
None => Ok(true),
Some(HostedMission::Running { handle, _repo_busy }) => {
let finished = handle.is_finished();
if !finished {
map.insert(
id.to_string(),
HostedMission::Running { handle, _repo_busy },
);
}
Ok(finished)
}
Some(HostedMission::Planning {
cell,
last_use,
pending_plan,
}) => match Arc::try_unwrap(cell) {
Ok(mutex) => {
drop(mutex.into_inner()); Ok(true)
}
Err(cell) => {
map.insert(
id.to_string(),
HostedMission::Planning {
cell,
last_use,
pending_plan,
},
);
Err(turn_in_flight())
}
},
}
}
fn sweep_idle_from(
missions: &Mutex<HashMap<String, HostedMission>>,
threshold: Duration,
) -> Vec<String> {
let idle_ids: Vec<String> = {
let map = missions.lock().expect("missions registry lock");
map.iter()
.filter_map(|(id, mission)| match mission {
HostedMission::Planning { last_use, .. } => {
let elapsed = last_use.lock().expect("last-use lock").elapsed();
(elapsed >= threshold).then(|| id.clone())
}
HostedMission::Running { .. } => None,
})
.collect()
};
idle_ids
.into_iter()
.filter(|id| matches!(release_from(missions, id), Ok(true)))
.collect()
}
fn try_lock(
cell: &EngineCell,
) -> Result<tokio::sync::MutexGuard<'_, Box<MissionEngine>>, ApiError> {
cell.try_lock().map_err(|_| turn_in_flight())
}
fn turn_in_flight() -> ApiError {
ApiError::conflict("a turn is in flight for this mission — wait for it to finish")
.with_code(ApiErrorCode::TurnInFlight)
}
fn prepend_seed(seed: Option<String>, reply: String) -> String {
match seed {
Some(seed) => format!("{seed}\n\n{reply}"),
None => reply,
}
}
fn estimate_json(estimate: &CostEstimate) -> Value {
let confidence = match estimate.confidence {
kranz_engine::cost::Confidence::High => "high",
kranz_engine::cost::Confidence::Low => "low",
};
json!({
"workerRuns": estimate.worker_runs,
"validatorRuns": estimate.validator_runs,
"lowUsd": estimate.low_usd,
"expectedUsd": estimate.expected_usd,
"highUsd": estimate.high_usd,
"confidence": confidence,
})
}
pub(crate) async fn create_mission(
State(server): State<Arc<ServerState>>,
body: Bytes,
) -> Result<impl IntoResponse, ApiError> {
let value = parse_body(&body)?;
let goal = value
.get("goal")
.and_then(Value::as_str)
.map(str::trim)
.filter(|goal| !goal.is_empty())
.ok_or_else(|| {
ApiError::bad_request(r#"body must be {"goal":"..."} with a non-empty goal"#)
})?;
let id = server.host.create(goal, value.get("config")).await?;
Ok((StatusCode::CREATED, Json(json!({ "id": id }))))
}
pub(crate) async fn planning_turn(
State(server): State<Arc<ServerState>>,
UrlPath(id): UrlPath<String>,
body: Bytes,
) -> Result<Json<Value>, ApiError> {
let id = valid_id(&server, &id)?;
let value = parse_body(&body)?;
let text = value
.get("text")
.and_then(Value::as_str)
.map(str::trim)
.filter(|text| !text.is_empty())
.ok_or_else(|| {
ApiError::bad_request(r#"body must be {"text":"..."} with non-empty text"#)
})?;
let reply = server.host.planning_turn(&id, text).await?;
Ok(Json(json!({ "reply": reply })))
}
pub(crate) async fn request_plan(
State(server): State<Arc<ServerState>>,
UrlPath(id): UrlPath<String>,
) -> Result<Json<Value>, ApiError> {
let id = valid_id(&server, &id)?;
Ok(Json(server.host.request_plan(&id).await?))
}
pub(crate) async fn approve_mission(
State(server): State<Arc<ServerState>>,
UrlPath(id): UrlPath<String>,
body: Bytes,
) -> Result<Json<Value>, ApiError> {
let id = valid_id(&server, &id)?;
let value = parse_body(&body)?;
let plan = value
.get("plan")
.cloned()
.ok_or_else(|| ApiError::bad_request(r#"body must be {"plan":{...}}"#))?;
let plan: Plan = serde_json::from_value(plan)
.map_err(|e| ApiError::bad_request(format!("'plan' is not a valid Plan: {e}")))?;
let branch = server.host.approve(&id, plan).await?;
Ok(Json(json!({ "branch": branch })))
}
pub(crate) async fn pending_plan_route(
axum::Extension(reads): axum::Extension<crate::read_work::ReadWork>,
State(server): State<Arc<ServerState>>,
UrlPath(id): UrlPath<String>,
) -> Result<Json<Value>, ApiError> {
reads
.run(move || {
let id = valid_id(&server, &id)?;
Ok(Json(match server.host.pending_plan(&id) {
Some(plan) => {
json!({ "pending": true, "planIdentity": plan_identity(&plan), "plan": plan })
}
None => json!({ "pending": false }),
}))
})
.await
}
pub(crate) async fn approve_pending_route(
State(server): State<Arc<ServerState>>,
UrlPath(id): UrlPath<String>,
body: Bytes,
) -> Result<Json<Value>, ApiError> {
let id = valid_id(&server, &id)?;
let value = parse_body(&body)?;
let start = value.get("start").and_then(Value::as_bool).unwrap_or(false);
let expected_identity = value.get("planIdentity").and_then(Value::as_str);
let branch = server.host.approve_pending(&id, expected_identity).await?;
if start {
server.host.start(&id).await?;
}
Ok(Json(json!({ "branch": branch, "started": start })))
}
pub(crate) async fn abandon_mission_route(
State(server): State<Arc<ServerState>>,
UrlPath(id): UrlPath<String>,
body: Bytes,
) -> Result<Json<Value>, ApiError> {
let id = valid_id(&server, &id)?;
let value = parse_body(&body)?;
let reason = value
.get("reason")
.and_then(Value::as_str)
.map(str::trim)
.filter(|r| !r.is_empty())
.unwrap_or("abandoned by operator");
server.host.abandon(&id, reason).await?;
Ok(Json(json!({ "abandoned": true })))
}
pub(crate) async fn release_mission_route(
State(server): State<Arc<ServerState>>,
UrlPath(id): UrlPath<String>,
body: Bytes,
) -> Result<Json<Value>, ApiError> {
let id = valid_id(&server, &id)?;
let _ = parse_body(&body)?;
if !MissionPaths::new(server.host.repo_root(), &id)
.events_file()
.is_file()
{
return Err(ApiError::not_found(format!("mission '{id}' not found")));
}
let released = server.host.release(&id)?;
Ok(Json(json!({ "released": released })))
}
pub(crate) async fn delete_mission_route(
State(server): State<Arc<ServerState>>,
UrlPath(id): UrlPath<String>,
body: Bytes,
) -> Result<Json<Value>, ApiError> {
let id = valid_id(&server, &id)?;
let value = parse_body(&body)?;
let all = value.get("all").and_then(Value::as_bool).unwrap_or(false);
server.host.clean(&id, all)?;
Ok(Json(json!({ "deleted": true })))
}
pub(crate) async fn start_mission(
State(server): State<Arc<ServerState>>,
UrlPath(id): UrlPath<String>,
) -> Result<impl IntoResponse, ApiError> {
let id = valid_id(&server, &id)?;
server.host.start(&id).await?;
Ok((StatusCode::ACCEPTED, Json(json!({ "running": true }))))
}
pub(crate) async fn merge_mission_route(
State(server): State<Arc<ServerState>>,
UrlPath(id): UrlPath<String>,
) -> Result<Json<Value>, ApiError> {
let id = valid_id(&server, &id)?;
Ok(Json(server.host.merge(&id).await?))
}
pub(crate) async fn drain_queue_route(
State(server): State<Arc<ServerState>>,
body: Bytes,
) -> Result<Json<Value>, ApiError> {
let _ = parse_body(&body)?;
Ok(Json(server.host.drain().await?))
}
pub(crate) async fn queue_state_route(
axum::Extension(reads): axum::Extension<crate::read_work::ReadWork>,
State(server): State<Arc<ServerState>>,
) -> Result<Json<Value>, ApiError> {
reads.run(move || Ok(Json(server.host.queue_state()))).await
}
fn valid_id(server: &ServerState, id: &str) -> Result<String, ApiError> {
crate::rest::mission_paths(server, id)?;
Ok(id.to_string())
}
pub(crate) fn parse_body(body: &Bytes) -> Result<Value, ApiError> {
if body.is_empty() {
return Ok(json!({}));
}
serde_json::from_slice(body)
.map_err(|e| ApiError::bad_request(format!("invalid JSON body: {e}")))
}
#[cfg(test)]
mod tests {
use super::*;
use axum::http::StatusCode;
use kranz_engine::backend_mock::{mock_init, mock_result_text, MockBackend, MockScript};
use std::process::Command;
use std::sync::Once;
static ENV_ISOLATION: Once = Once::new();
fn isolate_git_env() {
ENV_ISOLATION.call_once(|| {
let missing = std::env::temp_dir()
.join(format!("kranz-host-test-no-config-{}", std::process::id()));
std::env::set_var("GIT_CONFIG_GLOBAL", &missing);
std::env::set_var("GIT_CONFIG_SYSTEM", &missing);
if let Ok(ceiling) = std::fs::canonicalize(std::env::temp_dir()) {
std::env::set_var("GIT_CEILING_DIRECTORIES", ceiling);
}
let home =
std::env::temp_dir().join(format!("kranz-host-test-home-{}", std::process::id()));
let _ = std::fs::create_dir_all(&home);
std::env::set_var(if cfg!(windows) { "USERPROFILE" } else { "HOME" }, &home);
});
}
fn git(dir: &std::path::Path, args: &[&str]) {
let out = Command::new("git")
.args(args)
.current_dir(dir)
.output()
.expect("spawn git");
assert!(
out.status.success(),
"git {args:?}: {}",
String::from_utf8_lossy(&out.stderr)
);
}
fn init_repo() -> Option<(tempfile::TempDir, PathBuf)> {
isolate_git_env();
let git_works = Command::new("git")
.arg("--version")
.output()
.map(|o| o.status.success())
.unwrap_or(false);
if !git_works {
kranz_engine::test_capability::skip(
kranz_engine::test_capability::capability::GIT,
"git is not on PATH",
);
return None;
}
let dir = tempfile::tempdir().expect("tempdir");
let init = Command::new("git")
.args(["init", "-b", "main"])
.current_dir(dir.path())
.output()
.expect("spawn git init");
if !init.status.success() {
git(dir.path(), &["init"]);
git(dir.path(), &["symbolic-ref", "HEAD", "refs/heads/main"]);
}
git(dir.path(), &["config", "user.name", "test"]);
git(dir.path(), &["config", "user.email", "test@example.com"]);
std::fs::write(dir.path().join("README.md"), "seed\n").unwrap();
git(dir.path(), &["add", "-A"]);
git(dir.path(), &["commit", "-m", "seed"]);
let root = std::fs::canonicalize(dir.path()).expect("canonicalize");
Some((dir, root))
}
#[tokio::test]
async fn ask_runs_read_only_one_shot_without_creating_mission_state() {
let Some((_dir, root)) = init_repo() else {
return;
};
let backend = Arc::new(MockBackend::with_scripts(vec![MockScript::single_shot(
"Nothing is currently blocked.",
)]));
let host = MissionHost::with_backend(root.clone(), backend.clone());
let before = MissionPaths::list_missions(&root);
let value = host.ask("what is blocked?").await.unwrap();
assert_eq!(value["answer"], "Nothing is currently blocked.");
assert_eq!(
MissionPaths::list_missions(&root),
before,
"ask must not create or mutate mission directories"
);
let specs = backend.started_specs();
assert_eq!(specs.len(), 1);
assert!(!specs[0].writable, "ask session is read-only");
assert_eq!(specs[0].permission_mode.as_deref(), Some("plan"));
let prompt = match &specs[0].prompt {
PromptMode::SingleShot(prompt) => prompt,
other => panic!("ask must be one-shot, got {other:?}"),
};
assert!(prompt.contains("what is blocked?"));
assert!(prompt.contains("## Missions"));
}
#[tokio::test]
async fn http_api_error_codes_match_dashboard_wire_fixtures() {
use http_body_util::BodyExt as _;
let dir = tempfile::tempdir().unwrap();
let paths = MissionPaths::new(dir.path(), "m-fixture");
std::fs::create_dir_all(paths.mission_dir()).unwrap();
std::fs::write(paths.events_file(), "").unwrap();
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let mut host = MissionHost::with_backend(dir.path().to_path_buf(), backend);
host.global_run_permits = Some(Arc::new(Semaphore::new(0)));
let errors = [
("mission_not_hosted", host.not_hosted("m-fixture")),
("turn_in_flight", turn_in_flight()),
("repository_busy", host.try_global_run_permit().unwrap_err()),
(
"stale_plan",
host.approve_pending("m-fixture", None).await.unwrap_err(),
),
("legacy", ApiError::conflict("mission is not hosted")),
];
let mut actual = Vec::new();
for (name, error) in errors {
let response = error.into_response();
let status = response.status().as_u16();
assert_eq!(response.headers()["content-type"], "application/json");
let bytes = response.into_body().collect().await.unwrap().to_bytes();
let body: Value = serde_json::from_slice(&bytes).unwrap();
actual.push(json!({ "name": name, "status": status, "body": body }));
}
let fixture: Value = serde_json::from_str(include_str!(
"../../../apps/dashboard/src/lib/fixtures/api-errors.json"
))
.unwrap();
assert_eq!(json!(actual), fixture);
}
#[tokio::test]
async fn contended_planning_mutex_is_409_for_turns_and_start() {
let Some((_dir, root)) = init_repo() else {
return;
};
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let host = MissionHost::with_backend(root, backend);
let id = host.create("ship it", None).await.expect("create mission");
let cell = host.planning_cell(&id).expect("hosted planning cell");
let _guard = cell.try_lock().expect("uncontended lock");
let err = host
.planning_turn(&id, "hello")
.await
.expect_err("turn must 409");
assert_eq!(err.status, StatusCode::CONFLICT);
assert!(err.message.contains("turn is in flight"), "{}", err.message);
assert_eq!(err.code, Some(ApiErrorCode::TurnInFlight));
let err = host
.request_plan(&id)
.await
.expect_err("request-plan must 409");
assert_eq!(err.status, StatusCode::CONFLICT);
let err = host.start(&id).await.expect_err("start must 409");
assert_eq!(err.status, StatusCode::CONFLICT);
assert!(
host.planning_cell(&id).is_ok(),
"registry entry must survive"
);
}
#[tokio::test]
async fn start_without_an_approved_plan_is_409() {
let Some((_dir, root)) = init_repo() else {
return;
};
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let host = MissionHost::with_backend(root, backend);
let id = host.create("ship it", None).await.expect("create mission");
let err = host
.start(&id)
.await
.expect_err("start must 409 in planning");
assert_eq!(err.status, StatusCode::CONFLICT);
assert!(err.message.contains("no approved plan"), "{}", err.message);
assert!(host.planning_cell(&id).is_ok());
}
#[tokio::test]
async fn approve_pending_matching_refuses_a_different_plan_without_consuming_it() {
let Some((_dir, root)) = init_repo() else {
return;
};
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let host = MissionHost::with_backend(root.clone(), backend);
let id = host.create("ship it", None).await.expect("create mission");
let plan: Plan = serde_json::from_value(plan_json()).expect("plan");
let identity = plan_identity(&plan);
assert_eq!(
host.try_approve_pending_matching(&id, Some(&identity))
.await
.unwrap(),
PendingApproval::NothingParked,
"nothing parked is not an approval"
);
host.set_pending_plan(&id, Some(plan.clone()));
assert_eq!(
host.try_approve_pending_matching(&id, Some("an older plan"))
.await
.unwrap(),
PendingApproval::Mismatch {
parked: identity.clone()
},
"a card naming a different plan must be refused, naming the parked one"
);
assert!(
host.pending_plan(&id).is_some(),
"a refused approve must not consume the parked plan"
);
assert!(
matches!(
host.try_approve_pending_matching(&id, None).await.unwrap(),
PendingApproval::Mismatch { .. }
),
"a card that names no plan cannot match one"
);
assert!(host.pending_plan(&id).is_some());
assert_eq!(
host.try_approve_pending_matching(&id, Some(&identity))
.await
.unwrap(),
PendingApproval::Approved(format!("kranz/mission-{id}"))
);
assert!(
host.pending_plan(&id).is_none(),
"an approve consumes the parked plan"
);
assert_eq!(
host.try_approve_pending_matching(&id, Some(&identity))
.await
.unwrap(),
PendingApproval::NothingParked,
"a second click has nothing left to commit"
);
}
#[tokio::test]
async fn approve_pending_matching_leaves_pending_untouched_on_busy_or_failure() {
let Some((_dir, root)) = init_repo() else {
return;
};
let host = MissionHost::with_backend(root, Arc::new(MockBackend::new()));
let id = host.create("ship it", None).await.unwrap();
let mut plan: Plan = serde_json::from_value(plan_json()).unwrap();
host.set_pending_plan(&id, Some(plan.clone()));
let cell = host.planning_cell(&id).unwrap();
let guard = cell.try_lock().unwrap();
let identity = plan_identity(&plan);
let err = host
.try_approve_pending_matching(&id, Some(&identity))
.await
.unwrap_err();
assert_eq!(err.status, StatusCode::CONFLICT);
assert_eq!(plan_identity(&host.pending_plan(&id).unwrap()), identity);
drop(guard);
plan.milestones.clear();
let invalid_identity = plan_identity(&plan);
host.set_pending_plan(&id, Some(plan));
let err = host
.try_approve_pending_matching(&id, Some(&invalid_identity))
.await
.unwrap_err();
assert!(err.message.contains("no milestones"), "{}", err.message);
assert_eq!(
plan_identity(&host.pending_plan(&id).unwrap()),
invalid_identity
);
let replacement: Plan = serde_json::from_value(plan_json()).unwrap();
host.set_pending_plan(&id, Some(replacement));
assert_eq!(
host.try_approve_pending_matching(&id, Some(&invalid_identity))
.await
.unwrap(),
PendingApproval::Mismatch {
parked: identity.clone()
},
);
assert_eq!(plan_identity(&host.pending_plan(&id).unwrap()), identity);
}
#[tokio::test]
async fn start_is_409_when_repo_busy() {
let Some((_dir, root)) = init_repo() else {
return;
};
let _hold =
kranz_engine::queue::acquire_repo_busy(&root, "m-sibling").expect("sibling busy hold");
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let host = MissionHost::with_backend(root.clone(), backend);
let id = host.create("ship it", None).await.expect("create mission");
let plan: Plan = serde_json::from_value(plan_json()).expect("plan");
host.approve(&id, plan).await.expect("approve");
let err = host.start(&id).await.expect_err("start must 409 when busy");
assert_eq!(err.status, StatusCode::CONFLICT);
assert_eq!(err.code, Some(ApiErrorCode::RepositoryBusy));
assert!(
err.message.contains("busy"),
"expected busy conflict, got: {}",
err.message
);
assert!(host.planning_cell(&id).is_ok());
}
#[tokio::test]
async fn start_is_409_when_global_repository_limit_is_saturated() {
let Some((_dir, root)) = init_repo() else {
return;
};
let permits = Arc::new(Semaphore::new(1));
let _other_repo = Arc::clone(&permits).try_acquire_owned().unwrap();
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let mut host = MissionHost::with_backend(root, backend);
host.global_run_permits = Some(permits);
let id = host.create("ship it", None).await.expect("create mission");
let plan: Plan = serde_json::from_value(plan_json()).expect("plan");
host.approve(&id, plan).await.expect("approve");
let error = host.start(&id).await.expect_err("global cap must refuse");
assert_eq!(error.status, StatusCode::CONFLICT);
assert!(error.message.contains("maxConcurrentRepos"));
assert_eq!(error.code, Some(ApiErrorCode::RepositoryBusy));
assert!(
host.planning_cell(&id).is_ok(),
"refused start must restore the hosted engine"
);
}
#[tokio::test]
async fn global_run_permit_is_released_when_hosted_task_panics() {
let permits = Arc::new(Semaphore::new(1));
let permit = Arc::clone(&permits).try_acquire_owned().unwrap();
assert_eq!(permits.available_permits(), 0);
let handle = spawn_with_global_run_permit(Some(permit), async {
panic!("simulated hosted-run panic");
});
assert!(handle.await.unwrap_err().is_panic());
assert_eq!(permits.available_permits(), 1);
}
#[tokio::test]
async fn sweep_idle_leaves_a_mid_turn_mission_hosted() {
let Some((_dir, root)) = init_repo() else {
return;
};
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let host = MissionHost::with_backend(root, backend);
let id = host.create("ship it", None).await.expect("create mission");
let cell = host.planning_cell(&id).expect("hosted planning cell");
let _guard = cell.try_lock().expect("uncontended lock");
let released = host.sweep_idle(std::time::Duration::ZERO);
assert!(!released.contains(&id), "{released:?}");
assert!(
host.planning_cell(&id).is_ok(),
"mission must remain hosted"
);
}
#[tokio::test]
async fn release_route_is_409_mid_turn() {
use axum::body::Body;
use axum::http::Request;
use tower::ServiceExt;
let Some((_dir, root)) = init_repo() else {
return;
};
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let host = MissionHost::with_backend(root, backend);
let id = host.create("ship it", None).await.expect("create mission");
let cell = host.planning_cell(&id).expect("hosted planning cell");
let _guard = cell.try_lock().expect("uncontended lock");
let app =
crate::router_with_host(host, None, crate::MutationAuthority::new("tok").unwrap());
let response = app
.oneshot(
Request::builder()
.method("POST")
.uri(format!("/api/missions/{id}/release"))
.header("content-type", "application/json")
.header("x-kranz-token", "tok")
.body(Body::from("{}"))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CONFLICT);
}
#[tokio::test]
async fn bodyless_post_with_valid_token_is_not_rejected_as_unsupported_media_type() {
use axum::body::Body;
use axum::http::Request;
use tower::ServiceExt;
let Some((_dir, root)) = init_repo() else {
return;
};
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let host = MissionHost::with_backend(root, backend);
let id = host.create("ship it", None).await.expect("create mission");
let app =
crate::router_with_host(host, None, crate::MutationAuthority::new("tok").unwrap());
let response = app
.oneshot(
Request::builder()
.method("POST")
.uri(format!("/api/missions/{id}/start"))
.header("x-kranz-token", "tok")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_ne!(response.status(), StatusCode::UNSUPPORTED_MEDIA_TYPE);
assert_eq!(response.status(), StatusCode::CONFLICT);
}
#[tokio::test]
async fn bodyless_post_gate_still_rejects_non_empty_non_json_bodies() {
use axum::body::Body;
use axum::http::Request;
use tower::ServiceExt;
let Some((_dir, root)) = init_repo() else {
return;
};
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let host = MissionHost::with_backend(root, backend);
let id = host.create("ship it", None).await.expect("create mission");
let app =
crate::router_with_host(host, None, crate::MutationAuthority::new("tok").unwrap());
let payload = "not json";
let response = app
.oneshot(
Request::builder()
.method("POST")
.uri(format!("/api/missions/{id}/release"))
.header("content-type", "text/plain")
.header("content-length", payload.len().to_string())
.header("x-kranz-token", "tok")
.body(Body::from(payload))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::UNSUPPORTED_MEDIA_TYPE);
}
#[tokio::test]
async fn create_rejects_an_invalid_config_patch() {
let Some((_dir, root)) = init_repo() else {
return;
};
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let host = MissionHost::with_backend(root, backend);
let patch = json!({ "maxParallelWorkers": 9 });
let err = host
.create("ship it", Some(&patch))
.await
.expect_err("must reject");
assert_eq!(err.status, StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn empty_queue_drain_returns_ok_and_settles_idle() {
let Some((_dir, root)) = init_repo() else {
return;
};
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let host = MissionHost::with_backend(root, backend);
let body = host
.drain()
.await
.expect("drain must not error on an empty queue");
assert!(body.get("live").is_some(), "{body}");
let deadline = std::time::Instant::now() + Duration::from_secs(5);
loop {
let state = host.queue_state();
if state["drain"]["live"] == false {
break;
}
assert!(
std::time::Instant::now() < deadline,
"drain never settled idle: {state}"
);
tokio::time::sleep(Duration::from_millis(20)).await;
}
}
#[tokio::test]
async fn drain_is_409_when_global_repository_limit_is_saturated() {
let Some((_dir, root)) = init_repo() else {
return;
};
let permits = Arc::new(Semaphore::new(1));
let _other_repo = Arc::clone(&permits).try_acquire_owned().unwrap();
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let mut host = MissionHost::with_backend(root, backend);
host.global_run_permits = Some(permits);
let error = host.drain().await.expect_err("global cap must refuse");
assert_eq!(error.status, StatusCode::CONFLICT);
assert!(error.message.contains("maxConcurrentRepos"));
assert!(matches!(
&*host.drain.lock().expect("drain tracker lock"),
DrainSlot::Idle
));
}
#[tokio::test]
async fn queue_state_reports_global_concurrency_saturation() {
let Some((_dir, root)) = init_repo() else {
return;
};
let permits = Arc::new(Semaphore::new(1));
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let mut host = MissionHost::with_backend(root, backend);
host.global_run_permits = Some(Arc::clone(&permits));
let open = host.queue_state();
assert_eq!(open["maxConcurrentReposAvailable"], 1);
assert_eq!(open["maxConcurrentReposSaturated"], false);
let _hold = permits.try_acquire_owned().unwrap();
let saturated = host.queue_state();
assert_eq!(saturated["maxConcurrentReposAvailable"], 0);
assert_eq!(saturated["maxConcurrentReposSaturated"], true);
}
#[tokio::test]
async fn second_drain_while_live_returns_tracked_state_without_spawning_second() {
let Some((_dir, root)) = init_repo() else {
return;
};
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let host = MissionHost::with_backend(root, backend);
let state = Arc::new(Mutex::new(DrainState {
live: true,
current_mission_id: Some("m-fake".to_string()),
ran: vec!["m-earlier".to_string()],
parked: Vec::new(),
}));
let never_finishes = tokio::spawn(async {
std::future::pending::<()>().await;
});
*host.drain.lock().expect("drain tracker lock") = DrainSlot::Running(DrainHandle {
join: never_finishes,
state: Arc::clone(&state),
});
let before = Arc::as_ptr(&state);
let first = host.drain().await.expect("drain must not error");
let second = host.drain().await.expect("drain must not error");
assert_eq!(first, second);
assert_eq!(first["live"], true);
assert_eq!(first["currentMissionId"], "m-fake");
assert_eq!(first["ran"], json!(["m-earlier"]));
let after = {
let guard = host.drain.lock().expect("drain tracker lock");
match &*guard {
DrainSlot::Running(handle) => Arc::as_ptr(&handle.state),
_ => panic!("expected the tracker to still be Running"),
}
};
assert_eq!(before, after, "a second drain must not replace the tracker");
}
#[tokio::test]
async fn two_concurrent_cold_drains_spawn_exactly_one() {
let Some((_dir, root)) = init_repo() else {
return;
};
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let host = MissionHost::with_backend(root, backend);
let (first, second) = tokio::join!(host.drain(), host.drain());
let first = first.expect("first drain must not error");
let second = second.expect("second drain must not error");
assert_eq!(first["live"], true, "{first}");
assert_eq!(second["live"], true, "{second}");
match &*host.drain.lock().expect("drain tracker lock") {
DrainSlot::Running(_) | DrainSlot::Starting(_) => {}
DrainSlot::Idle => {
panic!("expected a live drain to be tracked after two concurrent calls")
}
}
let deadline = std::time::Instant::now() + Duration::from_secs(5);
loop {
let state = host.queue_state();
if state["drain"]["live"] == false {
break;
}
assert!(
std::time::Instant::now() < deadline,
"drain never settled idle: {state}"
);
tokio::time::sleep(Duration::from_millis(20)).await;
}
}
fn seed_one_queued(root: &Path, mission_id: &str) -> Arc<Mutex<DrainState>> {
kranz_engine::queue::enqueue(
root,
kranz_engine::queue::QueueEntry {
mission_id: mission_id.to_string(),
ticket_slug: None,
priority: 5,
seq: 0,
},
)
.expect("enqueue");
Arc::new(Mutex::new(DrainState::default()))
}
fn proceed_readiness(
_repo_root: &Path,
mission_id: &str,
) -> kranz_engine::error::Result<kranz_engine::backend_readiness::ReadinessReport> {
Ok(kranz_engine::backend_readiness::ReadinessReport {
mission_id: mission_id.to_string(),
roles: Vec::new(),
overall: kranz_engine::backend_readiness::ReadinessStatus::Ok,
warnings: Vec::new(),
})
}
#[tokio::test]
async fn auto_work_drain_mode_processes_only_one_queue_front() {
let Some((_dir, root)) = init_repo() else {
return;
};
let state = seed_one_queued(&root, "m-first");
kranz_engine::queue::enqueue(
&root,
kranz_engine::queue::QueueEntry {
mission_id: "m-second".to_string(),
ticket_slug: None,
priority: 5,
seq: 0,
},
)
.expect("enqueue second mission");
drain_task_with_probe(
root.clone(),
Arc::clone(&state),
true,
|_mission_id| async { Ok(0) },
proceed_readiness,
)
.await;
assert_eq!(state.lock().expect("drain state lock").ran, ["m-first"]);
let remaining = kranz_engine::queue::list(&root);
assert_eq!(remaining.len(), 1);
assert_eq!(remaining[0].mission_id, "m-second");
}
#[tokio::test]
async fn hosted_drain_restores_dispatch_checkout() {
let Some((_dir, root)) = init_repo() else {
return;
};
let state = seed_one_queued(&root, "m-restore");
let run_root = root.clone();
drain_task_with_probe(
root.clone(),
Arc::clone(&state),
false,
move |mission_id| {
let root = run_root.clone();
async move {
let git = GitRepo::open(&root)?;
let branch = format!("kranz/mission-{mission_id}");
git.create_branch(&branch, None)?;
git.checkout(&branch)?;
Ok(0)
}
},
proceed_readiness,
)
.await;
assert_eq!(
state.lock().expect("drain state lock").ran,
["m-restore"],
"the injected mission runner must execute"
);
let git = GitRepo::open(&root).expect("open repo");
assert_eq!(
git.current_branch().expect("current branch"),
"main",
"the operator's dispatch-time checkout must be restored on drain exit"
);
}
#[tokio::test]
async fn hosted_drain_restores_dispatch_checkout_on_err() {
let Some((_dir, root)) = init_repo() else {
return;
};
let state = seed_one_queued(&root, "m-err-restore");
let run_root = root.clone();
let runner_called = Arc::new(std::sync::atomic::AtomicBool::new(false));
let called = Arc::clone(&runner_called);
drain_task_with_probe(
root.clone(),
state,
false,
move |mission_id| {
let root = run_root.clone();
let called = Arc::clone(&called);
async move {
called.store(true, std::sync::atomic::Ordering::SeqCst);
let git = GitRepo::open(&root)?;
let branch = format!("kranz/mission-{mission_id}");
git.create_branch(&branch, None)?;
git.checkout(&branch)?;
Err(anyhow::anyhow!("simulated drain runner failure"))
}
},
proceed_readiness,
)
.await;
assert!(
runner_called.load(std::sync::atomic::Ordering::SeqCst),
"the injected mission runner must execute"
);
let git = GitRepo::open(&root).expect("open repo");
assert_eq!(
git.current_branch().expect("current branch"),
"main",
"an errored drain must still restore the operator's dispatch-time checkout"
);
}
#[tokio::test]
async fn hosted_drain_skips_restore_when_started_on_mission_branch() {
let Some((_dir, root)) = init_repo() else {
return;
};
{
let git = GitRepo::open(&root).expect("open repo");
git.create_branch("kranz/mission-existing", None)
.expect("create existing mission branch");
git.checkout("kranz/mission-existing")
.expect("checkout existing mission branch");
}
let state = seed_one_queued(&root, "m-skip");
drain_task_with_probe(
root.clone(),
Arc::clone(&state),
false,
|_mission_id| async { Ok(0) },
proceed_readiness,
)
.await;
assert_eq!(
state.lock().expect("drain state lock").ran,
["m-skip"],
"the injected mission runner must execute"
);
let git = GitRepo::open(&root).expect("open repo");
assert_eq!(
git.current_branch().expect("current branch"),
"kranz/mission-existing",
"started on a mission branch: no restore must be attempted"
);
}
#[tokio::test]
async fn hosted_drain_leaves_checkout_when_tracked_tree_dirty() {
let Some((_dir, root)) = init_repo() else {
return;
};
let state = seed_one_queued(&root, "m-dirty");
let run_root = root.clone();
drain_task_with_probe(
root.clone(),
Arc::clone(&state),
false,
move |mission_id| {
let root = run_root.clone();
async move {
let git = GitRepo::open(&root)?;
let branch = format!("kranz/mission-{mission_id}");
git.create_branch(&branch, None)?;
git.checkout(&branch)?;
std::fs::write(root.join("README.md"), "dirty tracked edit\n")?;
Ok(0)
}
},
proceed_readiness,
)
.await;
assert_eq!(
state.lock().expect("drain state lock").ran,
["m-dirty"],
"the injected mission runner must execute"
);
let git = GitRepo::open(&root).expect("open repo");
assert_eq!(
git.current_branch().expect("current branch"),
"kranz/mission-m-dirty",
"a dirty tracked tree must abort the restore, leaving the checkout on the mission \
branch"
);
}
#[tokio::test]
async fn hosted_drain_second_call_does_not_capture_or_restore() {
let Some((_dir, root)) = init_repo() else {
return;
};
{
let git = GitRepo::open(&root).expect("open repo");
git.create_branch("feature-branch", None)
.expect("create feature branch");
git.checkout("feature-branch")
.expect("checkout feature branch");
}
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let host = MissionHost::with_backend(root.clone(), backend);
let tracked_state = Arc::new(Mutex::new(DrainState {
live: true,
current_mission_id: Some("m-inflight".to_string()),
ran: Vec::new(),
parked: Vec::new(),
}));
let never_finishes = tokio::spawn(async {
std::future::pending::<()>().await;
});
*host.drain.lock().expect("drain tracker lock") = DrainSlot::Running(DrainHandle {
join: never_finishes,
state: Arc::clone(&tracked_state),
});
let result = host
.drain()
.await
.expect("second drain call must not error");
assert_eq!(result["live"], true, "{result}");
let git = GitRepo::open(&root).expect("open repo");
assert_eq!(
git.current_branch().expect("current branch"),
"feature-branch",
"the idempotent second drain() must not mutate the checkout"
);
match &*host.drain.lock().expect("drain tracker lock") {
DrainSlot::Running(handle) => {
assert_eq!(
Arc::as_ptr(&handle.state),
Arc::as_ptr(&tracked_state),
"a second drain must not replace the tracker or spawn a second task"
);
}
_ => panic!("expected the tracker to still be Running"),
};
}
#[tokio::test]
async fn starting_reservation_is_not_overwritten_or_double_spawned() {
let Some((_dir, root)) = init_repo() else {
return;
};
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let host = MissionHost::with_backend(root, backend);
let state = Arc::new(Mutex::new(DrainState {
live: true,
current_mission_id: Some("m-reserved".to_string()),
ran: Vec::new(),
parked: Vec::new(),
}));
*host.drain.lock().expect("drain tracker lock") = DrainSlot::Starting(Arc::clone(&state));
let before = Arc::as_ptr(&state);
let result = host.drain().await.expect("drain must not error");
assert_eq!(result["live"], true, "{result}");
assert_eq!(result["currentMissionId"], "m-reserved");
let after = match &*host.drain.lock().expect("drain tracker lock") {
DrainSlot::Starting(tracked) => Arc::as_ptr(tracked),
DrainSlot::Running(_) => panic!(
"the Starting reservation was upgraded/replaced by this call — the deflection \
arm was bypassed and a second drain was spawned"
),
DrainSlot::Idle => panic!("the Starting reservation was cleared by this call"),
};
assert_eq!(
before, after,
"drain() must return the SAME tracked reservation, not install a new one"
);
}
#[tokio::test]
async fn failed_drain_construction_clears_the_reservation_to_idle() {
let Some((_dir, root)) = init_repo() else {
return;
};
std::fs::create_dir_all(root.join(".kranz")).expect("mkdir .kranz");
std::fs::write(root.join(".kranz").join("config.json"), "not json")
.expect("write malformed config");
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let host = MissionHost::with_backend(root, backend);
host.drain()
.await
.expect_err("malformed config must fail drain construction");
let is_idle = matches!(
&*host.drain.lock().expect("drain tracker lock"),
DrainSlot::Idle
);
assert!(
is_idle,
"a failed drain construction must reset the tracker to Idle"
);
}
#[tokio::test]
async fn queue_state_reports_a_starting_reservation_as_live() {
let Some((_dir, root)) = init_repo() else {
return;
};
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let host = MissionHost::with_backend(root, backend);
let state = Arc::new(Mutex::new(DrainState {
live: true,
current_mission_id: Some("m-starting".to_string()),
ran: Vec::new(),
parked: Vec::new(),
}));
*host.drain.lock().expect("drain tracker lock") = DrainSlot::Starting(state);
let queue_state = host.queue_state();
assert_eq!(queue_state["drain"]["live"], true, "{queue_state}");
assert_eq!(queue_state["drain"]["currentMissionId"], "m-starting");
}
#[tokio::test]
async fn queue_drain_route_requires_token_but_queue_route_does_not() {
use axum::body::Body;
use axum::http::Request;
use tower::ServiceExt;
let Some((_dir, root)) = init_repo() else {
return;
};
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let host = MissionHost::with_backend(root, backend);
let app =
crate::router_with_host(host, None, crate::MutationAuthority::new("tok").unwrap());
let response = app
.clone()
.oneshot(
Request::builder()
.method("POST")
.uri("/api/queue/drain")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
let response = app
.clone()
.oneshot(
Request::builder()
.method("POST")
.uri("/api/queue/drain")
.header("x-kranz-token", "tok")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_ne!(response.status(), StatusCode::UNAUTHORIZED);
assert_eq!(response.status(), StatusCode::OK);
let response = app
.oneshot(
Request::builder()
.uri("/api/queue")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
}
#[test]
fn should_auto_drain_truth_table() {
assert!(should_auto_drain(true, true, false));
assert!(!should_auto_drain(false, true, false));
assert!(!should_auto_drain(false, false, false));
assert!(!should_auto_drain(true, false, false));
assert!(!should_auto_drain(true, true, true));
assert!(!should_auto_drain(false, false, true));
}
fn write_auto_work_config(root: &std::path::Path, enabled: bool) {
let dir = root.join(".kranz");
std::fs::create_dir_all(&dir).expect("create .kranz dir");
std::fs::write(
dir.join("config.json"),
json!({ "autoWork": enabled }).to_string(),
)
.expect("write config.json");
}
fn turn(reply: &str) -> Vec<kranz_engine::backend::AgentEvent> {
vec![
kranz_engine::backend_mock::mock_text(reply),
mock_result_text(reply),
]
}
fn preflight_authenticated_script() -> MockScript {
MockScript::single_shot("ack")
}
fn worker_pass() -> MockScript {
MockScript::single_shot_json(&json!({
"result": "pass",
"summary": "implemented and tested",
"filesTouched": [],
"testsAdded": [],
"testEvidence": "all green",
"commits": []
}))
}
fn plan_json() -> Value {
json!({
"goal": "ship the demo",
"validationContract": [],
"milestones": [{
"title": "M1",
"features": [{
"title": "F1",
"spec": "build the thing",
"validationCriteria": ["it works"]
}]
}]
})
}
#[tokio::test(flavor = "multi_thread")]
async fn auto_work_tick_drains_a_queued_mission_when_enabled() {
let Some((_dir, root)) = init_repo() else {
return;
};
write_auto_work_config(&root, true);
let judgement =
json!({ "decision": "complete", "guidance": "", "summary": "worker did the job" });
let orch = MockScript::streaming(vec![mock_init("orch-auto"), mock_result_text("seed-hi")])
.responding(vec![
turn("scoping the demo"),
turn(&plan_json().to_string()),
]);
let orch_run = MockScript::streaming(vec![
mock_init("orch-auto-run"),
mock_result_text("resumed"),
])
.responding(vec![turn(&judgement.to_string()), turn("NONE")]);
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::with_scripts(vec![
orch,
preflight_authenticated_script(),
worker_pass(),
orch_run,
]));
let host = MissionHost::with_backend(root.clone(), backend);
let id = host
.create(
"drain me via autoWork",
Some(&json!({ "skipScrutiny": true, "skipFunctional": true })),
)
.await
.expect("create mission");
host.planning_turn(&id, "go").await.expect("planning turn");
let plan_body = host.request_plan(&id).await.expect("request plan");
assert_eq!(plan_body["ready"], true, "{plan_body}");
let plan: Plan =
serde_json::from_value(plan_body["plan"].clone()).expect("plan deserializes");
host.approve(&id, plan).await.expect("approve");
host.release(&id).expect("release");
kranz_engine::queue::enqueue(
&root,
kranz_engine::queue::QueueEntry {
mission_id: id.clone(),
ticket_slug: None,
priority: 2,
seq: 0,
},
)
.expect("enqueue");
host.auto_work_tick().await;
assert!(
host.drain_is_live(),
"autoWork tick with autoWork=true and a non-empty queue must start a drain"
);
let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
loop {
let state = host.queue_state();
if state["entries"]
.as_array()
.map(|a| a.is_empty())
.unwrap_or(false)
&& state["drain"]["live"] == false
{
break;
}
assert!(
tokio::time::Instant::now() < deadline,
"autoWork drain never completed: {state}"
);
tokio::time::sleep(Duration::from_millis(50)).await;
}
}
#[tokio::test]
async fn auto_work_tick_leaves_the_queue_untouched_when_disabled() {
let Some((_dir, root)) = init_repo() else {
return;
};
let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
let host = MissionHost::with_backend(root.clone(), backend);
kranz_engine::queue::enqueue(
&root,
kranz_engine::queue::QueueEntry {
mission_id: "m-untouched".to_string(),
ticket_slug: None,
priority: 2,
seq: 0,
},
)
.expect("enqueue");
host.auto_work_tick().await;
assert!(
!host.drain_is_live(),
"autoWork=false must never start a drain"
);
let entries = kranz_engine::queue::list(&root);
assert_eq!(
entries.len(),
1,
"queue entry must be left untouched when autoWork is disabled: {entries:?}"
);
assert_eq!(entries[0].mission_id, "m-untouched");
}
}