1use crate::error::{ApiError, ApiErrorCode};
28use crate::ServerState;
29use axum::body::Bytes;
30use axum::extract::{Path as UrlPath, State};
31use axum::http::StatusCode;
32use axum::response::IntoResponse;
33use axum::Json;
34use kranz_engine::backend::{AgentBackend, AgentEvent, PromptMode, SessionExit, SessionSpec};
35use kranz_engine::backend_claude::ClaudeBackend;
36use kranz_engine::config;
37use kranz_engine::cost::{self, CostEstimate};
38use kranz_engine::deps;
39use kranz_engine::draft::{drive_draft, DraftOutcome};
40use kranz_engine::error::EngineError;
41use kranz_engine::event_log::{EventLog, LockForce};
42use kranz_engine::git_ops::GitRepo;
43use kranz_engine::git_ops::KranzCommitMetadata;
44use kranz_engine::merge::{
45 merge_mission_with_external_evidence, MergeReport, StandardsMergeEvidence,
46};
47use kranz_engine::orchestrator::{MissionEngine, PlanRequest};
48use kranz_engine::paths::MissionPaths;
49use kranz_engine::planning::plan_identity;
50use kranz_engine::queue;
51use kranz_engine::ticket::Ticket;
52use kranz_engine::types::{MissionConfig, MissionStatus, Plan, TokenUsage};
53use serde_json::{json, Value};
54use std::collections::HashMap;
55use std::path::{Path, PathBuf};
56use std::sync::{Arc, Mutex};
57use std::time::{Duration, Instant};
58use tokio::sync::{OwnedSemaphorePermit, Semaphore};
59
60type EngineCell = Arc<tokio::sync::Mutex<Box<MissionEngine>>>;
63
64enum HostedMission {
66 Planning {
70 cell: EngineCell,
71 last_use: Arc<Mutex<Instant>>,
72 pending_plan: Arc<Mutex<Option<Plan>>>,
77 },
78 Running {
83 handle: tokio::task::JoinHandle<()>,
84 _repo_busy: kranz_engine::queue::RepoBusyHold,
85 },
86}
87
88#[derive(Debug, Clone, PartialEq, Eq)]
94pub enum PendingApproval {
95 Approved(String),
98 NothingParked,
101 Mismatch { parked: String },
104}
105
106pub struct MissionHost {
108 repo_root: PathBuf,
109 backend: tokio::sync::OnceCell<Arc<dyn AgentBackend>>,
113 missions: Arc<Mutex<HashMap<String, HostedMission>>>,
115 sweeper: Mutex<Option<tokio::task::JoinHandle<()>>>,
118 drain: Mutex<DrainSlot>,
121 global_run_permits: Option<Arc<Semaphore>>,
125 gate_executor: GateExecutor,
129 readiness_front_cache: Mutex<Option<FrontReadinessCache>>,
132 readiness_probe: ReadinessProbe,
136}
137
138struct FrontReadinessCache {
140 mission_id: String,
141 report: Value,
142 at: Instant,
143}
144
145const READINESS_FRONT_CACHE_TTL: Duration = Duration::from_secs(5);
146
147#[derive(Debug, Clone, Default)]
151struct DrainState {
152 live: bool,
153 current_mission_id: Option<String>,
154 ran: Vec<String>,
155 parked: Vec<String>,
156}
157
158type GateExecutor = Arc<dyn Fn(&str, &Path) -> (bool, String) + Send + Sync>;
163
164type ReadinessProbe =
165 fn(
166 &Path,
167 &str,
168 ) -> kranz_engine::error::Result<kranz_engine::backend_readiness::ReadinessReport>;
169
170fn injected_backend_readiness(
171 _repo_root: &Path,
172 mission_id: &str,
173) -> kranz_engine::error::Result<kranz_engine::backend_readiness::ReadinessReport> {
174 Ok(kranz_engine::backend_readiness::ReadinessReport {
175 mission_id: mission_id.to_string(),
176 roles: Vec::new(),
177 overall: kranz_engine::backend_readiness::ReadinessStatus::Ok,
178 warnings: Vec::new(),
179 })
180}
181
182fn real_gate_executor() -> GateExecutor {
186 Arc::new(|command, cwd| kranz_engine::command_exec::run_bounded_gate_command(cwd, command))
187}
188
189fn should_auto_drain(auto_work: bool, queue_non_empty: bool, drain_live: bool) -> bool {
193 auto_work && queue_non_empty && !drain_live
194}
195
196fn drain_state_json(state: &DrainState) -> Value {
197 json!({
198 "live": state.live,
199 "currentMissionId": state.current_mission_id,
200 "ran": state.ran,
201 "parked": state.parked,
202 })
203}
204
205struct DrainHandle {
209 join: tokio::task::JoinHandle<()>,
210 state: Arc<Mutex<DrainState>>,
211}
212
213enum DrainSlot {
222 Idle,
223 Starting(Arc<Mutex<DrainState>>),
224 Running(DrainHandle),
225}
226
227impl MissionHost {
228 pub fn new(repo_root: PathBuf) -> Self {
230 MissionHost {
231 repo_root,
232 backend: tokio::sync::OnceCell::new(),
233 missions: Arc::new(Mutex::new(HashMap::new())),
234 sweeper: Mutex::new(None),
235 drain: Mutex::new(DrainSlot::Idle),
236 global_run_permits: None,
237 gate_executor: real_gate_executor(),
238 readiness_front_cache: Mutex::new(None),
239 readiness_probe: kranz_engine::backend_readiness::probe_mission,
240 }
241 }
242
243 pub fn with_backend(repo_root: PathBuf, backend: Arc<dyn AgentBackend>) -> Self {
246 MissionHost {
247 repo_root,
248 backend: tokio::sync::OnceCell::new_with(Some(backend)),
249 missions: Arc::new(Mutex::new(HashMap::new())),
250 sweeper: Mutex::new(None),
251 drain: Mutex::new(DrainSlot::Idle),
252 global_run_permits: None,
253 gate_executor: real_gate_executor(),
254 readiness_front_cache: Mutex::new(None),
255 readiness_probe: injected_backend_readiness,
256 }
257 }
258
259 pub fn with_gate_executor<F>(repo_root: PathBuf, gate_executor: F) -> Self
264 where
265 F: Fn(&str, &Path) -> (bool, String) + Send + Sync + 'static,
266 {
267 MissionHost {
268 repo_root,
269 backend: tokio::sync::OnceCell::new(),
270 missions: Arc::new(Mutex::new(HashMap::new())),
271 sweeper: Mutex::new(None),
272 drain: Mutex::new(DrainSlot::Idle),
273 global_run_permits: None,
274 gate_executor: Arc::new(gate_executor),
275 readiness_front_cache: Mutex::new(None),
276 readiness_probe: kranz_engine::backend_readiness::probe_mission,
277 }
278 }
279
280 pub(crate) fn new_with_global_run_permits(
282 repo_root: PathBuf,
283 global_run_permits: Arc<Semaphore>,
284 ) -> Self {
285 MissionHost {
286 repo_root,
287 backend: tokio::sync::OnceCell::new(),
288 missions: Arc::new(Mutex::new(HashMap::new())),
289 sweeper: Mutex::new(None),
290 drain: Mutex::new(DrainSlot::Idle),
291 global_run_permits: Some(global_run_permits),
292 gate_executor: real_gate_executor(),
293 readiness_front_cache: Mutex::new(None),
294 readiness_probe: kranz_engine::backend_readiness::probe_mission,
295 }
296 }
297
298 #[cfg(test)]
301 pub(crate) fn with_backend_and_global_run_permits(
302 repo_root: PathBuf,
303 backend: Arc<dyn AgentBackend>,
304 global_run_permits: Arc<Semaphore>,
305 ) -> Self {
306 MissionHost {
307 repo_root,
308 backend: tokio::sync::OnceCell::new_with(Some(backend)),
309 missions: Arc::new(Mutex::new(HashMap::new())),
310 sweeper: Mutex::new(None),
311 drain: Mutex::new(DrainSlot::Idle),
312 global_run_permits: Some(global_run_permits),
313 gate_executor: real_gate_executor(),
314 readiness_front_cache: Mutex::new(None),
315 readiness_probe: injected_backend_readiness,
316 }
317 }
318
319 pub fn repo_root(&self) -> &PathBuf {
321 &self.repo_root
322 }
323
324 pub(crate) fn try_global_run_permit(&self) -> Result<Option<OwnedSemaphorePermit>, ApiError> {
325 self.global_run_permits
326 .as_ref()
327 .map(|permits| {
328 Arc::clone(permits).try_acquire_owned().map_err(|_| {
329 ApiError::conflict(
330 "host.maxConcurrentRepos is saturated; retry when another repository finishes",
331 )
332 .with_code(ApiErrorCode::RepositoryBusy)
333 })
334 })
335 .transpose()
336 }
337
338 async fn backend(
340 &self,
341 claude_binary: Option<&str>,
342 ) -> Result<Arc<dyn AgentBackend>, ApiError> {
343 let configured = claude_binary.map(str::to_string);
344 self.backend
345 .get_or_try_init(|| async move {
346 let backend = ClaudeBackend::discover(configured.as_deref())?;
347 Ok::<Arc<dyn AgentBackend>, EngineError>(Arc::new(backend))
348 })
349 .await
350 .map(Arc::clone)
351 .map_err(ApiError::from)
352 }
353
354 pub async fn create(
364 &self,
365 goal: &str,
366 config_patch: Option<&Value>,
367 ) -> Result<String, ApiError> {
368 let mut cfg = config::load(&self.repo_root)?;
369 if let Some(patch) = config_patch {
370 if !patch.is_object() {
371 return Err(ApiError::bad_request("'config' must be a JSON object"));
372 }
373 let mut merged = serde_json::to_value(&cfg)
374 .map_err(|e| ApiError::internal(format!("config does not serialize: {e}")))?;
375 config::deep_merge(&mut merged, patch);
376 cfg = serde_json::from_value(merged).map_err(|e| {
377 ApiError::bad_request(format!("'config' patch does not deserialize: {e}"))
378 })?;
379 }
380 config::validate(&cfg)?;
381
382 let backend = self.backend(cfg.claude_binary.as_deref()).await?;
383 let engine = MissionEngine::create(backend, self.repo_root.clone(), goal, cfg)?;
384 let id = engine.mission_id().to_string();
385 self.missions
386 .lock()
387 .expect("missions registry lock")
388 .insert(id.clone(), new_planning(new_cell(Box::new(engine))));
389 self.ensure_sweeper_started();
390 Ok(id)
391 }
392
393 pub async fn draft(&self, slug: &str, then_enqueue: bool) -> Result<DraftOutcome, ApiError> {
405 Ticket::ensure_valid_slug(slug)?;
406 let ticket_path = Ticket::tickets_dir(&self.repo_root).join(format!("{slug}.md"));
407 if !ticket_path.is_file() {
408 return Err(ApiError::not_found(format!("ticket '{slug}' not found")));
409 }
410 let ticket = Ticket::load(&ticket_path)?;
411
412 let cfg = config_for_ticket(config::load(&self.repo_root)?, &ticket);
413 let backend = self.backend(cfg.claude_binary.as_deref()).await?;
414 let engine =
415 MissionEngine::create(backend, self.repo_root.clone(), &ticket.mission_goal(), cfg)?;
416 let id = engine.mission_id().to_string();
417 let cell = new_cell(Box::new(engine));
418 self.missions
419 .lock()
420 .expect("missions registry lock")
421 .insert(id.clone(), new_planning(Arc::clone(&cell)));
422 self.ensure_sweeper_started();
423
424 let drive_result = {
425 let mut engine = cell.lock().await;
426 drive_draft(&mut engine, &self.repo_root, &ticket, then_enqueue).await
427 };
428
429 self.missions
433 .lock()
434 .expect("missions registry lock")
435 .remove(&id);
436 drop(cell);
437
438 Ok(drive_result?.outcome)
439 }
440
441 pub async fn draft_async(&self, slug: &str, then_enqueue: bool) -> Result<String, ApiError> {
450 Ticket::ensure_valid_slug(slug)?;
451 let ticket_path = Ticket::tickets_dir(&self.repo_root).join(format!("{slug}.md"));
452 if !ticket_path.is_file() {
453 return Err(ApiError::not_found(format!("ticket '{slug}' not found")));
454 }
455 let ticket = Ticket::load(&ticket_path)?;
456
457 let cfg = config_for_ticket(config::load(&self.repo_root)?, &ticket);
458 let backend = self.backend(cfg.claude_binary.as_deref()).await?;
459 let engine =
460 MissionEngine::create(backend, self.repo_root.clone(), &ticket.mission_goal(), cfg)?;
461 let id = engine.mission_id().to_string();
462 let cell = new_cell(Box::new(engine));
463 self.missions
464 .lock()
465 .expect("missions registry lock")
466 .insert(id.clone(), new_planning(Arc::clone(&cell)));
467 self.ensure_sweeper_started();
468
469 let repo_root = self.repo_root.clone();
470 let missions = Arc::clone(&self.missions);
471 let mission_id = id.clone();
472 tokio::spawn(async move {
473 let drive_result = {
474 let mut engine = cell.lock().await;
475 drive_draft(&mut engine, &repo_root, &ticket, then_enqueue).await
476 };
477 missions
481 .lock()
482 .expect("missions registry lock")
483 .remove(&mission_id);
484 drop(cell);
485 if let Err(e) = drive_result {
486 tracing::error!(mission = %mission_id, error = %e, "hosted ticket draft errored");
487 }
488 });
489
490 Ok(id)
491 }
492
493 pub fn approve_ticket(
498 &self,
499 slug: &str,
500 force: bool,
501 ) -> Result<deps::ApprovedTicket, ApiError> {
502 deps::approve_ticket(&self.repo_root, slug, None, force).map_err(ApiError::from)
503 }
504
505 pub async fn planning_turn(&self, id: &str, text: &str) -> Result<String, ApiError> {
509 let cell = self.planning_cell_or_attach(id).await?;
510 let mut engine = try_lock(&cell)?;
511 let reply = engine.planning_turn(text).await?;
512 Ok(prepend_seed(engine.take_seed_reply(), reply))
513 }
514
515 pub async fn request_plan(&self, id: &str) -> Result<Value, ApiError> {
519 let cell = self.planning_cell_or_attach(id).await?;
520 let mut engine = try_lock(&cell)?;
521 let request = engine.request_plan().await?;
522 let seed = engine.take_seed_reply();
523 match request {
524 PlanRequest::Ready(plan) => {
525 let calibration = cost::calibrate(&self.repo_root);
528 let estimate = cost::estimate(&plan, &engine.state().config, &calibration.params);
529 let estimate = cost::apply_shape(estimate, &plan, &calibration);
530 self.set_pending_plan(id, Some(plan.clone()));
533 Ok(json!({
534 "ready": true,
535 "planIdentity": plan_identity(&plan),
536 "plan": plan,
537 "estimate": estimate_json(&estimate),
538 "calibration": { "missionsUsed": calibration.missions_used },
539 }))
540 }
541 PlanRequest::NotReady(reply) | PlanRequest::WrongPlan { reason: reply } => {
542 Ok(json!({ "ready": false, "reply": prepend_seed(seed, reply) }))
546 }
547 }
548 }
549
550 pub async fn approve(&self, id: &str, plan: Plan) -> Result<String, ApiError> {
553 let cell = self.planning_cell_or_attach(id).await?;
554 let mut engine = try_lock(&cell)?;
555 engine.approve_plan_as(
556 plan,
557 kranz_engine::live_permission::Actor::LocalMutationCapability,
558 )?;
559 self.set_pending_plan(id, None);
560 Ok(engine.state().mission.mission_branch.clone())
561 }
562
563 pub async fn start(&self, id: &str) -> Result<(), ApiError> {
568 let taken: Option<Box<MissionEngine>> = {
570 let mut map = self.missions.lock().expect("missions registry lock");
571 match map.remove(id) {
572 None => None,
573 Some(HostedMission::Running { handle, _repo_busy }) => {
574 if handle.is_finished() {
575 drop(_repo_busy);
579 None
580 } else {
581 map.insert(
582 id.to_string(),
583 HostedMission::Running { handle, _repo_busy },
584 );
585 return Err(ApiError::conflict(format!(
586 "mission '{id}' is already running — observe it via GET \
587 /api/missions/{id}/state or steer it via POST \
588 /api/missions/{id}/control"
589 )));
590 }
591 }
592 Some(HostedMission::Planning {
593 cell,
594 last_use,
595 pending_plan,
596 }) => match Arc::try_unwrap(cell) {
597 Err(cell) => {
598 map.insert(
601 id.to_string(),
602 HostedMission::Planning {
603 cell,
604 last_use,
605 pending_plan,
606 },
607 );
608 return Err(turn_in_flight());
609 }
610 Ok(mutex) => {
611 let engine = mutex.into_inner();
612 if engine.state().mission.status == MissionStatus::Planning {
613 map.insert(id.to_string(), new_planning(new_cell(engine)));
614 return Err(ApiError::conflict(format!(
615 "mission '{id}' has no approved plan yet — approve one via \
616 POST /api/missions/{id}/approve first"
617 )));
618 }
619 Some(engine)
620 }
621 },
622 }
623 };
624
625 let (engine, from_registry) = match taken {
626 Some(engine) => (engine, true),
627 None => {
628 if !MissionPaths::new(&self.repo_root, id)
632 .events_file()
633 .is_file()
634 {
635 return Err(ApiError::not_found(format!("unknown mission '{id}'")));
636 }
637 let cfg = config::load(&self.repo_root)?;
638 let backend = self.backend(cfg.claude_binary.as_deref()).await?;
639 let engine = Box::new(MissionEngine::resume(
640 backend,
641 self.repo_root.clone(),
642 id,
643 LockForce::No,
644 )?);
645 match engine.state().mission.status {
646 MissionStatus::Planning => {
647 return Err(ApiError::conflict(format!(
648 "mission '{id}' is still in planning — approve a plan first \
649 (POST /api/missions/{id}/approve, or `kranz plan`)"
650 )))
651 }
652 MissionStatus::Complete => {
653 return Err(ApiError::conflict(format!(
654 "mission '{id}' is already complete — nothing to run"
655 )))
656 }
657 MissionStatus::Failed => {
658 return Err(ApiError::conflict(format!(
659 "mission '{id}' has failed — inspect its log; there is nothing \
660 the engine can resume"
661 )))
662 }
663 _ => {}
664 }
665 (engine, false)
666 }
667 };
668
669 let global_run_permit = match self.try_global_run_permit() {
670 Ok(permit) => permit,
671 Err(error) => {
672 if from_registry {
673 self.missions
674 .lock()
675 .expect("missions registry lock")
676 .insert(id.to_string(), new_planning(new_cell(engine)));
677 }
678 return Err(error);
679 }
680 };
681
682 let repo_busy = match kranz_engine::queue::acquire_repo_busy(&self.repo_root, id) {
686 Ok(hold) => hold,
687 Err(e) => {
688 if from_registry {
689 self.missions
692 .lock()
693 .expect("missions registry lock")
694 .insert(id.to_string(), new_planning(new_cell(engine)));
695 }
696 return Err(match e {
698 e @ EngineError::LockHeld(_) => {
699 ApiError::from(e).with_code(ApiErrorCode::RepositoryBusy)
700 }
701 other => other.into(),
702 });
703 }
704 };
705
706 {
710 let mut map = self.missions.lock().expect("missions registry lock");
711 let missions = Arc::clone(&self.missions);
712 let mission_id = id.to_string();
713 let handle = spawn_with_global_run_permit(
714 global_run_permit,
715 run_to_end(engine, mission_id, missions),
716 );
717 map.insert(
718 id.to_string(),
719 HostedMission::Running {
720 handle,
721 _repo_busy: repo_busy,
722 },
723 );
724 }
725 Ok(())
726 }
727
728 pub async fn merge(&self, id: &str) -> Result<Value, ApiError> {
735 if !MissionPaths::is_safe_id(id) {
736 return Err(ApiError::not_found(format!("unknown mission '{id}'")));
737 }
738 let paths = MissionPaths::new(&self.repo_root, id);
739 if !paths.events_file().is_file() {
740 return Err(ApiError::not_found(format!("unknown mission '{id}'")));
741 }
742 let repo_busy = kranz_engine::queue::acquire_repo_busy(&self.repo_root, id).map_err(
749 |error| match error {
750 error @ EngineError::LockHeld(_) => {
751 ApiError::from(error).with_code(ApiErrorCode::RepositoryBusy)
752 }
753 other => other.into(),
754 },
755 )?;
756 let events = EventLog::read_events(&paths.events_file())?;
757 let state = kranz_engine::reducer::fold(&events).map_err(ApiError::from)?;
758 if state.mission.status != MissionStatus::Complete {
759 return Err(ApiError::conflict(format!(
760 "mission '{id}' is {:?}; only a complete mission can be merged",
761 state.mission.status
762 )));
763 }
764 let base_branch = state.mission.base_branch.clone();
765 let base_sha = state.mission.base_sha.clone().ok_or_else(|| {
766 ApiError::conflict(format!(
767 "mission '{id}' has no pinned base sha — approve a plan first"
768 ))
769 })?;
770 let mission_branch = state.mission.mission_branch.clone();
771 let standards_pin = state.mission.standards_manifest.clone();
776 let standards_coverage = kranz_engine::standards_coverage::standards_coverage(id, &events);
777 let standards_evidence = StandardsMergeEvidence::from_mission_events(
778 id,
779 standards_pin.as_ref(),
780 standards_coverage.as_ref(),
781 &events,
782 chrono::Utc::now(),
783 );
784 let metadata = KranzCommitMetadata {
785 mission_id: state.mission.id.clone(),
786 cost_usd: state.total_cost_usd,
787 tokens: state.totals.clone(),
788 };
789
790 let repo_root = self.repo_root.clone();
791 let gate_executor = Arc::clone(&self.gate_executor);
792 let gate_policy = kranz_engine::command_exec::MergeGatePolicy {
807 sandbox: kranz_engine::command_exec::worker_gate_sandbox(&state.config)?,
808 mission_dir: paths.mission_dir(),
809 };
810 if let Some(note) = gate_policy.degradation_note() {
817 tracing::warn!(mission = %id, note = %note, "merge gate sandbox cannot wrap; gates fail closed");
818 }
819 let merge_paths = paths.clone();
820 let report = tokio::task::spawn_blocking(move || {
821 let repo = GitRepo::open(&repo_root)?;
822 let report = merge_mission_with_external_evidence(
823 &repo,
824 &base_branch,
825 &base_sha,
826 &mission_branch,
827 Some(metadata),
828 standards_pin.as_ref(),
829 &standards_evidence,
830 |cmd, cwd| {
831 if gate_policy.enforces_on_this_host() {
832 kranz_engine::command_exec::run_bounded_gate_command_sandboxed(
833 cwd,
834 cmd,
835 &gate_policy,
836 )
837 } else {
838 gate_executor(cmd, cwd)
839 }
840 },
841 &merge_paths,
842 kranz_engine::live_permission::Actor::LocalMutationCapability,
843 );
844 drop(repo_busy);
847 report
848 })
849 .await
850 .map_err(|e| ApiError::internal(format!("merge task panicked: {e}")))?
851 .map_err(ApiError::from)?;
852
853 match report {
854 MergeReport::Merged { commit, stale_base } => Ok(json!({
855 "merged": true,
856 "commit": commit,
857 "staleBase": stale_base.map(|warning| json!({
858 "baseSha": warning.base_sha,
859 "liveBase": warning.live_base,
860 "mergeCommitsSinceBase": warning.merge_commits_since_base,
861 "message": format!(
862 "stale base: {} merge commit(s) landed on {} since the mission base; cross-branch semantic conflicts are more likely, and full gates have run",
863 warning.merge_commits_since_base,
864 warning.live_base,
865 ),
866 })),
867 })),
868 MergeReport::RefusedDirtyTree => Err(ApiError::conflict(
869 "refusing to merge: tracked working tree is dirty",
870 )),
871 MergeReport::GateFailed { gate, output } => Err(ApiError::unprocessable(
872 kranz_engine::scrub::scrub(&format!("{gate} failed:\n{output}")),
873 )),
874 MergeReport::GateConfigInvalid { detail } => Err(ApiError::unprocessable(format!(
875 "refusing to merge without a valid repo gate suite: {detail}"
876 ))),
877 MergeReport::SecretScanFailed { findings } => Err(ApiError::unprocessable(format!(
878 "secret scan failed; add a fingerprint to {} only for a reviewed false positive:\n{}",
879 kranz_engine::scrub::SECRET_ALLOWLIST_PATH,
880 kranz_engine::scrub::format_findings(&findings)
881 ))),
882 MergeReport::Conflict { files } => Err(ApiError::conflict(format!(
883 "merge conflicted in: {}",
884 files.join(", ")
885 ))),
886 MergeReport::RefusedPreMerge { detail } => Err(ApiError::conflict(format!(
887 "merge refused before it started: {detail}"
888 ))),
889 MergeReport::StandardsDrifted {
890 approved_digest,
891 current_digest,
892 changed_rules,
893 } => {
894 if let Err(error) = EventLog::acquire(
900 &paths,
901 id,
902 std::time::Duration::ZERO,
903 LockForce::No,
904 )
905 .and_then(|mut log| {
906 log.append(kranz_engine::events::EventKind::StandardsDrifted {
907 approved_digest: approved_digest.clone(),
908 current_digest: current_digest.clone(),
909 surface: "merge".to_string(),
910 changed_rules: changed_rules.clone(),
911 })
912 .map(|_| ())
913 }) {
914 tracing::warn!(mission = %id, %error, "standards.drifted event could not be appended; the merge refusal stands");
915 }
916 Err(ApiError::unprocessable(format!(
917 "refusing to merge: the live base Flight Rules policy drifted from the \
918 approved pin (approved sha256:{approved_digest}, current {}) — the \
919 applicable enforced set changed; revalidate and re-approve the mission:\n{}",
920 current_digest
921 .as_deref()
922 .map(|d| format!("sha256:{d}"))
923 .unwrap_or_else(|| "<unreadable>".to_string()),
924 changed_rules.join("\n")
925 )))
926 }
927 MergeReport::StandardsFailed {
928 rule_id,
929 checker,
930 output,
931 } => Err(ApiError::unprocessable(kranz_engine::scrub::scrub(
932 &format!(
933 "Flight Rules merge checker refused {rule_id} ({checker}):\n{output}"
934 ),
935 ))),
936 }
937 }
938
939 pub async fn ask(&self, question: &str) -> Result<Value, ApiError> {
944 let question = question.trim();
945 if question.is_empty() {
946 return Err(ApiError::bad_request("ask requires a question"));
947 }
948 let cfg = config::load(&self.repo_root)?;
949 config::validate(&cfg)?;
950 let role = cfg.validator_scrutiny.clone();
951 let backend = self.backend(cfg.claude_binary.as_deref()).await?;
952 let prompt = ask_prompt(question, &ask_context(&self.repo_root));
953 let spec = SessionSpec {
954 cwd: self.repo_root.clone(),
955 prompt: PromptMode::SingleShot(prompt),
956 append_system_prompt: Some(
957 "You answer read-only questions about this Kranz repository. \
958 Use only the supplied context; if it is insufficient, say what is missing. \
959 Do not modify files, run commands, create missions, enqueue work, approve, \
960 start, or merge anything."
961 .to_string(),
962 ),
963 model: role.model,
964 effort: role.reasoning_effort,
965 session_id: format!("ask-{}", uuid::Uuid::new_v4()),
966 resume: None,
967 permission_mode: Some("plan".to_string()),
968 allowed_tools: vec![],
969 disallowed_tools: vec![
970 "Bash(*)".to_string(),
971 "Edit(*)".to_string(),
972 "Write(*)".to_string(),
973 ],
974 tools: vec![
975 "Read".to_string(),
976 "Grep".to_string(),
977 "Glob".to_string(),
978 "LS".to_string(),
979 ],
980 writable: false,
981 settings_json: None,
982 json_schema: None,
983 max_budget_usd: role.max_budget_usd,
984 max_turns: role.max_turns,
985 env: HashMap::new(),
986 sandbox: None,
987 hook_status: None,
988 };
989 let outcome = run_ask_session(backend, spec).await?;
990 Ok(json!({
991 "answer": outcome.answer,
992 "costUsd": outcome.cost_usd,
993 "tokens": outcome.tokens,
994 }))
995 }
996
997 pub fn release(&self, id: &str) -> Result<bool, ApiError> {
1009 release_from(&self.missions, id)
1010 }
1011
1012 pub fn sweep_idle(&self, threshold: Duration) -> Vec<String> {
1018 sweep_idle_from(&self.missions, threshold)
1019 }
1020
1021 fn ensure_sweeper_started(&self) {
1027 let mut guard = self.sweeper.lock().expect("sweeper lock");
1028 if guard.is_some() {
1029 return;
1030 }
1031 let repo_root = self.repo_root.clone();
1032 let missions = Arc::clone(&self.missions);
1033 *guard = Some(tokio::spawn(async move {
1034 const SWEEP_INTERVAL: Duration = Duration::from_secs(60);
1035 loop {
1036 tokio::time::sleep(SWEEP_INTERVAL).await;
1037 let minutes = match config::load(&repo_root) {
1038 Ok(cfg) => cfg.planning_idle_release_minutes,
1039 Err(_) => continue,
1040 };
1041 if minutes == 0 {
1042 continue;
1043 }
1044 let threshold = Duration::from_secs(minutes * 60);
1045 let released = sweep_idle_from(&missions, threshold);
1046 for id in released {
1047 tracing::info!(mission = %id, "released idle planning engine");
1048 }
1049 }
1050 }));
1051 }
1052
1053 pub(crate) fn drain_is_live(&self) -> bool {
1057 match &*self.drain.lock().expect("drain tracker lock") {
1058 DrainSlot::Idle => false,
1059 DrainSlot::Starting(_) => true,
1060 DrainSlot::Running(handle) => !handle.join.is_finished(),
1061 }
1062 }
1063
1064 pub(crate) async fn auto_work_tick(&self) -> bool {
1071 let cfg = match config::load(&self.repo_root) {
1072 Ok(cfg) => cfg,
1073 Err(_) => return false,
1074 };
1075 let queue_non_empty = kranz_engine::queue::peek(&self.repo_root).is_some();
1076 if should_auto_drain(cfg.auto_work, queue_non_empty, self.drain_is_live()) {
1077 if kranz_engine::queue::is_repo_busy(&self.repo_root).is_some() {
1082 return false;
1083 }
1084 match self.drain_once().await {
1085 Ok(_) => return true,
1086 Err(e) if e.code == Some(ApiErrorCode::RepositoryBusy) => {}
1087 Err(e) => tracing::error!(error = %e.message, "autoWork drain failed"),
1088 }
1089 }
1090 false
1091 }
1092
1093 pub async fn abandon(&self, id: &str, reason: &str) -> Result<(), ApiError> {
1102 let taken = self
1103 .missions
1104 .lock()
1105 .expect("missions registry lock")
1106 .remove(id);
1107 match taken {
1108 None => {}
1109 Some(HostedMission::Planning {
1110 cell,
1111 last_use,
1112 pending_plan,
1113 }) => match Arc::try_unwrap(cell) {
1114 Ok(mutex) => drop(mutex.into_inner()),
1115 Err(cell) => {
1116 self.missions
1117 .lock()
1118 .expect("missions registry lock")
1119 .insert(
1120 id.to_string(),
1121 HostedMission::Planning {
1122 cell,
1123 last_use,
1124 pending_plan,
1125 },
1126 );
1127 return Err(turn_in_flight());
1128 }
1129 },
1130 Some(HostedMission::Running { handle, _repo_busy }) => {
1131 if !handle.is_finished() {
1132 handle.abort();
1133 }
1134 let _ = handle.await;
1139 drop(_repo_busy);
1140 }
1141 }
1142 kranz_engine::mission_catalog::abandon_mission(
1143 self.repo_root.clone(),
1144 id,
1145 reason,
1146 LockForce::No,
1147 )
1148 .map_err(ApiError::from)?;
1149 Ok(())
1150 }
1151
1152 pub fn clean(&self, id: &str, all: bool) -> Result<(), ApiError> {
1161 use kranz_engine::mission_catalog::{
1162 cleanable_class, mission_lock_is_live, prune_mission_index_file, CleanClass,
1163 };
1164 if self
1165 .missions
1166 .lock()
1167 .expect("missions registry lock")
1168 .contains_key(id)
1169 {
1170 return Err(ApiError::conflict(format!(
1171 "mission '{id}' is hosted by this server (attached or running) — abandon it \
1172 first, or let its run finish"
1173 )));
1174 }
1175 let paths = MissionPaths::new(&self.repo_root, id);
1176 if !paths.events_file().is_file() {
1177 return Err(ApiError::not_found(format!("unknown mission '{id}'")));
1178 }
1179 let events = EventLog::read_events(&paths.events_file())?;
1180 let state = kranz_engine::reducer::fold(&events).map_err(ApiError::from)?;
1181 let has_plan = paths.plan_file().is_file();
1182 match cleanable_class(state.mission.status, has_plan) {
1183 CleanClass::Keep => {
1184 return Err(ApiError::conflict(format!(
1185 "mission '{id}' is live ({:?}) — abandon it before deleting",
1186 state.mission.status
1187 )))
1188 }
1189 CleanClass::CompleteKeepByDefault if !all => {
1190 return Err(ApiError::conflict(format!(
1191 "mission '{id}' is Complete; completed missions feed the cost-calibration \
1192 corpus — pass \"all\": true to delete it anyway"
1193 )))
1194 }
1195 CleanClass::Stale | CleanClass::CompleteKeepByDefault => {}
1196 }
1197 if mission_lock_is_live(&paths) {
1200 return Err(ApiError::conflict(format!(
1201 "mission '{id}' became live — nothing was deleted"
1202 )));
1203 }
1204 kranz_engine::queue::remove(&self.repo_root, id);
1205 std::fs::remove_dir_all(paths.mission_dir())
1206 .map_err(|e| ApiError::internal(format!("removing mission '{id}': {e}")))?;
1207 prune_mission_index_file(&self.repo_root, id);
1208 Ok(())
1209 }
1210
1211 pub fn pending_plan(&self, id: &str) -> Option<Plan> {
1214 let pending = {
1215 let map = self.missions.lock().expect("missions registry lock");
1216 match map.get(id) {
1217 Some(HostedMission::Planning { pending_plan, .. }) => Arc::clone(pending_plan),
1218 _ => return None,
1219 }
1220 };
1221 let plan = pending.lock().expect("pending plan lock").clone();
1224 plan
1225 }
1226
1227 fn set_pending_plan(&self, id: &str, plan: Option<Plan>) {
1228 let map = self.missions.lock().expect("missions registry lock");
1229 if let Some(HostedMission::Planning { pending_plan, .. }) = map.get(id) {
1230 *pending_plan.lock().expect("pending plan lock") = plan;
1231 }
1232 }
1233
1234 pub async fn try_approve_pending(&self, id: &str) -> Result<Option<String>, ApiError> {
1237 match self.approve_parked(id, |_| true)? {
1238 PendingApproval::Approved(branch) => Ok(Some(branch)),
1239 PendingApproval::NothingParked => Ok(None),
1240 PendingApproval::Mismatch { .. } => unreachable!("unconditional approval"),
1241 }
1242 }
1243
1244 pub async fn try_approve_pending_matching(
1249 &self,
1250 id: &str,
1251 expected_identity: Option<&str>,
1252 ) -> Result<PendingApproval, ApiError> {
1253 self.approve_parked(id, |plan| {
1254 expected_identity == Some(plan_identity(plan).as_str())
1255 })
1256 }
1257
1258 fn approve_parked(
1259 &self,
1260 id: &str,
1261 matches: impl FnOnce(&Plan) -> bool,
1262 ) -> Result<PendingApproval, ApiError> {
1263 let (cell, pending) = {
1264 let map = self.missions.lock().expect("missions registry lock");
1265 let Some(HostedMission::Planning {
1266 cell,
1267 pending_plan,
1268 last_use,
1269 }) = map.get(id)
1270 else {
1271 return Ok(PendingApproval::NothingParked);
1272 };
1273 *last_use.lock().expect("last-use lock") = Instant::now();
1274 (Arc::clone(cell), Arc::clone(pending_plan))
1275 };
1276 let mut engine = try_lock(&cell)?;
1279 let mut parked = pending.lock().expect("pending plan lock");
1280 let Some(plan) = parked.as_ref() else {
1281 return Ok(PendingApproval::NothingParked);
1282 };
1283 if !matches(plan) {
1284 return Ok(PendingApproval::Mismatch {
1285 parked: plan_identity(plan),
1286 });
1287 }
1288 engine.approve_plan_as(
1289 plan.clone(),
1290 kranz_engine::live_permission::Actor::LocalMutationCapability,
1291 )?;
1292 parked.take();
1293 Ok(PendingApproval::Approved(
1294 engine.state().mission.mission_branch.clone(),
1295 ))
1296 }
1297
1298 pub async fn approve_pending(
1300 &self,
1301 id: &str,
1302 expected_identity: Option<&str>,
1303 ) -> Result<String, ApiError> {
1304 match self.try_approve_pending_matching(id, expected_identity).await? {
1305 PendingApproval::Approved(branch) => Ok(branch),
1306 PendingApproval::NothingParked => Err(ApiError::conflict(format!(
1307 "mission '{id}' has no reviewed plan pending — refresh the plan preview before approving"
1308 )).with_code(ApiErrorCode::StalePlan)),
1309 PendingApproval::Mismatch { .. } => Err(ApiError::conflict(
1310 "reviewed plan identity is missing or stale — refresh the plan preview before approving",
1311 ).with_code(ApiErrorCode::StalePlan)),
1312 }
1313 }
1314
1315 pub async fn drain(&self) -> Result<Value, ApiError> {
1326 self.drain_with_mode(false).await
1327 }
1328
1329 async fn drain_once(&self) -> Result<Value, ApiError> {
1332 self.drain_with_mode(true).await
1333 }
1334
1335 async fn drain_with_mode(&self, once: bool) -> Result<Value, ApiError> {
1336 {
1338 let guard = self.drain.lock().expect("drain tracker lock");
1339 match &*guard {
1340 DrainSlot::Starting(state) => {
1341 return Ok(drain_state_json(&state.lock().expect("drain state lock")));
1342 }
1343 DrainSlot::Running(handle) if !handle.join.is_finished() => {
1344 return Ok(drain_state_json(
1345 &handle.state.lock().expect("drain state lock"),
1346 ));
1347 }
1348 DrainSlot::Idle | DrainSlot::Running(_) => {}
1349 }
1350 }
1351
1352 let cfg = config::load(&self.repo_root)?;
1357 let backend = self.backend(cfg.claude_binary.as_deref()).await?;
1358 let repo_root = self.repo_root.clone();
1359
1360 let (state, global_run_permit) = {
1364 let mut guard = self.drain.lock().expect("drain tracker lock");
1365 match &*guard {
1366 DrainSlot::Starting(state) => {
1367 return Ok(drain_state_json(&state.lock().expect("drain state lock")));
1368 }
1369 DrainSlot::Running(handle) if !handle.join.is_finished() => {
1370 return Ok(drain_state_json(
1371 &handle.state.lock().expect("drain state lock"),
1372 ));
1373 }
1374 DrainSlot::Idle | DrainSlot::Running(_) => {}
1375 }
1376 let global_run_permit = self.try_global_run_permit()?;
1377 let state = Arc::new(Mutex::new(DrainState {
1378 live: true,
1379 current_mission_id: None,
1380 ran: Vec::new(),
1381 parked: Vec::new(),
1382 }));
1383 *guard = DrainSlot::Starting(Arc::clone(&state));
1384 (state, global_run_permit)
1385 };
1386
1387 let task_state = Arc::clone(&state);
1393 let readiness_probe = self.readiness_probe;
1394 let join = tokio::spawn(async move {
1395 let _global_run_permit = global_run_permit;
1396 drain_task(
1397 repo_root.clone(),
1398 task_state,
1399 once,
1400 move |mission_id| {
1401 let backend = Arc::clone(&backend);
1402 let repo_root = repo_root.clone();
1403 async move { run_mission_headless(backend, repo_root, mission_id).await }
1404 },
1405 readiness_probe,
1406 )
1407 .await;
1408 });
1409
1410 let initial = drain_state_json(&state.lock().expect("drain state lock"));
1411 *self.drain.lock().expect("drain tracker lock") =
1412 DrainSlot::Running(DrainHandle { join, state });
1413 Ok(initial)
1414 }
1415
1416 pub fn queue_state(&self) -> Value {
1423 let entries = kranz_engine::queue::list(&self.repo_root);
1424 let busy_with = kranz_engine::queue::is_repo_busy(&self.repo_root);
1425 let drain = match &*self.drain.lock().expect("drain tracker lock") {
1426 DrainSlot::Running(handle) => {
1427 drain_state_json(&handle.state.lock().expect("drain state lock"))
1428 }
1429 DrainSlot::Starting(state) => {
1430 drain_state_json(&state.lock().expect("drain state lock"))
1431 }
1432 DrainSlot::Idle => drain_state_json(&DrainState::default()),
1433 };
1434
1435 let front_readiness = entries.first().map(|e| {
1436 let mid = e.mission_id.as_str();
1437 {
1438 let cache = self
1439 .readiness_front_cache
1440 .lock()
1441 .expect("readiness front cache lock");
1442 if let Some(cached) = cache.as_ref() {
1443 if cached.mission_id == mid && cached.at.elapsed() < READINESS_FRONT_CACHE_TTL {
1444 return (mid.to_string(), cached.report.clone());
1445 }
1446 }
1447 }
1448 let report = (self.readiness_probe)(&self.repo_root, mid)
1449 .ok()
1450 .and_then(|r| serde_json::to_value(r).ok())
1451 .unwrap_or(Value::Null);
1452 *self
1453 .readiness_front_cache
1454 .lock()
1455 .expect("readiness front cache lock") = Some(FrontReadinessCache {
1456 mission_id: mid.to_string(),
1457 report: report.clone(),
1458 at: Instant::now(),
1459 });
1460 (mid.to_string(), report)
1461 });
1462
1463 let entries_json: Vec<Value> = entries
1464 .into_iter()
1465 .map(|e| {
1466 let readiness = front_readiness.as_ref().and_then(|(id, report)| {
1467 if id == &e.mission_id && !report.is_null() {
1468 Some(report.clone())
1469 } else {
1470 None
1471 }
1472 });
1473 json!({
1474 "missionId": e.mission_id,
1475 "ticketSlug": e.ticket_slug,
1476 "priority": e.priority,
1477 "seq": e.seq,
1478 "readiness": readiness,
1479 })
1480 })
1481 .collect();
1482 let mut state = json!({
1483 "entries": entries_json,
1484 "busyWith": busy_with,
1485 "drain": drain,
1486 });
1487 if let Some(permits) = &self.global_run_permits {
1491 let available = permits.available_permits();
1492 state["maxConcurrentReposAvailable"] = json!(available);
1493 state["maxConcurrentReposSaturated"] = json!(available == 0);
1494 }
1495 state
1496 }
1497
1498 fn planning_cell(&self, id: &str) -> Result<EngineCell, ApiError> {
1505 let map = self.missions.lock().expect("missions registry lock");
1506 match map.get(id) {
1507 Some(HostedMission::Planning { cell, last_use, .. }) => {
1508 *last_use.lock().expect("last-use lock") = Instant::now();
1509 Ok(Arc::clone(cell))
1510 }
1511 Some(HostedMission::Running { .. }) => Err(ApiError::conflict(format!(
1512 "mission '{id}' is running — steer it via POST /api/missions/{id}/control"
1513 ))),
1514 None => Err(self.not_hosted(id)),
1515 }
1516 }
1517
1518 async fn planning_cell_or_attach(&self, id: &str) -> Result<EngineCell, ApiError> {
1526 let miss = match self.planning_cell(id) {
1527 Ok(cell) => return Ok(cell),
1528 Err(miss) => miss,
1529 };
1530 if !MissionPaths::new(&self.repo_root, id)
1533 .events_file()
1534 .is_file()
1535 || self
1536 .missions
1537 .lock()
1538 .expect("missions registry lock")
1539 .contains_key(id)
1540 {
1541 return Err(miss);
1542 }
1543 let cfg = config::load(&self.repo_root)?;
1544 let backend = self.backend(cfg.claude_binary.as_deref()).await?;
1545 let engine = match MissionEngine::resume(backend, self.repo_root.clone(), id, LockForce::No)
1546 {
1547 Ok(engine) => Box::new(engine),
1548 Err(EngineError::LockHeld(holder)) => {
1551 return self
1552 .planning_cell(id)
1553 .map_err(|_| ApiError::from(EngineError::LockHeld(holder)))
1554 }
1555 Err(e) => return Err(e.into()),
1556 };
1557 if engine.state().mission.status != MissionStatus::Planning {
1558 return Err(ApiError::conflict(format!(
1560 "mission '{id}' is not in planning (status {:?}) — planning turns only \
1561 apply before a plan is approved",
1562 engine.state().mission.status
1563 )));
1564 }
1565 let cell = new_cell(engine);
1566 let mut map = self.missions.lock().expect("missions registry lock");
1567 map.insert(id.to_string(), new_planning(Arc::clone(&cell)));
1570 drop(map);
1571 self.ensure_sweeper_started();
1572 Ok(cell)
1573 }
1574
1575 fn not_hosted(&self, id: &str) -> ApiError {
1579 let paths = MissionPaths::new(&self.repo_root, id);
1580 if paths.events_file().is_file() {
1581 ApiError::conflict(format!(
1582 "mission '{id}' is not hosted by this server — resume planning with \
1583 `kranz plan --mission {id}`, or start execution via POST \
1584 /api/missions/{id}/start"
1585 ))
1586 .with_code(ApiErrorCode::MissionNotHosted)
1587 } else {
1588 ApiError::not_found(format!("unknown mission '{id}'"))
1589 }
1590 }
1591}
1592
1593fn spawn_with_global_run_permit<F>(
1598 global_run_permit: Option<OwnedSemaphorePermit>,
1599 task: F,
1600) -> tokio::task::JoinHandle<()>
1601where
1602 F: std::future::Future<Output = ()> + Send + 'static,
1603{
1604 tokio::spawn(async move {
1605 let _global_run_permit = global_run_permit;
1608 task.await;
1609 })
1610}
1611
1612async fn run_to_end(
1613 mut engine: Box<MissionEngine>,
1614 mission_id: String,
1615 missions: Arc<Mutex<HashMap<String, HostedMission>>>,
1616) {
1617 let repo_root = engine.paths().repo_root.clone();
1618 let result = engine.run().await;
1619 match &result {
1620 Ok(status) => {
1621 tracing::info!(mission = %mission_id, status = ?status, "hosted mission run ended")
1622 }
1623 Err(e) => {
1624 tracing::error!(mission = %mission_id, error = %e, "hosted mission run errored")
1625 }
1626 }
1627 drop(engine);
1628 if let Err(e) = kranz_engine::work::reconcile_ticket_for_mission(&repo_root, &mission_id) {
1632 tracing::warn!(mission = %mission_id, error = %e, "failed to reconcile linked ticket");
1633 }
1634 missions
1635 .lock()
1636 .expect("missions registry lock")
1637 .remove(&mission_id);
1638}
1639
1640struct AskRunOutcome {
1641 answer: String,
1642 cost_usd: f64,
1643 tokens: TokenUsage,
1644}
1645
1646async fn run_ask_session(
1647 backend: Arc<dyn AgentBackend>,
1648 spec: SessionSpec,
1649) -> Result<AskRunOutcome, ApiError> {
1650 let mut session = backend.start(spec).await.map_err(ApiError::from)?;
1651 let mut streamed_text = String::new();
1652 let mut result_text = None;
1653 let mut tokens = TokenUsage::default();
1654 let mut cost_usd = 0.0;
1655 let mut result_error = false;
1656 while let Some(event) = session.next_event().await.map_err(ApiError::from)? {
1657 match event {
1658 AgentEvent::Text { text, .. } => streamed_text.push_str(&text),
1659 AgentEvent::Result {
1660 text,
1661 is_error,
1662 usage,
1663 cost_usd: cost,
1664 ..
1665 } => {
1666 result_error |= is_error;
1667 tokens.add(&usage);
1668 cost_usd += cost.unwrap_or(0.0);
1669 if !text.trim().is_empty() {
1670 result_text = Some(text);
1671 }
1672 }
1673 _ => {}
1674 }
1675 }
1676 match session.exit_status() {
1677 Some(SessionExit::Completed) if !result_error => {
1678 let answer = result_text.unwrap_or(streamed_text).trim().to_string();
1679 if answer.is_empty() {
1680 return Err(ApiError::internal("ask turn produced an empty answer"));
1681 }
1682 Ok(AskRunOutcome {
1683 answer,
1684 cost_usd,
1685 tokens,
1686 })
1687 }
1688 Some(SessionExit::Completed) => Err(ApiError::internal("ask turn failed")),
1689 Some(SessionExit::Failed(reason)) => {
1690 Err(ApiError::internal(format!("ask turn failed: {reason}")))
1691 }
1692 Some(SessionExit::Aborted) => Err(ApiError::internal("ask turn aborted")),
1693 None => Err(ApiError::internal("ask turn ended without an exit status")),
1694 }
1695}
1696
1697fn ask_prompt(question: &str, context: &str) -> String {
1698 format!(
1699 "Answer this operator question about the Kranz repository.\n\n\
1700 Rules:\n\
1701 - Ground the answer only in the context below.\n\
1702 - If the context is insufficient, say what is missing.\n\
1703 - Keep the answer concise but specific, citing mission ids or ticket slugs when relevant.\n\
1704 - This is read-only: do not propose that you have changed state.\n\n\
1705 Question:\n{question}\n\nContext:\n{context}"
1706 )
1707}
1708
1709fn ask_context(repo_root: &Path) -> String {
1710 let mut out = String::new();
1711 out.push_str("## Missions\n");
1712 let mut ids = MissionPaths::list_missions(repo_root);
1713 ids.sort();
1714 ids.reverse();
1715 if ids.is_empty() {
1716 out.push_str("(none)\n");
1717 }
1718 for id in ids.into_iter().take(20) {
1719 let paths = MissionPaths::new(repo_root, &id);
1720 let Ok(events) = EventLog::read_events(&paths.events_file()) else {
1721 continue;
1722 };
1723 let Ok(state) = kranz_engine::reducer::fold(&events) else {
1724 continue;
1725 };
1726 out.push_str(&format!(
1727 "- {}: {:?}; goal: {}; branch: {}; cost: ${:.4}; tokens in/out/cacheRead/cacheWrite: {}/{}/{}/{}\n",
1728 state.mission.id,
1729 state.mission.status,
1730 one_line(&state.mission.goal),
1731 state.mission.mission_branch,
1732 state.total_cost_usd,
1733 state.totals.input,
1734 state.totals.output,
1735 state.totals.cache_read,
1736 state.totals.cache_write,
1737 ));
1738 for decision in state.recent_decisions.iter().rev().take(3) {
1739 out.push_str(&format!(" decision: {}\n", one_line(decision)));
1740 }
1741 let report = paths.report_file();
1747 let mut text = String::new();
1748 let read = kranz_engine::paths::open_read_nofollow(&report)
1749 .and_then(|mut file| {
1750 use std::io::Read as _;
1751 file.read_to_string(&mut text)?;
1752 Ok(())
1753 })
1754 .is_ok();
1755 if read {
1756 out.push_str(&format!(
1757 " report excerpt: {}\n",
1758 truncate(&one_line(&text), 500)
1759 ));
1760 }
1761 }
1762
1763 out.push_str("\n## Tickets\n");
1764 let tickets = Ticket::list(repo_root);
1765 if tickets.is_empty() {
1766 out.push_str("(none)\n");
1767 }
1768 for ticket in tickets.iter().take(40) {
1769 let state = Ticket::read_state(repo_root, &ticket.slug);
1770 out.push_str(&format!(
1771 "- {} [{:?}, p{}]: {}; blocked-by: {}\n",
1772 ticket.slug,
1773 state,
1774 ticket.priority,
1775 one_line(&ticket.title),
1776 if ticket.blocked_by.is_empty() {
1777 "none".to_string()
1778 } else {
1779 ticket.blocked_by.join(", ")
1780 }
1781 ));
1782 }
1783
1784 out.push_str("\n## Queue\n");
1785 let entries = queue::list(repo_root);
1786 if entries.is_empty() {
1787 out.push_str("(empty)\n");
1788 }
1789 for entry in entries.iter().take(20) {
1790 out.push_str(&format!(
1791 "- {} priority={} ticket={}\n",
1792 entry.mission_id,
1793 entry.priority,
1794 entry.ticket_slug.as_deref().unwrap_or("-")
1795 ));
1796 }
1797 out
1798}
1799
1800fn one_line(text: &str) -> String {
1801 text.split_whitespace().collect::<Vec<_>>().join(" ")
1802}
1803
1804fn truncate(text: &str, max: usize) -> String {
1805 if text.chars().count() <= max {
1806 return text.to_string();
1807 }
1808 let mut out: String = text.chars().take(max.saturating_sub(1)).collect();
1809 out.push('…');
1810 out
1811}
1812
1813async fn drain_task<R, Fut>(
1822 repo_root: PathBuf,
1823 state: Arc<Mutex<DrainState>>,
1824 once: bool,
1825 run_mission: R,
1826 readiness_probe: ReadinessProbe,
1827) where
1828 R: Fn(String) -> Fut,
1829 Fut: std::future::Future<Output = anyhow::Result<i32>>,
1830{
1831 drain_task_with_probe(repo_root, state, once, run_mission, readiness_probe).await;
1832}
1833
1834async fn drain_task_with_probe<R, Fut, P>(
1838 repo_root: PathBuf,
1839 state: Arc<Mutex<DrainState>>,
1840 once: bool,
1841 run_mission: R,
1842 readiness_probe: P,
1843) where
1844 R: Fn(String) -> Fut,
1845 Fut: std::future::Future<Output = anyhow::Result<i32>>,
1846 P: Fn(
1847 &Path,
1848 &str,
1849 ) -> kranz_engine::error::Result<kranz_engine::backend_readiness::ReadinessReport>,
1850{
1851 let dispatch_branch = GitRepo::open(&repo_root)
1854 .ok()
1855 .and_then(|g| g.current_branch().ok());
1856
1857 let result = kranz_engine::work::drain_queue_with_probe(
1858 &repo_root,
1859 once,
1860 |mission_id| {
1861 let state = Arc::clone(&state);
1862 let fut = run_mission(mission_id.clone());
1863 async move {
1864 state.lock().expect("drain state lock").current_mission_id =
1865 Some(mission_id.clone());
1866 let outcome = fut.await;
1867 let mut guard = state.lock().expect("drain state lock");
1868 guard.current_mission_id = None;
1869 if outcome.is_ok() {
1870 guard.ran.push(mission_id);
1871 }
1872 outcome
1873 }
1874 },
1875 readiness_probe,
1876 )
1877 .await;
1878
1879 match &result {
1880 Ok(report) if !report.stopped_busy => {
1881 {
1882 let mut guard = state.lock().expect("drain state lock");
1883 for id in &report.parked {
1884 if !guard.parked.contains(id) {
1885 guard.parked.push(id.clone());
1886 }
1887 }
1888 }
1889 restore_drain_checkout(&repo_root, dispatch_branch.as_deref());
1890 }
1891 Ok(report) => {
1892 let mut guard = state.lock().expect("drain state lock");
1893 for id in &report.parked {
1894 if !guard.parked.contains(id) {
1895 guard.parked.push(id.clone());
1896 }
1897 }
1898 }
1903 Err(e) => {
1904 tracing::error!(error = %e, "hosted queue drain errored");
1905 restore_drain_checkout(&repo_root, dispatch_branch.as_deref());
1908 }
1909 }
1910 state.lock().expect("drain state lock").live = false;
1911}
1912
1913fn restore_drain_checkout(repo_root: &Path, original: Option<&str>) {
1922 let Some(original) = original else { return };
1923 if original.starts_with("kranz/mission-") {
1924 return;
1925 }
1926 let Ok(git) = GitRepo::open(repo_root) else {
1927 return;
1928 };
1929 if git.current_branch().ok().as_deref() == Some(original) {
1930 return;
1931 }
1932 match git.is_clean_tracked() {
1933 Ok(true) => match git.checkout(original) {
1934 Ok(()) => tracing::info!(branch = %original, "hosted drain restored operator checkout"),
1935 Err(e) => {
1936 tracing::warn!(branch = %original, error = %e, "hosted drain could not restore checkout")
1937 }
1938 },
1939 Ok(false) => tracing::warn!(
1940 "hosted drain leaving checkout in place: tracked files have uncommitted changes"
1941 ),
1942 Err(e) => {
1943 tracing::warn!(error = %e, "hosted drain could not probe the working tree; checkout left in place")
1944 }
1945 }
1946}
1947
1948async fn run_mission_headless(
1954 backend: Arc<dyn AgentBackend>,
1955 repo_root: PathBuf,
1956 mission_id: String,
1957) -> anyhow::Result<i32> {
1958 let mut engine = MissionEngine::resume(backend, repo_root, &mission_id, LockForce::No)?;
1959 let status = engine.run().await?;
1960 Ok(exit_code_for(status))
1961}
1962
1963fn exit_code_for(status: MissionStatus) -> i32 {
1966 match status {
1967 MissionStatus::Complete => 0,
1968 MissionStatus::Blocked => 2,
1969 _ => 1,
1970 }
1971}
1972
1973fn config_for_ticket(mut cfg: MissionConfig, ticket: &Ticket) -> MissionConfig {
1977 if let Some(budget) = ticket.max_budget_usd {
1978 cfg.orchestrator.max_budget_usd = Some(budget);
1979 }
1980 cfg
1981}
1982
1983#[allow(dead_code)]
1988fn draft_outcome_json(outcome: &DraftOutcome) -> Value {
1989 match outcome {
1990 DraftOutcome::ParkedForReview {
1991 mission_id,
1992 mission_branch,
1993 } => json!({
1994 "outcome": "parkedForReview",
1995 "missionId": mission_id,
1996 "missionBranch": mission_branch,
1997 }),
1998 DraftOutcome::Enqueued { mission_id } => json!({
1999 "outcome": "enqueued",
2000 "missionId": mission_id,
2001 }),
2002 DraftOutcome::PlanAsProse { mission_id } => json!({
2003 "outcome": "planAsProse",
2004 "missionId": mission_id,
2005 "message": "the orchestrator produced a plan but emitted it as prose instead of \
2006 through the plan channel, so nothing was queued; re-run draft for \
2007 this ticket",
2008 }),
2009 DraftOutcome::NeedsContext {
2010 mission_id,
2011 questions,
2012 } => json!({
2013 "outcome": "needsContext",
2014 "missionId": mission_id,
2015 "questions": questions,
2016 }),
2017 DraftOutcome::WrongPlan { mission_id, reason } => json!({
2018 "outcome": "wrongPlan",
2019 "missionId": mission_id,
2020 "reason": reason,
2021 }),
2022 }
2023}
2024
2025fn new_cell(engine: Box<MissionEngine>) -> EngineCell {
2026 Arc::new(tokio::sync::Mutex::new(engine))
2027}
2028
2029fn new_planning(cell: EngineCell) -> HostedMission {
2031 HostedMission::Planning {
2032 cell,
2033 last_use: Arc::new(Mutex::new(Instant::now())),
2034 pending_plan: Arc::new(Mutex::new(None)),
2035 }
2036}
2037
2038fn release_from(
2042 missions: &Mutex<HashMap<String, HostedMission>>,
2043 id: &str,
2044) -> Result<bool, ApiError> {
2045 let mut map = missions.lock().expect("missions registry lock");
2046 match map.remove(id) {
2047 None => Ok(true),
2048 Some(HostedMission::Running { handle, _repo_busy }) => {
2049 let finished = handle.is_finished();
2050 if !finished {
2051 map.insert(
2052 id.to_string(),
2053 HostedMission::Running { handle, _repo_busy },
2054 );
2055 }
2056 Ok(finished)
2057 }
2058 Some(HostedMission::Planning {
2059 cell,
2060 last_use,
2061 pending_plan,
2062 }) => match Arc::try_unwrap(cell) {
2063 Ok(mutex) => {
2064 drop(mutex.into_inner()); Ok(true)
2066 }
2067 Err(cell) => {
2068 map.insert(
2069 id.to_string(),
2070 HostedMission::Planning {
2071 cell,
2072 last_use,
2073 pending_plan,
2074 },
2075 );
2076 Err(turn_in_flight())
2077 }
2078 },
2079 }
2080}
2081
2082fn sweep_idle_from(
2088 missions: &Mutex<HashMap<String, HostedMission>>,
2089 threshold: Duration,
2090) -> Vec<String> {
2091 let idle_ids: Vec<String> = {
2092 let map = missions.lock().expect("missions registry lock");
2093 map.iter()
2094 .filter_map(|(id, mission)| match mission {
2095 HostedMission::Planning { last_use, .. } => {
2096 let elapsed = last_use.lock().expect("last-use lock").elapsed();
2097 (elapsed >= threshold).then(|| id.clone())
2098 }
2099 HostedMission::Running { .. } => None,
2100 })
2101 .collect()
2102 };
2103 idle_ids
2104 .into_iter()
2105 .filter(|id| matches!(release_from(missions, id), Ok(true)))
2106 .collect()
2107}
2108
2109fn try_lock(
2111 cell: &EngineCell,
2112) -> Result<tokio::sync::MutexGuard<'_, Box<MissionEngine>>, ApiError> {
2113 cell.try_lock().map_err(|_| turn_in_flight())
2114}
2115
2116fn turn_in_flight() -> ApiError {
2117 ApiError::conflict("a turn is in flight for this mission — wait for it to finish")
2118 .with_code(ApiErrorCode::TurnInFlight)
2119}
2120
2121fn prepend_seed(seed: Option<String>, reply: String) -> String {
2124 match seed {
2125 Some(seed) => format!("{seed}\n\n{reply}"),
2126 None => reply,
2127 }
2128}
2129
2130fn estimate_json(estimate: &CostEstimate) -> Value {
2133 let confidence = match estimate.confidence {
2134 kranz_engine::cost::Confidence::High => "high",
2135 kranz_engine::cost::Confidence::Low => "low",
2136 };
2137 json!({
2138 "workerRuns": estimate.worker_runs,
2139 "validatorRuns": estimate.validator_runs,
2140 "lowUsd": estimate.low_usd,
2141 "expectedUsd": estimate.expected_usd,
2142 "highUsd": estimate.high_usd,
2143 "confidence": confidence,
2144 })
2145}
2146
2147pub(crate) async fn create_mission(
2154 State(server): State<Arc<ServerState>>,
2155 body: Bytes,
2156) -> Result<impl IntoResponse, ApiError> {
2157 let value = parse_body(&body)?;
2158 let goal = value
2159 .get("goal")
2160 .and_then(Value::as_str)
2161 .map(str::trim)
2162 .filter(|goal| !goal.is_empty())
2163 .ok_or_else(|| {
2164 ApiError::bad_request(r#"body must be {"goal":"..."} with a non-empty goal"#)
2165 })?;
2166 let id = server.host.create(goal, value.get("config")).await?;
2167 Ok((StatusCode::CREATED, Json(json!({ "id": id }))))
2168}
2169
2170pub(crate) async fn planning_turn(
2173 State(server): State<Arc<ServerState>>,
2174 UrlPath(id): UrlPath<String>,
2175 body: Bytes,
2176) -> Result<Json<Value>, ApiError> {
2177 let id = valid_id(&server, &id)?;
2178 let value = parse_body(&body)?;
2179 let text = value
2180 .get("text")
2181 .and_then(Value::as_str)
2182 .map(str::trim)
2183 .filter(|text| !text.is_empty())
2184 .ok_or_else(|| {
2185 ApiError::bad_request(r#"body must be {"text":"..."} with non-empty text"#)
2186 })?;
2187 let reply = server.host.planning_turn(&id, text).await?;
2188 Ok(Json(json!({ "reply": reply })))
2189}
2190
2191pub(crate) async fn request_plan(
2195 State(server): State<Arc<ServerState>>,
2196 UrlPath(id): UrlPath<String>,
2197) -> Result<Json<Value>, ApiError> {
2198 let id = valid_id(&server, &id)?;
2199 Ok(Json(server.host.request_plan(&id).await?))
2200}
2201
2202pub(crate) async fn approve_mission(
2205 State(server): State<Arc<ServerState>>,
2206 UrlPath(id): UrlPath<String>,
2207 body: Bytes,
2208) -> Result<Json<Value>, ApiError> {
2209 let id = valid_id(&server, &id)?;
2210 let value = parse_body(&body)?;
2211 let plan = value
2212 .get("plan")
2213 .cloned()
2214 .ok_or_else(|| ApiError::bad_request(r#"body must be {"plan":{...}}"#))?;
2215 let plan: Plan = serde_json::from_value(plan)
2216 .map_err(|e| ApiError::bad_request(format!("'plan' is not a valid Plan: {e}")))?;
2217 let branch = server.host.approve(&id, plan).await?;
2218 Ok(Json(json!({ "branch": branch })))
2219}
2220
2221pub(crate) async fn pending_plan_route(
2225 axum::Extension(reads): axum::Extension<crate::read_work::ReadWork>,
2226 State(server): State<Arc<ServerState>>,
2227 UrlPath(id): UrlPath<String>,
2228) -> Result<Json<Value>, ApiError> {
2229 reads
2230 .run(move || {
2231 let id = valid_id(&server, &id)?;
2232 Ok(Json(match server.host.pending_plan(&id) {
2233 Some(plan) => {
2234 json!({ "pending": true, "planIdentity": plan_identity(&plan), "plan": plan })
2235 }
2236 None => json!({ "pending": false }),
2237 }))
2238 })
2239 .await
2240}
2241
2242pub(crate) async fn approve_pending_route(
2246 State(server): State<Arc<ServerState>>,
2247 UrlPath(id): UrlPath<String>,
2248 body: Bytes,
2249) -> Result<Json<Value>, ApiError> {
2250 let id = valid_id(&server, &id)?;
2251 let value = parse_body(&body)?;
2252 let start = value.get("start").and_then(Value::as_bool).unwrap_or(false);
2253 let expected_identity = value.get("planIdentity").and_then(Value::as_str);
2254 let branch = server.host.approve_pending(&id, expected_identity).await?;
2255 if start {
2256 server.host.start(&id).await?;
2257 }
2258 Ok(Json(json!({ "branch": branch, "started": start })))
2259}
2260
2261pub(crate) async fn abandon_mission_route(
2264 State(server): State<Arc<ServerState>>,
2265 UrlPath(id): UrlPath<String>,
2266 body: Bytes,
2267) -> Result<Json<Value>, ApiError> {
2268 let id = valid_id(&server, &id)?;
2269 let value = parse_body(&body)?;
2270 let reason = value
2271 .get("reason")
2272 .and_then(Value::as_str)
2273 .map(str::trim)
2274 .filter(|r| !r.is_empty())
2275 .unwrap_or("abandoned by operator");
2276 server.host.abandon(&id, reason).await?;
2277 Ok(Json(json!({ "abandoned": true })))
2278}
2279
2280pub(crate) async fn release_mission_route(
2285 State(server): State<Arc<ServerState>>,
2286 UrlPath(id): UrlPath<String>,
2287 body: Bytes,
2288) -> Result<Json<Value>, ApiError> {
2289 let id = valid_id(&server, &id)?;
2290 let _ = parse_body(&body)?;
2291 if !MissionPaths::new(server.host.repo_root(), &id)
2292 .events_file()
2293 .is_file()
2294 {
2295 return Err(ApiError::not_found(format!("mission '{id}' not found")));
2296 }
2297 let released = server.host.release(&id)?;
2298 Ok(Json(json!({ "released": released })))
2299}
2300
2301pub(crate) async fn delete_mission_route(
2306 State(server): State<Arc<ServerState>>,
2307 UrlPath(id): UrlPath<String>,
2308 body: Bytes,
2309) -> Result<Json<Value>, ApiError> {
2310 let id = valid_id(&server, &id)?;
2311 let value = parse_body(&body)?;
2312 let all = value.get("all").and_then(Value::as_bool).unwrap_or(false);
2313 server.host.clean(&id, all)?;
2314 Ok(Json(json!({ "deleted": true })))
2315}
2316
2317pub(crate) async fn start_mission(
2319 State(server): State<Arc<ServerState>>,
2320 UrlPath(id): UrlPath<String>,
2321) -> Result<impl IntoResponse, ApiError> {
2322 let id = valid_id(&server, &id)?;
2323 server.host.start(&id).await?;
2324 Ok((StatusCode::ACCEPTED, Json(json!({ "running": true }))))
2325}
2326
2327pub(crate) async fn merge_mission_route(
2331 State(server): State<Arc<ServerState>>,
2332 UrlPath(id): UrlPath<String>,
2333) -> Result<Json<Value>, ApiError> {
2334 let id = valid_id(&server, &id)?;
2335 Ok(Json(server.host.merge(&id).await?))
2336}
2337
2338pub(crate) async fn drain_queue_route(
2341 State(server): State<Arc<ServerState>>,
2342 body: Bytes,
2343) -> Result<Json<Value>, ApiError> {
2344 let _ = parse_body(&body)?;
2345 Ok(Json(server.host.drain().await?))
2346}
2347
2348pub(crate) async fn queue_state_route(
2351 axum::Extension(reads): axum::Extension<crate::read_work::ReadWork>,
2352 State(server): State<Arc<ServerState>>,
2353) -> Result<Json<Value>, ApiError> {
2354 reads.run(move || Ok(Json(server.host.queue_state()))).await
2355}
2356
2357fn valid_id(server: &ServerState, id: &str) -> Result<String, ApiError> {
2359 crate::rest::mission_paths(server, id)?;
2360 Ok(id.to_string())
2361}
2362
2363pub(crate) fn parse_body(body: &Bytes) -> Result<Value, ApiError> {
2364 if body.is_empty() {
2365 return Ok(json!({}));
2366 }
2367 serde_json::from_slice(body)
2368 .map_err(|e| ApiError::bad_request(format!("invalid JSON body: {e}")))
2369}
2370
2371#[cfg(test)]
2378mod tests {
2379 use super::*;
2380 use axum::http::StatusCode;
2381 use kranz_engine::backend_mock::{mock_init, mock_result_text, MockBackend, MockScript};
2382 use std::process::Command;
2383 use std::sync::Once;
2384
2385 static ENV_ISOLATION: Once = Once::new();
2386
2387 fn isolate_git_env() {
2392 ENV_ISOLATION.call_once(|| {
2393 let missing = std::env::temp_dir()
2394 .join(format!("kranz-host-test-no-config-{}", std::process::id()));
2395 std::env::set_var("GIT_CONFIG_GLOBAL", &missing);
2396 std::env::set_var("GIT_CONFIG_SYSTEM", &missing);
2397 if let Ok(ceiling) = std::fs::canonicalize(std::env::temp_dir()) {
2398 std::env::set_var("GIT_CEILING_DIRECTORIES", ceiling);
2399 }
2400 let home =
2401 std::env::temp_dir().join(format!("kranz-host-test-home-{}", std::process::id()));
2402 let _ = std::fs::create_dir_all(&home);
2403 std::env::set_var(if cfg!(windows) { "USERPROFILE" } else { "HOME" }, &home);
2404 });
2405 }
2406
2407 fn git(dir: &std::path::Path, args: &[&str]) {
2408 let out = Command::new("git")
2409 .args(args)
2410 .current_dir(dir)
2411 .output()
2412 .expect("spawn git");
2413 assert!(
2414 out.status.success(),
2415 "git {args:?}: {}",
2416 String::from_utf8_lossy(&out.stderr)
2417 );
2418 }
2419
2420 fn init_repo() -> Option<(tempfile::TempDir, PathBuf)> {
2422 isolate_git_env();
2423 let git_works = Command::new("git")
2424 .arg("--version")
2425 .output()
2426 .map(|o| o.status.success())
2427 .unwrap_or(false);
2428 if !git_works {
2429 kranz_engine::test_capability::skip(
2430 kranz_engine::test_capability::capability::GIT,
2431 "git is not on PATH",
2432 );
2433 return None;
2434 }
2435 let dir = tempfile::tempdir().expect("tempdir");
2436 let init = Command::new("git")
2437 .args(["init", "-b", "main"])
2438 .current_dir(dir.path())
2439 .output()
2440 .expect("spawn git init");
2441 if !init.status.success() {
2442 git(dir.path(), &["init"]);
2443 git(dir.path(), &["symbolic-ref", "HEAD", "refs/heads/main"]);
2444 }
2445 git(dir.path(), &["config", "user.name", "test"]);
2446 git(dir.path(), &["config", "user.email", "test@example.com"]);
2447 std::fs::write(dir.path().join("README.md"), "seed\n").unwrap();
2448 git(dir.path(), &["add", "-A"]);
2449 git(dir.path(), &["commit", "-m", "seed"]);
2450 let root = std::fs::canonicalize(dir.path()).expect("canonicalize");
2451 Some((dir, root))
2452 }
2453
2454 #[tokio::test]
2455 async fn ask_runs_read_only_one_shot_without_creating_mission_state() {
2456 let Some((_dir, root)) = init_repo() else {
2457 return;
2458 };
2459 let backend = Arc::new(MockBackend::with_scripts(vec![MockScript::single_shot(
2460 "Nothing is currently blocked.",
2461 )]));
2462 let host = MissionHost::with_backend(root.clone(), backend.clone());
2463
2464 let before = MissionPaths::list_missions(&root);
2465 let value = host.ask("what is blocked?").await.unwrap();
2466
2467 assert_eq!(value["answer"], "Nothing is currently blocked.");
2468 assert_eq!(
2469 MissionPaths::list_missions(&root),
2470 before,
2471 "ask must not create or mutate mission directories"
2472 );
2473 let specs = backend.started_specs();
2474 assert_eq!(specs.len(), 1);
2475 assert!(!specs[0].writable, "ask session is read-only");
2476 assert_eq!(specs[0].permission_mode.as_deref(), Some("plan"));
2477 let prompt = match &specs[0].prompt {
2478 PromptMode::SingleShot(prompt) => prompt,
2479 other => panic!("ask must be one-shot, got {other:?}"),
2480 };
2481 assert!(prompt.contains("what is blocked?"));
2482 assert!(prompt.contains("## Missions"));
2483 }
2484
2485 #[tokio::test]
2486 async fn http_api_error_codes_match_dashboard_wire_fixtures() {
2487 use http_body_util::BodyExt as _;
2488
2489 let dir = tempfile::tempdir().unwrap();
2490 let paths = MissionPaths::new(dir.path(), "m-fixture");
2491 std::fs::create_dir_all(paths.mission_dir()).unwrap();
2492 std::fs::write(paths.events_file(), "").unwrap();
2493 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
2494 let mut host = MissionHost::with_backend(dir.path().to_path_buf(), backend);
2495 host.global_run_permits = Some(Arc::new(Semaphore::new(0)));
2496
2497 let errors = [
2501 ("mission_not_hosted", host.not_hosted("m-fixture")),
2502 ("turn_in_flight", turn_in_flight()),
2503 ("repository_busy", host.try_global_run_permit().unwrap_err()),
2504 (
2505 "stale_plan",
2506 host.approve_pending("m-fixture", None).await.unwrap_err(),
2507 ),
2508 ("legacy", ApiError::conflict("mission is not hosted")),
2509 ];
2510 let mut actual = Vec::new();
2511 for (name, error) in errors {
2512 let response = error.into_response();
2513 let status = response.status().as_u16();
2514 assert_eq!(response.headers()["content-type"], "application/json");
2515 let bytes = response.into_body().collect().await.unwrap().to_bytes();
2516 let body: Value = serde_json::from_slice(&bytes).unwrap();
2517 actual.push(json!({ "name": name, "status": status, "body": body }));
2518 }
2519 let fixture: Value = serde_json::from_str(include_str!(
2520 "../../../apps/dashboard/src/lib/fixtures/api-errors.json"
2521 ))
2522 .unwrap();
2523 assert_eq!(json!(actual), fixture);
2524 }
2525
2526 #[tokio::test]
2527 async fn contended_planning_mutex_is_409_for_turns_and_start() {
2528 let Some((_dir, root)) = init_repo() else {
2529 return;
2530 };
2531 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
2532 let host = MissionHost::with_backend(root, backend);
2533 let id = host.create("ship it", None).await.expect("create mission");
2534
2535 let cell = host.planning_cell(&id).expect("hosted planning cell");
2537 let _guard = cell.try_lock().expect("uncontended lock");
2538
2539 let err = host
2540 .planning_turn(&id, "hello")
2541 .await
2542 .expect_err("turn must 409");
2543 assert_eq!(err.status, StatusCode::CONFLICT);
2544 assert!(err.message.contains("turn is in flight"), "{}", err.message);
2545 assert_eq!(err.code, Some(ApiErrorCode::TurnInFlight));
2546
2547 let err = host
2548 .request_plan(&id)
2549 .await
2550 .expect_err("request-plan must 409");
2551 assert_eq!(err.status, StatusCode::CONFLICT);
2552
2553 let err = host.start(&id).await.expect_err("start must 409");
2556 assert_eq!(err.status, StatusCode::CONFLICT);
2557 assert!(
2558 host.planning_cell(&id).is_ok(),
2559 "registry entry must survive"
2560 );
2561 }
2562
2563 #[tokio::test]
2564 async fn start_without_an_approved_plan_is_409() {
2565 let Some((_dir, root)) = init_repo() else {
2566 return;
2567 };
2568 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
2569 let host = MissionHost::with_backend(root, backend);
2570 let id = host.create("ship it", None).await.expect("create mission");
2571
2572 let err = host
2573 .start(&id)
2574 .await
2575 .expect_err("start must 409 in planning");
2576 assert_eq!(err.status, StatusCode::CONFLICT);
2577 assert!(err.message.contains("no approved plan"), "{}", err.message);
2578 assert!(host.planning_cell(&id).is_ok());
2580 }
2581
2582 #[tokio::test]
2589 async fn approve_pending_matching_refuses_a_different_plan_without_consuming_it() {
2590 let Some((_dir, root)) = init_repo() else {
2591 return;
2592 };
2593 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
2594 let host = MissionHost::with_backend(root.clone(), backend);
2595 let id = host.create("ship it", None).await.expect("create mission");
2596 let plan: Plan = serde_json::from_value(plan_json()).expect("plan");
2597 let identity = plan_identity(&plan);
2598
2599 assert_eq!(
2600 host.try_approve_pending_matching(&id, Some(&identity))
2601 .await
2602 .unwrap(),
2603 PendingApproval::NothingParked,
2604 "nothing parked is not an approval"
2605 );
2606
2607 host.set_pending_plan(&id, Some(plan.clone()));
2608
2609 assert_eq!(
2610 host.try_approve_pending_matching(&id, Some("an older plan"))
2611 .await
2612 .unwrap(),
2613 PendingApproval::Mismatch {
2614 parked: identity.clone()
2615 },
2616 "a card naming a different plan must be refused, naming the parked one"
2617 );
2618 assert!(
2619 host.pending_plan(&id).is_some(),
2620 "a refused approve must not consume the parked plan"
2621 );
2622
2623 assert!(
2624 matches!(
2625 host.try_approve_pending_matching(&id, None).await.unwrap(),
2626 PendingApproval::Mismatch { .. }
2627 ),
2628 "a card that names no plan cannot match one"
2629 );
2630 assert!(host.pending_plan(&id).is_some());
2631
2632 assert_eq!(
2633 host.try_approve_pending_matching(&id, Some(&identity))
2634 .await
2635 .unwrap(),
2636 PendingApproval::Approved(format!("kranz/mission-{id}"))
2637 );
2638 assert!(
2639 host.pending_plan(&id).is_none(),
2640 "an approve consumes the parked plan"
2641 );
2642 assert_eq!(
2643 host.try_approve_pending_matching(&id, Some(&identity))
2644 .await
2645 .unwrap(),
2646 PendingApproval::NothingParked,
2647 "a second click has nothing left to commit"
2648 );
2649 }
2650
2651 #[tokio::test]
2652 async fn approve_pending_matching_leaves_pending_untouched_on_busy_or_failure() {
2653 let Some((_dir, root)) = init_repo() else {
2654 return;
2655 };
2656 let host = MissionHost::with_backend(root, Arc::new(MockBackend::new()));
2657 let id = host.create("ship it", None).await.unwrap();
2658 let mut plan: Plan = serde_json::from_value(plan_json()).unwrap();
2659 host.set_pending_plan(&id, Some(plan.clone()));
2660 let cell = host.planning_cell(&id).unwrap();
2661 let guard = cell.try_lock().unwrap();
2662 let identity = plan_identity(&plan);
2663 let err = host
2664 .try_approve_pending_matching(&id, Some(&identity))
2665 .await
2666 .unwrap_err();
2667 assert_eq!(err.status, StatusCode::CONFLICT);
2668 assert_eq!(plan_identity(&host.pending_plan(&id).unwrap()), identity);
2669 drop(guard);
2670
2671 plan.milestones.clear();
2672 let invalid_identity = plan_identity(&plan);
2673 host.set_pending_plan(&id, Some(plan));
2674 let err = host
2675 .try_approve_pending_matching(&id, Some(&invalid_identity))
2676 .await
2677 .unwrap_err();
2678 assert!(err.message.contains("no milestones"), "{}", err.message);
2679 assert_eq!(
2680 plan_identity(&host.pending_plan(&id).unwrap()),
2681 invalid_identity
2682 );
2683
2684 let replacement: Plan = serde_json::from_value(plan_json()).unwrap();
2686 host.set_pending_plan(&id, Some(replacement));
2687 assert_eq!(
2688 host.try_approve_pending_matching(&id, Some(&invalid_identity))
2689 .await
2690 .unwrap(),
2691 PendingApproval::Mismatch {
2692 parked: identity.clone()
2693 },
2694 );
2695 assert_eq!(plan_identity(&host.pending_plan(&id).unwrap()), identity);
2696 }
2697
2698 #[tokio::test]
2699 async fn start_is_409_when_repo_busy() {
2700 let Some((_dir, root)) = init_repo() else {
2701 return;
2702 };
2703 let _hold =
2707 kranz_engine::queue::acquire_repo_busy(&root, "m-sibling").expect("sibling busy hold");
2708 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
2709 let host = MissionHost::with_backend(root.clone(), backend);
2710 let id = host.create("ship it", None).await.expect("create mission");
2711 let plan: Plan = serde_json::from_value(plan_json()).expect("plan");
2712 host.approve(&id, plan).await.expect("approve");
2713
2714 let err = host.start(&id).await.expect_err("start must 409 when busy");
2715 assert_eq!(err.status, StatusCode::CONFLICT);
2716 assert_eq!(err.code, Some(ApiErrorCode::RepositoryBusy));
2717 assert!(
2718 err.message.contains("busy"),
2719 "expected busy conflict, got: {}",
2720 err.message
2721 );
2722 assert!(host.planning_cell(&id).is_ok());
2724 }
2725
2726 #[tokio::test]
2727 async fn start_is_409_when_global_repository_limit_is_saturated() {
2728 let Some((_dir, root)) = init_repo() else {
2729 return;
2730 };
2731 let permits = Arc::new(Semaphore::new(1));
2732 let _other_repo = Arc::clone(&permits).try_acquire_owned().unwrap();
2733 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
2734 let mut host = MissionHost::with_backend(root, backend);
2735 host.global_run_permits = Some(permits);
2736 let id = host.create("ship it", None).await.expect("create mission");
2737 let plan: Plan = serde_json::from_value(plan_json()).expect("plan");
2738 host.approve(&id, plan).await.expect("approve");
2739
2740 let error = host.start(&id).await.expect_err("global cap must refuse");
2741
2742 assert_eq!(error.status, StatusCode::CONFLICT);
2743 assert!(error.message.contains("maxConcurrentRepos"));
2744 assert_eq!(error.code, Some(ApiErrorCode::RepositoryBusy));
2745 assert!(
2746 host.planning_cell(&id).is_ok(),
2747 "refused start must restore the hosted engine"
2748 );
2749 }
2750
2751 #[tokio::test]
2752 async fn global_run_permit_is_released_when_hosted_task_panics() {
2753 let permits = Arc::new(Semaphore::new(1));
2754 let permit = Arc::clone(&permits).try_acquire_owned().unwrap();
2755 assert_eq!(permits.available_permits(), 0);
2756
2757 let handle = spawn_with_global_run_permit(Some(permit), async {
2758 panic!("simulated hosted-run panic");
2759 });
2760 assert!(handle.await.unwrap_err().is_panic());
2761
2762 assert_eq!(permits.available_permits(), 1);
2763 }
2764
2765 #[tokio::test]
2766 async fn sweep_idle_leaves_a_mid_turn_mission_hosted() {
2767 let Some((_dir, root)) = init_repo() else {
2768 return;
2769 };
2770 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
2771 let host = MissionHost::with_backend(root, backend);
2772 let id = host.create("ship it", None).await.expect("create mission");
2773
2774 let cell = host.planning_cell(&id).expect("hosted planning cell");
2776 let _guard = cell.try_lock().expect("uncontended lock");
2777
2778 let released = host.sweep_idle(std::time::Duration::ZERO);
2779 assert!(!released.contains(&id), "{released:?}");
2780 assert!(
2781 host.planning_cell(&id).is_ok(),
2782 "mission must remain hosted"
2783 );
2784 }
2785
2786 #[tokio::test]
2787 async fn release_route_is_409_mid_turn() {
2788 use axum::body::Body;
2789 use axum::http::Request;
2790 use tower::ServiceExt;
2791
2792 let Some((_dir, root)) = init_repo() else {
2793 return;
2794 };
2795 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
2796 let host = MissionHost::with_backend(root, backend);
2797 let id = host.create("ship it", None).await.expect("create mission");
2798
2799 let cell = host.planning_cell(&id).expect("hosted planning cell");
2801 let _guard = cell.try_lock().expect("uncontended lock");
2802
2803 let app =
2804 crate::router_with_host(host, None, crate::MutationAuthority::new("tok").unwrap());
2805 let response = app
2806 .oneshot(
2807 Request::builder()
2808 .method("POST")
2809 .uri(format!("/api/missions/{id}/release"))
2810 .header("content-type", "application/json")
2811 .header("x-kranz-token", "tok")
2812 .body(Body::from("{}"))
2813 .unwrap(),
2814 )
2815 .await
2816 .unwrap();
2817 assert_eq!(response.status(), StatusCode::CONFLICT);
2818 }
2819
2820 #[tokio::test]
2821 async fn bodyless_post_with_valid_token_is_not_rejected_as_unsupported_media_type() {
2822 use axum::body::Body;
2823 use axum::http::Request;
2824 use tower::ServiceExt;
2825
2826 let Some((_dir, root)) = init_repo() else {
2827 return;
2828 };
2829 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
2830 let host = MissionHost::with_backend(root, backend);
2831 let id = host.create("ship it", None).await.expect("create mission");
2832
2833 let app =
2834 crate::router_with_host(host, None, crate::MutationAuthority::new("tok").unwrap());
2835 let response = app
2836 .oneshot(
2837 Request::builder()
2838 .method("POST")
2839 .uri(format!("/api/missions/{id}/start"))
2840 .header("x-kranz-token", "tok")
2844 .body(Body::empty())
2845 .unwrap(),
2846 )
2847 .await
2848 .unwrap();
2849 assert_ne!(response.status(), StatusCode::UNSUPPORTED_MEDIA_TYPE);
2853 assert_eq!(response.status(), StatusCode::CONFLICT);
2854 }
2855
2856 #[tokio::test]
2857 async fn bodyless_post_gate_still_rejects_non_empty_non_json_bodies() {
2858 use axum::body::Body;
2859 use axum::http::Request;
2860 use tower::ServiceExt;
2861
2862 let Some((_dir, root)) = init_repo() else {
2863 return;
2864 };
2865 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
2866 let host = MissionHost::with_backend(root, backend);
2867 let id = host.create("ship it", None).await.expect("create mission");
2868
2869 let app =
2870 crate::router_with_host(host, None, crate::MutationAuthority::new("tok").unwrap());
2871 let payload = "not json";
2872 let response = app
2873 .oneshot(
2874 Request::builder()
2875 .method("POST")
2876 .uri(format!("/api/missions/{id}/release"))
2877 .header("content-type", "text/plain")
2878 .header("content-length", payload.len().to_string())
2879 .header("x-kranz-token", "tok")
2880 .body(Body::from(payload))
2881 .unwrap(),
2882 )
2883 .await
2884 .unwrap();
2885 assert_eq!(response.status(), StatusCode::UNSUPPORTED_MEDIA_TYPE);
2886 }
2887
2888 #[tokio::test]
2889 async fn create_rejects_an_invalid_config_patch() {
2890 let Some((_dir, root)) = init_repo() else {
2891 return;
2892 };
2893 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
2894 let host = MissionHost::with_backend(root, backend);
2895
2896 let patch = json!({ "maxParallelWorkers": 9 });
2900 let err = host
2901 .create("ship it", Some(&patch))
2902 .await
2903 .expect_err("must reject");
2904 assert_eq!(err.status, StatusCode::BAD_REQUEST);
2905 }
2906
2907 #[tokio::test]
2912 async fn empty_queue_drain_returns_ok_and_settles_idle() {
2913 let Some((_dir, root)) = init_repo() else {
2914 return;
2915 };
2916 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
2917 let host = MissionHost::with_backend(root, backend);
2918
2919 let body = host
2920 .drain()
2921 .await
2922 .expect("drain must not error on an empty queue");
2923 assert!(body.get("live").is_some(), "{body}");
2924
2925 let deadline = std::time::Instant::now() + Duration::from_secs(5);
2927 loop {
2928 let state = host.queue_state();
2929 if state["drain"]["live"] == false {
2930 break;
2931 }
2932 assert!(
2933 std::time::Instant::now() < deadline,
2934 "drain never settled idle: {state}"
2935 );
2936 tokio::time::sleep(Duration::from_millis(20)).await;
2937 }
2938 }
2939
2940 #[tokio::test]
2941 async fn drain_is_409_when_global_repository_limit_is_saturated() {
2942 let Some((_dir, root)) = init_repo() else {
2943 return;
2944 };
2945 let permits = Arc::new(Semaphore::new(1));
2946 let _other_repo = Arc::clone(&permits).try_acquire_owned().unwrap();
2947 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
2948 let mut host = MissionHost::with_backend(root, backend);
2949 host.global_run_permits = Some(permits);
2950
2951 let error = host.drain().await.expect_err("global cap must refuse");
2952
2953 assert_eq!(error.status, StatusCode::CONFLICT);
2954 assert!(error.message.contains("maxConcurrentRepos"));
2955 assert!(matches!(
2956 &*host.drain.lock().expect("drain tracker lock"),
2957 DrainSlot::Idle
2958 ));
2959 }
2960
2961 #[tokio::test]
2962 async fn queue_state_reports_global_concurrency_saturation() {
2963 let Some((_dir, root)) = init_repo() else {
2964 return;
2965 };
2966 let permits = Arc::new(Semaphore::new(1));
2967 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
2968 let mut host = MissionHost::with_backend(root, backend);
2969 host.global_run_permits = Some(Arc::clone(&permits));
2970
2971 let open = host.queue_state();
2972 assert_eq!(open["maxConcurrentReposAvailable"], 1);
2973 assert_eq!(open["maxConcurrentReposSaturated"], false);
2974
2975 let _hold = permits.try_acquire_owned().unwrap();
2976 let saturated = host.queue_state();
2977 assert_eq!(saturated["maxConcurrentReposAvailable"], 0);
2978 assert_eq!(saturated["maxConcurrentReposSaturated"], true);
2979 }
2980
2981 #[tokio::test]
2982 async fn second_drain_while_live_returns_tracked_state_without_spawning_second() {
2983 let Some((_dir, root)) = init_repo() else {
2984 return;
2985 };
2986 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
2987 let host = MissionHost::with_backend(root, backend);
2988
2989 let state = Arc::new(Mutex::new(DrainState {
2992 live: true,
2993 current_mission_id: Some("m-fake".to_string()),
2994 ran: vec!["m-earlier".to_string()],
2995 parked: Vec::new(),
2996 }));
2997 let never_finishes = tokio::spawn(async {
2998 std::future::pending::<()>().await;
2999 });
3000 *host.drain.lock().expect("drain tracker lock") = DrainSlot::Running(DrainHandle {
3001 join: never_finishes,
3002 state: Arc::clone(&state),
3003 });
3004 let before = Arc::as_ptr(&state);
3005
3006 let first = host.drain().await.expect("drain must not error");
3007 let second = host.drain().await.expect("drain must not error");
3008 assert_eq!(first, second);
3009 assert_eq!(first["live"], true);
3010 assert_eq!(first["currentMissionId"], "m-fake");
3011 assert_eq!(first["ran"], json!(["m-earlier"]));
3012
3013 let after = {
3016 let guard = host.drain.lock().expect("drain tracker lock");
3017 match &*guard {
3018 DrainSlot::Running(handle) => Arc::as_ptr(&handle.state),
3019 _ => panic!("expected the tracker to still be Running"),
3020 }
3021 };
3022 assert_eq!(before, after, "a second drain must not replace the tracker");
3023 }
3024
3025 #[tokio::test]
3026 async fn two_concurrent_cold_drains_spawn_exactly_one() {
3027 let Some((_dir, root)) = init_repo() else {
3028 return;
3029 };
3030 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
3031 let host = MissionHost::with_backend(root, backend);
3032
3033 let (first, second) = tokio::join!(host.drain(), host.drain());
3049 let first = first.expect("first drain must not error");
3050 let second = second.expect("second drain must not error");
3051 assert_eq!(first["live"], true, "{first}");
3052 assert_eq!(second["live"], true, "{second}");
3053
3054 match &*host.drain.lock().expect("drain tracker lock") {
3057 DrainSlot::Running(_) | DrainSlot::Starting(_) => {}
3058 DrainSlot::Idle => {
3059 panic!("expected a live drain to be tracked after two concurrent calls")
3060 }
3061 }
3062
3063 let deadline = std::time::Instant::now() + Duration::from_secs(5);
3066 loop {
3067 let state = host.queue_state();
3068 if state["drain"]["live"] == false {
3069 break;
3070 }
3071 assert!(
3072 std::time::Instant::now() < deadline,
3073 "drain never settled idle: {state}"
3074 );
3075 tokio::time::sleep(Duration::from_millis(20)).await;
3076 }
3077 }
3078
3079 fn seed_one_queued(root: &Path, mission_id: &str) -> Arc<Mutex<DrainState>> {
3088 kranz_engine::queue::enqueue(
3089 root,
3090 kranz_engine::queue::QueueEntry {
3091 mission_id: mission_id.to_string(),
3092 ticket_slug: None,
3093 priority: 5,
3094 seq: 0,
3095 },
3096 )
3097 .expect("enqueue");
3098 Arc::new(Mutex::new(DrainState::default()))
3099 }
3100
3101 fn proceed_readiness(
3102 _repo_root: &Path,
3103 mission_id: &str,
3104 ) -> kranz_engine::error::Result<kranz_engine::backend_readiness::ReadinessReport> {
3105 Ok(kranz_engine::backend_readiness::ReadinessReport {
3106 mission_id: mission_id.to_string(),
3107 roles: Vec::new(),
3108 overall: kranz_engine::backend_readiness::ReadinessStatus::Ok,
3109 warnings: Vec::new(),
3110 })
3111 }
3112
3113 #[tokio::test]
3114 async fn auto_work_drain_mode_processes_only_one_queue_front() {
3115 let Some((_dir, root)) = init_repo() else {
3116 return;
3117 };
3118 let state = seed_one_queued(&root, "m-first");
3119 kranz_engine::queue::enqueue(
3120 &root,
3121 kranz_engine::queue::QueueEntry {
3122 mission_id: "m-second".to_string(),
3123 ticket_slug: None,
3124 priority: 5,
3125 seq: 0,
3126 },
3127 )
3128 .expect("enqueue second mission");
3129
3130 drain_task_with_probe(
3131 root.clone(),
3132 Arc::clone(&state),
3133 true,
3134 |_mission_id| async { Ok(0) },
3135 proceed_readiness,
3136 )
3137 .await;
3138
3139 assert_eq!(state.lock().expect("drain state lock").ran, ["m-first"]);
3140 let remaining = kranz_engine::queue::list(&root);
3141 assert_eq!(remaining.len(), 1);
3142 assert_eq!(remaining[0].mission_id, "m-second");
3143 }
3144
3145 #[tokio::test]
3146 async fn hosted_drain_restores_dispatch_checkout() {
3147 let Some((_dir, root)) = init_repo() else {
3148 return;
3149 };
3150 let state = seed_one_queued(&root, "m-restore");
3151
3152 let run_root = root.clone();
3153 drain_task_with_probe(
3154 root.clone(),
3155 Arc::clone(&state),
3156 false,
3157 move |mission_id| {
3158 let root = run_root.clone();
3159 async move {
3160 let git = GitRepo::open(&root)?;
3161 let branch = format!("kranz/mission-{mission_id}");
3162 git.create_branch(&branch, None)?;
3163 git.checkout(&branch)?;
3164 Ok(0)
3165 }
3166 },
3167 proceed_readiness,
3168 )
3169 .await;
3170
3171 assert_eq!(
3172 state.lock().expect("drain state lock").ran,
3173 ["m-restore"],
3174 "the injected mission runner must execute"
3175 );
3176
3177 let git = GitRepo::open(&root).expect("open repo");
3178 assert_eq!(
3179 git.current_branch().expect("current branch"),
3180 "main",
3181 "the operator's dispatch-time checkout must be restored on drain exit"
3182 );
3183 }
3184
3185 #[tokio::test]
3186 async fn hosted_drain_restores_dispatch_checkout_on_err() {
3187 let Some((_dir, root)) = init_repo() else {
3188 return;
3189 };
3190 let state = seed_one_queued(&root, "m-err-restore");
3191
3192 let run_root = root.clone();
3193 let runner_called = Arc::new(std::sync::atomic::AtomicBool::new(false));
3194 let called = Arc::clone(&runner_called);
3195 drain_task_with_probe(
3196 root.clone(),
3197 state,
3198 false,
3199 move |mission_id| {
3200 let root = run_root.clone();
3201 let called = Arc::clone(&called);
3202 async move {
3203 called.store(true, std::sync::atomic::Ordering::SeqCst);
3204 let git = GitRepo::open(&root)?;
3205 let branch = format!("kranz/mission-{mission_id}");
3206 git.create_branch(&branch, None)?;
3207 git.checkout(&branch)?;
3208 Err(anyhow::anyhow!("simulated drain runner failure"))
3209 }
3210 },
3211 proceed_readiness,
3212 )
3213 .await;
3214
3215 assert!(
3216 runner_called.load(std::sync::atomic::Ordering::SeqCst),
3217 "the injected mission runner must execute"
3218 );
3219
3220 let git = GitRepo::open(&root).expect("open repo");
3221 assert_eq!(
3222 git.current_branch().expect("current branch"),
3223 "main",
3224 "an errored drain must still restore the operator's dispatch-time checkout"
3225 );
3226 }
3227
3228 #[tokio::test]
3229 async fn hosted_drain_skips_restore_when_started_on_mission_branch() {
3230 let Some((_dir, root)) = init_repo() else {
3231 return;
3232 };
3233 {
3234 let git = GitRepo::open(&root).expect("open repo");
3235 git.create_branch("kranz/mission-existing", None)
3236 .expect("create existing mission branch");
3237 git.checkout("kranz/mission-existing")
3238 .expect("checkout existing mission branch");
3239 }
3240 let state = seed_one_queued(&root, "m-skip");
3241
3242 drain_task_with_probe(
3243 root.clone(),
3244 Arc::clone(&state),
3245 false,
3246 |_mission_id| async { Ok(0) },
3247 proceed_readiness,
3248 )
3249 .await;
3250
3251 assert_eq!(
3252 state.lock().expect("drain state lock").ran,
3253 ["m-skip"],
3254 "the injected mission runner must execute"
3255 );
3256
3257 let git = GitRepo::open(&root).expect("open repo");
3258 assert_eq!(
3259 git.current_branch().expect("current branch"),
3260 "kranz/mission-existing",
3261 "started on a mission branch: no restore must be attempted"
3262 );
3263 }
3264
3265 #[tokio::test]
3266 async fn hosted_drain_leaves_checkout_when_tracked_tree_dirty() {
3267 let Some((_dir, root)) = init_repo() else {
3268 return;
3269 };
3270 let state = seed_one_queued(&root, "m-dirty");
3271
3272 let run_root = root.clone();
3273 drain_task_with_probe(
3274 root.clone(),
3275 Arc::clone(&state),
3276 false,
3277 move |mission_id| {
3278 let root = run_root.clone();
3279 async move {
3280 let git = GitRepo::open(&root)?;
3281 let branch = format!("kranz/mission-{mission_id}");
3282 git.create_branch(&branch, None)?;
3283 git.checkout(&branch)?;
3284 std::fs::write(root.join("README.md"), "dirty tracked edit\n")?;
3285 Ok(0)
3286 }
3287 },
3288 proceed_readiness,
3289 )
3290 .await;
3291
3292 assert_eq!(
3293 state.lock().expect("drain state lock").ran,
3294 ["m-dirty"],
3295 "the injected mission runner must execute"
3296 );
3297
3298 let git = GitRepo::open(&root).expect("open repo");
3299 assert_eq!(
3300 git.current_branch().expect("current branch"),
3301 "kranz/mission-m-dirty",
3302 "a dirty tracked tree must abort the restore, leaving the checkout on the mission \
3303 branch"
3304 );
3305 }
3306
3307 #[tokio::test]
3308 async fn hosted_drain_second_call_does_not_capture_or_restore() {
3309 let Some((_dir, root)) = init_repo() else {
3310 return;
3311 };
3312 {
3313 let git = GitRepo::open(&root).expect("open repo");
3314 git.create_branch("feature-branch", None)
3315 .expect("create feature branch");
3316 git.checkout("feature-branch")
3317 .expect("checkout feature branch");
3318 }
3319 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
3320 let host = MissionHost::with_backend(root.clone(), backend);
3321
3322 let tracked_state = Arc::new(Mutex::new(DrainState {
3327 live: true,
3328 current_mission_id: Some("m-inflight".to_string()),
3329 ran: Vec::new(),
3330 parked: Vec::new(),
3331 }));
3332 let never_finishes = tokio::spawn(async {
3333 std::future::pending::<()>().await;
3334 });
3335 *host.drain.lock().expect("drain tracker lock") = DrainSlot::Running(DrainHandle {
3336 join: never_finishes,
3337 state: Arc::clone(&tracked_state),
3338 });
3339
3340 let result = host
3341 .drain()
3342 .await
3343 .expect("second drain call must not error");
3344 assert_eq!(result["live"], true, "{result}");
3345
3346 let git = GitRepo::open(&root).expect("open repo");
3349 assert_eq!(
3350 git.current_branch().expect("current branch"),
3351 "feature-branch",
3352 "the idempotent second drain() must not mutate the checkout"
3353 );
3354
3355 match &*host.drain.lock().expect("drain tracker lock") {
3358 DrainSlot::Running(handle) => {
3359 assert_eq!(
3360 Arc::as_ptr(&handle.state),
3361 Arc::as_ptr(&tracked_state),
3362 "a second drain must not replace the tracker or spawn a second task"
3363 );
3364 }
3365 _ => panic!("expected the tracker to still be Running"),
3366 };
3367 }
3368
3369 #[tokio::test]
3378 async fn starting_reservation_is_not_overwritten_or_double_spawned() {
3379 let Some((_dir, root)) = init_repo() else {
3380 return;
3381 };
3382 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
3383 let host = MissionHost::with_backend(root, backend);
3384
3385 let state = Arc::new(Mutex::new(DrainState {
3386 live: true,
3387 current_mission_id: Some("m-reserved".to_string()),
3388 ran: Vec::new(),
3389 parked: Vec::new(),
3390 }));
3391 *host.drain.lock().expect("drain tracker lock") = DrainSlot::Starting(Arc::clone(&state));
3392 let before = Arc::as_ptr(&state);
3393
3394 let result = host.drain().await.expect("drain must not error");
3395 assert_eq!(result["live"], true, "{result}");
3396 assert_eq!(result["currentMissionId"], "m-reserved");
3397
3398 let after = match &*host.drain.lock().expect("drain tracker lock") {
3401 DrainSlot::Starting(tracked) => Arc::as_ptr(tracked),
3402 DrainSlot::Running(_) => panic!(
3403 "the Starting reservation was upgraded/replaced by this call — the deflection \
3404 arm was bypassed and a second drain was spawned"
3405 ),
3406 DrainSlot::Idle => panic!("the Starting reservation was cleared by this call"),
3407 };
3408 assert_eq!(
3409 before, after,
3410 "drain() must return the SAME tracked reservation, not install a new one"
3411 );
3412 }
3413
3414 #[tokio::test]
3419 async fn failed_drain_construction_clears_the_reservation_to_idle() {
3420 let Some((_dir, root)) = init_repo() else {
3421 return;
3422 };
3423 std::fs::create_dir_all(root.join(".kranz")).expect("mkdir .kranz");
3424 std::fs::write(root.join(".kranz").join("config.json"), "not json")
3425 .expect("write malformed config");
3426
3427 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
3428 let host = MissionHost::with_backend(root, backend);
3429
3430 host.drain()
3431 .await
3432 .expect_err("malformed config must fail drain construction");
3433
3434 let is_idle = matches!(
3435 &*host.drain.lock().expect("drain tracker lock"),
3436 DrainSlot::Idle
3437 );
3438 assert!(
3439 is_idle,
3440 "a failed drain construction must reset the tracker to Idle"
3441 );
3442 }
3443
3444 #[tokio::test]
3448 async fn queue_state_reports_a_starting_reservation_as_live() {
3449 let Some((_dir, root)) = init_repo() else {
3450 return;
3451 };
3452 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
3453 let host = MissionHost::with_backend(root, backend);
3454
3455 let state = Arc::new(Mutex::new(DrainState {
3456 live: true,
3457 current_mission_id: Some("m-starting".to_string()),
3458 ran: Vec::new(),
3459 parked: Vec::new(),
3460 }));
3461 *host.drain.lock().expect("drain tracker lock") = DrainSlot::Starting(state);
3462
3463 let queue_state = host.queue_state();
3464 assert_eq!(queue_state["drain"]["live"], true, "{queue_state}");
3465 assert_eq!(queue_state["drain"]["currentMissionId"], "m-starting");
3466 }
3467
3468 #[tokio::test]
3469 async fn queue_drain_route_requires_token_but_queue_route_does_not() {
3470 use axum::body::Body;
3471 use axum::http::Request;
3472 use tower::ServiceExt;
3473
3474 let Some((_dir, root)) = init_repo() else {
3475 return;
3476 };
3477 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
3478 let host = MissionHost::with_backend(root, backend);
3479 let app =
3480 crate::router_with_host(host, None, crate::MutationAuthority::new("tok").unwrap());
3481
3482 let response = app
3483 .clone()
3484 .oneshot(
3485 Request::builder()
3486 .method("POST")
3487 .uri("/api/queue/drain")
3488 .body(Body::empty())
3489 .unwrap(),
3490 )
3491 .await
3492 .unwrap();
3493 assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
3494
3495 let response = app
3496 .clone()
3497 .oneshot(
3498 Request::builder()
3499 .method("POST")
3500 .uri("/api/queue/drain")
3501 .header("x-kranz-token", "tok")
3502 .body(Body::empty())
3503 .unwrap(),
3504 )
3505 .await
3506 .unwrap();
3507 assert_ne!(response.status(), StatusCode::UNAUTHORIZED);
3508 assert_eq!(response.status(), StatusCode::OK);
3509
3510 let response = app
3511 .oneshot(
3512 Request::builder()
3513 .uri("/api/queue")
3514 .body(Body::empty())
3515 .unwrap(),
3516 )
3517 .await
3518 .unwrap();
3519 assert_eq!(response.status(), StatusCode::OK);
3520 }
3521
3522 #[test]
3527 fn should_auto_drain_truth_table() {
3528 assert!(should_auto_drain(true, true, false));
3530 assert!(!should_auto_drain(false, true, false));
3532 assert!(!should_auto_drain(false, false, false));
3533 assert!(!should_auto_drain(true, false, false));
3535 assert!(!should_auto_drain(true, true, true));
3537 assert!(!should_auto_drain(false, false, true));
3538 }
3539
3540 fn write_auto_work_config(root: &std::path::Path, enabled: bool) {
3544 let dir = root.join(".kranz");
3545 std::fs::create_dir_all(&dir).expect("create .kranz dir");
3546 std::fs::write(
3547 dir.join("config.json"),
3548 json!({ "autoWork": enabled }).to_string(),
3549 )
3550 .expect("write config.json");
3551 }
3552
3553 fn turn(reply: &str) -> Vec<kranz_engine::backend::AgentEvent> {
3556 vec![
3557 kranz_engine::backend_mock::mock_text(reply),
3558 mock_result_text(reply),
3559 ]
3560 }
3561
3562 fn preflight_authenticated_script() -> MockScript {
3566 MockScript::single_shot("ack")
3567 }
3568
3569 fn worker_pass() -> MockScript {
3571 MockScript::single_shot_json(&json!({
3572 "result": "pass",
3573 "summary": "implemented and tested",
3574 "filesTouched": [],
3575 "testsAdded": [],
3576 "testEvidence": "all green",
3577 "commits": []
3578 }))
3579 }
3580
3581 fn plan_json() -> Value {
3583 json!({
3584 "goal": "ship the demo",
3585 "validationContract": [],
3586 "milestones": [{
3587 "title": "M1",
3588 "features": [{
3589 "title": "F1",
3590 "spec": "build the thing",
3591 "validationCriteria": ["it works"]
3592 }]
3593 }]
3594 })
3595 }
3596
3597 #[tokio::test(flavor = "multi_thread")]
3598 async fn auto_work_tick_drains_a_queued_mission_when_enabled() {
3599 let Some((_dir, root)) = init_repo() else {
3600 return;
3601 };
3602 write_auto_work_config(&root, true);
3603
3604 let judgement =
3605 json!({ "decision": "complete", "guidance": "", "summary": "worker did the job" });
3606 let orch = MockScript::streaming(vec![mock_init("orch-auto"), mock_result_text("seed-hi")])
3607 .responding(vec![
3608 turn("scoping the demo"),
3609 turn(&plan_json().to_string()),
3610 ]);
3611 let orch_run = MockScript::streaming(vec![
3612 mock_init("orch-auto-run"),
3613 mock_result_text("resumed"),
3614 ])
3615 .responding(vec![turn(&judgement.to_string()), turn("NONE")]);
3616 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::with_scripts(vec![
3617 orch,
3618 preflight_authenticated_script(),
3619 worker_pass(),
3620 orch_run,
3621 ]));
3622 let host = MissionHost::with_backend(root.clone(), backend);
3623
3624 let id = host
3625 .create(
3626 "drain me via autoWork",
3627 Some(&json!({ "skipScrutiny": true, "skipFunctional": true })),
3628 )
3629 .await
3630 .expect("create mission");
3631 host.planning_turn(&id, "go").await.expect("planning turn");
3632 let plan_body = host.request_plan(&id).await.expect("request plan");
3633 assert_eq!(plan_body["ready"], true, "{plan_body}");
3634 let plan: Plan =
3635 serde_json::from_value(plan_body["plan"].clone()).expect("plan deserializes");
3636 host.approve(&id, plan).await.expect("approve");
3637 host.release(&id).expect("release");
3638
3639 kranz_engine::queue::enqueue(
3640 &root,
3641 kranz_engine::queue::QueueEntry {
3642 mission_id: id.clone(),
3643 ticket_slug: None,
3644 priority: 2,
3645 seq: 0,
3646 },
3647 )
3648 .expect("enqueue");
3649
3650 host.auto_work_tick().await;
3653 assert!(
3654 host.drain_is_live(),
3655 "autoWork tick with autoWork=true and a non-empty queue must start a drain"
3656 );
3657
3658 let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
3659 loop {
3660 let state = host.queue_state();
3661 if state["entries"]
3662 .as_array()
3663 .map(|a| a.is_empty())
3664 .unwrap_or(false)
3665 && state["drain"]["live"] == false
3666 {
3667 break;
3668 }
3669 assert!(
3670 tokio::time::Instant::now() < deadline,
3671 "autoWork drain never completed: {state}"
3672 );
3673 tokio::time::sleep(Duration::from_millis(50)).await;
3674 }
3675 }
3676
3677 #[tokio::test]
3678 async fn auto_work_tick_leaves_the_queue_untouched_when_disabled() {
3679 let Some((_dir, root)) = init_repo() else {
3680 return;
3681 };
3682 let backend: Arc<dyn AgentBackend> = Arc::new(MockBackend::new());
3687 let host = MissionHost::with_backend(root.clone(), backend);
3688
3689 kranz_engine::queue::enqueue(
3690 &root,
3691 kranz_engine::queue::QueueEntry {
3692 mission_id: "m-untouched".to_string(),
3693 ticket_slug: None,
3694 priority: 2,
3695 seq: 0,
3696 },
3697 )
3698 .expect("enqueue");
3699
3700 host.auto_work_tick().await;
3701
3702 assert!(
3703 !host.drain_is_live(),
3704 "autoWork=false must never start a drain"
3705 );
3706 let entries = kranz_engine::queue::list(&root);
3707 assert_eq!(
3708 entries.len(),
3709 1,
3710 "queue entry must be left untouched when autoWork is disabled: {entries:?}"
3711 );
3712 assert_eq!(entries[0].mission_id, "m-untouched");
3713 }
3714}