1use std::collections::{BTreeMap, HashMap, HashSet};
2use std::fs::{self, OpenOptions};
3use std::io::{self, BufRead, BufReader, Read, Write};
4use std::os::unix::fs::PermissionsExt;
5use std::os::unix::net::UnixStream as StdUnixStream;
6use std::os::unix::process::CommandExt;
7use std::path::{Path, PathBuf};
8use std::process::{Child, Command, Stdio};
9use std::sync::Arc;
10use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
11use std::time::{Duration, Instant};
12
13use base64::Engine;
14use base64::engine::general_purpose::URL_SAFE_NO_PAD as BASE64_TOKEN;
15use fs2::FileExt;
16use serde_json::json;
17use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader as AsyncBufReader};
18use tokio::net::{UnixListener, UnixStream};
19use tokio::sync::{mpsc, oneshot};
20
21use crate::clock::{Clock, FileClock, SystemClock, format_timestamp};
22use crate::config::{AgentConfig, Config, ConfigError, Project, RunningHours, expand_agent_cmd};
23use crate::flow::Flow;
24use crate::frontmatter::{Frontmatter, FrontmatterError};
25use crate::ids::{IdError, next_id};
26use crate::logging::{LogLevel, OperationalLog};
27use crate::outcome::{
28 MergeOutcome, RunEvidence, StageOutcome, classify_exit, derive_outcome, wants_merge,
29 wants_tests,
30};
31use crate::protocol::{
32 Capability, ErrorBody, ErrorCode, Request, RequestEnvelope, RequestId, ResponseEnvelope,
33};
34use crate::run_log::{OutputSource, OutputStream, RunLogWriter};
35use crate::store::{
36 ActivationKind, ClaimRequest, EvidenceRecord, NewActivation, QueuedActivation, RecoverableRun,
37 StageRecord, Store, StoreError, TicketState,
38};
39
40const MAX_ENVELOPE_BYTES: u64 = 1024 * 1024;
41const STARTUP_TIMEOUT: Duration = Duration::from_secs(5);
42const CLIENT_TIMEOUT: Duration = Duration::from_secs(5);
43const DISPATCH_CHANNEL_CAPACITY: usize = 64;
44const DEFAULT_LEASE_MS: i64 = 10 * 60 * 1000;
45pub(crate) const WORKER_BOOTSTRAP_PROMPT: &str =
46 include_str!("worker-instructions.md").trim_ascii();
47const LOGS_PAGE_LIMIT: usize = 64;
50
51static NEXT_REQUEST_ID: AtomicU64 = AtomicU64::new(1);
52static MERGE_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
53
54pub struct ClientResponse {
55 pub response: ResponseEnvelope,
56 pub started: bool,
57}
58
59pub fn request(request: Request) -> Result<ClientResponse, DaemonError> {
60 let cwd = std::env::current_dir().map_err(DaemonError::CurrentDirectory)?;
61 let project = Project::discover(&cwd)?;
62 Config::load(&project)?;
63
64 if let Ok(response) = send_existing(&project, request.clone()) {
65 return Ok(ClientResponse {
66 response,
67 started: false,
68 });
69 }
70
71 spawn_daemon(&project)?;
72 let deadline = Instant::now() + STARTUP_TIMEOUT;
73 loop {
74 match send_existing(&project, request.clone()) {
75 Ok(response) => {
76 return Ok(ClientResponse {
77 response,
78 started: true,
79 });
80 }
81 Err(error) if Instant::now() >= deadline => return Err(error),
82 Err(_) => std::thread::sleep(Duration::from_millis(20)),
83 }
84 }
85}
86
87pub fn request_running(request: Request) -> Result<Option<ResponseEnvelope>, DaemonError> {
90 let cwd = std::env::current_dir().map_err(DaemonError::CurrentDirectory)?;
91 let project = Project::discover(&cwd)?;
92 Config::load(&project)?;
93 match send_existing(&project, request) {
94 Ok(response) => Ok(Some(response)),
95 Err(DaemonError::Connect(_)) => Ok(None),
96 Err(error) => Err(error),
97 }
98}
99
100pub fn serve_current_project() -> Result<(), DaemonError> {
101 let cwd = std::env::current_dir().map_err(DaemonError::CurrentDirectory)?;
102 let project = Project::discover(&cwd)?;
103 let config = Config::load(&project)?;
104 fs::create_dir_all(&project.state_dir).map_err(|source| DaemonError::Io {
105 path: project.state_dir.clone(),
106 source,
107 })?;
108 fs::set_permissions(&project.state_dir, fs::Permissions::from_mode(0o700)).map_err(
109 |source| DaemonError::Io {
110 path: project.state_dir.clone(),
111 source,
112 },
113 )?;
114 let runtime_root = project
115 .runtime_dir
116 .parent()
117 .expect("repository runtime directories have a parent");
118 fs::create_dir_all(runtime_root).map_err(|source| DaemonError::Io {
119 path: runtime_root.to_path_buf(),
120 source,
121 })?;
122 fs::set_permissions(runtime_root, fs::Permissions::from_mode(0o700)).map_err(|source| {
123 DaemonError::Io {
124 path: runtime_root.to_path_buf(),
125 source,
126 }
127 })?;
128 fs::create_dir(&project.runtime_dir)
129 .or_else(|source| {
130 if source.kind() == io::ErrorKind::AlreadyExists {
131 Ok(())
132 } else {
133 Err(source)
134 }
135 })
136 .map_err(|source| DaemonError::Io {
137 path: project.runtime_dir.clone(),
138 source,
139 })?;
140 fs::set_permissions(&project.runtime_dir, fs::Permissions::from_mode(0o700)).map_err(
141 |source| DaemonError::Io {
142 path: project.runtime_dir.clone(),
143 source,
144 },
145 )?;
146
147 let lock = OpenOptions::new()
148 .create(true)
149 .truncate(false)
150 .read(true)
151 .write(true)
152 .open(&project.lock_path)
153 .map_err(|source| DaemonError::Io {
154 path: project.lock_path.clone(),
155 source,
156 })?;
157 lock.try_lock_exclusive().map_err(|source| {
158 if source.kind() == io::ErrorKind::WouldBlock {
159 DaemonError::AlreadyRunning
160 } else {
161 DaemonError::Io {
162 path: project.lock_path.clone(),
163 source,
164 }
165 }
166 })?;
167 let legacy_lock_path = project.runtime_dir.join("daemon.lock");
171 let legacy_lock = OpenOptions::new()
172 .create(true)
173 .truncate(false)
174 .read(true)
175 .write(true)
176 .open(&legacy_lock_path)
177 .map_err(|source| DaemonError::Io {
178 path: legacy_lock_path.clone(),
179 source,
180 })?;
181 legacy_lock.try_lock_exclusive().map_err(|source| {
182 if source.kind() == io::ErrorKind::WouldBlock {
183 DaemonError::AlreadyRunning
184 } else {
185 DaemonError::Io {
186 path: legacy_lock_path,
187 source,
188 }
189 }
190 })?;
191 let identity = json!({
194 "pid": std::process::id(),
195 "started_at_ms": process_start_time(std::process::id()),
196 "socket": project.operator_socket,
197 });
198 let _ = lock.set_len(0);
199 let _ = {
200 use std::io::Write as _;
201 writeln!(&lock, "{identity}")
202 };
203
204 let clock: Arc<dyn Clock> = match std::env::var_os("SLOOP_TEST_CLOCK_PATH") {
205 Some(path) => Arc::new(FileClock::new(path.into())),
206 None => Arc::new(SystemClock),
207 };
208 let store = Store::open(&project.db_path, clock.now_ms()).map_err(DaemonError::Store)?;
209 if let Some(agent) = &config.agent {
210 store
211 .backfill_ticket_targets(&agent.default_target, clock.now_ms())
212 .map_err(DaemonError::Store)?;
213 }
214 index_projects(
215 &project.root,
216 &config.project_dir,
217 &store,
218 clock.now_ms(),
219 &config.project_prefix,
220 )?;
221 reconcile_tickets(
222 &project.root,
223 &store,
224 clock.now_ms(),
225 config.delete_missing_after_ms,
226 )?;
227
228 let runtime = tokio::runtime::Builder::new_multi_thread()
229 .enable_all()
230 .build()
231 .map_err(DaemonError::Runtime)?;
232 runtime.block_on(serve(project, config, store, lock, legacy_lock, clock))
233}
234
235#[derive(Debug, Clone, Copy, PartialEq, Eq)]
236pub(crate) enum OrphanDisposition {
237 MarkMissing,
238 Delete,
239 Keep,
240}
241
242fn orphan_disposition(
246 missing_since_ms: Option<i64>,
247 is_referenced: bool,
248 now_ms: i64,
249 delete_missing_after_ms: i64,
250) -> OrphanDisposition {
251 match missing_since_ms {
252 None => OrphanDisposition::MarkMissing,
253 Some(since) if now_ms - since >= delete_missing_after_ms && !is_referenced => {
254 OrphanDisposition::Delete
255 }
256 Some(_) => OrphanDisposition::Keep,
257 }
258}
259
260fn reconcile_tickets(
263 root: &Path,
264 store: &Store,
265 now_ms: i64,
266 delete_missing_after_ms: i64,
267) -> Result<(), DaemonError> {
268 for ticket in store.local_ticket_files().map_err(DaemonError::Store)? {
269 if root.join(&ticket.file_path).is_file() {
270 if ticket.missing_at_ms.is_some() {
271 store
272 .clear_ticket_missing(&ticket.id, now_ms)
273 .map_err(DaemonError::Store)?;
274 }
275 continue;
276 }
277 let is_referenced = store
278 .ticket_is_referenced(&ticket.id)
279 .map_err(DaemonError::Store)?;
280 match orphan_disposition(
281 ticket.missing_at_ms,
282 is_referenced,
283 now_ms,
284 delete_missing_after_ms,
285 ) {
286 OrphanDisposition::MarkMissing => {
287 store
288 .mark_ticket_missing(&ticket.id, now_ms)
289 .map_err(DaemonError::Store)?;
290 }
291 OrphanDisposition::Delete => {
292 store
293 .delete_ticket(&ticket.id)
294 .map_err(DaemonError::Store)?;
295 }
296 OrphanDisposition::Keep => {}
297 }
298 }
299 Ok(())
300}
301
302fn index_projects(
305 root: &Path,
306 project_dir: &Path,
307 store: &Store,
308 now_ms: i64,
309 project_prefix: &str,
310) -> Result<(), DaemonError> {
311 let directory = root.join(project_dir);
312 let entries = match fs::read_dir(&directory) {
313 Ok(entries) => entries,
314 Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
315 Err(source) => {
316 return Err(DaemonError::Io {
317 path: directory,
318 source,
319 });
320 }
321 };
322
323 let mut paths = Vec::new();
324 for entry in entries {
325 let path = entry
326 .map_err(|source| DaemonError::Io {
327 path: directory.clone(),
328 source,
329 })?
330 .path();
331 if path.extension().and_then(|extension| extension.to_str()) == Some("md") {
332 paths.push(path);
333 }
334 }
335 paths.sort();
336
337 struct ProjectFile {
338 path: PathBuf,
339 content: String,
340 stem: String,
341 frontmatter: Frontmatter,
342 }
343
344 let mut projects = Vec::new();
345 for path in paths {
346 let content = fs::read_to_string(&path).map_err(|source| DaemonError::Io {
347 path: path.clone(),
348 source,
349 })?;
350 let stem = path
351 .file_stem()
352 .map(|stem| stem.to_string_lossy().into_owned())
353 .unwrap_or_default();
354 let Ok(frontmatter) = crate::frontmatter::parse(&content) else {
357 continue;
358 };
359 projects.push(ProjectFile {
360 path,
361 content,
362 stem,
363 frontmatter,
364 });
365 }
366
367 let mut ids: Vec<String> = projects
370 .iter()
371 .filter_map(|project| project.frontmatter.id.clone())
372 .collect();
373 for project in projects {
374 let id = match project.frontmatter.id {
375 Some(id) => id,
376 None => {
377 let id = next_id(project_prefix, ids.iter().map(String::as_str))?;
378 let updated =
379 crate::frontmatter::stamp_id(&project.content, &id).map_err(|error| {
380 DaemonError::Frontmatter {
381 path: project.path.clone(),
382 error,
383 }
384 })?;
385 fs::write(
386 &project.path,
387 updated.expect("idless project always needs an ID stamp"),
388 )
389 .map_err(|source| DaemonError::Io {
390 path: project.path.clone(),
391 source,
392 })?;
393 ids.push(id.clone());
394 id
395 }
396 };
397 let title = project.frontmatter.title.unwrap_or(project.stem);
398 let relative = project
399 .path
400 .strip_prefix(root)
401 .unwrap_or(&project.path)
402 .to_string_lossy()
403 .into_owned();
404 store
405 .upsert_local_project(&id, &relative, &title, now_ms)
406 .map_err(DaemonError::Store)?;
407 }
408 Ok(())
409}
410
411async fn serve(
412 project: Project,
413 config: Config,
414 store: Store,
415 _lock: fs::File,
416 _legacy_lock: fs::File,
417 clock: Arc<dyn Clock>,
418) -> Result<(), DaemonError> {
419 if project.operator_socket.exists() {
420 fs::remove_file(&project.operator_socket).map_err(|source| DaemonError::Io {
421 path: project.operator_socket.clone(),
422 source,
423 })?;
424 }
425
426 let listener =
427 UnixListener::bind(&project.operator_socket).map_err(|source| DaemonError::Io {
428 path: project.operator_socket.clone(),
429 source,
430 })?;
431 fs::set_permissions(&project.operator_socket, fs::Permissions::from_mode(0o600)).map_err(
432 |source| DaemonError::Io {
433 path: project.operator_socket.clone(),
434 source,
435 },
436 )?;
437
438 let log = OperationalLog::open(&project.daemon_log).map_err(|source| DaemonError::Io {
439 path: project.daemon_log.clone(),
440 source,
441 })?;
442 log.emit(LogLevel::Info, "sloop::daemon", "daemon_started");
443
444 let paused = store.paused().map_err(DaemonError::Store)?;
445 let (dispatcher_tx, dispatcher_rx) = mpsc::channel(DISPATCH_CHANNEL_CAPACITY);
446 let (events_tx, events_rx) = mpsc::channel(DISPATCH_CHANNEL_CAPACITY);
447 let (shutdown_tx, mut shutdown_rx) = mpsc::channel::<()>(1);
448 let shutdown_flag = Arc::new(AtomicBool::new(false));
449 let mut state = DispatcherState {
450 pid: std::process::id(),
451 paused,
452 max_agents: config.max_parallel_tasks,
453 ticket_prefix: config.ticket_prefix.clone(),
454 running_hours: config.running_hours.clone(),
455 agent: config.agent.clone(),
456 flows: config.flows.clone(),
457 default_flow: config.default_flow.clone(),
458 aftercare_test_cmd: config.aftercare_test_cmd.clone(),
459 root: project.root.clone(),
460 ticket_dir: config.ticket_dir.clone(),
461 worktree_dir: project.root.join(&config.worktree_dir),
462 state_dir: project.state_dir.clone(),
463 runtime_dir: project.runtime_dir.clone(),
464 socket: project.operator_socket.clone(),
465 daemon_log: project.daemon_log.clone(),
466 store,
467 active: HashSet::new(),
468 cancelling: HashSet::new(),
469 worker_tokens: HashMap::new(),
470 worker_listeners: HashMap::new(),
471 worker_socket_paths: HashMap::new(),
472 pending_exits: HashMap::new(),
473 requests_tx: dispatcher_tx.clone(),
474 log: log.clone(),
475 clock,
476 shutdown: shutdown_tx.clone(),
477 shutdown_flag: shutdown_flag.clone(),
478 };
479 recover_inflight_runs(&mut state, &events_tx, &log)?;
480 tokio::spawn(run_dispatcher(
481 state,
482 dispatcher_rx,
483 events_rx,
484 events_tx,
485 log.clone(),
486 ));
487
488 loop {
489 tokio::select! {
490 accepted = listener.accept() => {
491 let (stream, _) = accepted.map_err(|source| DaemonError::Io {
492 path: project.operator_socket.clone(),
493 source,
494 })?;
495 let dispatcher_tx = dispatcher_tx.clone();
496 let log = log.clone();
497 let shutdown = shutdown_tx.clone();
498 tokio::spawn(async move {
499 if let Err(error) = handle_connection(stream, dispatcher_tx, shutdown).await {
500 log.emit_with_fields(
501 LogLevel::Error,
502 "sloop::socket",
503 "connection_failed",
504 json!({"error": error.to_string()}),
505 );
506 }
507 });
508 }
509 _ = shutdown_rx.recv() => {
510 shutdown_flag.store(true, Ordering::Release);
511 log.emit(LogLevel::Info, "sloop::daemon", "daemon_stopped");
512 let _ = fs::remove_file(&project.operator_socket);
513 return Ok(());
514 }
515 }
516 }
517}
518
519async fn handle_connection(
520 stream: UnixStream,
521 dispatcher: mpsc::Sender<DispatcherMessage>,
522 shutdown: mpsc::Sender<()>,
523) -> io::Result<()> {
524 let reader = AsyncBufReader::new(stream);
525 let mut limited = reader.take(MAX_ENVELOPE_BYTES + 1);
526 let mut bytes = Vec::new();
527 let read = limited.read_until(b'\n', &mut bytes).await?;
528 if read == 0 {
529 return Ok(());
530 }
531
532 let mut stream = limited.into_inner().into_inner();
533 let envelope = if bytes.len() as u64 > MAX_ENVELOPE_BYTES {
534 Err(protocol_error("request envelope is too large"))
535 } else {
536 std::str::from_utf8(&bytes)
537 .map_err(|_| protocol_error("request envelope must be UTF-8"))
538 .and_then(|line| RequestEnvelope::decode(line.trim_end()).map_err(|error| error.body))
539 };
540
541 let is_stop = matches!(
542 &envelope,
543 Ok(envelope) if matches!(envelope.request, Request::Stop(_))
544 );
545 let response = match envelope {
546 Ok(envelope) if envelope.token.is_some() => ResponseEnvelope::failure(
547 Some(envelope.id),
548 unauthorized("operator socket does not accept worker tokens"),
549 ),
550 Ok(envelope)
551 if !matches!(
552 envelope.request.capability(),
553 Capability::Operator | Capability::Both
554 ) =>
555 {
556 ResponseEnvelope::failure(
557 Some(envelope.id),
558 unauthorized("worker verbs are not available on the operator socket"),
559 )
560 }
561 Ok(envelope) => dispatch_envelope(envelope, RequestOrigin::Operator, &dispatcher).await,
562 Err(error) => ResponseEnvelope::failure(None, error),
563 };
564
565 let stopping = is_stop && response.ok;
568 let encoded = serde_json::to_vec(&response).map_err(io::Error::other)?;
569 stream.write_all(&encoded).await?;
570 stream.write_all(b"\n").await?;
571 stream.shutdown().await?;
572 if stopping {
573 let _ = shutdown.send(()).await;
574 }
575 Ok(())
576}
577
578async fn handle_worker_connection(
582 stream: UnixStream,
583 run_id: String,
584 dispatcher: mpsc::Sender<DispatcherMessage>,
585) -> io::Result<()> {
586 let reader = AsyncBufReader::new(stream);
587 let mut limited = reader.take(MAX_ENVELOPE_BYTES + 1);
588 let mut bytes = Vec::new();
589 let read = limited.read_until(b'\n', &mut bytes).await?;
590 if read == 0 {
591 return Ok(());
592 }
593
594 let mut stream = limited.into_inner().into_inner();
595 let envelope = if bytes.len() as u64 > MAX_ENVELOPE_BYTES {
596 Err(protocol_error("request envelope is too large"))
597 } else {
598 std::str::from_utf8(&bytes)
599 .map_err(|_| protocol_error("request envelope must be UTF-8"))
600 .and_then(|line| RequestEnvelope::decode(line.trim_end()).map_err(|error| error.body))
601 };
602
603 let response = match envelope {
604 Ok(envelope)
605 if !matches!(
606 envelope.request.capability(),
607 Capability::Worker | Capability::Both
608 ) =>
609 {
610 ResponseEnvelope::failure(
611 Some(envelope.id),
612 unauthorized("operator verbs are not available on a worker socket"),
613 )
614 }
615 Ok(envelope) => {
616 let token = envelope.token.clone();
617 dispatch_envelope(
618 envelope,
619 RequestOrigin::Worker { run_id, token },
620 &dispatcher,
621 )
622 .await
623 }
624 Err(error) => ResponseEnvelope::failure(None, error),
625 };
626
627 let encoded = serde_json::to_vec(&response).map_err(io::Error::other)?;
628 stream.write_all(&encoded).await?;
629 stream.write_all(b"\n").await?;
630 stream.shutdown().await
631}
632
633async fn dispatch_envelope(
634 envelope: RequestEnvelope,
635 origin: RequestOrigin,
636 dispatcher: &mpsc::Sender<DispatcherMessage>,
637) -> ResponseEnvelope {
638 let (reply_tx, reply_rx) = oneshot::channel();
639 let id = envelope.id;
640 if dispatcher
641 .send(DispatcherMessage {
642 id: id.clone(),
643 request: envelope.request,
644 origin,
645 reply: reply_tx,
646 })
647 .await
648 .is_err()
649 {
650 ResponseEnvelope::failure(Some(id), internal("dispatcher is unavailable"))
651 } else {
652 reply_rx.await.unwrap_or_else(|_| {
653 ResponseEnvelope::failure(Some(id), internal("dispatcher dropped response"))
654 })
655 }
656}
657
658struct DispatcherMessage {
659 id: RequestId,
660 request: Request,
661 origin: RequestOrigin,
662 reply: oneshot::Sender<ResponseEnvelope>,
663}
664
665enum RequestOrigin {
669 Operator,
670 Worker {
671 run_id: String,
672 token: Option<String>,
673 },
674}
675
676struct DispatcherState {
677 pid: u32,
678 paused: bool,
679 max_agents: usize,
680 ticket_prefix: String,
681 running_hours: Option<RunningHours>,
682 agent: Option<AgentConfig>,
683 flows: BTreeMap<String, Flow>,
684 default_flow: String,
685 aftercare_test_cmd: Option<Vec<String>>,
686 root: PathBuf,
687 ticket_dir: PathBuf,
688 worktree_dir: PathBuf,
689 state_dir: PathBuf,
690 runtime_dir: PathBuf,
691 socket: PathBuf,
692 daemon_log: PathBuf,
693 store: Store,
694 active: HashSet<String>,
696 cancelling: HashSet<String>,
699 worker_tokens: HashMap<String, String>,
702 worker_listeners: HashMap<String, tokio::task::JoinHandle<()>>,
704 worker_socket_paths: HashMap<String, PathBuf>,
705 pending_exits: HashMap<String, RunEvent>,
708 requests_tx: mpsc::Sender<DispatcherMessage>,
711 log: OperationalLog,
712 clock: Arc<dyn Clock>,
713 shutdown: mpsc::Sender<()>,
716 shutdown_flag: Arc<AtomicBool>,
717}
718
719struct StageResult {
721 outcome: StageOutcome,
722 exit_code: Option<i32>,
723 started_at_ms: i64,
724 finished_at_ms: i64,
725}
726
727enum RunEvent {
729 Exited {
730 run_id: String,
731 exit_code: Option<i32>,
732 capture_complete: bool,
735 commits: Vec<String>,
738 tests: Option<StageResult>,
739 merge: Option<MergeOutcome>,
740 recovery: Option<RecoveryClassification>,
741 },
742}
743
744#[derive(Debug, Clone, Copy, PartialEq, Eq)]
745enum RecoveryClassification {
746 Aftercare,
747 Orphaned,
748}
749
750async fn run_dispatcher(
751 mut state: DispatcherState,
752 mut requests: mpsc::Receiver<DispatcherMessage>,
753 mut events: mpsc::Receiver<RunEvent>,
754 events_tx: mpsc::Sender<RunEvent>,
755 log: OperationalLog,
756) {
757 reconcile(&mut state, &events_tx, &log);
758 loop {
759 let deadline = next_dispatch_deadline(&state);
760 let clock = state.clock.clone();
761 tokio::select! {
762 message = requests.recv() => {
763 let Some(message) = message else { break };
764 let response = match message.origin {
765 RequestOrigin::Operator => dispatch(&mut state, message.id, message.request),
766 RequestOrigin::Worker { run_id, token } => dispatch_worker(
767 &mut state,
768 message.id,
769 message.request,
770 &run_id,
771 token.as_deref(),
772 ),
773 };
774 let _ = message.reply.send(response);
775 log.emit(LogLevel::Info, "sloop::dispatcher", "request_handled");
776 }
777 event = events.recv() => {
778 let Some(event) = event else { break };
779 settle_run_exit(&mut state, event, &log);
780 }
781 () = wait_for_deadline(clock, deadline) => {
782 log.emit(LogLevel::Info, "sloop::dispatcher", "timer_fired");
783 }
784 () = tokio::time::sleep(std::time::Duration::from_secs(2)) => {
787 if !state.root.join(".git").exists() {
788 log.emit(LogLevel::Error, "sloop::dispatcher", "project_root_missing");
789 let _ = state.shutdown.send(()).await;
790 break;
791 }
792 }
793 }
794 reconcile(&mut state, &events_tx, &log);
795 }
796}
797
798async fn wait_for_deadline(clock: Arc<dyn Clock>, deadline: Option<i64>) {
799 match deadline {
800 Some(deadline) => clock.sleep_until(deadline).await,
801 None => std::future::pending().await,
802 }
803}
804
805fn settle_run_exit(state: &mut DispatcherState, event: RunEvent, log: &OperationalLog) {
810 let run_id = match &event {
811 RunEvent::Exited { run_id, .. } => run_id.clone(),
812 };
813 state.pending_exits.insert(run_id, event);
814 settle_pending_exits(state, log);
815}
816
817fn settle_pending_exits(state: &mut DispatcherState, log: &OperationalLog) {
818 let run_ids: Vec<String> = state.pending_exits.keys().cloned().collect();
819 for run_id in run_ids {
820 let Some(event) = state.pending_exits.remove(&run_id) else {
821 continue;
822 };
823 match try_settle_run_exit(state, &event) {
824 Ok(outcome) => {
825 state.cancelling.remove(&run_id);
826 state.active.remove(&run_id);
827 close_worker_socket(state, &run_id);
828 log.emit_with_fields(
829 LogLevel::Info,
830 "sloop::dispatcher",
831 "run_exited",
832 json!({"run_id": run_id, "outcome": outcome.as_str()}),
833 );
834 }
835 Err(error) => {
836 log.emit_with_fields(
837 LogLevel::Error,
838 "sloop::dispatcher",
839 "run_exit_persist_failed",
840 json!({"run_id": run_id, "error": error.to_string()}),
841 );
842 state.pending_exits.insert(run_id, event);
843 }
844 }
845 }
846}
847
848fn try_settle_run_exit(
849 state: &mut DispatcherState,
850 event: &RunEvent,
851) -> Result<crate::outcome::Outcome, StoreError> {
852 let RunEvent::Exited {
853 run_id,
854 exit_code,
855 capture_complete,
856 commits,
857 tests,
858 merge,
859 recovery,
860 } = event;
861
862 let cancelled =
863 state.cancelling.contains(run_id) || state.store.cancellation_requested(run_id)?;
864 let commit_count = commits.len() as u64;
865 let evidence = RunEvidence {
866 cancelled,
867 exit: classify_exit(*exit_code),
868 commit_count,
869 tests: tests.as_ref().map(|stage| stage.outcome),
870 merge: *merge,
871 };
872 let outcome = if *recovery == Some(RecoveryClassification::Orphaned) && !cancelled {
873 crate::outcome::Outcome::Orphaned
874 } else {
875 derive_outcome(&evidence)
876 };
877
878 let mut records = vec![
879 EvidenceRecord {
880 kind: "exit_classified",
881 data_json: json!({"exit_code": exit_code}).to_string(),
882 },
883 EvidenceRecord {
884 kind: "commits_observed",
885 data_json: json!({"count": commit_count, "oids": commits}).to_string(),
886 },
887 ];
888 if let Some(classification) = recovery {
889 records.push(EvidenceRecord {
890 kind: "recovery_classified",
891 data_json: json!({
892 "classification": match classification {
893 RecoveryClassification::Aftercare => "aftercare",
894 RecoveryClassification::Orphaned => "orphaned",
895 }
896 })
897 .to_string(),
898 });
899 }
900 if let Some(stage) = &tests {
901 records.push(EvidenceRecord {
902 kind: "test_result",
903 data_json: json!({
904 "passed": stage.outcome == StageOutcome::Passed,
905 "exit_code": stage.exit_code,
906 })
907 .to_string(),
908 });
909 }
910 if let Some(merge) = *merge {
911 records.push(EvidenceRecord {
912 kind: "merge_result",
913 data_json: json!({"merged": merge == MergeOutcome::Merged}).to_string(),
914 });
915 }
916 if !capture_complete {
917 records.push(EvidenceRecord {
918 kind: "capture_incomplete",
919 data_json: json!({}).to_string(),
920 });
921 }
922 let stage_row = tests.as_ref().map(|stage| StageRecord {
923 stage: "test",
924 state: match stage.outcome {
925 StageOutcome::Passed => "passed",
926 StageOutcome::Failed => "failed",
927 },
928 started_at_ms: stage.started_at_ms,
929 finished_at_ms: stage.finished_at_ms,
930 exit_code: stage.exit_code,
931 });
932
933 let ticket_id = state
934 .store
935 .run(run_id)?
936 .ok_or_else(|| StoreError::RunNotFound {
937 run_id: run_id.clone(),
938 })?
939 .ticket_id;
940 state.store.finish_run(
941 run_id,
942 &ticket_id,
943 *exit_code,
944 outcome,
945 &records,
946 stage_row.as_ref(),
947 state.clock.now_ms(),
948 )?;
949 Ok(outcome)
950}
951
952fn close_worker_socket(state: &mut DispatcherState, run_id: &str) {
956 state.worker_tokens.remove(run_id);
957 if let Some(listener) = state.worker_listeners.remove(run_id) {
958 listener.abort();
959 }
960 let socket_path = state
961 .worker_socket_paths
962 .remove(run_id)
963 .unwrap_or_else(|| worker_socket_path(&state.runtime_dir, run_id));
964 let _ = fs::remove_file(socket_path);
965}
966
967fn recover_inflight_runs(
971 state: &mut DispatcherState,
972 events: &mpsc::Sender<RunEvent>,
973 log: &OperationalLog,
974) -> Result<(), DaemonError> {
975 let runs = state.store.recoverable_runs().map_err(DaemonError::Store)?;
976 for run in runs {
977 state.active.insert(run.id.clone());
980 if recoverable_process_matches(&run) {
981 let cancellation_requested = state
982 .store
983 .cancellation_requested(&run.id)
984 .map_err(DaemonError::Store)?;
985 if cancellation_requested {
986 state.cancelling.insert(run.id.clone());
987 if recoverable_process_matches(&run)
988 && let Some(group) = run.process_group_id
989 {
990 unsafe {
991 libc::kill(-(group as libc::pid_t), libc::SIGKILL);
992 }
993 }
994 }
995 if let Err(error) = restore_worker_socket(state, &run) {
996 log.emit_with_fields(
997 LogLevel::Error,
998 "sloop::recovery",
999 "worker_socket_restore_failed",
1000 json!({"run_id": run.id, "error": error}),
1001 );
1002 }
1003 monitor_recovered_run(state, events.clone(), run.clone());
1004 log.emit_with_fields(
1005 LogLevel::Info,
1006 "sloop::recovery",
1007 "run_readopted",
1008 json!({"run_id": run.id, "ticket_id": run.ticket_id}),
1009 );
1010 } else {
1011 if run.state == "aftercare" {
1012 spawn_aftercare_recovery(state, events.clone(), run, log.clone());
1013 } else {
1014 spawn_dead_run_recovery(state, events.clone(), run, log.clone());
1015 }
1016 }
1017 }
1018 Ok(())
1019}
1020
1021fn restore_worker_socket(state: &mut DispatcherState, run: &RecoverableRun) -> Result<(), String> {
1022 let token = run
1023 .worker_token
1024 .as_ref()
1025 .ok_or_else(|| "the persisted run has no worker token".to_owned())?;
1026 let socket_path = run
1027 .worker_socket_path
1028 .as_deref()
1029 .map(PathBuf::from)
1030 .unwrap_or_else(|| worker_socket_path(&state.runtime_dir, &run.id));
1031 fs::create_dir_all(socket_path.parent().expect("worker sockets have a parent"))
1032 .map_err(|error| error.to_string())?;
1033 let _ = fs::remove_file(&socket_path);
1034 let listener = UnixListener::bind(&socket_path).map_err(|error| error.to_string())?;
1035 fs::set_permissions(&socket_path, fs::Permissions::from_mode(0o600))
1036 .map_err(|error| error.to_string())?;
1037 state.worker_tokens.insert(run.id.clone(), token.clone());
1038 state
1039 .worker_socket_paths
1040 .insert(run.id.clone(), socket_path.clone());
1041 let accept_loop = tokio::spawn(serve_worker_socket(
1042 listener,
1043 run.id.clone(),
1044 state.requests_tx.clone(),
1045 state.log.clone(),
1046 ));
1047 state.worker_listeners.insert(run.id.clone(), accept_loop);
1048 Ok(())
1049}
1050
1051fn monitor_recovered_run(
1052 state: &DispatcherState,
1053 events: mpsc::Sender<RunEvent>,
1054 run: RecoverableRun,
1055) {
1056 let root = state.root.clone();
1057 let log = state.log.clone();
1058 let shutdown = state.shutdown_flag.clone();
1059 tokio::task::spawn_blocking(move || {
1060 while recoverable_process_matches(&run) {
1061 if shutdown.load(Ordering::Acquire) {
1062 return;
1063 }
1064 std::thread::sleep(Duration::from_millis(100));
1065 }
1066 while !shutdown.load(Ordering::Acquire) {
1067 match recovered_exit_event(&root, &run) {
1068 Ok(event) => {
1069 let _ = events.blocking_send(event);
1070 break;
1071 }
1072 Err(error) => {
1073 log.emit_with_fields(
1074 LogLevel::Error,
1075 "sloop::recovery",
1076 "run_observation_failed",
1077 json!({"run_id": run.id, "error": error}),
1078 );
1079 std::thread::sleep(Duration::from_secs(1));
1080 }
1081 }
1082 }
1083 });
1084}
1085
1086fn spawn_dead_run_recovery(
1087 state: &DispatcherState,
1088 events: mpsc::Sender<RunEvent>,
1089 run: RecoverableRun,
1090 log: OperationalLog,
1091) {
1092 let root = state.root.clone();
1093 let shutdown = state.shutdown_flag.clone();
1094 tokio::task::spawn_blocking(move || {
1095 while !shutdown.load(Ordering::Acquire) {
1096 match recovered_exit_event(&root, &run) {
1097 Ok(event) => {
1098 let _ = events.blocking_send(event);
1099 break;
1100 }
1101 Err(error) => {
1102 log.emit_with_fields(
1103 LogLevel::Error,
1104 "sloop::recovery",
1105 "run_observation_failed",
1106 json!({"run_id": run.id, "error": error}),
1107 );
1108 std::thread::sleep(Duration::from_secs(1));
1109 }
1110 }
1111 }
1112 });
1113}
1114
1115fn recovered_exit_event(root: &Path, run: &RecoverableRun) -> Result<RunEvent, String> {
1116 let commits = run
1117 .branch
1118 .as_deref()
1119 .map(|branch| try_commits_on_branch(root, branch))
1120 .transpose()?
1121 .unwrap_or_default();
1122 let recovery = if commits.is_empty() {
1123 RecoveryClassification::Orphaned
1124 } else {
1125 RecoveryClassification::Aftercare
1126 };
1127 Ok(RunEvent::Exited {
1128 run_id: run.id.clone(),
1129 exit_code: None,
1130 capture_complete: false,
1131 commits,
1132 tests: None,
1133 merge: None,
1134 recovery: Some(recovery),
1135 })
1136}
1137
1138fn spawn_aftercare_recovery(
1139 state: &DispatcherState,
1140 events: mpsc::Sender<RunEvent>,
1141 run: RecoverableRun,
1142 log: OperationalLog,
1143) {
1144 let root = state.root.clone();
1145 let state_dir = state.state_dir.clone();
1146 let test_cmd = state.aftercare_test_cmd.clone();
1147 let clock = state.clock.clone();
1148 let db_path = state.state_dir.join("sloop.db");
1149 let shutdown = state.shutdown_flag.clone();
1150 tokio::task::spawn_blocking(move || {
1151 while !shutdown.load(Ordering::Acquire) {
1152 let result = Store::open(&db_path, clock.now_ms())
1153 .map_err(|error| error.to_string())
1154 .and_then(|store| {
1155 resume_aftercare(
1156 &root,
1157 &state_dir,
1158 test_cmd.as_deref(),
1159 clock.as_ref(),
1160 &store,
1161 &run,
1162 &log,
1163 )
1164 });
1165 match result {
1166 Ok(event) => {
1167 let _ = events.blocking_send(event);
1168 break;
1169 }
1170 Err(error) => {
1171 log.emit_with_fields(
1172 LogLevel::Error,
1173 "sloop::recovery",
1174 "aftercare_resume_failed",
1175 json!({"run_id": run.id, "error": error}),
1176 );
1177 std::thread::sleep(Duration::from_secs(1));
1178 }
1179 }
1180 }
1181 });
1182}
1183
1184#[allow(clippy::too_many_arguments)]
1185fn resume_aftercare(
1186 root: &Path,
1187 state_dir: &Path,
1188 test_cmd: Option<&[String]>,
1189 clock: &dyn Clock,
1190 store: &Store,
1191 run: &RecoverableRun,
1192 log: &OperationalLog,
1193) -> Result<RunEvent, String> {
1194 let rows = store
1195 .run_evidence(&run.id)
1196 .map_err(|error| error.to_string())?;
1197 let value = |kind: &str| {
1198 rows.iter()
1199 .find(|(candidate, _)| candidate == kind)
1200 .and_then(|(_, data)| serde_json::from_str::<serde_json::Value>(data).ok())
1201 };
1202 let commits = value("commits_observed")
1203 .and_then(|data| {
1204 data["oids"].as_array().map(|oids| {
1205 oids.iter()
1206 .filter_map(|oid| oid.as_str().map(str::to_owned))
1207 .collect::<Vec<_>>()
1208 })
1209 })
1210 .ok_or_else(|| "the aftercare checkpoint has no valid commit evidence".to_owned())?;
1211 let exit_code = run.exit_code.and_then(|code| i32::try_from(code).ok());
1212 let exit = classify_exit(exit_code);
1213 let output_log = RunLogWriter::open(&run_output_path(state_dir, &run.id))
1214 .map_err(|error| format!("cannot open run output: {error}"))?;
1215
1216 let mut tests = value("test_result").map(|data| StageResult {
1217 outcome: if data["passed"].as_bool() == Some(true) {
1218 StageOutcome::Passed
1219 } else {
1220 StageOutcome::Failed
1221 },
1222 exit_code: data["exit_code"]
1223 .as_i64()
1224 .and_then(|code| i32::try_from(code).ok()),
1225 started_at_ms: data["started_at_ms"]
1226 .as_i64()
1227 .unwrap_or_else(|| clock.now_ms()),
1228 finished_at_ms: data["finished_at_ms"]
1229 .as_i64()
1230 .unwrap_or_else(|| clock.now_ms()),
1231 });
1232 if aftercare_cancelled(store, &run.id, log) {
1233 return Ok(RunEvent::Exited {
1234 run_id: run.id.clone(),
1235 exit_code,
1236 capture_complete: !rows.iter().any(|(kind, _)| kind == "capture_incomplete"),
1237 commits,
1238 tests,
1239 merge: None,
1240 recovery: Some(RecoveryClassification::Aftercare),
1241 });
1242 }
1243 if tests.is_none()
1244 && wants_tests(exit, commits.len() as u64)
1245 && let Some(cmd) = test_cmd
1246 {
1247 stop_interrupted_process(&rows, "test_process")?;
1248 let worktree = run
1249 .worktree_path
1250 .as_deref()
1251 .ok_or_else(|| "the aftercare checkpoint has no worktree".to_owned())?;
1252 let stage = run_test_stage(
1253 Path::new(worktree),
1254 cmd,
1255 &output_log,
1256 clock,
1257 Some(store),
1258 &run.id,
1259 log,
1260 );
1261 store
1262 .record_aftercare_evidence(
1263 &run.id,
1264 "test_result",
1265 &test_result_json(&stage),
1266 clock.now_ms(),
1267 )
1268 .map_err(|error| error.to_string())?;
1269 tests = Some(stage);
1270 }
1271
1272 let mut merge = value("merge_result").map(|data| {
1273 if data["merged"].as_bool() == Some(true) {
1274 MergeOutcome::Merged
1275 } else {
1276 MergeOutcome::Diverged
1277 }
1278 });
1279 if !aftercare_cancelled(store, &run.id, log)
1280 && merge.is_none()
1281 && wants_merge(
1282 exit,
1283 commits.len() as u64,
1284 tests.as_ref().map(|stage| stage.outcome),
1285 )
1286 && let Some(branch) = run.branch.as_deref()
1287 {
1288 stop_interrupted_process(&rows, "merge_process")?;
1289 let outcome = attempt_merge(root, branch, Some(store), &run.id, clock, log);
1290 store
1291 .record_aftercare_evidence(
1292 &run.id,
1293 "merge_result",
1294 &merge_result_json(outcome),
1295 clock.now_ms(),
1296 )
1297 .map_err(|error| error.to_string())?;
1298 merge = Some(outcome);
1299 }
1300
1301 Ok(RunEvent::Exited {
1302 run_id: run.id.clone(),
1303 exit_code,
1304 capture_complete: !rows.iter().any(|(kind, _)| kind == "capture_incomplete"),
1305 commits,
1306 tests,
1307 merge,
1308 recovery: Some(RecoveryClassification::Aftercare),
1309 })
1310}
1311
1312fn stop_interrupted_process(rows: &[(String, String)], kind: &str) -> Result<(), String> {
1313 let Some((pid, start_time, group)) = aftercare_process_identity(rows, kind)? else {
1314 return Ok(());
1315 };
1316 if !process_identity_matches(pid, Some(start_time)) {
1317 return Ok(());
1318 }
1319 unsafe {
1320 libc::kill(-(group as libc::pid_t), libc::SIGKILL);
1321 }
1322 let deadline = Instant::now() + Duration::from_secs(5);
1323 while process_identity_matches(pid, Some(start_time)) {
1324 if Instant::now() >= deadline {
1325 return Err("the interrupted test process did not exit".into());
1326 }
1327 std::thread::sleep(Duration::from_millis(20));
1328 }
1329 Ok(())
1330}
1331
1332fn aftercare_process_identity(
1333 rows: &[(String, String)],
1334 kind: &str,
1335) -> Result<Option<(u32, i64, i64)>, String> {
1336 let Some(data) = rows
1337 .iter()
1338 .find(|(candidate, _)| candidate == kind)
1339 .and_then(|(_, data)| serde_json::from_str::<serde_json::Value>(data).ok())
1340 else {
1341 return Ok(None);
1342 };
1343 let pid = data["pid"]
1344 .as_u64()
1345 .and_then(|pid| u32::try_from(pid).ok())
1346 .ok_or_else(|| "the interrupted test has no valid pid".to_owned())?;
1347 let start_time = data["pid_start_time"]
1348 .as_i64()
1349 .ok_or_else(|| "the interrupted test has no valid start time".to_owned())?;
1350 let group = data["process_group_id"]
1351 .as_i64()
1352 .ok_or_else(|| "the interrupted test has no valid process group".to_owned())?;
1353 Ok(Some((pid, start_time, group)))
1354}
1355
1356fn recoverable_process_matches(run: &RecoverableRun) -> bool {
1357 if run.state != "running" {
1358 return false;
1359 }
1360 let Some(pid) = run.pid.and_then(|pid| u32::try_from(pid).ok()) else {
1361 return false;
1362 };
1363 process_identity_matches(pid, run.pid_start_time)
1364}
1365
1366fn process_identity_matches(pid: u32, expected_start_time: Option<i64>) -> bool {
1367 matches!(
1368 (expected_start_time, process_start_time(pid)),
1369 (Some(expected), Some(actual)) if expected == actual
1370 )
1371}
1372
1373fn worker_socket_path(runtime_dir: &Path, run_id: &str) -> PathBuf {
1374 runtime_dir.join("workers").join(format!("{run_id}.sock"))
1375}
1376
1377fn reconcile(state: &mut DispatcherState, events: &mpsc::Sender<RunEvent>, log: &OperationalLog) {
1381 settle_pending_exits(state, log);
1382 let now_ms = state.clock.now_ms();
1383 if state.paused || state.agent.is_none() || !running_hours_open(state, now_ms) {
1384 return;
1385 }
1386 let activations = match state.store.queued_dispatchable_activations() {
1387 Ok(activations) => activations,
1388 Err(error) => {
1389 log.emit_with_fields(
1390 LogLevel::Error,
1391 "sloop::dispatcher",
1392 "activation_scan_failed",
1393 json!({"error": error.to_string()}),
1394 );
1395 return;
1396 }
1397 };
1398
1399 for activation in activations {
1400 if state.active.len() >= state.max_agents {
1401 break;
1402 }
1403 let Some(ticket_id) = eligible_ticket(&state.store, &activation) else {
1404 continue;
1405 };
1406
1407 let now_ms = state.clock.now_ms();
1408 let Ok(run_ordinal) = state.store.next_run_ordinal() else {
1409 continue;
1410 };
1411 let run_id = format!("R{run_ordinal}");
1412 let owner = format!("daemon-{}", state.pid);
1413 let claim = ClaimRequest {
1414 ticket_id: &ticket_id,
1415 run_id: &run_id,
1416 activation_id: &activation.id,
1417 owner_id: &owner,
1418 lease_ms: DEFAULT_LEASE_MS,
1419 };
1420 let claimed = match state.store.claim_ticket(&claim, now_ms) {
1421 Ok(claimed) => claimed,
1422 Err(StoreError::TicketNotReady { .. }) => continue,
1424 Err(error) => {
1425 log.emit_with_fields(
1426 LogLevel::Error,
1427 "sloop::dispatcher",
1428 "claim_failed",
1429 json!({
1430 "activation_id": activation.id,
1431 "ticket_id": ticket_id,
1432 "run_id": run_id,
1433 "error": error.to_string(),
1434 }),
1435 );
1436 continue;
1437 }
1438 };
1439 if let Err(error) = state.store.complete_activation(&activation.id, now_ms) {
1442 log.emit_with_fields(
1443 LogLevel::Error,
1444 "sloop::dispatcher",
1445 "activation_update_failed",
1446 json!({"activation_id": activation.id, "error": error.to_string()}),
1447 );
1448 }
1449
1450 match launch_agent(state, &run_id, &ticket_id, claimed.attempt) {
1451 Ok(launched) => {
1452 state.active.insert(run_id.clone());
1453 let events = events.clone();
1454 let exited_run = run_id.clone();
1455 let root = state.root.clone();
1456 let test_cmd = state.aftercare_test_cmd.clone();
1457 let clock = state.clock.clone();
1458 let supervisor_log = log.clone();
1459 let db_path = state.state_dir.join("sloop.db");
1460 let LaunchedRun {
1461 mut child,
1462 readers,
1463 worktree,
1464 branch,
1465 output_log,
1466 worker_listener,
1467 worker_token,
1468 worker_socket_path,
1469 } = launched;
1470 state.worker_tokens.insert(run_id.clone(), worker_token);
1471 state
1472 .worker_socket_paths
1473 .insert(run_id.clone(), worker_socket_path);
1474 let accept_loop = tokio::spawn(serve_worker_socket(
1475 worker_listener,
1476 run_id.clone(),
1477 state.requests_tx.clone(),
1478 state.log.clone(),
1479 ));
1480 state.worker_listeners.insert(run_id.clone(), accept_loop);
1481 let pid = child.id();
1482 tokio::task::spawn_blocking(move || {
1483 let exit_code = match child.wait() {
1484 Ok(status) => status.code(),
1485 Err(error) => {
1486 supervisor_log.emit_with_fields(
1487 LogLevel::Error,
1488 "sloop::supervisor",
1489 "agent_wait_failed",
1490 json!({"run_id": exited_run, "error": error.to_string()}),
1491 );
1492 None
1493 }
1494 };
1495 let capture_complete = readers
1498 .into_iter()
1499 .all(|reader| reader.join().unwrap_or(false));
1500 let mut checkpoint_store = match Store::open(&db_path, clock.now_ms()) {
1501 Ok(store) => Some(store),
1502 Err(error) => {
1503 supervisor_log.emit_with_fields(
1504 LogLevel::Error,
1505 "sloop::supervisor",
1506 "aftercare_checkpoint_open_failed",
1507 json!({"run_id": exited_run, "error": error.to_string()}),
1508 );
1509 None
1510 }
1511 };
1512 let (commits, tests, merge) = gather_exit_evidence(
1513 &exited_run,
1514 &root,
1515 &worktree,
1516 &branch,
1517 test_cmd.as_deref(),
1518 clock.as_ref(),
1519 &output_log,
1520 exit_code,
1521 capture_complete,
1522 checkpoint_store.as_mut(),
1523 &supervisor_log,
1524 );
1525 let _ = events.blocking_send(RunEvent::Exited {
1526 run_id: exited_run,
1527 exit_code,
1528 capture_complete,
1529 commits,
1530 tests,
1531 merge,
1532 recovery: None,
1533 });
1534 });
1535 log.emit_with_fields(
1536 LogLevel::Info,
1537 "sloop::dispatcher",
1538 "run_started",
1539 json!({"run_id": run_id, "ticket_id": ticket_id, "pid": pid}),
1540 );
1541 }
1542 Err(error) => {
1543 if let Err(abort_error) =
1544 state
1545 .store
1546 .abort_claim(&run_id, &ticket_id, state.clock.now_ms())
1547 {
1548 log.emit_with_fields(
1549 LogLevel::Error,
1550 "sloop::dispatcher",
1551 "claim_abort_failed",
1552 json!({
1553 "run_id": run_id,
1554 "ticket_id": ticket_id,
1555 "error": abort_error.to_string(),
1556 }),
1557 );
1558 }
1559 close_worker_socket(state, &run_id);
1561 log.emit_with_fields(
1562 LogLevel::Error,
1563 "sloop::dispatcher",
1564 "run_launch_failed",
1565 json!({
1566 "run_id": run_id,
1567 "ticket_id": ticket_id,
1568 "error": error,
1569 }),
1570 );
1571 }
1572 }
1573 }
1574}
1575
1576fn running_hours_open(state: &DispatcherState, now_ms: i64) -> bool {
1577 state
1578 .running_hours
1579 .as_ref()
1580 .is_none_or(|hours| hours.is_open(state.clock.local_minute(now_ms)))
1581}
1582
1583fn next_dispatch_deadline(state: &DispatcherState) -> Option<i64> {
1584 let hours = state.running_hours.as_ref()?;
1585 let now_ms = state.clock.now_ms();
1586 if hours.is_open(state.clock.local_minute(now_ms)) {
1587 return None;
1588 }
1589 let has_demand = state
1590 .store
1591 .queued_dispatchable_activations()
1592 .is_ok_and(|activations| !activations.is_empty());
1593 has_demand.then(|| hours.next_opening_ms(state.clock.as_ref(), now_ms))
1594}
1595
1596fn eligible_ticket(store: &Store, activation: &QueuedActivation) -> Option<String> {
1597 match &activation.ticket_id {
1598 Some(ticket) => Some(ticket.clone()),
1599 None => store.select_ready_ticket(activation).ok().flatten(),
1600 }
1601}
1602
1603struct LaunchedRun {
1609 child: Child,
1610 readers: Vec<std::thread::JoinHandle<bool>>,
1611 worktree: PathBuf,
1612 branch: String,
1613 output_log: RunLogWriter,
1614 worker_listener: UnixListener,
1617 worker_token: String,
1618 worker_socket_path: PathBuf,
1619}
1620
1621#[allow(clippy::too_many_arguments)]
1626fn gather_exit_evidence(
1627 run_id: &str,
1628 root: &Path,
1629 worktree: &Path,
1630 branch: &str,
1631 test_cmd: Option<&[String]>,
1632 clock: &dyn Clock,
1633 output_log: &RunLogWriter,
1634 exit_code: Option<i32>,
1635 capture_complete: bool,
1636 mut checkpoint_store: Option<&mut Store>,
1637 operational_log: &OperationalLog,
1638) -> (Vec<String>, Option<StageResult>, Option<MergeOutcome>) {
1639 let exit = classify_exit(exit_code);
1640 let commits = commits_on_branch(root, branch);
1641 let commit_count = commits.len() as u64;
1642 let checkpointed = if let Some(store) = checkpoint_store.as_deref_mut() {
1643 match store.record_agent_exit(
1644 run_id,
1645 exit_code,
1646 capture_complete,
1647 &json!({"count": commit_count, "oids": commits}).to_string(),
1648 clock.now_ms(),
1649 ) {
1650 Ok(()) => true,
1651 Err(error) => {
1652 operational_log.emit_with_fields(
1653 LogLevel::Error,
1654 "sloop::supervisor",
1655 "agent_exit_checkpoint_failed",
1656 json!({"run_id": run_id, "error": error.to_string()}),
1657 );
1658 false
1659 }
1660 }
1661 } else {
1662 false
1663 };
1664 if !checkpointed
1668 || checkpoint_store
1669 .as_deref()
1670 .is_some_and(|store| aftercare_cancelled(store, run_id, operational_log))
1671 {
1672 return (commits, None, None);
1673 }
1674
1675 let tests = match test_cmd {
1676 Some(cmd) if wants_tests(exit, commit_count) => {
1677 let stage = run_test_stage(
1678 worktree,
1679 cmd,
1680 output_log,
1681 clock,
1682 checkpoint_store.as_deref(),
1683 run_id,
1684 operational_log,
1685 );
1686 let test_checkpointed = if let Some(store) = checkpoint_store.as_deref()
1687 && let Err(error) = store.record_aftercare_evidence(
1688 run_id,
1689 "test_result",
1690 &test_result_json(&stage),
1691 clock.now_ms(),
1692 ) {
1693 operational_log.emit_with_fields(
1694 LogLevel::Error,
1695 "sloop::supervisor",
1696 "aftercare_checkpoint_failed",
1697 json!({"run_id": run_id, "stage": "test", "error": error.to_string()}),
1698 );
1699 false
1700 } else {
1701 true
1702 };
1703 if !test_checkpointed {
1704 return (commits, Some(stage), None);
1705 }
1706 Some(stage)
1707 }
1708 _ => None,
1709 };
1710
1711 let merge = if !checkpoint_store
1712 .as_deref()
1713 .is_some_and(|store| aftercare_cancelled(store, run_id, operational_log))
1714 && wants_merge(
1715 exit,
1716 commit_count,
1717 tests.as_ref().map(|stage| stage.outcome),
1718 ) {
1719 let outcome = attempt_merge(
1720 root,
1721 branch,
1722 checkpoint_store.as_deref(),
1723 run_id,
1724 clock,
1725 operational_log,
1726 );
1727 if let Some(store) = checkpoint_store.as_deref()
1728 && let Err(error) = store.record_aftercare_evidence(
1729 run_id,
1730 "merge_result",
1731 &merge_result_json(outcome),
1732 clock.now_ms(),
1733 )
1734 {
1735 operational_log.emit_with_fields(
1736 LogLevel::Error,
1737 "sloop::supervisor",
1738 "aftercare_checkpoint_failed",
1739 json!({"run_id": run_id, "stage": "merge", "error": error.to_string()}),
1740 );
1741 }
1742 Some(outcome)
1743 } else {
1744 None
1745 };
1746 (commits, tests, merge)
1747}
1748
1749fn aftercare_cancelled(store: &Store, run_id: &str, log: &OperationalLog) -> bool {
1750 match store.cancellation_requested(run_id) {
1751 Ok(cancelled) => cancelled,
1752 Err(error) => {
1753 log.emit_with_fields(
1754 LogLevel::Error,
1755 "sloop::supervisor",
1756 "cancellation_read_failed",
1757 json!({"run_id": run_id, "error": error.to_string()}),
1758 );
1759 true
1760 }
1761 }
1762}
1763
1764fn test_result_json(stage: &StageResult) -> String {
1765 json!({
1766 "passed": stage.outcome == StageOutcome::Passed,
1767 "exit_code": stage.exit_code,
1768 "started_at_ms": stage.started_at_ms,
1769 "finished_at_ms": stage.finished_at_ms,
1770 })
1771 .to_string()
1772}
1773
1774fn merge_result_json(outcome: MergeOutcome) -> String {
1775 json!({"merged": outcome == MergeOutcome::Merged}).to_string()
1776}
1777
1778fn commits_on_branch(root: &Path, branch: &str) -> Vec<String> {
1781 try_commits_on_branch(root, branch).unwrap_or_default()
1782}
1783
1784fn try_commits_on_branch(root: &Path, branch: &str) -> Result<Vec<String>, String> {
1785 let output = Command::new("git")
1786 .args(["rev-list", "--reverse", &format!("HEAD..{branch}")])
1787 .current_dir(root)
1788 .output()
1789 .map_err(|error| error.to_string())?;
1790 match output {
1791 output if output.status.success() => Ok(String::from_utf8_lossy(&output.stdout)
1792 .lines()
1793 .map(str::to_owned)
1794 .collect()),
1795 output => Err(format!(
1796 "git rev-list failed: {}",
1797 String::from_utf8_lossy(&output.stderr).trim()
1798 )),
1799 }
1800}
1801
1802fn run_test_stage(
1805 worktree: &Path,
1806 cmd: &[String],
1807 output_log: &RunLogWriter,
1808 clock: &dyn Clock,
1809 checkpoint_store: Option<&Store>,
1810 run_id: &str,
1811 operational_log: &OperationalLog,
1812) -> StageResult {
1813 let started_at_ms = clock.now_ms();
1814 let failed = |finished_at_ms| StageResult {
1815 outcome: StageOutcome::Failed,
1816 exit_code: None,
1817 started_at_ms,
1818 finished_at_ms,
1819 };
1820
1821 let mut command = Command::new(&cmd[0]);
1822 command
1823 .args(&cmd[1..])
1824 .current_dir(worktree)
1825 .stdin(Stdio::null())
1826 .stdout(Stdio::piped())
1827 .stderr(Stdio::piped())
1828 .process_group(0);
1829 let Ok(mut child) = command.spawn() else {
1830 return failed(clock.now_ms());
1831 };
1832 let pid = child.id();
1833 let Some(pid_start_time) = process_start_time(pid) else {
1834 unsafe {
1835 libc::kill(-(pid as libc::pid_t), libc::SIGKILL);
1836 }
1837 let _ = child.wait();
1838 return failed(clock.now_ms());
1839 };
1840 let readers = vec![
1841 spawn_output_reader(
1842 child.stdout.take().expect("stdout was piped"),
1843 output_log.clone(),
1844 OutputSource::Aftercare,
1845 Some("test"),
1846 OutputStream::Stdout,
1847 ),
1848 spawn_output_reader(
1849 child.stderr.take().expect("stderr was piped"),
1850 output_log.clone(),
1851 OutputSource::Aftercare,
1852 Some("test"),
1853 OutputStream::Stderr,
1854 ),
1855 ];
1856 wait_for_test_hook("before-test-process-checkpoint");
1857 if let Some(store) = checkpoint_store
1858 && let Err(error) = store.record_aftercare_evidence(
1859 run_id,
1860 "test_process",
1861 &json!({
1862 "pid": pid,
1863 "pid_start_time": pid_start_time,
1864 "process_group_id": pid,
1865 })
1866 .to_string(),
1867 clock.now_ms(),
1868 )
1869 {
1870 operational_log.emit_with_fields(
1871 LogLevel::Error,
1872 "sloop::supervisor",
1873 "aftercare_process_checkpoint_failed",
1874 json!({"run_id": run_id, "stage": "test", "error": error.to_string()}),
1875 );
1876 if process_identity_matches(pid, Some(pid_start_time)) {
1877 unsafe {
1878 libc::kill(-(pid as libc::pid_t), libc::SIGKILL);
1879 }
1880 }
1881 let _ = child.wait();
1882 for reader in readers {
1883 let _ = reader.join();
1884 }
1885 return failed(clock.now_ms());
1886 }
1887 if checkpoint_store.is_some_and(|store| aftercare_cancelled(store, run_id, operational_log)) {
1888 if process_identity_matches(pid, Some(pid_start_time)) {
1889 unsafe {
1890 libc::kill(-(pid as libc::pid_t), libc::SIGKILL);
1891 }
1892 }
1893 let _ = child.wait();
1894 for reader in readers {
1895 let _ = reader.join();
1896 }
1897 return failed(clock.now_ms());
1898 }
1899
1900 let status = child.wait();
1901 for reader in readers {
1902 let _ = reader.join();
1903 }
1904 let Ok(status) = status else {
1905 return failed(clock.now_ms());
1906 };
1907 StageResult {
1908 outcome: if status.success() {
1909 StageOutcome::Passed
1910 } else {
1911 StageOutcome::Failed
1912 },
1913 exit_code: status.code(),
1914 started_at_ms,
1915 finished_at_ms: clock.now_ms(),
1916 }
1917}
1918
1919#[allow(clippy::too_many_arguments)]
1924fn attempt_merge(
1925 root: &Path,
1926 branch: &str,
1927 checkpoint_store: Option<&Store>,
1928 run_id: &str,
1929 clock: &dyn Clock,
1930 operational_log: &OperationalLog,
1931) -> MergeOutcome {
1932 let Ok(_guard) = MERGE_LOCK.lock() else {
1933 return MergeOutcome::Diverged;
1934 };
1935 let message = format!("Merge run branch '{branch}'");
1936 let mut command = Command::new("git");
1940 command
1941 .args([
1942 "-c",
1943 "user.name=sloop",
1944 "-c",
1945 "user.email=sloop@sloop.invalid",
1946 "merge",
1947 "--quiet",
1948 "-m",
1949 &message,
1950 branch,
1951 ])
1952 .current_dir(root)
1953 .stdin(Stdio::null())
1954 .stdout(Stdio::null())
1955 .stderr(Stdio::null())
1956 .process_group(0);
1957 let Ok(mut child) = command.spawn() else {
1958 return MergeOutcome::Diverged;
1959 };
1960 let pid = child.id();
1961 let Some(pid_start_time) = process_start_time(pid) else {
1962 unsafe {
1963 libc::kill(-(pid as libc::pid_t), libc::SIGKILL);
1964 }
1965 let _ = child.wait();
1966 return MergeOutcome::Diverged;
1967 };
1968 if let Some(store) = checkpoint_store {
1969 let checkpoint = json!({
1970 "pid": pid,
1971 "pid_start_time": pid_start_time,
1972 "process_group_id": pid,
1973 })
1974 .to_string();
1975 if let Err(error) =
1976 store.record_aftercare_evidence(run_id, "merge_process", &checkpoint, clock.now_ms())
1977 {
1978 operational_log.emit_with_fields(
1979 LogLevel::Error,
1980 "sloop::supervisor",
1981 "aftercare_process_checkpoint_failed",
1982 json!({"run_id": run_id, "stage": "merge", "error": error.to_string()}),
1983 );
1984 unsafe {
1985 libc::kill(-(pid as libc::pid_t), libc::SIGKILL);
1986 }
1987 let _ = child.wait();
1988 return MergeOutcome::Diverged;
1989 }
1990 if aftercare_cancelled(store, run_id, operational_log) {
1991 unsafe {
1992 libc::kill(-(pid as libc::pid_t), libc::SIGKILL);
1993 }
1994 let _ = child.wait();
1995 return MergeOutcome::Diverged;
1996 }
1997 }
1998 match child.wait() {
1999 Ok(status) if status.success() => MergeOutcome::Merged,
2000 _ => {
2001 let _ = Command::new("git")
2002 .args(["merge", "--abort"])
2003 .current_dir(root)
2004 .output();
2005 MergeOutcome::Diverged
2006 }
2007 }
2008}
2009
2010fn run_output_path(state_dir: &Path, run_id: &str) -> PathBuf {
2011 state_dir.join("runs").join(run_id).join("output.ndjson")
2012}
2013
2014fn spawn_output_reader(
2018 pipe: impl Read + Send + 'static,
2019 log: RunLogWriter,
2020 source: OutputSource,
2021 stage: Option<&'static str>,
2022 stream: OutputStream,
2023) -> std::thread::JoinHandle<bool> {
2024 std::thread::spawn(move || {
2025 let mut pipe = pipe;
2026 let mut buffer = [0u8; 8192];
2027 loop {
2028 match pipe.read(&mut buffer) {
2029 Ok(0) => return true,
2030 Ok(read) => {
2031 if log.append(source, stage, stream, &buffer[..read]).is_err() {
2032 return false;
2033 }
2034 }
2035 Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
2036 Err(_) => return false,
2037 }
2038 }
2039 })
2040}
2041
2042fn launch_agent(
2045 state: &DispatcherState,
2046 run_id: &str,
2047 ticket_id: &str,
2048 attempt: i64,
2049) -> Result<LaunchedRun, String> {
2050 let agent = state
2051 .agent
2052 .as_ref()
2053 .ok_or_else(|| "no agent targets configured".to_owned())?;
2054 let ticket = state
2055 .store
2056 .ticket(ticket_id)
2057 .map_err(|error| error.to_string())?
2058 .ok_or_else(|| format!("ticket `{ticket_id}` no longer exists"))?;
2059 let target = ticket
2060 .target
2061 .as_deref()
2062 .ok_or_else(|| format!("ticket `{ticket_id}` does not specify an agent target"))?;
2063 let template = agent
2064 .targets
2065 .get(target)
2066 .ok_or_else(|| format!("ticket `{ticket_id}` names unknown agent target `{target}`"))?;
2067 let prompt = compose_worker_prompt(&state.root)?;
2068 let cmd = expand_agent_cmd(
2069 template,
2070 ticket.model.as_deref(),
2071 ticket.effort.as_deref(),
2072 &prompt,
2073 )
2074 .map_err(|error| format!("ticket `{ticket_id}` {error}"))?;
2075 let executable = std::env::current_exe()
2076 .map_err(|error| format!("cannot locate sloop executable: {error}"))?;
2077 let executable_dir = executable
2078 .parent()
2079 .ok_or_else(|| "sloop executable has no parent directory".to_owned())?;
2080 let mut path_entries = vec![executable_dir.to_path_buf()];
2081 if let Some(path) = std::env::var_os("PATH") {
2082 path_entries.extend(std::env::split_paths(&path));
2083 }
2084 let path = std::env::join_paths(path_entries)
2085 .map_err(|error| format!("cannot construct agent PATH: {error}"))?;
2086 let branch = format!("sloop/{ticket_id}-a{attempt}-{run_id}");
2089 fs::create_dir_all(&state.worktree_dir).map_err(|error| error.to_string())?;
2090 let worktree = state.worktree_dir.join(run_id);
2091
2092 let git = Command::new("git")
2093 .args(["worktree", "add", "--quiet", "-b", &branch])
2094 .arg(&worktree)
2095 .current_dir(&state.root)
2096 .output()
2097 .map_err(|error| error.to_string())?;
2098 if !git.status.success() {
2099 return Err(format!(
2100 "git worktree add failed: {}",
2101 String::from_utf8_lossy(&git.stderr).trim()
2102 ));
2103 }
2104
2105 let output_log = RunLogWriter::open(&run_output_path(&state.state_dir, run_id))
2106 .map_err(|error| error.to_string())?;
2107
2108 let worker_token = generate_worker_token()?;
2109 let socket_path = worker_socket_path(&state.runtime_dir, run_id);
2110 fs::create_dir_all(socket_path.parent().expect("worker sockets have a parent"))
2111 .map_err(|error| error.to_string())?;
2112 let _ = fs::remove_file(&socket_path);
2113 let worker_listener = UnixListener::bind(&socket_path).map_err(|error| error.to_string())?;
2114 fs::set_permissions(&socket_path, fs::Permissions::from_mode(0o600))
2115 .map_err(|error| error.to_string())?;
2116
2117 let mut command = Command::new(&cmd[0]);
2118 command
2119 .args(&cmd[1..])
2120 .current_dir(&worktree)
2121 .env("SLOOP_RUN_ID", run_id)
2122 .env("SLOOP_TICKET_ID", ticket_id)
2123 .env("SLOOP_BIN", &executable)
2124 .env("PATH", path)
2125 .env("SLOOP_SOCKET", &socket_path)
2126 .env("SLOOP_TOKEN", &worker_token)
2127 .stdin(Stdio::null())
2128 .stdout(Stdio::piped())
2129 .stderr(Stdio::piped())
2130 .process_group(0);
2131 let mut child = command.spawn().map_err(|error| error.to_string())?;
2132 let readers = vec![
2133 spawn_output_reader(
2134 child.stdout.take().expect("stdout was piped"),
2135 output_log.clone(),
2136 OutputSource::Agent,
2137 None,
2138 OutputStream::Stdout,
2139 ),
2140 spawn_output_reader(
2141 child.stderr.take().expect("stderr was piped"),
2142 output_log.clone(),
2143 OutputSource::Agent,
2144 None,
2145 OutputStream::Stderr,
2146 ),
2147 ];
2148
2149 let pid = child.id();
2150 if let Err(error) = state.store.mark_run_running(
2151 run_id,
2152 &branch,
2153 &worktree.to_string_lossy(),
2154 pid,
2155 process_start_time(pid),
2156 pid, &worker_token,
2158 &socket_path.to_string_lossy(),
2159 state.clock.now_ms(),
2160 ) {
2161 unsafe {
2162 libc::kill(-(pid as libc::pid_t), libc::SIGKILL);
2163 }
2164 let _ = child.wait();
2165 for reader in readers {
2166 let _ = reader.join();
2167 }
2168 return Err(error.to_string());
2169 }
2170 Ok(LaunchedRun {
2171 child,
2172 readers,
2173 worktree,
2174 branch,
2175 output_log,
2176 worker_listener,
2177 worker_token,
2178 worker_socket_path: socket_path,
2179 })
2180}
2181
2182fn compose_worker_prompt(root: &Path) -> Result<String, String> {
2183 let path = root.join(".agents/sloop/instructions.md");
2184 match fs::read_to_string(&path) {
2185 Ok(instructions) => Ok(format!("{WORKER_BOOTSTRAP_PROMPT}\n\n{instructions}")),
2186 Err(error) if error.kind() == io::ErrorKind::NotFound => {
2187 Ok(WORKER_BOOTSTRAP_PROMPT.to_owned())
2188 }
2189 Err(error) => Err(format!("cannot read {}: {error}", path.display())),
2190 }
2191}
2192
2193fn generate_worker_token() -> Result<String, String> {
2197 let mut bytes = [0u8; 32];
2198 let mut urandom = fs::File::open("/dev/urandom").map_err(|error| error.to_string())?;
2199 urandom
2200 .read_exact(&mut bytes)
2201 .map_err(|error| error.to_string())?;
2202 Ok(BASE64_TOKEN.encode(bytes))
2203}
2204
2205#[derive(Debug, Clone, PartialEq, Eq)]
2208pub struct LockIdentity {
2209 pub pid: u32,
2210 pub started_at_ms: Option<i64>,
2211 pub socket: Option<PathBuf>,
2212}
2213
2214pub fn read_lock_identity(path: &Path) -> Option<LockIdentity> {
2215 let content = fs::read_to_string(path).ok()?;
2216 let value: serde_json::Value = serde_json::from_str(content.trim()).ok()?;
2217 Some(LockIdentity {
2218 pid: u32::try_from(value["pid"].as_u64()?).ok()?,
2219 started_at_ms: value["started_at_ms"].as_i64(),
2220 socket: value["socket"].as_str().map(PathBuf::from),
2221 })
2222}
2223
2224fn process_start_time(pid: u32) -> Option<i64> {
2228 #[cfg(target_os = "linux")]
2229 {
2230 let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
2231 let after_command = &stat[stat.rfind(')')? + 1..];
2232 after_command
2233 .split_whitespace()
2234 .nth(19)
2235 .and_then(|field| field.parse().ok())
2236 }
2237
2238 #[cfg(not(target_os = "linux"))]
2239 {
2240 let output = Command::new("ps")
2241 .args(["-o", "lstart=", "-p", &pid.to_string()])
2242 .output()
2243 .ok()?;
2244 if !output.status.success() || output.stdout.iter().all(u8::is_ascii_whitespace) {
2245 return None;
2246 }
2247 let mut hash = 0xcbf29ce484222325_u64;
2248 for byte in output.stdout {
2249 hash ^= u64::from(byte);
2250 hash = hash.wrapping_mul(0x100000001b3);
2251 }
2252 Some(hash as i64)
2253 }
2254}
2255
2256#[cfg(debug_assertions)]
2257fn wait_for_test_hook(name: &str) {
2258 let Some(directory) = std::env::var_os("SLOOP_TEST_HOOK_DIR").map(PathBuf::from) else {
2259 return;
2260 };
2261 let armed = directory.join(format!("{name}.armed"));
2262 if !armed.is_file() {
2263 return;
2264 }
2265 let reached = directory.join(format!("{name}.reached"));
2266 let release = directory.join(format!("{name}.release"));
2267 if fs::write(&reached, b"").is_err() {
2268 return;
2269 }
2270 while !release.is_file() {
2271 std::thread::sleep(Duration::from_millis(10));
2272 }
2273}
2274
2275#[cfg(not(debug_assertions))]
2276fn wait_for_test_hook(_name: &str) {}
2277
2278async fn serve_worker_socket(
2282 listener: UnixListener,
2283 run_id: String,
2284 dispatcher: mpsc::Sender<DispatcherMessage>,
2285 log: OperationalLog,
2286) {
2287 loop {
2288 let Ok((stream, _)) = listener.accept().await else {
2289 return;
2290 };
2291 let run_id = run_id.clone();
2292 let dispatcher = dispatcher.clone();
2293 let log = log.clone();
2294 tokio::spawn(async move {
2295 if let Err(error) = handle_worker_connection(stream, run_id.clone(), dispatcher).await {
2296 log.emit_with_fields(
2297 LogLevel::Error,
2298 "sloop::socket",
2299 "worker_connection_failed",
2300 json!({"run_id": run_id, "error": error.to_string()}),
2301 );
2302 }
2303 });
2304 }
2305}
2306
2307fn dispatch_worker(
2311 state: &mut DispatcherState,
2312 id: RequestId,
2313 request: Request,
2314 run_id: &str,
2315 token: Option<&str>,
2316) -> ResponseEnvelope {
2317 let valid = token.is_some_and(|presented| {
2318 state
2319 .worker_tokens
2320 .get(run_id)
2321 .is_some_and(|issued| issued == presented)
2322 });
2323 if !valid {
2324 return ResponseEnvelope::failure(
2325 Some(id),
2326 unauthorized("the presented token is not valid for this run"),
2327 );
2328 }
2329
2330 let data = match request {
2331 Request::Brief(_) => handle_brief(state, run_id),
2332 Request::Show(args) => handle_show(state, run_id, &args.reference),
2333 Request::Note(args) => handle_note(state, run_id, &args.text),
2334 _ => Err(unauthorized(
2336 "operator verbs are not available on a worker socket",
2337 )),
2338 };
2339 match data {
2340 Ok(data) => ResponseEnvelope::success(Some(id), data),
2341 Err(error) => ResponseEnvelope::failure(Some(id), error),
2342 }
2343}
2344
2345fn handle_brief(state: &DispatcherState, run_id: &str) -> Result<serde_json::Value, ErrorBody> {
2349 let run = lookup(state, |store| store.run(run_id))?
2350 .ok_or_else(|| internal("the run for this token no longer exists"))?;
2351 let ticket = lookup(state, |store| store.ticket(&run.ticket_id))?
2352 .ok_or_else(|| internal("the ticket for this run no longer exists"))?;
2353 let body = ticket
2354 .file_path
2355 .as_ref()
2356 .and_then(|file_path| fs::read_to_string(state.root.join(file_path)).ok())
2357 .unwrap_or_default();
2358
2359 let mut definition_of_done = vec!["Commit your work to the run branch".to_owned()];
2360 if state.aftercare_test_cmd.is_some() {
2361 definition_of_done.push("The configured test command passes".to_owned());
2362 }
2363
2364 Ok(json!({
2365 "run": run_id,
2366 "ticket": {
2367 "id": ticket.id,
2368 "name": ticket.name,
2369 "blocked_by": ticket.blocked_by,
2370 "worktree": ticket.worktree,
2371 "body": body,
2372 "acceptance": [],
2373 "target": ticket.target,
2374 "model": ticket.model,
2375 "effort": ticket.effort,
2376 },
2377 "worktree": run.worktree_path,
2378 "branch": run.branch,
2379 "definition_of_done": definition_of_done,
2380 }))
2381}
2382
2383fn handle_show(
2387 state: &DispatcherState,
2388 run_id: &str,
2389 reference: &str,
2390) -> Result<serde_json::Value, ErrorBody> {
2391 let run = lookup(state, |store| store.run(run_id))?
2392 .ok_or_else(|| internal("the run for this token no longer exists"))?;
2393 if reference != run.ticket_id {
2394 return Err(unauthorized("workers may only show their own run's ticket"));
2395 }
2396 let ticket = lookup(state, |store| store.ticket(&run.ticket_id))?
2397 .ok_or_else(|| internal("the ticket for this run no longer exists"))?;
2398 Ok(ticket_show(reference, &ticket))
2399}
2400
2401fn ticket_show(reference: &str, ticket: &crate::store::TicketRecord) -> serde_json::Value {
2402 json!({
2403 "ref": reference,
2404 "kind": "ticket",
2405 "value": {
2406 "id": ticket.id,
2407 "project": ticket.project_id,
2408 "state": ticket.state,
2409 "file": ticket.file_path,
2410 "name": ticket.name,
2411 "blocked_by": ticket.blocked_by,
2412 "worktree": ticket.worktree,
2413 "target": ticket.target,
2414 "model": ticket.model,
2415 "effort": ticket.effort,
2416 },
2417 })
2418}
2419
2420fn handle_operator_show(
2421 state: &DispatcherState,
2422 reference: &str,
2423) -> Result<serde_json::Value, ErrorBody> {
2424 if let Some(ticket) = lookup(state, |store| store.ticket(reference))? {
2425 return Ok(ticket_show(reference, &ticket));
2426 }
2427 let project = lookup(state, |store| store.project(reference))?
2428 .ok_or_else(|| not_found(&format!("reference `{reference}` is not indexed")))?;
2429 let tickets = lookup(state, |store| store.tickets_for_project(reference))?;
2430
2431 let mut notes: HashMap<String, Vec<serde_json::Value>> = HashMap::new();
2432 for note in lookup(state, |store| store.notes_for_project(reference))? {
2433 notes.entry(note.ticket_id).or_default().push(json!({
2434 "id": note.id,
2435 "run": note.run_id,
2436 "text": note.text,
2437 "recorded_at_ms": note.recorded_at_ms,
2438 }));
2439 }
2440
2441 let mut commits: HashMap<String, Vec<serde_json::Value>> = HashMap::new();
2442 for evidence in lookup(state, |store| store.commit_evidence_for_project(reference))? {
2443 let data: serde_json::Value = serde_json::from_str(&evidence.data_json)
2444 .map_err(|error| internal(&format!("cannot decode commit evidence: {error}")))?;
2445 for oid in data["oids"]
2446 .as_array()
2447 .map(Vec::as_slice)
2448 .unwrap_or_default()
2449 .iter()
2450 .filter_map(serde_json::Value::as_str)
2451 {
2452 let (short_hash, message) = git_commit(&state.root, oid)?;
2453 commits
2454 .entry(evidence.ticket_id.clone())
2455 .or_default()
2456 .push(json!({
2457 "run": evidence.run_id.clone(),
2458 "hash": short_hash,
2459 "message": message,
2460 }));
2461 }
2462 }
2463
2464 let activity = tickets
2465 .into_iter()
2466 .map(|ticket| {
2467 let ticket_notes = notes.remove(&ticket.id).unwrap_or_default();
2468 let ticket_commits = commits.remove(&ticket.id).unwrap_or_default();
2469 json!({
2470 "id": ticket.id,
2471 "name": ticket.name,
2472 "state": ticket.state,
2473 "notes": ticket_notes,
2474 "commits": ticket_commits,
2475 })
2476 })
2477 .collect::<Vec<_>>();
2478
2479 Ok(json!({
2480 "ref": reference,
2481 "kind": "project",
2482 "value": {
2483 "id": project.id,
2484 "title": project.title,
2485 "file": project.file_path,
2486 "tickets": activity,
2487 },
2488 }))
2489}
2490
2491fn git_commit(root: &Path, oid: &str) -> Result<(String, String), ErrorBody> {
2492 let output = Command::new("git")
2493 .args(["show", "--no-patch", "--format=%h%x00%s", oid, "--"])
2494 .current_dir(root)
2495 .output()
2496 .map_err(|error| internal(&format!("cannot read commit `{oid}`: {error}")))?;
2497 if !output.status.success() {
2498 return Err(internal(&format!(
2499 "cannot read commit `{oid}`: {}",
2500 String::from_utf8_lossy(&output.stderr).trim()
2501 )));
2502 }
2503 let rendered = String::from_utf8_lossy(&output.stdout);
2504 let (hash, message) = rendered
2505 .trim_end()
2506 .split_once('\0')
2507 .ok_or_else(|| internal(&format!("Git returned malformed data for commit `{oid}`")))?;
2508 Ok((hash.to_owned(), message.to_owned()))
2509}
2510
2511fn handle_note(
2514 state: &DispatcherState,
2515 run_id: &str,
2516 text: &str,
2517) -> Result<serde_json::Value, ErrorBody> {
2518 let ordinal = lookup(state, |store| store.next_note_ordinal())?;
2519 let note_id = format!("N{ordinal}");
2520 state
2521 .store
2522 .insert_note(¬e_id, run_id, text, state.clock.now_ms())
2523 .map_err(|error| internal(&format!("cannot record note: {error}")))?;
2524 Ok(json!({"note": {"id": note_id, "run": run_id, "text": text}}))
2525}
2526
2527fn dispatch(state: &mut DispatcherState, id: RequestId, request: Request) -> ResponseEnvelope {
2528 let data = match request {
2529 Request::Show(args) => match handle_operator_show(state, &args.reference) {
2530 Ok(data) => data,
2531 Err(error) => return ResponseEnvelope::failure(Some(id), error),
2532 },
2533 Request::Run(args) => match handle_run(state, &args) {
2534 Ok(data) => data,
2535 Err(error) => return ResponseEnvelope::failure(Some(id), error),
2536 },
2537 Request::Daemon(_) => json!({
2538 "pid": state.pid,
2539 "socket": state.socket.to_string_lossy(),
2540 "state_dir": state.state_dir.to_string_lossy(),
2541 "log": state.daemon_log.to_string_lossy(),
2542 "version": env!("CARGO_PKG_VERSION"),
2543 "started": false
2544 }),
2545 Request::Post(args) => {
2546 match crate::post::handle(
2547 &state.root,
2548 &state.ticket_dir,
2549 &state.store,
2550 &args,
2551 state.clock.now_ms(),
2552 &state.ticket_prefix,
2553 state.agent.as_ref(),
2554 &state.flows,
2555 &state.default_flow,
2556 ) {
2557 Ok(data) => data,
2558 Err(error) => {
2559 return ResponseEnvelope::failure(Some(id), post_error_body(&error));
2560 }
2561 }
2562 }
2563 Request::List(_) => match handle_list(state) {
2564 Ok(data) => data,
2565 Err(error) => return ResponseEnvelope::failure(Some(id), error),
2566 },
2567 Request::Status(_) => {
2568 let tickets = match state.store.ticket_counts() {
2569 Ok(counts) => counts,
2570 Err(error) => {
2571 return ResponseEnvelope::failure(
2572 Some(id),
2573 internal(&format!("cannot read ticket counts: {error}")),
2574 );
2575 }
2576 };
2577 let runs: Vec<_> = match state.store.active_runs() {
2578 Ok(runs) => runs
2579 .into_iter()
2580 .map(|run| {
2581 json!({
2582 "id": run.id,
2583 "project": run.project_id,
2584 "ticket": run.ticket_id,
2585 "state": run.state,
2586 })
2587 })
2588 .collect(),
2589 Err(error) => {
2590 return ResponseEnvelope::failure(
2591 Some(id),
2592 internal(&format!("cannot read active runs: {error}")),
2593 );
2594 }
2595 };
2596 let queued: Vec<_> = match state.store.queued_dispatchable_activations() {
2597 Ok(activations) => activations
2598 .into_iter()
2599 .map(|activation| {
2600 json!({
2601 "id": activation.id,
2602 "ticket": activation.ticket_id,
2603 "project": activation.project_id,
2604 "state": "queued",
2605 })
2606 })
2607 .collect(),
2608 Err(error) => {
2609 return ResponseEnvelope::failure(
2610 Some(id),
2611 internal(&format!("cannot read queued activations: {error}")),
2612 );
2613 }
2614 };
2615 let now_ms = state.clock.now_ms();
2616 let mut gate = json!({
2617 "active_agents": state.active.len(),
2618 "max_agents": state.max_agents,
2619 });
2620 if let Some(hours) = &state.running_hours {
2621 gate["running_hours"] = json!({
2622 "start": hours.start,
2623 "end": hours.end,
2624 "open": hours.is_open(state.clock.local_minute(now_ms)),
2625 });
2626 }
2627 let mut snapshot = json!({
2628 "daemon": {"pid": state.pid, "paused": state.paused},
2629 "gate": gate,
2630 "runs": runs,
2631 "queued_activations": queued,
2632 "tickets": {
2633 "ready": tickets.ready,
2634 "held": tickets.held,
2635 "blocked": tickets.blocked,
2636 "claimed": tickets.claimed,
2637 "merged": tickets.merged,
2638 "failed": tickets.failed,
2639 "needs_review": tickets.needs_review
2640 }
2641 });
2642 if let Some(deadline) = next_dispatch_deadline(state)
2643 && let Some(formatted) = format_timestamp(deadline)
2644 {
2645 snapshot["next_wake"] = json!(formatted);
2646 }
2647 snapshot
2648 }
2649 Request::Pause(_) => {
2650 if let Err(error) = state.store.set_paused(true, state.clock.now_ms()) {
2651 return ResponseEnvelope::failure(
2652 Some(id),
2653 internal(&format!("cannot pause scheduler: {error}")),
2654 );
2655 }
2656 state.paused = true;
2657 json!({"paused": true})
2658 }
2659 Request::Resume(_) => {
2660 if let Err(error) = state.store.set_paused(false, state.clock.now_ms()) {
2661 return ResponseEnvelope::failure(
2662 Some(id),
2663 internal(&format!("cannot resume scheduler: {error}")),
2664 );
2665 }
2666 state.paused = false;
2667 json!({"paused": false})
2668 }
2669 Request::Hold(args) => match handle_hold(state, &args) {
2670 Ok(data) => data,
2671 Err(error) => return ResponseEnvelope::failure(Some(id), error),
2672 },
2673 Request::Ready(args) => match handle_ready(state, &args) {
2674 Ok(data) => data,
2675 Err(error) => return ResponseEnvelope::failure(Some(id), error),
2676 },
2677 Request::Retry(args) => match handle_retry(state, &args) {
2678 Ok(data) => data,
2679 Err(error) => return ResponseEnvelope::failure(Some(id), error),
2680 },
2681 Request::Logs(args) => match handle_logs(state, &args) {
2682 Ok(data) => data,
2683 Err(error) => return ResponseEnvelope::failure(Some(id), error),
2684 },
2685 Request::Cancel(args) => match handle_cancel(state, &args) {
2686 Ok(data) => data,
2687 Err(error) => return ResponseEnvelope::failure(Some(id), error),
2688 },
2689 Request::Stop(args) => match handle_stop(state, &args) {
2690 Ok(data) => data,
2691 Err(error) => return ResponseEnvelope::failure(Some(id), error),
2692 },
2693 Request::Wait(args) => match handle_wait(state, &args) {
2694 Ok(data) => data,
2695 Err(error) => return ResponseEnvelope::failure(Some(id), error),
2696 },
2697 request => {
2698 return ResponseEnvelope::failure(
2699 Some(id),
2700 ErrorBody {
2701 code: ErrorCode::InvalidRequest,
2702 message: format!("verb `{}` is not implemented by the daemon", request.verb()),
2703 details: json!({"verb": request.verb()}),
2704 },
2705 );
2706 }
2707 };
2708 ResponseEnvelope::success(Some(id), data)
2709}
2710
2711fn handle_list(state: &DispatcherState) -> Result<serde_json::Value, ErrorBody> {
2712 let now_ms = state.clock.now_ms();
2713 let gates = crate::eligibility::Gates {
2714 paused: state.paused,
2715 agent_configured: state.agent.is_some(),
2716 hours_open: running_hours_open(state, now_ms),
2717 at_capacity: state.active.len() >= state.max_agents,
2718 has_queued_activation: !lookup(state, Store::queued_dispatchable_activations)?.is_empty(),
2719 };
2720 let mut rows = Vec::new();
2721 for ticket in lookup(state, Store::tickets)? {
2722 let active_run = lookup(state, |store| store.active_run_for_ticket(&ticket.id))?;
2723 let reason = crate::eligibility::ticket_ineligibility(
2724 &ticket.state,
2725 ticket.attempts,
2726 active_run.as_deref(),
2727 &gates,
2728 )
2729 .map(|reason| reason.describe());
2730 rows.push(json!({
2731 "id": ticket.id,
2732 "project": ticket.project_id,
2733 "state": ticket.state,
2734 "run": active_run,
2735 "reason": reason,
2736 }));
2737 }
2738 Ok(json!({"tickets": rows}))
2739}
2740
2741fn spawn_daemon(project: &Project) -> Result<(), DaemonError> {
2742 let executable = std::env::current_exe().map_err(DaemonError::CurrentExecutable)?;
2743 Command::new(executable)
2744 .args(["daemon", "--foreground"])
2745 .current_dir(&project.root)
2746 .stdin(Stdio::null())
2747 .stdout(Stdio::null())
2748 .stderr(Stdio::null())
2749 .spawn()
2750 .map(|_| ())
2751 .map_err(DaemonError::Spawn)
2752}
2753
2754fn send_existing(project: &Project, request: Request) -> Result<ResponseEnvelope, DaemonError> {
2755 match send(&project.operator_socket, request.clone()) {
2756 Ok(response) => Ok(response),
2757 Err(current_error) => {
2758 let Some(identity) = read_lock_identity(&project.lock_path) else {
2759 return Err(current_error);
2760 };
2761 if !process_identity_matches(identity.pid, identity.started_at_ms) {
2762 return Err(current_error);
2763 }
2764 let Some(socket) = identity.socket else {
2765 return Err(current_error);
2766 };
2767 if socket == project.operator_socket {
2768 return Err(current_error);
2769 }
2770 send(&socket, request)
2771 }
2772 }
2773}
2774
2775fn send(socket: &Path, request: Request) -> Result<ResponseEnvelope, DaemonError> {
2776 let mut stream = StdUnixStream::connect(socket).map_err(DaemonError::Connect)?;
2777 stream
2778 .set_read_timeout(Some(CLIENT_TIMEOUT))
2779 .map_err(DaemonError::Connect)?;
2780 stream
2781 .set_write_timeout(Some(CLIENT_TIMEOUT))
2782 .map_err(DaemonError::Connect)?;
2783
2784 let sequence = NEXT_REQUEST_ID.fetch_add(1, Ordering::Relaxed);
2785 let envelope = RequestEnvelope::new(
2786 RequestId::new(format!("req-{}-{sequence}", std::process::id())),
2787 request,
2788 None,
2789 );
2790 serde_json::to_writer(&mut stream, &envelope).map_err(DaemonError::Encode)?;
2791 stream.write_all(b"\n").map_err(DaemonError::Write)?;
2792
2793 let mut line = String::new();
2794 let mut reader = BufReader::new(stream).take(MAX_ENVELOPE_BYTES + 1);
2795 reader.read_line(&mut line).map_err(DaemonError::Read)?;
2796 if line.len() as u64 > MAX_ENVELOPE_BYTES {
2797 return Err(DaemonError::InvalidResponse(
2798 "response envelope is too large".into(),
2799 ));
2800 }
2801 serde_json::from_str(line.trim_end()).map_err(DaemonError::Decode)
2802}
2803
2804fn handle_run(
2807 state: &mut DispatcherState,
2808 args: &crate::protocol::RunArgs,
2809) -> Result<serde_json::Value, ErrorBody> {
2810 use crate::protocol::RunActivation;
2811
2812 if args.ticket.is_some() && args.project.is_some() {
2813 return Err(invalid_arguments(
2814 "a run may target a ticket or a project, not both",
2815 ));
2816 }
2817 if let Some(ticket_id) = &args.ticket {
2818 let Some(ticket) = lookup(state, |store| store.ticket(ticket_id))? else {
2819 return Err(not_found(&format!(
2820 "ticket `{ticket_id}` is not registered"
2821 )));
2822 };
2823 if ticket.state == TicketState::Held.as_str() {
2824 return Err(conflict(&format!(
2825 "ticket `{ticket_id}` is held; release it with `sloop ready {ticket_id}`"
2826 )));
2827 }
2828 }
2829 if let Some(project) = &args.project
2830 && !lookup(state, |store| store.project_exists(project))?
2831 {
2832 return Err(not_found(&format!("project `{project}` is not indexed")));
2833 }
2834 for only in &args.only {
2835 let Some(ticket) = lookup(state, |store| store.ticket(only))? else {
2836 return Err(not_found(&format!("ticket `{only}` is not registered")));
2837 };
2838 if let Some(project) = &args.project
2839 && &ticket.project_id != project
2840 {
2841 return Err(invalid_arguments(&format!(
2842 "ticket `{only}` belongs to project `{}`, not `{project}`",
2843 ticket.project_id
2844 )));
2845 }
2846 }
2847
2848 let (kind, echo_kind, interval_ms) = match &args.activation {
2849 RunActivation::Now => (ActivationKind::Immediate, "now", None),
2850 RunActivation::At { .. } => (ActivationKind::At, "at", None),
2851 RunActivation::Every { interval_ms } => {
2852 (ActivationKind::Every, "every", Some(*interval_ms as i64))
2853 }
2854 RunActivation::Overnight => (ActivationKind::Overnight, "overnight", None),
2855 };
2856
2857 let now_ms = state.clock.now_ms();
2858 let activation_id = format!(
2859 "A{}",
2860 lookup(state, |store| store.next_activation_ordinal())?
2861 );
2862 lookup(state, |store| {
2863 store.insert_activation(
2864 &NewActivation {
2865 id: &activation_id,
2866 kind,
2867 ticket_id: args.ticket.as_deref(),
2868 project_id: args.project.as_deref(),
2869 eligible_at_ms: None,
2872 interval_ms,
2873 },
2874 now_ms,
2875 )
2876 })?;
2877 for only in &args.only {
2878 lookup(state, |store| {
2879 store.insert_activation_filter(&activation_id, only)
2880 })?;
2881 }
2882
2883 let mut activation = json!({
2884 "id": activation_id,
2885 "kind": echo_kind,
2886 "state": "queued",
2887 });
2888 if let Some(ticket) = &args.ticket {
2889 activation["ticket"] = json!(ticket);
2890 }
2891 if let Some(project) = &args.project {
2892 activation["project"] = json!(project);
2893 }
2894 Ok(json!({"activation": activation}))
2895}
2896
2897fn handle_hold(
2898 state: &mut DispatcherState,
2899 args: &crate::protocol::TicketReferenceArgs,
2900) -> Result<serde_json::Value, ErrorBody> {
2901 let requested = TicketState::Held;
2902 let previous = state
2903 .store
2904 .set_ticket_hold(&args.ticket, requested, state.clock.now_ms())
2905 .map_err(|error| match error {
2906 StoreError::TicketNotFound { .. } => not_found(&error.to_string()),
2907 StoreError::TicketStateConflict { .. } => conflict(&error.to_string()),
2908 _ => internal(&error.to_string()),
2909 })?;
2910 Ok(json!({
2911 "ticket": args.ticket,
2912 "previous_state": previous,
2913 "state": requested.as_str(),
2914 "overridden": previous != requested.as_str(),
2915 }))
2916}
2917
2918fn handle_ready(
2919 state: &mut DispatcherState,
2920 args: &crate::protocol::TicketReferenceArgs,
2921) -> Result<serde_json::Value, ErrorBody> {
2922 let requested = TicketState::Ready;
2923 let previous = state
2924 .store
2925 .set_ticket_hold(&args.ticket, requested, state.clock.now_ms())
2926 .map_err(|error| match error {
2927 StoreError::TicketNotFound { .. } => not_found(&error.to_string()),
2928 StoreError::TicketStateConflict { .. } => conflict(&error.to_string()),
2929 _ => internal(&error.to_string()),
2930 })?;
2931 Ok(json!({
2932 "ticket": args.ticket,
2933 "previous_state": previous,
2934 "state": requested.as_str(),
2935 "overridden": previous != requested.as_str(),
2936 }))
2937}
2938
2939fn handle_retry(
2940 state: &mut DispatcherState,
2941 args: &crate::protocol::TicketReferenceArgs,
2942) -> Result<serde_json::Value, ErrorBody> {
2943 let previous = state
2944 .store
2945 .retry_ticket(&args.ticket, state.clock.now_ms())
2946 .map_err(|error| match error {
2947 StoreError::TicketNotFound { .. } => not_found(&error.to_string()),
2948 StoreError::TicketStateConflict { .. } => conflict(&error.to_string()),
2949 _ => internal(&error.to_string()),
2950 })?;
2951 Ok(json!({
2952 "ticket": args.ticket,
2953 "previous_state": previous,
2954 "state": TicketState::Ready.as_str(),
2955 }))
2956}
2957
2958fn handle_wait(
2961 state: &DispatcherState,
2962 args: &crate::protocol::RunReferenceArgs,
2963) -> Result<serde_json::Value, ErrorBody> {
2964 let Some(run) = lookup(state, |store| store.run(&args.run))? else {
2965 return Err(not_found(&format!("run `{}` does not exist", args.run)));
2966 };
2967 let terminal = matches!(
2968 run.state.as_str(),
2969 "merged" | "failed" | "needs_review" | "cancelled" | "orphaned" | "aborted"
2970 );
2971 Ok(json!({
2972 "run": run.id,
2973 "state": run.state,
2974 "terminal": terminal,
2975 "exit_code": run.exit_code,
2976 }))
2977}
2978
2979fn handle_logs(
2982 state: &DispatcherState,
2983 args: &crate::protocol::RunReferenceArgs,
2984) -> Result<serde_json::Value, ErrorBody> {
2985 if lookup(state, |store| store.run(&args.run))?.is_none() {
2986 return Err(not_found(&format!("run `{}` does not exist", args.run)));
2987 }
2988 let page = crate::run_log::read_page(
2989 &run_output_path(&state.state_dir, &args.run),
2990 0,
2991 LOGS_PAGE_LIMIT,
2992 )
2993 .map_err(|error| internal(&format!("cannot read run log: {error}")))?;
2994 let entries = page
2995 .entries
2996 .iter()
2997 .map(serde_json::to_value)
2998 .collect::<Result<Vec<_>, _>>()
2999 .map_err(|error| internal(&format!("cannot encode run log: {error}")))?;
3000 Ok(json!({
3001 "run": args.run,
3002 "entries": entries,
3003 "next_cursor": page.next_cursor,
3004 "complete": page.complete,
3005 }))
3006}
3007
3008fn handle_cancel(
3013 state: &mut DispatcherState,
3014 args: &crate::protocol::RunReferenceArgs,
3015) -> Result<serde_json::Value, ErrorBody> {
3016 let Some(run) = lookup(state, |store| store.run(&args.run))? else {
3017 return Err(not_found(&format!("run `{}` does not exist", args.run)));
3018 };
3019 if !matches!(run.state.as_str(), "running" | "aftercare") || run.exited_at_ms.is_some() {
3020 return Err(conflict(&format!(
3021 "run `{}` is `{}` and cannot be cancelled",
3022 run.id, run.state
3023 )));
3024 }
3025
3026 lookup(state, |store| {
3029 store.record_cancel_requested(&run.id, state.clock.now_ms())
3030 })?;
3031 state.cancelling.insert(run.id.clone());
3032
3033 if run.state == "aftercare" {
3034 let rows = lookup(state, |store| store.run_evidence(&run.id))?;
3035 for kind in ["merge_process", "test_process"] {
3036 if let Some((pid, start_time, group)) =
3037 aftercare_process_identity(&rows, kind).map_err(|error| internal(&error))?
3038 && process_identity_matches(pid, Some(start_time))
3039 {
3040 unsafe {
3041 libc::kill(-(group as libc::pid_t), libc::SIGKILL);
3042 }
3043 break;
3044 }
3045 }
3046 } else {
3047 let process_matches = run
3048 .pid
3049 .and_then(|pid| u32::try_from(pid).ok())
3050 .is_some_and(|pid| process_identity_matches(pid, run.pid_start_time));
3051 if process_matches && let Some(group) = run.process_group_id {
3052 unsafe {
3056 libc::kill(-(group as libc::pid_t), libc::SIGKILL);
3057 }
3058 }
3059 }
3060
3061 Ok(json!({
3062 "run": run.id,
3063 "state": "cancelling",
3064 "worktree": run.worktree_path,
3065 "preserved": true,
3066 }))
3067}
3068
3069fn handle_stop(
3073 state: &mut DispatcherState,
3074 args: &crate::protocol::StopArgs,
3075) -> Result<serde_json::Value, ErrorBody> {
3076 let mut active: Vec<String> = state.active.iter().cloned().collect();
3077 active.sort();
3078 if !active.is_empty() && !args.force {
3079 return Err(conflict(&format!(
3080 "{} active run(s): {}; stop --force cancels them",
3081 active.len(),
3082 active.join(", "),
3083 )));
3084 }
3085 let mut cancelled = Vec::new();
3086 for run_id in active {
3087 if handle_cancel(
3088 state,
3089 &crate::protocol::RunReferenceArgs {
3090 run: run_id.clone(),
3091 },
3092 )
3093 .is_ok()
3094 {
3095 cancelled.push(run_id);
3096 }
3097 }
3098 Ok(json!({
3099 "stopping": true,
3100 "pid": state.pid,
3101 "cancelled_runs": cancelled,
3102 }))
3103}
3104
3105fn lookup<T>(
3106 state: &DispatcherState,
3107 query: impl FnOnce(&Store) -> Result<T, StoreError>,
3108) -> Result<T, ErrorBody> {
3109 query(&state.store).map_err(|error| internal(&error.to_string()))
3110}
3111
3112fn invalid_arguments(message: &str) -> ErrorBody {
3113 ErrorBody {
3114 code: ErrorCode::InvalidArguments,
3115 message: message.into(),
3116 details: json!({}),
3117 }
3118}
3119
3120fn not_found(message: &str) -> ErrorBody {
3121 ErrorBody {
3122 code: ErrorCode::NotFound,
3123 message: message.into(),
3124 details: json!({}),
3125 }
3126}
3127
3128fn conflict(message: &str) -> ErrorBody {
3129 ErrorBody {
3130 code: ErrorCode::Conflict,
3131 message: message.into(),
3132 details: json!({}),
3133 }
3134}
3135
3136fn post_error_body(error: &crate::post::PostError) -> ErrorBody {
3137 use crate::post::PostError;
3138 let code = match error {
3139 PostError::TicketFileNotFound(_)
3140 | PostError::UnknownProject(_)
3141 | PostError::UnknownFlow { .. }
3142 | PostError::UnknownBlockedBy { .. } => ErrorCode::NotFound,
3143 PostError::OutsideRepository(_)
3144 | PostError::OutsideTicketDirectory { .. }
3145 | PostError::InvalidTicket { .. }
3146 | PostError::MissingName { .. }
3147 | PostError::MissingBlockedBy { .. }
3148 | PostError::InvalidBlockedBy { .. }
3149 | PostError::EmptyBody { .. }
3150 | PostError::UnknownTarget(_)
3151 | PostError::MissingTargetValue { .. } => ErrorCode::InvalidArguments,
3152 PostError::ProjectConflict { .. }
3153 | PostError::FlowConflict { .. }
3154 | PostError::TicketIdTaken { .. }
3155 | PostError::DependencyCycle(_) => ErrorCode::Conflict,
3156 PostError::Io { .. } | PostError::Store(_) | PostError::IdAllocation(_) => {
3157 ErrorCode::Internal
3158 }
3159 };
3160 ErrorBody {
3161 code,
3162 message: error.to_string(),
3163 details: json!({}),
3164 }
3165}
3166
3167fn protocol_error(message: &str) -> ErrorBody {
3168 ErrorBody {
3169 code: ErrorCode::InvalidRequest,
3170 message: message.into(),
3171 details: json!({}),
3172 }
3173}
3174
3175fn unauthorized(message: &str) -> ErrorBody {
3176 ErrorBody {
3177 code: ErrorCode::Unauthorized,
3178 message: message.into(),
3179 details: json!({}),
3180 }
3181}
3182
3183fn internal(message: &str) -> ErrorBody {
3184 ErrorBody {
3185 code: ErrorCode::Internal,
3186 message: message.into(),
3187 details: json!({}),
3188 }
3189}
3190
3191#[derive(Debug)]
3192pub enum DaemonError {
3193 Config(ConfigError),
3194 Store(StoreError),
3195 CurrentDirectory(io::Error),
3196 CurrentExecutable(io::Error),
3197 Io {
3198 path: PathBuf,
3199 source: io::Error,
3200 },
3201 AlreadyRunning,
3202 Runtime(io::Error),
3203 Spawn(io::Error),
3204 Connect(io::Error),
3205 Write(io::Error),
3206 Read(io::Error),
3207 Encode(serde_json::Error),
3208 Decode(serde_json::Error),
3209 InvalidResponse(String),
3210 Frontmatter {
3211 path: PathBuf,
3212 error: FrontmatterError,
3213 },
3214 IdAllocation(IdError),
3215}
3216
3217impl DaemonError {
3218 pub fn error_body(&self) -> ErrorBody {
3219 let code = match self {
3220 Self::Config(_) => ErrorCode::InvalidArguments,
3221 _ => ErrorCode::DaemonUnavailable,
3222 };
3223 ErrorBody {
3224 code,
3225 message: self.to_string(),
3226 details: json!({}),
3227 }
3228 }
3229}
3230
3231impl From<ConfigError> for DaemonError {
3232 fn from(error: ConfigError) -> Self {
3233 Self::Config(error)
3234 }
3235}
3236
3237impl From<IdError> for DaemonError {
3238 fn from(error: IdError) -> Self {
3239 Self::IdAllocation(error)
3240 }
3241}
3242
3243impl std::fmt::Display for DaemonError {
3244 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
3245 match self {
3246 Self::Config(error) => error.fmt(formatter),
3247 Self::Store(error) => error.fmt(formatter),
3248 Self::CurrentDirectory(error) => {
3249 write!(formatter, "cannot read current directory: {error}")
3250 }
3251 Self::CurrentExecutable(error) => {
3252 write!(formatter, "cannot locate sloop executable: {error}")
3253 }
3254 Self::Io { path, source } => write!(formatter, "{}: {source}", path.display()),
3255 Self::AlreadyRunning => formatter.write_str("another sloop daemon holds the lock"),
3256 Self::Runtime(error) => write!(formatter, "cannot start async runtime: {error}"),
3257 Self::Spawn(error) => write!(formatter, "cannot spawn daemon: {error}"),
3258 Self::Connect(error) => write!(formatter, "cannot connect to daemon: {error}"),
3259 Self::Write(error) => write!(formatter, "cannot write daemon request: {error}"),
3260 Self::Read(error) => write!(formatter, "cannot read daemon response: {error}"),
3261 Self::Encode(error) => write!(formatter, "cannot encode daemon request: {error}"),
3262 Self::Decode(error) => write!(formatter, "cannot decode daemon response: {error}"),
3263 Self::InvalidResponse(message) => formatter.write_str(message),
3264 Self::Frontmatter { path, error } => write!(formatter, "{}: {error}", path.display()),
3265 Self::IdAllocation(error) => error.fmt(formatter),
3266 }
3267 }
3268}
3269
3270impl std::error::Error for DaemonError {}
3271
3272#[cfg(test)]
3273mod tests {
3274 use std::fs;
3275
3276 use tempfile::tempdir;
3277
3278 use super::{
3279 WORKER_BOOTSTRAP_PROMPT, compose_worker_prompt, index_projects, process_start_time,
3280 recoverable_process_matches,
3281 };
3282 use crate::config::expand_agent_cmd;
3283 use crate::store::{RecoverableRun, Store};
3284
3285 fn recoverable_current_process(start_time: Option<i64>) -> RecoverableRun {
3286 RecoverableRun {
3287 id: "R1".into(),
3288 ticket_id: "T1".into(),
3289 state: "running".into(),
3290 branch: None,
3291 worktree_path: None,
3292 pid: Some(i64::from(std::process::id())),
3293 pid_start_time: start_time,
3294 process_group_id: None,
3295 worker_token: None,
3296 worker_socket_path: None,
3297 exit_code: None,
3298 lease_expires_at_ms: 1,
3299 }
3300 }
3301
3302 #[test]
3303 fn recovery_requires_both_pid_and_start_time_to_match() {
3304 let Some(start_time) = process_start_time(std::process::id()) else {
3305 return;
3306 };
3307 assert!(recoverable_process_matches(&recoverable_current_process(
3308 Some(start_time)
3309 )));
3310 assert!(!recoverable_process_matches(&recoverable_current_process(
3311 Some(start_time + 1)
3312 )));
3313 assert!(!recoverable_process_matches(&recoverable_current_process(
3314 None
3315 )));
3316 }
3317
3318 #[test]
3319 fn agent_command_expands_ticket_model_and_effort() {
3320 let template = vec![
3321 "agent".to_owned(),
3322 "--model={model}".to_owned(),
3323 "--effort".to_owned(),
3324 "{effort}".to_owned(),
3325 "prompt={prompt}".to_owned(),
3326 ];
3327
3328 assert_eq!(
3329 expand_agent_cmd(&template, Some("sonnet"), Some("medium"), "assignment").unwrap(),
3330 [
3331 "agent",
3332 "--model=sonnet",
3333 "--effort",
3334 "medium",
3335 "prompt=assignment"
3336 ]
3337 );
3338 }
3339
3340 #[test]
3341 fn agent_command_rejects_a_missing_ticket_field() {
3342 let template = vec!["agent".to_owned(), "{model}".to_owned()];
3343
3344 assert_eq!(
3345 expand_agent_cmd(&template, None, Some("medium"), "assignment"),
3346 Err("does not specify `model`".to_owned())
3347 );
3348 }
3349
3350 #[test]
3351 fn worker_prompt_uses_the_builtin_when_instructions_are_absent() {
3352 let root = tempdir().unwrap();
3353
3354 assert_eq!(
3355 compose_worker_prompt(root.path()).unwrap(),
3356 WORKER_BOOTSTRAP_PROMPT
3357 );
3358 }
3359
3360 #[test]
3361 fn worker_prompt_appends_repository_instructions() {
3362 let root = tempdir().unwrap();
3363 fs::create_dir_all(root.path().join(".agents/sloop")).unwrap();
3364 fs::write(
3365 root.path().join(".agents/sloop/instructions.md"),
3366 "Use repository conventions.\n",
3367 )
3368 .unwrap();
3369
3370 assert_eq!(
3371 compose_worker_prompt(root.path()).unwrap(),
3372 format!("{WORKER_BOOTSTRAP_PROMPT}\n\nUse repository conventions.\n")
3373 );
3374 }
3375
3376 #[test]
3377 fn orphan_disposition_stamps_waits_then_deletes_unreferenced_rows() {
3378 use super::OrphanDisposition::{Delete, Keep, MarkMissing};
3379 use super::orphan_disposition;
3380
3381 assert_eq!(orphan_disposition(None, false, 1_000, 100), MarkMissing);
3382 assert_eq!(orphan_disposition(Some(950), false, 1_000, 100), Keep);
3383 assert_eq!(orphan_disposition(Some(900), false, 1_000, 100), Delete);
3384 assert_eq!(orphan_disposition(Some(900), true, 1_000, 100), Keep);
3385 }
3386
3387 #[test]
3388 fn reconcile_stamps_deletes_and_restores_tickets() {
3389 use crate::store::TicketState;
3390
3391 let root = tempdir().unwrap();
3392 let tickets = root.path().join(".agents/sloop/tickets");
3393 fs::create_dir_all(&tickets).unwrap();
3394 fs::write(tickets.join("present.md"), "# Present\n").unwrap();
3395 let store = Store::open(&root.path().join("sloop.db"), 1_000).unwrap();
3396 store
3397 .insert_local_project(
3398 "default",
3399 ".agents/sloop/projects/default.md",
3400 "Default",
3401 1_000,
3402 )
3403 .unwrap();
3404 let insert = |id: &str, file: &str, blocked_by: &[String]| {
3405 store
3406 .insert_local_ticket(
3407 id,
3408 "default",
3409 &format!(".agents/sloop/tickets/{file}"),
3410 id,
3411 blocked_by,
3412 &format!("sloop/{id}"),
3413 None,
3414 None,
3415 None,
3416 "default",
3417 TicketState::Ready,
3418 1_000,
3419 )
3420 .unwrap();
3421 };
3422 insert("T1", "present.md", &[]);
3423 insert("T2", "gone.md", &[]);
3424 insert("T3", "blocked-gone.md", &[]);
3425 insert("T4", "dependent.md", &["T3".into()]);
3426 fs::write(tickets.join("dependent.md"), "# Dependent\n").unwrap();
3427
3428 let window = 100;
3429 let stamps = |store: &Store| -> Vec<(String, Option<i64>)> {
3430 store
3431 .local_ticket_files()
3432 .unwrap()
3433 .into_iter()
3434 .map(|ticket| (ticket.id, ticket.missing_at_ms))
3435 .collect()
3436 };
3437
3438 super::reconcile_tickets(root.path(), &store, 2_000, window).unwrap();
3440 assert_eq!(
3441 stamps(&store),
3442 vec![
3443 ("T1".into(), None),
3444 ("T2".into(), Some(2_000)),
3445 ("T3".into(), Some(2_000)),
3446 ("T4".into(), None),
3447 ]
3448 );
3449
3450 super::reconcile_tickets(root.path(), &store, 2_050, window).unwrap();
3452 assert_eq!(stamps(&store)[1], ("T2".into(), Some(2_000)));
3453
3454 super::reconcile_tickets(root.path(), &store, 2_100, window).unwrap();
3457 assert_eq!(
3458 stamps(&store),
3459 vec![
3460 ("T1".into(), None),
3461 ("T3".into(), Some(2_000)),
3462 ("T4".into(), None),
3463 ]
3464 );
3465
3466 fs::write(tickets.join("blocked-gone.md"), "# Returned\n").unwrap();
3468 super::reconcile_tickets(root.path(), &store, 3_000, window).unwrap();
3469 assert_eq!(stamps(&store)[1], ("T3".into(), None));
3470 }
3471
3472 #[test]
3473 fn project_allocation_uses_sorted_paths_after_explicit_high_water_marks() {
3474 let root = tempdir().unwrap();
3475 let projects = root.path().join(".agents/sloop/projects");
3476 fs::create_dir_all(&projects).unwrap();
3477 fs::write(projects.join("zeta.md"), "# Zeta\n").unwrap();
3478 fs::write(projects.join("alpha.md"), "# Alpha\n").unwrap();
3479 fs::write(
3480 projects.join("middle.md"),
3481 "---\nid: PROJ-7\ntitle: Explicit\n---\n",
3482 )
3483 .unwrap();
3484 let store = Store::open(&root.path().join("sloop.db"), 1_000).unwrap();
3485
3486 index_projects(
3487 root.path(),
3488 std::path::Path::new(".agents/sloop/projects"),
3489 &store,
3490 1_000,
3491 "PROJ",
3492 )
3493 .unwrap();
3494
3495 assert!(
3496 fs::read_to_string(projects.join("alpha.md"))
3497 .unwrap()
3498 .contains("id: PROJ-8")
3499 );
3500 assert!(
3501 fs::read_to_string(projects.join("zeta.md"))
3502 .unwrap()
3503 .contains("id: PROJ-9")
3504 );
3505 }
3506}