Skip to main content

sloop/
daemon.rs

1use std::collections::{BTreeMap, HashMap, HashSet};
2use std::fs::{self, OpenOptions};
3use std::io::{self, BufRead, BufReader, Read, Write};
4use std::os::unix::fs::PermissionsExt;
5use std::os::unix::net::UnixStream as StdUnixStream;
6use std::os::unix::process::CommandExt;
7use std::path::{Path, PathBuf};
8use std::process::{Child, Command, Stdio};
9use std::sync::Arc;
10use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
11use std::time::{Duration, Instant};
12
13use base64::Engine;
14use base64::engine::general_purpose::URL_SAFE_NO_PAD as BASE64_TOKEN;
15use fs2::FileExt;
16use serde_json::json;
17use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader as AsyncBufReader};
18use tokio::net::{UnixListener, UnixStream};
19use tokio::sync::{mpsc, oneshot};
20
21use crate::clock::{Clock, FileClock, SystemClock, format_timestamp};
22use crate::config::{AgentConfig, Config, ConfigError, Project, RunningHours, expand_agent_cmd};
23use crate::flow::Flow;
24use crate::frontmatter::{Frontmatter, FrontmatterError};
25use crate::ids::{IdError, next_id};
26use crate::logging::{LogLevel, OperationalLog};
27use crate::outcome::{
28    MergeOutcome, RunEvidence, StageOutcome, classify_exit, derive_outcome, wants_merge,
29    wants_tests,
30};
31use crate::protocol::{
32    Capability, ErrorBody, ErrorCode, Request, RequestEnvelope, RequestId, ResponseEnvelope,
33};
34use crate::run_log::{OutputSource, OutputStream, RunLogWriter};
35use crate::store::{
36    ActivationKind, ClaimRequest, EvidenceRecord, NewActivation, QueuedActivation, RecoverableRun,
37    StageRecord, Store, StoreError, TicketState,
38};
39
40const MAX_ENVELOPE_BYTES: u64 = 1024 * 1024;
41const STARTUP_TIMEOUT: Duration = Duration::from_secs(5);
42const CLIENT_TIMEOUT: Duration = Duration::from_secs(5);
43const DISPATCH_CHANNEL_CAPACITY: usize = 64;
44const DEFAULT_LEASE_MS: i64 = 10 * 60 * 1000;
45pub(crate) const WORKER_BOOTSTRAP_PROMPT: &str =
46    include_str!("worker-instructions.md").trim_ascii();
47/// One `logs` page; chunks are ≤8KiB, so a page stays well inside the
48/// envelope limit once cursor arguments arrive.
49const LOGS_PAGE_LIMIT: usize = 64;
50
51static NEXT_REQUEST_ID: AtomicU64 = AtomicU64::new(1);
52static MERGE_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
53
54pub struct ClientResponse {
55    pub response: ResponseEnvelope,
56    pub started: bool,
57}
58
59pub fn request(request: Request) -> Result<ClientResponse, DaemonError> {
60    let cwd = std::env::current_dir().map_err(DaemonError::CurrentDirectory)?;
61    let project = Project::discover(&cwd)?;
62    Config::load(&project)?;
63
64    if let Ok(response) = send_existing(&project, request.clone()) {
65        return Ok(ClientResponse {
66            response,
67            started: false,
68        });
69    }
70
71    spawn_daemon(&project)?;
72    let deadline = Instant::now() + STARTUP_TIMEOUT;
73    loop {
74        match send_existing(&project, request.clone()) {
75            Ok(response) => {
76                return Ok(ClientResponse {
77                    response,
78                    started: true,
79                });
80            }
81            Err(error) if Instant::now() >= deadline => return Err(error),
82            Err(_) => std::thread::sleep(Duration::from_millis(20)),
83        }
84    }
85}
86
87/// Sends a request only if a daemon is already listening; `Ok(None)` means
88/// no daemon. Never spawns one.
89pub fn request_running(request: Request) -> Result<Option<ResponseEnvelope>, DaemonError> {
90    let cwd = std::env::current_dir().map_err(DaemonError::CurrentDirectory)?;
91    let project = Project::discover(&cwd)?;
92    Config::load(&project)?;
93    match send_existing(&project, request) {
94        Ok(response) => Ok(Some(response)),
95        Err(DaemonError::Connect(_)) => Ok(None),
96        Err(error) => Err(error),
97    }
98}
99
100pub fn serve_current_project() -> Result<(), DaemonError> {
101    let cwd = std::env::current_dir().map_err(DaemonError::CurrentDirectory)?;
102    let project = Project::discover(&cwd)?;
103    let config = Config::load(&project)?;
104    fs::create_dir_all(&project.state_dir).map_err(|source| DaemonError::Io {
105        path: project.state_dir.clone(),
106        source,
107    })?;
108    fs::set_permissions(&project.state_dir, fs::Permissions::from_mode(0o700)).map_err(
109        |source| DaemonError::Io {
110            path: project.state_dir.clone(),
111            source,
112        },
113    )?;
114    let runtime_root = project
115        .runtime_dir
116        .parent()
117        .expect("repository runtime directories have a parent");
118    fs::create_dir_all(runtime_root).map_err(|source| DaemonError::Io {
119        path: runtime_root.to_path_buf(),
120        source,
121    })?;
122    fs::set_permissions(runtime_root, fs::Permissions::from_mode(0o700)).map_err(|source| {
123        DaemonError::Io {
124            path: runtime_root.to_path_buf(),
125            source,
126        }
127    })?;
128    fs::create_dir(&project.runtime_dir)
129        .or_else(|source| {
130            if source.kind() == io::ErrorKind::AlreadyExists {
131                Ok(())
132            } else {
133                Err(source)
134            }
135        })
136        .map_err(|source| DaemonError::Io {
137            path: project.runtime_dir.clone(),
138            source,
139        })?;
140    fs::set_permissions(&project.runtime_dir, fs::Permissions::from_mode(0o700)).map_err(
141        |source| DaemonError::Io {
142            path: project.runtime_dir.clone(),
143            source,
144        },
145    )?;
146
147    let lock = OpenOptions::new()
148        .create(true)
149        .truncate(false)
150        .read(true)
151        .write(true)
152        .open(&project.lock_path)
153        .map_err(|source| DaemonError::Io {
154            path: project.lock_path.clone(),
155            source,
156        })?;
157    lock.try_lock_exclusive().map_err(|source| {
158        if source.kind() == io::ErrorKind::WouldBlock {
159            DaemonError::AlreadyRunning
160        } else {
161            DaemonError::Io {
162                path: project.lock_path.clone(),
163                source,
164            }
165        }
166    })?;
167    // Hold the pre-v7 runtime lock as well during the lock-location
168    // transition, preventing an already-running older daemon in this runtime
169    // root from sharing the database with the new process.
170    let legacy_lock_path = project.runtime_dir.join("daemon.lock");
171    let legacy_lock = OpenOptions::new()
172        .create(true)
173        .truncate(false)
174        .read(true)
175        .write(true)
176        .open(&legacy_lock_path)
177        .map_err(|source| DaemonError::Io {
178            path: legacy_lock_path.clone(),
179            source,
180        })?;
181    legacy_lock.try_lock_exclusive().map_err(|source| {
182        if source.kind() == io::ErrorKind::WouldBlock {
183            DaemonError::AlreadyRunning
184        } else {
185            DaemonError::Io {
186                path: legacy_lock_path,
187                source,
188            }
189        }
190    })?;
191    // Identity is advisory; the flock is the guard, so write errors are
192    // ignored rather than fatal.
193    let identity = json!({
194        "pid": std::process::id(),
195        "started_at_ms": process_start_time(std::process::id()),
196        "socket": project.operator_socket,
197    });
198    let _ = lock.set_len(0);
199    let _ = {
200        use std::io::Write as _;
201        writeln!(&lock, "{identity}")
202    };
203
204    let clock: Arc<dyn Clock> = match std::env::var_os("SLOOP_TEST_CLOCK_PATH") {
205        Some(path) => Arc::new(FileClock::new(path.into())),
206        None => Arc::new(SystemClock),
207    };
208    let store = Store::open(&project.db_path, clock.now_ms()).map_err(DaemonError::Store)?;
209    if let Some(agent) = &config.agent {
210        store
211            .backfill_ticket_targets(&agent.default_target, clock.now_ms())
212            .map_err(DaemonError::Store)?;
213    }
214    index_projects(
215        &project.root,
216        &config.project_dir,
217        &store,
218        clock.now_ms(),
219        &config.project_prefix,
220    )?;
221    reconcile_tickets(
222        &project.root,
223        &store,
224        clock.now_ms(),
225        config.delete_missing_after_ms,
226    )?;
227
228    let runtime = tokio::runtime::Builder::new_multi_thread()
229        .enable_all()
230        .build()
231        .map_err(DaemonError::Runtime)?;
232    runtime.block_on(serve(project, config, store, lock, legacy_lock, clock))
233}
234
235#[derive(Debug, Clone, Copy, PartialEq, Eq)]
236pub(crate) enum OrphanDisposition {
237    MarkMissing,
238    Delete,
239    Keep,
240}
241
242/// Policy for a DB ticket whose file has disappeared: stamp first, delete
243/// once the stamp outlives the grace window, and never delete a row
244/// something else still references — such rows just stay stamped.
245fn orphan_disposition(
246    missing_since_ms: Option<i64>,
247    is_referenced: bool,
248    now_ms: i64,
249    delete_missing_after_ms: i64,
250) -> OrphanDisposition {
251    match missing_since_ms {
252        None => OrphanDisposition::MarkMissing,
253        Some(since) if now_ms - since >= delete_missing_after_ms && !is_referenced => {
254            OrphanDisposition::Delete
255        }
256        Some(_) => OrphanDisposition::Keep,
257    }
258}
259
260/// Reconciles local ticket rows against their committed files. Runs at
261/// startup; `reindex` will share it.
262fn reconcile_tickets(
263    root: &Path,
264    store: &Store,
265    now_ms: i64,
266    delete_missing_after_ms: i64,
267) -> Result<(), DaemonError> {
268    for ticket in store.local_ticket_files().map_err(DaemonError::Store)? {
269        if root.join(&ticket.file_path).is_file() {
270            if ticket.missing_at_ms.is_some() {
271                store
272                    .clear_ticket_missing(&ticket.id, now_ms)
273                    .map_err(DaemonError::Store)?;
274            }
275            continue;
276        }
277        let is_referenced = store
278            .ticket_is_referenced(&ticket.id)
279            .map_err(DaemonError::Store)?;
280        match orphan_disposition(
281            ticket.missing_at_ms,
282            is_referenced,
283            now_ms,
284            delete_missing_after_ms,
285        ) {
286            OrphanDisposition::MarkMissing => {
287                store
288                    .mark_ticket_missing(&ticket.id, now_ms)
289                    .map_err(DaemonError::Store)?;
290            }
291            OrphanDisposition::Delete => {
292                store
293                    .delete_ticket(&ticket.id)
294                    .map_err(DaemonError::Store)?;
295            }
296            OrphanDisposition::Keep => {}
297        }
298    }
299    Ok(())
300}
301
302/// Indexes committed project Markdown files into the store so ticket membership
303/// can be validated. Runs at startup; `reindex` will share it.
304fn index_projects(
305    root: &Path,
306    project_dir: &Path,
307    store: &Store,
308    now_ms: i64,
309    project_prefix: &str,
310) -> Result<(), DaemonError> {
311    let directory = root.join(project_dir);
312    let entries = match fs::read_dir(&directory) {
313        Ok(entries) => entries,
314        Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
315        Err(source) => {
316            return Err(DaemonError::Io {
317                path: directory,
318                source,
319            });
320        }
321    };
322
323    let mut paths = Vec::new();
324    for entry in entries {
325        let path = entry
326            .map_err(|source| DaemonError::Io {
327                path: directory.clone(),
328                source,
329            })?
330            .path();
331        if path.extension().and_then(|extension| extension.to_str()) == Some("md") {
332            paths.push(path);
333        }
334    }
335    paths.sort();
336
337    struct ProjectFile {
338        path: PathBuf,
339        content: String,
340        stem: String,
341        frontmatter: Frontmatter,
342    }
343
344    let mut projects = Vec::new();
345    for path in paths {
346        let content = fs::read_to_string(&path).map_err(|source| DaemonError::Io {
347            path: path.clone(),
348            source,
349        })?;
350        let stem = path
351            .file_stem()
352            .map(|stem| stem.to_string_lossy().into_owned())
353            .unwrap_or_default();
354        // A malformed project file must not keep the daemon from starting;
355        // it is simply not indexed until fixed.
356        let Ok(frontmatter) = crate::frontmatter::parse(&content) else {
357            continue;
358        };
359        projects.push(ProjectFile {
360            path,
361            content,
362            stem,
363            frontmatter,
364        });
365    }
366
367    // Explicit IDs in every file establish the high-water mark before sorted
368    // idless files are assigned, regardless of where those explicit files sort.
369    let mut ids: Vec<String> = projects
370        .iter()
371        .filter_map(|project| project.frontmatter.id.clone())
372        .collect();
373    for project in projects {
374        let id = match project.frontmatter.id {
375            Some(id) => id,
376            None => {
377                let id = next_id(project_prefix, ids.iter().map(String::as_str))?;
378                let updated =
379                    crate::frontmatter::stamp_id(&project.content, &id).map_err(|error| {
380                        DaemonError::Frontmatter {
381                            path: project.path.clone(),
382                            error,
383                        }
384                    })?;
385                fs::write(
386                    &project.path,
387                    updated.expect("idless project always needs an ID stamp"),
388                )
389                .map_err(|source| DaemonError::Io {
390                    path: project.path.clone(),
391                    source,
392                })?;
393                ids.push(id.clone());
394                id
395            }
396        };
397        let title = project.frontmatter.title.unwrap_or(project.stem);
398        let relative = project
399            .path
400            .strip_prefix(root)
401            .unwrap_or(&project.path)
402            .to_string_lossy()
403            .into_owned();
404        store
405            .upsert_local_project(&id, &relative, &title, now_ms)
406            .map_err(DaemonError::Store)?;
407    }
408    Ok(())
409}
410
411async fn serve(
412    project: Project,
413    config: Config,
414    store: Store,
415    _lock: fs::File,
416    _legacy_lock: fs::File,
417    clock: Arc<dyn Clock>,
418) -> Result<(), DaemonError> {
419    if project.operator_socket.exists() {
420        fs::remove_file(&project.operator_socket).map_err(|source| DaemonError::Io {
421            path: project.operator_socket.clone(),
422            source,
423        })?;
424    }
425
426    let listener =
427        UnixListener::bind(&project.operator_socket).map_err(|source| DaemonError::Io {
428            path: project.operator_socket.clone(),
429            source,
430        })?;
431    fs::set_permissions(&project.operator_socket, fs::Permissions::from_mode(0o600)).map_err(
432        |source| DaemonError::Io {
433            path: project.operator_socket.clone(),
434            source,
435        },
436    )?;
437
438    let log = OperationalLog::open(&project.daemon_log).map_err(|source| DaemonError::Io {
439        path: project.daemon_log.clone(),
440        source,
441    })?;
442    log.emit(LogLevel::Info, "sloop::daemon", "daemon_started");
443
444    let paused = store.paused().map_err(DaemonError::Store)?;
445    let (dispatcher_tx, dispatcher_rx) = mpsc::channel(DISPATCH_CHANNEL_CAPACITY);
446    let (events_tx, events_rx) = mpsc::channel(DISPATCH_CHANNEL_CAPACITY);
447    let (shutdown_tx, mut shutdown_rx) = mpsc::channel::<()>(1);
448    let shutdown_flag = Arc::new(AtomicBool::new(false));
449    let mut state = DispatcherState {
450        pid: std::process::id(),
451        paused,
452        max_agents: config.max_parallel_tasks,
453        ticket_prefix: config.ticket_prefix.clone(),
454        running_hours: config.running_hours.clone(),
455        agent: config.agent.clone(),
456        flows: config.flows.clone(),
457        default_flow: config.default_flow.clone(),
458        aftercare_test_cmd: config.aftercare_test_cmd.clone(),
459        root: project.root.clone(),
460        ticket_dir: config.ticket_dir.clone(),
461        worktree_dir: project.root.join(&config.worktree_dir),
462        state_dir: project.state_dir.clone(),
463        runtime_dir: project.runtime_dir.clone(),
464        socket: project.operator_socket.clone(),
465        daemon_log: project.daemon_log.clone(),
466        store,
467        active: HashSet::new(),
468        cancelling: HashSet::new(),
469        worker_tokens: HashMap::new(),
470        worker_listeners: HashMap::new(),
471        worker_socket_paths: HashMap::new(),
472        pending_exits: HashMap::new(),
473        requests_tx: dispatcher_tx.clone(),
474        log: log.clone(),
475        clock,
476        shutdown: shutdown_tx.clone(),
477        shutdown_flag: shutdown_flag.clone(),
478    };
479    recover_inflight_runs(&mut state, &events_tx, &log)?;
480    tokio::spawn(run_dispatcher(
481        state,
482        dispatcher_rx,
483        events_rx,
484        events_tx,
485        log.clone(),
486    ));
487
488    loop {
489        tokio::select! {
490            accepted = listener.accept() => {
491                let (stream, _) = accepted.map_err(|source| DaemonError::Io {
492                    path: project.operator_socket.clone(),
493                    source,
494                })?;
495                let dispatcher_tx = dispatcher_tx.clone();
496                let log = log.clone();
497                let shutdown = shutdown_tx.clone();
498                tokio::spawn(async move {
499                    if let Err(error) = handle_connection(stream, dispatcher_tx, shutdown).await {
500                        log.emit_with_fields(
501                            LogLevel::Error,
502                            "sloop::socket",
503                            "connection_failed",
504                            json!({"error": error.to_string()}),
505                        );
506                    }
507                });
508            }
509            _ = shutdown_rx.recv() => {
510                shutdown_flag.store(true, Ordering::Release);
511                log.emit(LogLevel::Info, "sloop::daemon", "daemon_stopped");
512                let _ = fs::remove_file(&project.operator_socket);
513                return Ok(());
514            }
515        }
516    }
517}
518
519async fn handle_connection(
520    stream: UnixStream,
521    dispatcher: mpsc::Sender<DispatcherMessage>,
522    shutdown: mpsc::Sender<()>,
523) -> io::Result<()> {
524    let reader = AsyncBufReader::new(stream);
525    let mut limited = reader.take(MAX_ENVELOPE_BYTES + 1);
526    let mut bytes = Vec::new();
527    let read = limited.read_until(b'\n', &mut bytes).await?;
528    if read == 0 {
529        return Ok(());
530    }
531
532    let mut stream = limited.into_inner().into_inner();
533    let envelope = if bytes.len() as u64 > MAX_ENVELOPE_BYTES {
534        Err(protocol_error("request envelope is too large"))
535    } else {
536        std::str::from_utf8(&bytes)
537            .map_err(|_| protocol_error("request envelope must be UTF-8"))
538            .and_then(|line| RequestEnvelope::decode(line.trim_end()).map_err(|error| error.body))
539    };
540
541    let is_stop = matches!(
542        &envelope,
543        Ok(envelope) if matches!(envelope.request, Request::Stop(_))
544    );
545    let response = match envelope {
546        Ok(envelope) if envelope.token.is_some() => ResponseEnvelope::failure(
547            Some(envelope.id),
548            unauthorized("operator socket does not accept worker tokens"),
549        ),
550        Ok(envelope)
551            if !matches!(
552                envelope.request.capability(),
553                Capability::Operator | Capability::Both
554            ) =>
555        {
556            ResponseEnvelope::failure(
557                Some(envelope.id),
558                unauthorized("worker verbs are not available on the operator socket"),
559            )
560        }
561        Ok(envelope) => dispatch_envelope(envelope, RequestOrigin::Operator, &dispatcher).await,
562        Err(error) => ResponseEnvelope::failure(None, error),
563    };
564
565    // The reply must be flushed before the daemon exits, so the connection
566    // handler owns the shutdown signal for an accepted stop.
567    let stopping = is_stop && response.ok;
568    let encoded = serde_json::to_vec(&response).map_err(io::Error::other)?;
569    stream.write_all(&encoded).await?;
570    stream.write_all(b"\n").await?;
571    stream.shutdown().await?;
572    if stopping {
573        let _ = shutdown.send(()).await;
574    }
575    Ok(())
576}
577
578/// Reads one request line from a per-run worker socket, enforces the verb
579/// split at the boundary, and funnels the request through the dispatcher
580/// with the presented token for validation against the run's issued one.
581async fn handle_worker_connection(
582    stream: UnixStream,
583    run_id: String,
584    dispatcher: mpsc::Sender<DispatcherMessage>,
585) -> io::Result<()> {
586    let reader = AsyncBufReader::new(stream);
587    let mut limited = reader.take(MAX_ENVELOPE_BYTES + 1);
588    let mut bytes = Vec::new();
589    let read = limited.read_until(b'\n', &mut bytes).await?;
590    if read == 0 {
591        return Ok(());
592    }
593
594    let mut stream = limited.into_inner().into_inner();
595    let envelope = if bytes.len() as u64 > MAX_ENVELOPE_BYTES {
596        Err(protocol_error("request envelope is too large"))
597    } else {
598        std::str::from_utf8(&bytes)
599            .map_err(|_| protocol_error("request envelope must be UTF-8"))
600            .and_then(|line| RequestEnvelope::decode(line.trim_end()).map_err(|error| error.body))
601    };
602
603    let response = match envelope {
604        Ok(envelope)
605            if !matches!(
606                envelope.request.capability(),
607                Capability::Worker | Capability::Both
608            ) =>
609        {
610            ResponseEnvelope::failure(
611                Some(envelope.id),
612                unauthorized("operator verbs are not available on a worker socket"),
613            )
614        }
615        Ok(envelope) => {
616            let token = envelope.token.clone();
617            dispatch_envelope(
618                envelope,
619                RequestOrigin::Worker { run_id, token },
620                &dispatcher,
621            )
622            .await
623        }
624        Err(error) => ResponseEnvelope::failure(None, error),
625    };
626
627    let encoded = serde_json::to_vec(&response).map_err(io::Error::other)?;
628    stream.write_all(&encoded).await?;
629    stream.write_all(b"\n").await?;
630    stream.shutdown().await
631}
632
633async fn dispatch_envelope(
634    envelope: RequestEnvelope,
635    origin: RequestOrigin,
636    dispatcher: &mpsc::Sender<DispatcherMessage>,
637) -> ResponseEnvelope {
638    let (reply_tx, reply_rx) = oneshot::channel();
639    let id = envelope.id;
640    if dispatcher
641        .send(DispatcherMessage {
642            id: id.clone(),
643            request: envelope.request,
644            origin,
645            reply: reply_tx,
646        })
647        .await
648        .is_err()
649    {
650        ResponseEnvelope::failure(Some(id), internal("dispatcher is unavailable"))
651    } else {
652        reply_rx.await.unwrap_or_else(|_| {
653            ResponseEnvelope::failure(Some(id), internal("dispatcher dropped response"))
654        })
655    }
656}
657
658struct DispatcherMessage {
659    id: RequestId,
660    request: Request,
661    origin: RequestOrigin,
662    reply: oneshot::Sender<ResponseEnvelope>,
663}
664
665/// Which socket a request arrived on. Worker requests carry the run whose
666/// socket accepted the connection plus the token the caller presented; the
667/// dispatcher owns the comparison against the run's issued token.
668enum RequestOrigin {
669    Operator,
670    Worker {
671        run_id: String,
672        token: Option<String>,
673    },
674}
675
676struct DispatcherState {
677    pid: u32,
678    paused: bool,
679    max_agents: usize,
680    ticket_prefix: String,
681    running_hours: Option<RunningHours>,
682    agent: Option<AgentConfig>,
683    flows: BTreeMap<String, Flow>,
684    default_flow: String,
685    aftercare_test_cmd: Option<Vec<String>>,
686    root: PathBuf,
687    ticket_dir: PathBuf,
688    worktree_dir: PathBuf,
689    state_dir: PathBuf,
690    runtime_dir: PathBuf,
691    socket: PathBuf,
692    daemon_log: PathBuf,
693    store: Store,
694    /// Run IDs with a live supervised process; its size is the capacity gate.
695    active: HashSet<String>,
696    /// Run IDs whose cancellation was requested but whose exit has not been
697    /// resolved yet; mirrors the durable `cancel_requested` evidence.
698    cancelling: HashSet<String>,
699    /// Tokens issued to live runs; a worker request must present its run's
700    /// token exactly. Entries die with the run.
701    worker_tokens: HashMap<String, String>,
702    /// Accept-loop tasks for live per-run worker sockets, aborted at settle.
703    worker_listeners: HashMap<String, tokio::task::JoinHandle<()>>,
704    worker_socket_paths: HashMap<String, PathBuf>,
705    /// Exit evidence remains here until its atomic store transaction commits.
706    /// The dispatcher retries these records on every reconciliation pass.
707    pending_exits: HashMap<String, RunEvent>,
708    /// The dispatcher's own request channel, cloned into each worker
709    /// accept loop so every request funnels through the single owner.
710    requests_tx: mpsc::Sender<DispatcherMessage>,
711    log: OperationalLog,
712    clock: Arc<dyn Clock>,
713    /// Signals the accept loop to end the process; used by daemon-side
714    /// exits such as the project-root liveness check.
715    shutdown: mpsc::Sender<()>,
716    shutdown_flag: Arc<AtomicBool>,
717}
718
719/// One executed test stage as observed by the supervisor.
720struct StageResult {
721    outcome: StageOutcome,
722    exit_code: Option<i32>,
723    started_at_ms: i64,
724    finished_at_ms: i64,
725}
726
727/// Internal dispatcher events reported by effect tasks, never by clients.
728enum RunEvent {
729    Exited {
730        run_id: String,
731        exit_code: Option<i32>,
732        /// False when a pipe reader failed to durably record every chunk;
733        /// the loss becomes explicit run evidence instead of silence.
734        capture_complete: bool,
735        /// Commits on the run branch that were not on the default branch at
736        /// agent exit, before aftercare could fast-forward the default branch.
737        commits: Vec<String>,
738        tests: Option<StageResult>,
739        merge: Option<MergeOutcome>,
740        recovery: Option<RecoveryClassification>,
741    },
742}
743
744#[derive(Debug, Clone, Copy, PartialEq, Eq)]
745enum RecoveryClassification {
746    Aftercare,
747    Orphaned,
748}
749
750async fn run_dispatcher(
751    mut state: DispatcherState,
752    mut requests: mpsc::Receiver<DispatcherMessage>,
753    mut events: mpsc::Receiver<RunEvent>,
754    events_tx: mpsc::Sender<RunEvent>,
755    log: OperationalLog,
756) {
757    reconcile(&mut state, &events_tx, &log);
758    loop {
759        let deadline = next_dispatch_deadline(&state);
760        let clock = state.clock.clone();
761        tokio::select! {
762            message = requests.recv() => {
763                let Some(message) = message else { break };
764                let response = match message.origin {
765                    RequestOrigin::Operator => dispatch(&mut state, message.id, message.request),
766                    RequestOrigin::Worker { run_id, token } => dispatch_worker(
767                        &mut state,
768                        message.id,
769                        message.request,
770                        &run_id,
771                        token.as_deref(),
772                    ),
773                };
774                let _ = message.reply.send(response);
775                log.emit(LogLevel::Info, "sloop::dispatcher", "request_handled");
776            }
777            event = events.recv() => {
778                let Some(event) = event else { break };
779                settle_run_exit(&mut state, event, &log);
780            }
781            () = wait_for_deadline(clock, deadline) => {
782                log.emit(LogLevel::Info, "sloop::dispatcher", "timer_fired");
783            }
784            // Wall-clock is deliberate: this is a liveness probe, not
785            // decision logic, so the manual test clock must not gate it.
786            () = tokio::time::sleep(std::time::Duration::from_secs(2)) => {
787                if !state.root.join(".git").exists() {
788                    log.emit(LogLevel::Error, "sloop::dispatcher", "project_root_missing");
789                    let _ = state.shutdown.send(()).await;
790                    break;
791                }
792            }
793        }
794        reconcile(&mut state, &events_tx, &log);
795    }
796}
797
798async fn wait_for_deadline(clock: Arc<dyn Clock>, deadline: Option<i64>) {
799    match deadline {
800        Some(deadline) => clock.sleep_until(deadline).await,
801        None => std::future::pending().await,
802    }
803}
804
805/// Resolves one finished run: derives the outcome from the gathered evidence
806/// and commits the whole settlement in one store transaction. Cancellation
807/// intent recorded before the exit wins over every other reading, keeping a
808/// racing `cancel` and natural exit idempotent.
809fn settle_run_exit(state: &mut DispatcherState, event: RunEvent, log: &OperationalLog) {
810    let run_id = match &event {
811        RunEvent::Exited { run_id, .. } => run_id.clone(),
812    };
813    state.pending_exits.insert(run_id, event);
814    settle_pending_exits(state, log);
815}
816
817fn settle_pending_exits(state: &mut DispatcherState, log: &OperationalLog) {
818    let run_ids: Vec<String> = state.pending_exits.keys().cloned().collect();
819    for run_id in run_ids {
820        let Some(event) = state.pending_exits.remove(&run_id) else {
821            continue;
822        };
823        match try_settle_run_exit(state, &event) {
824            Ok(outcome) => {
825                state.cancelling.remove(&run_id);
826                state.active.remove(&run_id);
827                close_worker_socket(state, &run_id);
828                log.emit_with_fields(
829                    LogLevel::Info,
830                    "sloop::dispatcher",
831                    "run_exited",
832                    json!({"run_id": run_id, "outcome": outcome.as_str()}),
833                );
834            }
835            Err(error) => {
836                log.emit_with_fields(
837                    LogLevel::Error,
838                    "sloop::dispatcher",
839                    "run_exit_persist_failed",
840                    json!({"run_id": run_id, "error": error.to_string()}),
841                );
842                state.pending_exits.insert(run_id, event);
843            }
844        }
845    }
846}
847
848fn try_settle_run_exit(
849    state: &mut DispatcherState,
850    event: &RunEvent,
851) -> Result<crate::outcome::Outcome, StoreError> {
852    let RunEvent::Exited {
853        run_id,
854        exit_code,
855        capture_complete,
856        commits,
857        tests,
858        merge,
859        recovery,
860    } = event;
861
862    let cancelled =
863        state.cancelling.contains(run_id) || state.store.cancellation_requested(run_id)?;
864    let commit_count = commits.len() as u64;
865    let evidence = RunEvidence {
866        cancelled,
867        exit: classify_exit(*exit_code),
868        commit_count,
869        tests: tests.as_ref().map(|stage| stage.outcome),
870        merge: *merge,
871    };
872    let outcome = if *recovery == Some(RecoveryClassification::Orphaned) && !cancelled {
873        crate::outcome::Outcome::Orphaned
874    } else {
875        derive_outcome(&evidence)
876    };
877
878    let mut records = vec![
879        EvidenceRecord {
880            kind: "exit_classified",
881            data_json: json!({"exit_code": exit_code}).to_string(),
882        },
883        EvidenceRecord {
884            kind: "commits_observed",
885            data_json: json!({"count": commit_count, "oids": commits}).to_string(),
886        },
887    ];
888    if let Some(classification) = recovery {
889        records.push(EvidenceRecord {
890            kind: "recovery_classified",
891            data_json: json!({
892                "classification": match classification {
893                    RecoveryClassification::Aftercare => "aftercare",
894                    RecoveryClassification::Orphaned => "orphaned",
895                }
896            })
897            .to_string(),
898        });
899    }
900    if let Some(stage) = &tests {
901        records.push(EvidenceRecord {
902            kind: "test_result",
903            data_json: json!({
904                "passed": stage.outcome == StageOutcome::Passed,
905                "exit_code": stage.exit_code,
906            })
907            .to_string(),
908        });
909    }
910    if let Some(merge) = *merge {
911        records.push(EvidenceRecord {
912            kind: "merge_result",
913            data_json: json!({"merged": merge == MergeOutcome::Merged}).to_string(),
914        });
915    }
916    if !capture_complete {
917        records.push(EvidenceRecord {
918            kind: "capture_incomplete",
919            data_json: json!({}).to_string(),
920        });
921    }
922    let stage_row = tests.as_ref().map(|stage| StageRecord {
923        stage: "test",
924        state: match stage.outcome {
925            StageOutcome::Passed => "passed",
926            StageOutcome::Failed => "failed",
927        },
928        started_at_ms: stage.started_at_ms,
929        finished_at_ms: stage.finished_at_ms,
930        exit_code: stage.exit_code,
931    });
932
933    let ticket_id = state
934        .store
935        .run(run_id)?
936        .ok_or_else(|| StoreError::RunNotFound {
937            run_id: run_id.clone(),
938        })?
939        .ticket_id;
940    state.store.finish_run(
941        run_id,
942        &ticket_id,
943        *exit_code,
944        outcome,
945        &records,
946        stage_row.as_ref(),
947        state.clock.now_ms(),
948    )?;
949    Ok(outcome)
950}
951
952/// Tears down a run's worker boundary: the token stops validating, the
953/// accept loop ends, and the socket file disappears. Idempotent, so crash
954/// recovery and racing settlements can call it freely.
955fn close_worker_socket(state: &mut DispatcherState, run_id: &str) {
956    state.worker_tokens.remove(run_id);
957    if let Some(listener) = state.worker_listeners.remove(run_id) {
958        listener.abort();
959    }
960    let socket_path = state
961        .worker_socket_paths
962        .remove(run_id)
963        .unwrap_or_else(|| worker_socket_path(&state.runtime_dir, run_id));
964    let _ = fs::remove_file(socket_path);
965}
966
967/// Classifies every durable lease before normal dispatch. Matching processes
968/// consume capacity and are monitored by identity; dead or reused PIDs are
969/// settled from the work preserved in their branches.
970fn recover_inflight_runs(
971    state: &mut DispatcherState,
972    events: &mpsc::Sender<RunEvent>,
973    log: &OperationalLog,
974) -> Result<(), DaemonError> {
975    let runs = state.store.recoverable_runs().map_err(DaemonError::Store)?;
976    for run in runs {
977        // Every durable lease consumes capacity until adoption or settlement
978        // succeeds; a transient database error must never permit double-spawn.
979        state.active.insert(run.id.clone());
980        if recoverable_process_matches(&run) {
981            let cancellation_requested = state
982                .store
983                .cancellation_requested(&run.id)
984                .map_err(DaemonError::Store)?;
985            if cancellation_requested {
986                state.cancelling.insert(run.id.clone());
987                if recoverable_process_matches(&run)
988                    && let Some(group) = run.process_group_id
989                {
990                    unsafe {
991                        libc::kill(-(group as libc::pid_t), libc::SIGKILL);
992                    }
993                }
994            }
995            if let Err(error) = restore_worker_socket(state, &run) {
996                log.emit_with_fields(
997                    LogLevel::Error,
998                    "sloop::recovery",
999                    "worker_socket_restore_failed",
1000                    json!({"run_id": run.id, "error": error}),
1001                );
1002            }
1003            monitor_recovered_run(state, events.clone(), run.clone());
1004            log.emit_with_fields(
1005                LogLevel::Info,
1006                "sloop::recovery",
1007                "run_readopted",
1008                json!({"run_id": run.id, "ticket_id": run.ticket_id}),
1009            );
1010        } else {
1011            if run.state == "aftercare" {
1012                spawn_aftercare_recovery(state, events.clone(), run, log.clone());
1013            } else {
1014                spawn_dead_run_recovery(state, events.clone(), run, log.clone());
1015            }
1016        }
1017    }
1018    Ok(())
1019}
1020
1021fn restore_worker_socket(state: &mut DispatcherState, run: &RecoverableRun) -> Result<(), String> {
1022    let token = run
1023        .worker_token
1024        .as_ref()
1025        .ok_or_else(|| "the persisted run has no worker token".to_owned())?;
1026    let socket_path = run
1027        .worker_socket_path
1028        .as_deref()
1029        .map(PathBuf::from)
1030        .unwrap_or_else(|| worker_socket_path(&state.runtime_dir, &run.id));
1031    fs::create_dir_all(socket_path.parent().expect("worker sockets have a parent"))
1032        .map_err(|error| error.to_string())?;
1033    let _ = fs::remove_file(&socket_path);
1034    let listener = UnixListener::bind(&socket_path).map_err(|error| error.to_string())?;
1035    fs::set_permissions(&socket_path, fs::Permissions::from_mode(0o600))
1036        .map_err(|error| error.to_string())?;
1037    state.worker_tokens.insert(run.id.clone(), token.clone());
1038    state
1039        .worker_socket_paths
1040        .insert(run.id.clone(), socket_path.clone());
1041    let accept_loop = tokio::spawn(serve_worker_socket(
1042        listener,
1043        run.id.clone(),
1044        state.requests_tx.clone(),
1045        state.log.clone(),
1046    ));
1047    state.worker_listeners.insert(run.id.clone(), accept_loop);
1048    Ok(())
1049}
1050
1051fn monitor_recovered_run(
1052    state: &DispatcherState,
1053    events: mpsc::Sender<RunEvent>,
1054    run: RecoverableRun,
1055) {
1056    let root = state.root.clone();
1057    let log = state.log.clone();
1058    let shutdown = state.shutdown_flag.clone();
1059    tokio::task::spawn_blocking(move || {
1060        while recoverable_process_matches(&run) {
1061            if shutdown.load(Ordering::Acquire) {
1062                return;
1063            }
1064            std::thread::sleep(Duration::from_millis(100));
1065        }
1066        while !shutdown.load(Ordering::Acquire) {
1067            match recovered_exit_event(&root, &run) {
1068                Ok(event) => {
1069                    let _ = events.blocking_send(event);
1070                    break;
1071                }
1072                Err(error) => {
1073                    log.emit_with_fields(
1074                        LogLevel::Error,
1075                        "sloop::recovery",
1076                        "run_observation_failed",
1077                        json!({"run_id": run.id, "error": error}),
1078                    );
1079                    std::thread::sleep(Duration::from_secs(1));
1080                }
1081            }
1082        }
1083    });
1084}
1085
1086fn spawn_dead_run_recovery(
1087    state: &DispatcherState,
1088    events: mpsc::Sender<RunEvent>,
1089    run: RecoverableRun,
1090    log: OperationalLog,
1091) {
1092    let root = state.root.clone();
1093    let shutdown = state.shutdown_flag.clone();
1094    tokio::task::spawn_blocking(move || {
1095        while !shutdown.load(Ordering::Acquire) {
1096            match recovered_exit_event(&root, &run) {
1097                Ok(event) => {
1098                    let _ = events.blocking_send(event);
1099                    break;
1100                }
1101                Err(error) => {
1102                    log.emit_with_fields(
1103                        LogLevel::Error,
1104                        "sloop::recovery",
1105                        "run_observation_failed",
1106                        json!({"run_id": run.id, "error": error}),
1107                    );
1108                    std::thread::sleep(Duration::from_secs(1));
1109                }
1110            }
1111        }
1112    });
1113}
1114
1115fn recovered_exit_event(root: &Path, run: &RecoverableRun) -> Result<RunEvent, String> {
1116    let commits = run
1117        .branch
1118        .as_deref()
1119        .map(|branch| try_commits_on_branch(root, branch))
1120        .transpose()?
1121        .unwrap_or_default();
1122    let recovery = if commits.is_empty() {
1123        RecoveryClassification::Orphaned
1124    } else {
1125        RecoveryClassification::Aftercare
1126    };
1127    Ok(RunEvent::Exited {
1128        run_id: run.id.clone(),
1129        exit_code: None,
1130        capture_complete: false,
1131        commits,
1132        tests: None,
1133        merge: None,
1134        recovery: Some(recovery),
1135    })
1136}
1137
1138fn spawn_aftercare_recovery(
1139    state: &DispatcherState,
1140    events: mpsc::Sender<RunEvent>,
1141    run: RecoverableRun,
1142    log: OperationalLog,
1143) {
1144    let root = state.root.clone();
1145    let state_dir = state.state_dir.clone();
1146    let test_cmd = state.aftercare_test_cmd.clone();
1147    let clock = state.clock.clone();
1148    let db_path = state.state_dir.join("sloop.db");
1149    let shutdown = state.shutdown_flag.clone();
1150    tokio::task::spawn_blocking(move || {
1151        while !shutdown.load(Ordering::Acquire) {
1152            let result = Store::open(&db_path, clock.now_ms())
1153                .map_err(|error| error.to_string())
1154                .and_then(|store| {
1155                    resume_aftercare(
1156                        &root,
1157                        &state_dir,
1158                        test_cmd.as_deref(),
1159                        clock.as_ref(),
1160                        &store,
1161                        &run,
1162                        &log,
1163                    )
1164                });
1165            match result {
1166                Ok(event) => {
1167                    let _ = events.blocking_send(event);
1168                    break;
1169                }
1170                Err(error) => {
1171                    log.emit_with_fields(
1172                        LogLevel::Error,
1173                        "sloop::recovery",
1174                        "aftercare_resume_failed",
1175                        json!({"run_id": run.id, "error": error}),
1176                    );
1177                    std::thread::sleep(Duration::from_secs(1));
1178                }
1179            }
1180        }
1181    });
1182}
1183
1184#[allow(clippy::too_many_arguments)]
1185fn resume_aftercare(
1186    root: &Path,
1187    state_dir: &Path,
1188    test_cmd: Option<&[String]>,
1189    clock: &dyn Clock,
1190    store: &Store,
1191    run: &RecoverableRun,
1192    log: &OperationalLog,
1193) -> Result<RunEvent, String> {
1194    let rows = store
1195        .run_evidence(&run.id)
1196        .map_err(|error| error.to_string())?;
1197    let value = |kind: &str| {
1198        rows.iter()
1199            .find(|(candidate, _)| candidate == kind)
1200            .and_then(|(_, data)| serde_json::from_str::<serde_json::Value>(data).ok())
1201    };
1202    let commits = value("commits_observed")
1203        .and_then(|data| {
1204            data["oids"].as_array().map(|oids| {
1205                oids.iter()
1206                    .filter_map(|oid| oid.as_str().map(str::to_owned))
1207                    .collect::<Vec<_>>()
1208            })
1209        })
1210        .ok_or_else(|| "the aftercare checkpoint has no valid commit evidence".to_owned())?;
1211    let exit_code = run.exit_code.and_then(|code| i32::try_from(code).ok());
1212    let exit = classify_exit(exit_code);
1213    let output_log = RunLogWriter::open(&run_output_path(state_dir, &run.id))
1214        .map_err(|error| format!("cannot open run output: {error}"))?;
1215
1216    let mut tests = value("test_result").map(|data| StageResult {
1217        outcome: if data["passed"].as_bool() == Some(true) {
1218            StageOutcome::Passed
1219        } else {
1220            StageOutcome::Failed
1221        },
1222        exit_code: data["exit_code"]
1223            .as_i64()
1224            .and_then(|code| i32::try_from(code).ok()),
1225        started_at_ms: data["started_at_ms"]
1226            .as_i64()
1227            .unwrap_or_else(|| clock.now_ms()),
1228        finished_at_ms: data["finished_at_ms"]
1229            .as_i64()
1230            .unwrap_or_else(|| clock.now_ms()),
1231    });
1232    if aftercare_cancelled(store, &run.id, log) {
1233        return Ok(RunEvent::Exited {
1234            run_id: run.id.clone(),
1235            exit_code,
1236            capture_complete: !rows.iter().any(|(kind, _)| kind == "capture_incomplete"),
1237            commits,
1238            tests,
1239            merge: None,
1240            recovery: Some(RecoveryClassification::Aftercare),
1241        });
1242    }
1243    if tests.is_none()
1244        && wants_tests(exit, commits.len() as u64)
1245        && let Some(cmd) = test_cmd
1246    {
1247        stop_interrupted_process(&rows, "test_process")?;
1248        let worktree = run
1249            .worktree_path
1250            .as_deref()
1251            .ok_or_else(|| "the aftercare checkpoint has no worktree".to_owned())?;
1252        let stage = run_test_stage(
1253            Path::new(worktree),
1254            cmd,
1255            &output_log,
1256            clock,
1257            Some(store),
1258            &run.id,
1259            log,
1260        );
1261        store
1262            .record_aftercare_evidence(
1263                &run.id,
1264                "test_result",
1265                &test_result_json(&stage),
1266                clock.now_ms(),
1267            )
1268            .map_err(|error| error.to_string())?;
1269        tests = Some(stage);
1270    }
1271
1272    let mut merge = value("merge_result").map(|data| {
1273        if data["merged"].as_bool() == Some(true) {
1274            MergeOutcome::Merged
1275        } else {
1276            MergeOutcome::Diverged
1277        }
1278    });
1279    if !aftercare_cancelled(store, &run.id, log)
1280        && merge.is_none()
1281        && wants_merge(
1282            exit,
1283            commits.len() as u64,
1284            tests.as_ref().map(|stage| stage.outcome),
1285        )
1286        && let Some(branch) = run.branch.as_deref()
1287    {
1288        stop_interrupted_process(&rows, "merge_process")?;
1289        let outcome = attempt_merge(root, branch, Some(store), &run.id, clock, log);
1290        store
1291            .record_aftercare_evidence(
1292                &run.id,
1293                "merge_result",
1294                &merge_result_json(outcome),
1295                clock.now_ms(),
1296            )
1297            .map_err(|error| error.to_string())?;
1298        merge = Some(outcome);
1299    }
1300
1301    Ok(RunEvent::Exited {
1302        run_id: run.id.clone(),
1303        exit_code,
1304        capture_complete: !rows.iter().any(|(kind, _)| kind == "capture_incomplete"),
1305        commits,
1306        tests,
1307        merge,
1308        recovery: Some(RecoveryClassification::Aftercare),
1309    })
1310}
1311
1312fn stop_interrupted_process(rows: &[(String, String)], kind: &str) -> Result<(), String> {
1313    let Some((pid, start_time, group)) = aftercare_process_identity(rows, kind)? else {
1314        return Ok(());
1315    };
1316    if !process_identity_matches(pid, Some(start_time)) {
1317        return Ok(());
1318    }
1319    unsafe {
1320        libc::kill(-(group as libc::pid_t), libc::SIGKILL);
1321    }
1322    let deadline = Instant::now() + Duration::from_secs(5);
1323    while process_identity_matches(pid, Some(start_time)) {
1324        if Instant::now() >= deadline {
1325            return Err("the interrupted test process did not exit".into());
1326        }
1327        std::thread::sleep(Duration::from_millis(20));
1328    }
1329    Ok(())
1330}
1331
1332fn aftercare_process_identity(
1333    rows: &[(String, String)],
1334    kind: &str,
1335) -> Result<Option<(u32, i64, i64)>, String> {
1336    let Some(data) = rows
1337        .iter()
1338        .find(|(candidate, _)| candidate == kind)
1339        .and_then(|(_, data)| serde_json::from_str::<serde_json::Value>(data).ok())
1340    else {
1341        return Ok(None);
1342    };
1343    let pid = data["pid"]
1344        .as_u64()
1345        .and_then(|pid| u32::try_from(pid).ok())
1346        .ok_or_else(|| "the interrupted test has no valid pid".to_owned())?;
1347    let start_time = data["pid_start_time"]
1348        .as_i64()
1349        .ok_or_else(|| "the interrupted test has no valid start time".to_owned())?;
1350    let group = data["process_group_id"]
1351        .as_i64()
1352        .ok_or_else(|| "the interrupted test has no valid process group".to_owned())?;
1353    Ok(Some((pid, start_time, group)))
1354}
1355
1356fn recoverable_process_matches(run: &RecoverableRun) -> bool {
1357    if run.state != "running" {
1358        return false;
1359    }
1360    let Some(pid) = run.pid.and_then(|pid| u32::try_from(pid).ok()) else {
1361        return false;
1362    };
1363    process_identity_matches(pid, run.pid_start_time)
1364}
1365
1366fn process_identity_matches(pid: u32, expected_start_time: Option<i64>) -> bool {
1367    matches!(
1368        (expected_start_time, process_start_time(pid)),
1369        (Some(expected), Some(actual)) if expected == actual
1370    )
1371}
1372
1373fn worker_socket_path(runtime_dir: &Path, run_id: &str) -> PathBuf {
1374    runtime_dir.join("workers").join(format!("{run_id}.sock"))
1375}
1376
1377/// The single spawn decision point: every queued activation passes the same
1378/// pause and capacity gates, selects deterministically, claims conditionally,
1379/// and only then touches Git and processes.
1380fn reconcile(state: &mut DispatcherState, events: &mpsc::Sender<RunEvent>, log: &OperationalLog) {
1381    settle_pending_exits(state, log);
1382    let now_ms = state.clock.now_ms();
1383    if state.paused || state.agent.is_none() || !running_hours_open(state, now_ms) {
1384        return;
1385    }
1386    let activations = match state.store.queued_dispatchable_activations() {
1387        Ok(activations) => activations,
1388        Err(error) => {
1389            log.emit_with_fields(
1390                LogLevel::Error,
1391                "sloop::dispatcher",
1392                "activation_scan_failed",
1393                json!({"error": error.to_string()}),
1394            );
1395            return;
1396        }
1397    };
1398
1399    for activation in activations {
1400        if state.active.len() >= state.max_agents {
1401            break;
1402        }
1403        let Some(ticket_id) = eligible_ticket(&state.store, &activation) else {
1404            continue;
1405        };
1406
1407        let now_ms = state.clock.now_ms();
1408        let Ok(run_ordinal) = state.store.next_run_ordinal() else {
1409            continue;
1410        };
1411        let run_id = format!("R{run_ordinal}");
1412        let owner = format!("daemon-{}", state.pid);
1413        let claim = ClaimRequest {
1414            ticket_id: &ticket_id,
1415            run_id: &run_id,
1416            activation_id: &activation.id,
1417            owner_id: &owner,
1418            lease_ms: DEFAULT_LEASE_MS,
1419        };
1420        let claimed = match state.store.claim_ticket(&claim, now_ms) {
1421            Ok(claimed) => claimed,
1422            // Not ready right now; the activation stays queued for later.
1423            Err(StoreError::TicketNotReady { .. }) => continue,
1424            Err(error) => {
1425                log.emit_with_fields(
1426                    LogLevel::Error,
1427                    "sloop::dispatcher",
1428                    "claim_failed",
1429                    json!({
1430                        "activation_id": activation.id,
1431                        "ticket_id": ticket_id,
1432                        "run_id": run_id,
1433                        "error": error.to_string(),
1434                    }),
1435                );
1436                continue;
1437            }
1438        };
1439        // Immediate and auto activations are one-shot: producing a run
1440        // consumes them, even if the launch below fails.
1441        if let Err(error) = state.store.complete_activation(&activation.id, now_ms) {
1442            log.emit_with_fields(
1443                LogLevel::Error,
1444                "sloop::dispatcher",
1445                "activation_update_failed",
1446                json!({"activation_id": activation.id, "error": error.to_string()}),
1447            );
1448        }
1449
1450        match launch_agent(state, &run_id, &ticket_id, claimed.attempt) {
1451            Ok(launched) => {
1452                state.active.insert(run_id.clone());
1453                let events = events.clone();
1454                let exited_run = run_id.clone();
1455                let root = state.root.clone();
1456                let test_cmd = state.aftercare_test_cmd.clone();
1457                let clock = state.clock.clone();
1458                let supervisor_log = log.clone();
1459                let db_path = state.state_dir.join("sloop.db");
1460                let LaunchedRun {
1461                    mut child,
1462                    readers,
1463                    worktree,
1464                    branch,
1465                    output_log,
1466                    worker_listener,
1467                    worker_token,
1468                    worker_socket_path,
1469                } = launched;
1470                state.worker_tokens.insert(run_id.clone(), worker_token);
1471                state
1472                    .worker_socket_paths
1473                    .insert(run_id.clone(), worker_socket_path);
1474                let accept_loop = tokio::spawn(serve_worker_socket(
1475                    worker_listener,
1476                    run_id.clone(),
1477                    state.requests_tx.clone(),
1478                    state.log.clone(),
1479                ));
1480                state.worker_listeners.insert(run_id.clone(), accept_loop);
1481                let pid = child.id();
1482                tokio::task::spawn_blocking(move || {
1483                    let exit_code = match child.wait() {
1484                        Ok(status) => status.code(),
1485                        Err(error) => {
1486                            supervisor_log.emit_with_fields(
1487                                LogLevel::Error,
1488                                "sloop::supervisor",
1489                                "agent_wait_failed",
1490                                json!({"run_id": exited_run, "error": error.to_string()}),
1491                            );
1492                            None
1493                        }
1494                    };
1495                    // Capture must be complete on disk before the exit is
1496                    // reported; the readers end when the pipes close.
1497                    let capture_complete = readers
1498                        .into_iter()
1499                        .all(|reader| reader.join().unwrap_or(false));
1500                    let mut checkpoint_store = match Store::open(&db_path, clock.now_ms()) {
1501                        Ok(store) => Some(store),
1502                        Err(error) => {
1503                            supervisor_log.emit_with_fields(
1504                                LogLevel::Error,
1505                                "sloop::supervisor",
1506                                "aftercare_checkpoint_open_failed",
1507                                json!({"run_id": exited_run, "error": error.to_string()}),
1508                            );
1509                            None
1510                        }
1511                    };
1512                    let (commits, tests, merge) = gather_exit_evidence(
1513                        &exited_run,
1514                        &root,
1515                        &worktree,
1516                        &branch,
1517                        test_cmd.as_deref(),
1518                        clock.as_ref(),
1519                        &output_log,
1520                        exit_code,
1521                        capture_complete,
1522                        checkpoint_store.as_mut(),
1523                        &supervisor_log,
1524                    );
1525                    let _ = events.blocking_send(RunEvent::Exited {
1526                        run_id: exited_run,
1527                        exit_code,
1528                        capture_complete,
1529                        commits,
1530                        tests,
1531                        merge,
1532                        recovery: None,
1533                    });
1534                });
1535                log.emit_with_fields(
1536                    LogLevel::Info,
1537                    "sloop::dispatcher",
1538                    "run_started",
1539                    json!({"run_id": run_id, "ticket_id": ticket_id, "pid": pid}),
1540                );
1541            }
1542            Err(error) => {
1543                if let Err(abort_error) =
1544                    state
1545                        .store
1546                        .abort_claim(&run_id, &ticket_id, state.clock.now_ms())
1547                {
1548                    log.emit_with_fields(
1549                        LogLevel::Error,
1550                        "sloop::dispatcher",
1551                        "claim_abort_failed",
1552                        json!({
1553                            "run_id": run_id,
1554                            "ticket_id": ticket_id,
1555                            "error": abort_error.to_string(),
1556                        }),
1557                    );
1558                }
1559                // A launch can fail after the worker socket was bound.
1560                close_worker_socket(state, &run_id);
1561                log.emit_with_fields(
1562                    LogLevel::Error,
1563                    "sloop::dispatcher",
1564                    "run_launch_failed",
1565                    json!({
1566                        "run_id": run_id,
1567                        "ticket_id": ticket_id,
1568                        "error": error,
1569                    }),
1570                );
1571            }
1572        }
1573    }
1574}
1575
1576fn running_hours_open(state: &DispatcherState, now_ms: i64) -> bool {
1577    state
1578        .running_hours
1579        .as_ref()
1580        .is_none_or(|hours| hours.is_open(state.clock.local_minute(now_ms)))
1581}
1582
1583fn next_dispatch_deadline(state: &DispatcherState) -> Option<i64> {
1584    let hours = state.running_hours.as_ref()?;
1585    let now_ms = state.clock.now_ms();
1586    if hours.is_open(state.clock.local_minute(now_ms)) {
1587        return None;
1588    }
1589    let has_demand = state
1590        .store
1591        .queued_dispatchable_activations()
1592        .is_ok_and(|activations| !activations.is_empty());
1593    has_demand.then(|| hours.next_opening_ms(state.clock.as_ref(), now_ms))
1594}
1595
1596fn eligible_ticket(store: &Store, activation: &QueuedActivation) -> Option<String> {
1597    match &activation.ticket_id {
1598        Some(ticket) => Some(ticket.clone()),
1599        None => store.select_ready_ticket(activation).ok().flatten(),
1600    }
1601}
1602
1603/// A supervised agent process plus the reader threads draining its pipes;
1604/// the readers finish when the pipes close, and joining them guarantees
1605/// capture is flushed before exit evidence is reported. Worktree, branch,
1606/// and log writer stay with the supervisor for evidence gathering after
1607/// the exit.
1608struct LaunchedRun {
1609    child: Child,
1610    readers: Vec<std::thread::JoinHandle<bool>>,
1611    worktree: PathBuf,
1612    branch: String,
1613    output_log: RunLogWriter,
1614    /// The bound per-run worker socket, handed to an accept-loop task once
1615    /// the launch is registered.
1616    worker_listener: UnixListener,
1617    worker_token: String,
1618    worker_socket_path: PathBuf,
1619}
1620
1621/// Gathers post-exit evidence in the supervisor thread, keeping slow Git and
1622/// test work out of the dispatcher: commits on the run branch, the configured
1623/// test stage when the exit and commits justify it, and the merge attempt
1624/// when policy allows.
1625#[allow(clippy::too_many_arguments)]
1626fn gather_exit_evidence(
1627    run_id: &str,
1628    root: &Path,
1629    worktree: &Path,
1630    branch: &str,
1631    test_cmd: Option<&[String]>,
1632    clock: &dyn Clock,
1633    output_log: &RunLogWriter,
1634    exit_code: Option<i32>,
1635    capture_complete: bool,
1636    mut checkpoint_store: Option<&mut Store>,
1637    operational_log: &OperationalLog,
1638) -> (Vec<String>, Option<StageResult>, Option<MergeOutcome>) {
1639    let exit = classify_exit(exit_code);
1640    let commits = commits_on_branch(root, branch);
1641    let commit_count = commits.len() as u64;
1642    let checkpointed = if let Some(store) = checkpoint_store.as_deref_mut() {
1643        match store.record_agent_exit(
1644            run_id,
1645            exit_code,
1646            capture_complete,
1647            &json!({"count": commit_count, "oids": commits}).to_string(),
1648            clock.now_ms(),
1649        ) {
1650            Ok(()) => true,
1651            Err(error) => {
1652                operational_log.emit_with_fields(
1653                    LogLevel::Error,
1654                    "sloop::supervisor",
1655                    "agent_exit_checkpoint_failed",
1656                    json!({"run_id": run_id, "error": error.to_string()}),
1657                );
1658                false
1659            }
1660        }
1661    } else {
1662        false
1663    };
1664    // Tests and merge can have side effects. Without the pre-aftercare
1665    // checkpoint, preserve committed work for review rather than performing
1666    // an action that recovery could no longer prove or resume.
1667    if !checkpointed
1668        || checkpoint_store
1669            .as_deref()
1670            .is_some_and(|store| aftercare_cancelled(store, run_id, operational_log))
1671    {
1672        return (commits, None, None);
1673    }
1674
1675    let tests = match test_cmd {
1676        Some(cmd) if wants_tests(exit, commit_count) => {
1677            let stage = run_test_stage(
1678                worktree,
1679                cmd,
1680                output_log,
1681                clock,
1682                checkpoint_store.as_deref(),
1683                run_id,
1684                operational_log,
1685            );
1686            let test_checkpointed = if let Some(store) = checkpoint_store.as_deref()
1687                && let Err(error) = store.record_aftercare_evidence(
1688                    run_id,
1689                    "test_result",
1690                    &test_result_json(&stage),
1691                    clock.now_ms(),
1692                ) {
1693                operational_log.emit_with_fields(
1694                    LogLevel::Error,
1695                    "sloop::supervisor",
1696                    "aftercare_checkpoint_failed",
1697                    json!({"run_id": run_id, "stage": "test", "error": error.to_string()}),
1698                );
1699                false
1700            } else {
1701                true
1702            };
1703            if !test_checkpointed {
1704                return (commits, Some(stage), None);
1705            }
1706            Some(stage)
1707        }
1708        _ => None,
1709    };
1710
1711    let merge = if !checkpoint_store
1712        .as_deref()
1713        .is_some_and(|store| aftercare_cancelled(store, run_id, operational_log))
1714        && wants_merge(
1715            exit,
1716            commit_count,
1717            tests.as_ref().map(|stage| stage.outcome),
1718        ) {
1719        let outcome = attempt_merge(
1720            root,
1721            branch,
1722            checkpoint_store.as_deref(),
1723            run_id,
1724            clock,
1725            operational_log,
1726        );
1727        if let Some(store) = checkpoint_store.as_deref()
1728            && let Err(error) = store.record_aftercare_evidence(
1729                run_id,
1730                "merge_result",
1731                &merge_result_json(outcome),
1732                clock.now_ms(),
1733            )
1734        {
1735            operational_log.emit_with_fields(
1736                LogLevel::Error,
1737                "sloop::supervisor",
1738                "aftercare_checkpoint_failed",
1739                json!({"run_id": run_id, "stage": "merge", "error": error.to_string()}),
1740            );
1741        }
1742        Some(outcome)
1743    } else {
1744        None
1745    };
1746    (commits, tests, merge)
1747}
1748
1749fn aftercare_cancelled(store: &Store, run_id: &str, log: &OperationalLog) -> bool {
1750    match store.cancellation_requested(run_id) {
1751        Ok(cancelled) => cancelled,
1752        Err(error) => {
1753            log.emit_with_fields(
1754                LogLevel::Error,
1755                "sloop::supervisor",
1756                "cancellation_read_failed",
1757                json!({"run_id": run_id, "error": error.to_string()}),
1758            );
1759            true
1760        }
1761    }
1762}
1763
1764fn test_result_json(stage: &StageResult) -> String {
1765    json!({
1766        "passed": stage.outcome == StageOutcome::Passed,
1767        "exit_code": stage.exit_code,
1768        "started_at_ms": stage.started_at_ms,
1769        "finished_at_ms": stage.finished_at_ms,
1770    })
1771    .to_string()
1772}
1773
1774fn merge_result_json(outcome: MergeOutcome) -> String {
1775    json!({"merged": outcome == MergeOutcome::Merged}).to_string()
1776}
1777
1778/// Commits on the run branch that the root checkout does not have. A Git
1779/// failure reads as no commits: evidence that cannot be observed is not claimed.
1780fn commits_on_branch(root: &Path, branch: &str) -> Vec<String> {
1781    try_commits_on_branch(root, branch).unwrap_or_default()
1782}
1783
1784fn try_commits_on_branch(root: &Path, branch: &str) -> Result<Vec<String>, String> {
1785    let output = Command::new("git")
1786        .args(["rev-list", "--reverse", &format!("HEAD..{branch}")])
1787        .current_dir(root)
1788        .output()
1789        .map_err(|error| error.to_string())?;
1790    match output {
1791        output if output.status.success() => Ok(String::from_utf8_lossy(&output.stdout)
1792            .lines()
1793            .map(str::to_owned)
1794            .collect()),
1795        output => Err(format!(
1796            "git rev-list failed: {}",
1797            String::from_utf8_lossy(&output.stderr).trim()
1798        )),
1799    }
1800}
1801
1802/// Runs the configured test command in the run's worktree, capturing its
1803/// output as `aftercare` evidence in the same ordered run log.
1804fn run_test_stage(
1805    worktree: &Path,
1806    cmd: &[String],
1807    output_log: &RunLogWriter,
1808    clock: &dyn Clock,
1809    checkpoint_store: Option<&Store>,
1810    run_id: &str,
1811    operational_log: &OperationalLog,
1812) -> StageResult {
1813    let started_at_ms = clock.now_ms();
1814    let failed = |finished_at_ms| StageResult {
1815        outcome: StageOutcome::Failed,
1816        exit_code: None,
1817        started_at_ms,
1818        finished_at_ms,
1819    };
1820
1821    let mut command = Command::new(&cmd[0]);
1822    command
1823        .args(&cmd[1..])
1824        .current_dir(worktree)
1825        .stdin(Stdio::null())
1826        .stdout(Stdio::piped())
1827        .stderr(Stdio::piped())
1828        .process_group(0);
1829    let Ok(mut child) = command.spawn() else {
1830        return failed(clock.now_ms());
1831    };
1832    let pid = child.id();
1833    let Some(pid_start_time) = process_start_time(pid) else {
1834        unsafe {
1835            libc::kill(-(pid as libc::pid_t), libc::SIGKILL);
1836        }
1837        let _ = child.wait();
1838        return failed(clock.now_ms());
1839    };
1840    let readers = vec![
1841        spawn_output_reader(
1842            child.stdout.take().expect("stdout was piped"),
1843            output_log.clone(),
1844            OutputSource::Aftercare,
1845            Some("test"),
1846            OutputStream::Stdout,
1847        ),
1848        spawn_output_reader(
1849            child.stderr.take().expect("stderr was piped"),
1850            output_log.clone(),
1851            OutputSource::Aftercare,
1852            Some("test"),
1853            OutputStream::Stderr,
1854        ),
1855    ];
1856    wait_for_test_hook("before-test-process-checkpoint");
1857    if let Some(store) = checkpoint_store
1858        && let Err(error) = store.record_aftercare_evidence(
1859            run_id,
1860            "test_process",
1861            &json!({
1862                "pid": pid,
1863                "pid_start_time": pid_start_time,
1864                "process_group_id": pid,
1865            })
1866            .to_string(),
1867            clock.now_ms(),
1868        )
1869    {
1870        operational_log.emit_with_fields(
1871            LogLevel::Error,
1872            "sloop::supervisor",
1873            "aftercare_process_checkpoint_failed",
1874            json!({"run_id": run_id, "stage": "test", "error": error.to_string()}),
1875        );
1876        if process_identity_matches(pid, Some(pid_start_time)) {
1877            unsafe {
1878                libc::kill(-(pid as libc::pid_t), libc::SIGKILL);
1879            }
1880        }
1881        let _ = child.wait();
1882        for reader in readers {
1883            let _ = reader.join();
1884        }
1885        return failed(clock.now_ms());
1886    }
1887    if checkpoint_store.is_some_and(|store| aftercare_cancelled(store, run_id, operational_log)) {
1888        if process_identity_matches(pid, Some(pid_start_time)) {
1889            unsafe {
1890                libc::kill(-(pid as libc::pid_t), libc::SIGKILL);
1891            }
1892        }
1893        let _ = child.wait();
1894        for reader in readers {
1895            let _ = reader.join();
1896        }
1897        return failed(clock.now_ms());
1898    }
1899
1900    let status = child.wait();
1901    for reader in readers {
1902        let _ = reader.join();
1903    }
1904    let Ok(status) = status else {
1905        return failed(clock.now_ms());
1906    };
1907    StageResult {
1908        outcome: if status.success() {
1909            StageOutcome::Passed
1910        } else {
1911            StageOutcome::Failed
1912        },
1913        exit_code: status.code(),
1914        started_at_ms,
1915        finished_at_ms: clock.now_ms(),
1916    }
1917}
1918
1919/// Attempts the policy merge into the default branch: fast-forward when
1920/// possible, otherwise a merge commit. Only a textual conflict needs a
1921/// human; the merge is aborted so the checkout stays clean and the run
1922/// branch survives as evidence.
1923#[allow(clippy::too_many_arguments)]
1924fn attempt_merge(
1925    root: &Path,
1926    branch: &str,
1927    checkpoint_store: Option<&Store>,
1928    run_id: &str,
1929    clock: &dyn Clock,
1930    operational_log: &OperationalLog,
1931) -> MergeOutcome {
1932    let Ok(_guard) = MERGE_LOCK.lock() else {
1933        return MergeOutcome::Diverged;
1934    };
1935    let message = format!("Merge run branch '{branch}'");
1936    // The merge commit is sloop's own action, not the operator's or the
1937    // agent's, so it carries sloop's identity; a fast-forward creates no
1938    // commit and ignores these.
1939    let mut command = Command::new("git");
1940    command
1941        .args([
1942            "-c",
1943            "user.name=sloop",
1944            "-c",
1945            "user.email=sloop@sloop.invalid",
1946            "merge",
1947            "--quiet",
1948            "-m",
1949            &message,
1950            branch,
1951        ])
1952        .current_dir(root)
1953        .stdin(Stdio::null())
1954        .stdout(Stdio::null())
1955        .stderr(Stdio::null())
1956        .process_group(0);
1957    let Ok(mut child) = command.spawn() else {
1958        return MergeOutcome::Diverged;
1959    };
1960    let pid = child.id();
1961    let Some(pid_start_time) = process_start_time(pid) else {
1962        unsafe {
1963            libc::kill(-(pid as libc::pid_t), libc::SIGKILL);
1964        }
1965        let _ = child.wait();
1966        return MergeOutcome::Diverged;
1967    };
1968    if let Some(store) = checkpoint_store {
1969        let checkpoint = json!({
1970            "pid": pid,
1971            "pid_start_time": pid_start_time,
1972            "process_group_id": pid,
1973        })
1974        .to_string();
1975        if let Err(error) =
1976            store.record_aftercare_evidence(run_id, "merge_process", &checkpoint, clock.now_ms())
1977        {
1978            operational_log.emit_with_fields(
1979                LogLevel::Error,
1980                "sloop::supervisor",
1981                "aftercare_process_checkpoint_failed",
1982                json!({"run_id": run_id, "stage": "merge", "error": error.to_string()}),
1983            );
1984            unsafe {
1985                libc::kill(-(pid as libc::pid_t), libc::SIGKILL);
1986            }
1987            let _ = child.wait();
1988            return MergeOutcome::Diverged;
1989        }
1990        if aftercare_cancelled(store, run_id, operational_log) {
1991            unsafe {
1992                libc::kill(-(pid as libc::pid_t), libc::SIGKILL);
1993            }
1994            let _ = child.wait();
1995            return MergeOutcome::Diverged;
1996        }
1997    }
1998    match child.wait() {
1999        Ok(status) if status.success() => MergeOutcome::Merged,
2000        _ => {
2001            let _ = Command::new("git")
2002                .args(["merge", "--abort"])
2003                .current_dir(root)
2004                .output();
2005            MergeOutcome::Diverged
2006        }
2007    }
2008}
2009
2010fn run_output_path(state_dir: &Path, run_id: &str) -> PathBuf {
2011    state_dir.join("runs").join(run_id).join("output.ndjson")
2012}
2013
2014/// Continuously drains one child pipe into the run log so a verbose agent
2015/// can never fill the pipe and deadlock. Returns whether every chunk was
2016/// durably captured.
2017fn spawn_output_reader(
2018    pipe: impl Read + Send + 'static,
2019    log: RunLogWriter,
2020    source: OutputSource,
2021    stage: Option<&'static str>,
2022    stream: OutputStream,
2023) -> std::thread::JoinHandle<bool> {
2024    std::thread::spawn(move || {
2025        let mut pipe = pipe;
2026        let mut buffer = [0u8; 8192];
2027        loop {
2028            match pipe.read(&mut buffer) {
2029                Ok(0) => return true,
2030                Ok(read) => {
2031                    if log.append(source, stage, stream, &buffer[..read]).is_err() {
2032                        return false;
2033                    }
2034                }
2035                Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
2036                Err(_) => return false,
2037            }
2038        }
2039    })
2040}
2041
2042/// Creates the run's branch and isolated worktree, starts the configured
2043/// agent as its own process group, and records durable process identity.
2044fn launch_agent(
2045    state: &DispatcherState,
2046    run_id: &str,
2047    ticket_id: &str,
2048    attempt: i64,
2049) -> Result<LaunchedRun, String> {
2050    let agent = state
2051        .agent
2052        .as_ref()
2053        .ok_or_else(|| "no agent targets configured".to_owned())?;
2054    let ticket = state
2055        .store
2056        .ticket(ticket_id)
2057        .map_err(|error| error.to_string())?
2058        .ok_or_else(|| format!("ticket `{ticket_id}` no longer exists"))?;
2059    let target = ticket
2060        .target
2061        .as_deref()
2062        .ok_or_else(|| format!("ticket `{ticket_id}` does not specify an agent target"))?;
2063    let template = agent
2064        .targets
2065        .get(target)
2066        .ok_or_else(|| format!("ticket `{ticket_id}` names unknown agent target `{target}`"))?;
2067    let prompt = compose_worker_prompt(&state.root)?;
2068    let cmd = expand_agent_cmd(
2069        template,
2070        ticket.model.as_deref(),
2071        ticket.effort.as_deref(),
2072        &prompt,
2073    )
2074    .map_err(|error| format!("ticket `{ticket_id}` {error}"))?;
2075    let executable = std::env::current_exe()
2076        .map_err(|error| format!("cannot locate sloop executable: {error}"))?;
2077    let executable_dir = executable
2078        .parent()
2079        .ok_or_else(|| "sloop executable has no parent directory".to_owned())?;
2080    let mut path_entries = vec![executable_dir.to_path_buf()];
2081    if let Some(path) = std::env::var_os("PATH") {
2082        path_entries.extend(std::env::split_paths(&path));
2083    }
2084    let path = std::env::join_paths(path_entries)
2085        .map_err(|error| format!("cannot construct agent PATH: {error}"))?;
2086    // `retry` resets attempts, so the run ID keeps preserved failed branches
2087    // from colliding with a later run's first attempt.
2088    let branch = format!("sloop/{ticket_id}-a{attempt}-{run_id}");
2089    fs::create_dir_all(&state.worktree_dir).map_err(|error| error.to_string())?;
2090    let worktree = state.worktree_dir.join(run_id);
2091
2092    let git = Command::new("git")
2093        .args(["worktree", "add", "--quiet", "-b", &branch])
2094        .arg(&worktree)
2095        .current_dir(&state.root)
2096        .output()
2097        .map_err(|error| error.to_string())?;
2098    if !git.status.success() {
2099        return Err(format!(
2100            "git worktree add failed: {}",
2101            String::from_utf8_lossy(&git.stderr).trim()
2102        ));
2103    }
2104
2105    let output_log = RunLogWriter::open(&run_output_path(&state.state_dir, run_id))
2106        .map_err(|error| error.to_string())?;
2107
2108    let worker_token = generate_worker_token()?;
2109    let socket_path = worker_socket_path(&state.runtime_dir, run_id);
2110    fs::create_dir_all(socket_path.parent().expect("worker sockets have a parent"))
2111        .map_err(|error| error.to_string())?;
2112    let _ = fs::remove_file(&socket_path);
2113    let worker_listener = UnixListener::bind(&socket_path).map_err(|error| error.to_string())?;
2114    fs::set_permissions(&socket_path, fs::Permissions::from_mode(0o600))
2115        .map_err(|error| error.to_string())?;
2116
2117    let mut command = Command::new(&cmd[0]);
2118    command
2119        .args(&cmd[1..])
2120        .current_dir(&worktree)
2121        .env("SLOOP_RUN_ID", run_id)
2122        .env("SLOOP_TICKET_ID", ticket_id)
2123        .env("SLOOP_BIN", &executable)
2124        .env("PATH", path)
2125        .env("SLOOP_SOCKET", &socket_path)
2126        .env("SLOOP_TOKEN", &worker_token)
2127        .stdin(Stdio::null())
2128        .stdout(Stdio::piped())
2129        .stderr(Stdio::piped())
2130        .process_group(0);
2131    let mut child = command.spawn().map_err(|error| error.to_string())?;
2132    let readers = vec![
2133        spawn_output_reader(
2134            child.stdout.take().expect("stdout was piped"),
2135            output_log.clone(),
2136            OutputSource::Agent,
2137            None,
2138            OutputStream::Stdout,
2139        ),
2140        spawn_output_reader(
2141            child.stderr.take().expect("stderr was piped"),
2142            output_log.clone(),
2143            OutputSource::Agent,
2144            None,
2145            OutputStream::Stderr,
2146        ),
2147    ];
2148
2149    let pid = child.id();
2150    if let Err(error) = state.store.mark_run_running(
2151        run_id,
2152        &branch,
2153        &worktree.to_string_lossy(),
2154        pid,
2155        process_start_time(pid),
2156        pid, // process_group(0) makes the child its own group leader
2157        &worker_token,
2158        &socket_path.to_string_lossy(),
2159        state.clock.now_ms(),
2160    ) {
2161        unsafe {
2162            libc::kill(-(pid as libc::pid_t), libc::SIGKILL);
2163        }
2164        let _ = child.wait();
2165        for reader in readers {
2166            let _ = reader.join();
2167        }
2168        return Err(error.to_string());
2169    }
2170    Ok(LaunchedRun {
2171        child,
2172        readers,
2173        worktree,
2174        branch,
2175        output_log,
2176        worker_listener,
2177        worker_token,
2178        worker_socket_path: socket_path,
2179    })
2180}
2181
2182fn compose_worker_prompt(root: &Path) -> Result<String, String> {
2183    let path = root.join(".agents/sloop/instructions.md");
2184    match fs::read_to_string(&path) {
2185        Ok(instructions) => Ok(format!("{WORKER_BOOTSTRAP_PROMPT}\n\n{instructions}")),
2186        Err(error) if error.kind() == io::ErrorKind::NotFound => {
2187            Ok(WORKER_BOOTSTRAP_PROMPT.to_owned())
2188        }
2189        Err(error) => Err(format!("cannot read {}: {error}", path.display())),
2190    }
2191}
2192
2193/// 32 random bytes from the kernel, encoded for an environment variable.
2194/// Guessing it stops accidents, which is the threat model; same-uid
2195/// isolation needs a real sandbox.
2196fn generate_worker_token() -> Result<String, String> {
2197    let mut bytes = [0u8; 32];
2198    let mut urandom = fs::File::open("/dev/urandom").map_err(|error| error.to_string())?;
2199    urandom
2200        .read_exact(&mut bytes)
2201        .map_err(|error| error.to_string())?;
2202    Ok(BASE64_TOKEN.encode(bytes))
2203}
2204
2205/// Identity of the daemon owning a project's lockfile: PID plus process start
2206/// time, mirroring the identity rule used for supervised agents.
2207#[derive(Debug, Clone, PartialEq, Eq)]
2208pub struct LockIdentity {
2209    pub pid: u32,
2210    pub started_at_ms: Option<i64>,
2211    pub socket: Option<PathBuf>,
2212}
2213
2214pub fn read_lock_identity(path: &Path) -> Option<LockIdentity> {
2215    let content = fs::read_to_string(path).ok()?;
2216    let value: serde_json::Value = serde_json::from_str(content.trim()).ok()?;
2217    Some(LockIdentity {
2218        pid: u32::try_from(value["pid"].as_u64()?).ok()?,
2219        started_at_ms: value["started_at_ms"].as_i64(),
2220        socket: value["socket"].as_str().map(PathBuf::from),
2221    })
2222}
2223
2224/// Reads a stable start-time token for a PID, the second half of the durable
2225/// process identity. Linux exposes clock ticks directly; other Unix systems
2226/// use a stable hash of `ps`'s process start timestamp.
2227fn process_start_time(pid: u32) -> Option<i64> {
2228    #[cfg(target_os = "linux")]
2229    {
2230        let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
2231        let after_command = &stat[stat.rfind(')')? + 1..];
2232        after_command
2233            .split_whitespace()
2234            .nth(19)
2235            .and_then(|field| field.parse().ok())
2236    }
2237
2238    #[cfg(not(target_os = "linux"))]
2239    {
2240        let output = Command::new("ps")
2241            .args(["-o", "lstart=", "-p", &pid.to_string()])
2242            .output()
2243            .ok()?;
2244        if !output.status.success() || output.stdout.iter().all(u8::is_ascii_whitespace) {
2245            return None;
2246        }
2247        let mut hash = 0xcbf29ce484222325_u64;
2248        for byte in output.stdout {
2249            hash ^= u64::from(byte);
2250            hash = hash.wrapping_mul(0x100000001b3);
2251        }
2252        Some(hash as i64)
2253    }
2254}
2255
2256#[cfg(debug_assertions)]
2257fn wait_for_test_hook(name: &str) {
2258    let Some(directory) = std::env::var_os("SLOOP_TEST_HOOK_DIR").map(PathBuf::from) else {
2259        return;
2260    };
2261    let armed = directory.join(format!("{name}.armed"));
2262    if !armed.is_file() {
2263        return;
2264    }
2265    let reached = directory.join(format!("{name}.reached"));
2266    let release = directory.join(format!("{name}.release"));
2267    if fs::write(&reached, b"").is_err() {
2268        return;
2269    }
2270    while !release.is_file() {
2271        std::thread::sleep(Duration::from_millis(10));
2272    }
2273}
2274
2275#[cfg(not(debug_assertions))]
2276fn wait_for_test_hook(_name: &str) {}
2277
2278/// Accepts connections on one run's worker socket until the settle path
2279/// aborts this task. Each connection is one request, mirroring the
2280/// operator socket.
2281async fn serve_worker_socket(
2282    listener: UnixListener,
2283    run_id: String,
2284    dispatcher: mpsc::Sender<DispatcherMessage>,
2285    log: OperationalLog,
2286) {
2287    loop {
2288        let Ok((stream, _)) = listener.accept().await else {
2289            return;
2290        };
2291        let run_id = run_id.clone();
2292        let dispatcher = dispatcher.clone();
2293        let log = log.clone();
2294        tokio::spawn(async move {
2295            if let Err(error) = handle_worker_connection(stream, run_id.clone(), dispatcher).await {
2296                log.emit_with_fields(
2297                    LogLevel::Error,
2298                    "sloop::socket",
2299                    "worker_connection_failed",
2300                    json!({"run_id": run_id, "error": error.to_string()}),
2301                );
2302            }
2303        });
2304    }
2305}
2306
2307/// Serves a worker verb after proving the caller holds the run's token.
2308/// Everything an agent can reach flows through here: `brief` and `show` are
2309/// scoped reads, `note` is the only write and moves nothing.
2310fn dispatch_worker(
2311    state: &mut DispatcherState,
2312    id: RequestId,
2313    request: Request,
2314    run_id: &str,
2315    token: Option<&str>,
2316) -> ResponseEnvelope {
2317    let valid = token.is_some_and(|presented| {
2318        state
2319            .worker_tokens
2320            .get(run_id)
2321            .is_some_and(|issued| issued == presented)
2322    });
2323    if !valid {
2324        return ResponseEnvelope::failure(
2325            Some(id),
2326            unauthorized("the presented token is not valid for this run"),
2327        );
2328    }
2329
2330    let data = match request {
2331        Request::Brief(_) => handle_brief(state, run_id),
2332        Request::Show(args) => handle_show(state, run_id, &args.reference),
2333        Request::Note(args) => handle_note(state, run_id, &args.text),
2334        // The connection handler already rejected operator verbs.
2335        _ => Err(unauthorized(
2336            "operator verbs are not available on a worker socket",
2337        )),
2338    };
2339    match data {
2340        Ok(data) => ResponseEnvelope::success(Some(id), data),
2341        Err(error) => ResponseEnvelope::failure(Some(id), error),
2342    }
2343}
2344
2345/// Everything the agent needs to work, re-readable after a compaction: the
2346/// ticket body from its committed file, the isolated workspace, and the
2347/// evidence-based definition of done.
2348fn handle_brief(state: &DispatcherState, run_id: &str) -> Result<serde_json::Value, ErrorBody> {
2349    let run = lookup(state, |store| store.run(run_id))?
2350        .ok_or_else(|| internal("the run for this token no longer exists"))?;
2351    let ticket = lookup(state, |store| store.ticket(&run.ticket_id))?
2352        .ok_or_else(|| internal("the ticket for this run no longer exists"))?;
2353    let body = ticket
2354        .file_path
2355        .as_ref()
2356        .and_then(|file_path| fs::read_to_string(state.root.join(file_path)).ok())
2357        .unwrap_or_default();
2358
2359    let mut definition_of_done = vec!["Commit your work to the run branch".to_owned()];
2360    if state.aftercare_test_cmd.is_some() {
2361        definition_of_done.push("The configured test command passes".to_owned());
2362    }
2363
2364    Ok(json!({
2365        "run": run_id,
2366        "ticket": {
2367            "id": ticket.id,
2368            "name": ticket.name,
2369            "blocked_by": ticket.blocked_by,
2370            "worktree": ticket.worktree,
2371            "body": body,
2372            "acceptance": [],
2373            "target": ticket.target,
2374            "model": ticket.model,
2375            "effort": ticket.effort,
2376        },
2377        "worktree": run.worktree_path,
2378        "branch": run.branch,
2379        "definition_of_done": definition_of_done,
2380    }))
2381}
2382
2383/// Read-only lookup, scoped to the run's own ticket. Whether a foreign
2384/// reference exists is not the worker's to learn: everything else is
2385/// uniformly unauthorized.
2386fn handle_show(
2387    state: &DispatcherState,
2388    run_id: &str,
2389    reference: &str,
2390) -> Result<serde_json::Value, ErrorBody> {
2391    let run = lookup(state, |store| store.run(run_id))?
2392        .ok_or_else(|| internal("the run for this token no longer exists"))?;
2393    if reference != run.ticket_id {
2394        return Err(unauthorized("workers may only show their own run's ticket"));
2395    }
2396    let ticket = lookup(state, |store| store.ticket(&run.ticket_id))?
2397        .ok_or_else(|| internal("the ticket for this run no longer exists"))?;
2398    Ok(ticket_show(reference, &ticket))
2399}
2400
2401fn ticket_show(reference: &str, ticket: &crate::store::TicketRecord) -> serde_json::Value {
2402    json!({
2403        "ref": reference,
2404        "kind": "ticket",
2405        "value": {
2406            "id": ticket.id,
2407            "project": ticket.project_id,
2408            "state": ticket.state,
2409            "file": ticket.file_path,
2410            "name": ticket.name,
2411            "blocked_by": ticket.blocked_by,
2412            "worktree": ticket.worktree,
2413            "target": ticket.target,
2414            "model": ticket.model,
2415            "effort": ticket.effort,
2416        },
2417    })
2418}
2419
2420fn handle_operator_show(
2421    state: &DispatcherState,
2422    reference: &str,
2423) -> Result<serde_json::Value, ErrorBody> {
2424    if let Some(ticket) = lookup(state, |store| store.ticket(reference))? {
2425        return Ok(ticket_show(reference, &ticket));
2426    }
2427    let project = lookup(state, |store| store.project(reference))?
2428        .ok_or_else(|| not_found(&format!("reference `{reference}` is not indexed")))?;
2429    let tickets = lookup(state, |store| store.tickets_for_project(reference))?;
2430
2431    let mut notes: HashMap<String, Vec<serde_json::Value>> = HashMap::new();
2432    for note in lookup(state, |store| store.notes_for_project(reference))? {
2433        notes.entry(note.ticket_id).or_default().push(json!({
2434            "id": note.id,
2435            "run": note.run_id,
2436            "text": note.text,
2437            "recorded_at_ms": note.recorded_at_ms,
2438        }));
2439    }
2440
2441    let mut commits: HashMap<String, Vec<serde_json::Value>> = HashMap::new();
2442    for evidence in lookup(state, |store| store.commit_evidence_for_project(reference))? {
2443        let data: serde_json::Value = serde_json::from_str(&evidence.data_json)
2444            .map_err(|error| internal(&format!("cannot decode commit evidence: {error}")))?;
2445        for oid in data["oids"]
2446            .as_array()
2447            .map(Vec::as_slice)
2448            .unwrap_or_default()
2449            .iter()
2450            .filter_map(serde_json::Value::as_str)
2451        {
2452            let (short_hash, message) = git_commit(&state.root, oid)?;
2453            commits
2454                .entry(evidence.ticket_id.clone())
2455                .or_default()
2456                .push(json!({
2457                    "run": evidence.run_id.clone(),
2458                    "hash": short_hash,
2459                    "message": message,
2460                }));
2461        }
2462    }
2463
2464    let activity = tickets
2465        .into_iter()
2466        .map(|ticket| {
2467            let ticket_notes = notes.remove(&ticket.id).unwrap_or_default();
2468            let ticket_commits = commits.remove(&ticket.id).unwrap_or_default();
2469            json!({
2470                "id": ticket.id,
2471                "name": ticket.name,
2472                "state": ticket.state,
2473                "notes": ticket_notes,
2474                "commits": ticket_commits,
2475            })
2476        })
2477        .collect::<Vec<_>>();
2478
2479    Ok(json!({
2480        "ref": reference,
2481        "kind": "project",
2482        "value": {
2483            "id": project.id,
2484            "title": project.title,
2485            "file": project.file_path,
2486            "tickets": activity,
2487        },
2488    }))
2489}
2490
2491fn git_commit(root: &Path, oid: &str) -> Result<(String, String), ErrorBody> {
2492    let output = Command::new("git")
2493        .args(["show", "--no-patch", "--format=%h%x00%s", oid, "--"])
2494        .current_dir(root)
2495        .output()
2496        .map_err(|error| internal(&format!("cannot read commit `{oid}`: {error}")))?;
2497    if !output.status.success() {
2498        return Err(internal(&format!(
2499            "cannot read commit `{oid}`: {}",
2500            String::from_utf8_lossy(&output.stderr).trim()
2501        )));
2502    }
2503    let rendered = String::from_utf8_lossy(&output.stdout);
2504    let (hash, message) = rendered
2505        .trim_end()
2506        .split_once('\0')
2507        .ok_or_else(|| internal(&format!("Git returned malformed data for commit `{oid}`")))?;
2508    Ok((hash.to_owned(), message.to_owned()))
2509}
2510
2511/// The agent's only write: an advisory note recorded against its run. It
2512/// transitions nothing.
2513fn handle_note(
2514    state: &DispatcherState,
2515    run_id: &str,
2516    text: &str,
2517) -> Result<serde_json::Value, ErrorBody> {
2518    let ordinal = lookup(state, |store| store.next_note_ordinal())?;
2519    let note_id = format!("N{ordinal}");
2520    state
2521        .store
2522        .insert_note(&note_id, run_id, text, state.clock.now_ms())
2523        .map_err(|error| internal(&format!("cannot record note: {error}")))?;
2524    Ok(json!({"note": {"id": note_id, "run": run_id, "text": text}}))
2525}
2526
2527fn dispatch(state: &mut DispatcherState, id: RequestId, request: Request) -> ResponseEnvelope {
2528    let data = match request {
2529        Request::Show(args) => match handle_operator_show(state, &args.reference) {
2530            Ok(data) => data,
2531            Err(error) => return ResponseEnvelope::failure(Some(id), error),
2532        },
2533        Request::Run(args) => match handle_run(state, &args) {
2534            Ok(data) => data,
2535            Err(error) => return ResponseEnvelope::failure(Some(id), error),
2536        },
2537        Request::Daemon(_) => json!({
2538            "pid": state.pid,
2539            "socket": state.socket.to_string_lossy(),
2540            "state_dir": state.state_dir.to_string_lossy(),
2541            "log": state.daemon_log.to_string_lossy(),
2542            "version": env!("CARGO_PKG_VERSION"),
2543            "started": false
2544        }),
2545        Request::Post(args) => {
2546            match crate::post::handle(
2547                &state.root,
2548                &state.ticket_dir,
2549                &state.store,
2550                &args,
2551                state.clock.now_ms(),
2552                &state.ticket_prefix,
2553                state.agent.as_ref(),
2554                &state.flows,
2555                &state.default_flow,
2556            ) {
2557                Ok(data) => data,
2558                Err(error) => {
2559                    return ResponseEnvelope::failure(Some(id), post_error_body(&error));
2560                }
2561            }
2562        }
2563        Request::List(_) => match handle_list(state) {
2564            Ok(data) => data,
2565            Err(error) => return ResponseEnvelope::failure(Some(id), error),
2566        },
2567        Request::Status(_) => {
2568            let tickets = match state.store.ticket_counts() {
2569                Ok(counts) => counts,
2570                Err(error) => {
2571                    return ResponseEnvelope::failure(
2572                        Some(id),
2573                        internal(&format!("cannot read ticket counts: {error}")),
2574                    );
2575                }
2576            };
2577            let runs: Vec<_> = match state.store.active_runs() {
2578                Ok(runs) => runs
2579                    .into_iter()
2580                    .map(|run| {
2581                        json!({
2582                            "id": run.id,
2583                            "project": run.project_id,
2584                            "ticket": run.ticket_id,
2585                            "state": run.state,
2586                        })
2587                    })
2588                    .collect(),
2589                Err(error) => {
2590                    return ResponseEnvelope::failure(
2591                        Some(id),
2592                        internal(&format!("cannot read active runs: {error}")),
2593                    );
2594                }
2595            };
2596            let queued: Vec<_> = match state.store.queued_dispatchable_activations() {
2597                Ok(activations) => activations
2598                    .into_iter()
2599                    .map(|activation| {
2600                        json!({
2601                            "id": activation.id,
2602                            "ticket": activation.ticket_id,
2603                            "project": activation.project_id,
2604                            "state": "queued",
2605                        })
2606                    })
2607                    .collect(),
2608                Err(error) => {
2609                    return ResponseEnvelope::failure(
2610                        Some(id),
2611                        internal(&format!("cannot read queued activations: {error}")),
2612                    );
2613                }
2614            };
2615            let now_ms = state.clock.now_ms();
2616            let mut gate = json!({
2617                "active_agents": state.active.len(),
2618                "max_agents": state.max_agents,
2619            });
2620            if let Some(hours) = &state.running_hours {
2621                gate["running_hours"] = json!({
2622                    "start": hours.start,
2623                    "end": hours.end,
2624                    "open": hours.is_open(state.clock.local_minute(now_ms)),
2625                });
2626            }
2627            let mut snapshot = json!({
2628                "daemon": {"pid": state.pid, "paused": state.paused},
2629                "gate": gate,
2630                "runs": runs,
2631                "queued_activations": queued,
2632                "tickets": {
2633                    "ready": tickets.ready,
2634                    "held": tickets.held,
2635                    "blocked": tickets.blocked,
2636                    "claimed": tickets.claimed,
2637                    "merged": tickets.merged,
2638                    "failed": tickets.failed,
2639                    "needs_review": tickets.needs_review
2640                }
2641            });
2642            if let Some(deadline) = next_dispatch_deadline(state)
2643                && let Some(formatted) = format_timestamp(deadline)
2644            {
2645                snapshot["next_wake"] = json!(formatted);
2646            }
2647            snapshot
2648        }
2649        Request::Pause(_) => {
2650            if let Err(error) = state.store.set_paused(true, state.clock.now_ms()) {
2651                return ResponseEnvelope::failure(
2652                    Some(id),
2653                    internal(&format!("cannot pause scheduler: {error}")),
2654                );
2655            }
2656            state.paused = true;
2657            json!({"paused": true})
2658        }
2659        Request::Resume(_) => {
2660            if let Err(error) = state.store.set_paused(false, state.clock.now_ms()) {
2661                return ResponseEnvelope::failure(
2662                    Some(id),
2663                    internal(&format!("cannot resume scheduler: {error}")),
2664                );
2665            }
2666            state.paused = false;
2667            json!({"paused": false})
2668        }
2669        Request::Hold(args) => match handle_hold(state, &args) {
2670            Ok(data) => data,
2671            Err(error) => return ResponseEnvelope::failure(Some(id), error),
2672        },
2673        Request::Ready(args) => match handle_ready(state, &args) {
2674            Ok(data) => data,
2675            Err(error) => return ResponseEnvelope::failure(Some(id), error),
2676        },
2677        Request::Retry(args) => match handle_retry(state, &args) {
2678            Ok(data) => data,
2679            Err(error) => return ResponseEnvelope::failure(Some(id), error),
2680        },
2681        Request::Logs(args) => match handle_logs(state, &args) {
2682            Ok(data) => data,
2683            Err(error) => return ResponseEnvelope::failure(Some(id), error),
2684        },
2685        Request::Cancel(args) => match handle_cancel(state, &args) {
2686            Ok(data) => data,
2687            Err(error) => return ResponseEnvelope::failure(Some(id), error),
2688        },
2689        Request::Stop(args) => match handle_stop(state, &args) {
2690            Ok(data) => data,
2691            Err(error) => return ResponseEnvelope::failure(Some(id), error),
2692        },
2693        Request::Wait(args) => match handle_wait(state, &args) {
2694            Ok(data) => data,
2695            Err(error) => return ResponseEnvelope::failure(Some(id), error),
2696        },
2697        request => {
2698            return ResponseEnvelope::failure(
2699                Some(id),
2700                ErrorBody {
2701                    code: ErrorCode::InvalidRequest,
2702                    message: format!("verb `{}` is not implemented by the daemon", request.verb()),
2703                    details: json!({"verb": request.verb()}),
2704                },
2705            );
2706        }
2707    };
2708    ResponseEnvelope::success(Some(id), data)
2709}
2710
2711fn handle_list(state: &DispatcherState) -> Result<serde_json::Value, ErrorBody> {
2712    let now_ms = state.clock.now_ms();
2713    let gates = crate::eligibility::Gates {
2714        paused: state.paused,
2715        agent_configured: state.agent.is_some(),
2716        hours_open: running_hours_open(state, now_ms),
2717        at_capacity: state.active.len() >= state.max_agents,
2718        has_queued_activation: !lookup(state, Store::queued_dispatchable_activations)?.is_empty(),
2719    };
2720    let mut rows = Vec::new();
2721    for ticket in lookup(state, Store::tickets)? {
2722        let active_run = lookup(state, |store| store.active_run_for_ticket(&ticket.id))?;
2723        let reason = crate::eligibility::ticket_ineligibility(
2724            &ticket.state,
2725            ticket.attempts,
2726            active_run.as_deref(),
2727            &gates,
2728        )
2729        .map(|reason| reason.describe());
2730        rows.push(json!({
2731            "id": ticket.id,
2732            "project": ticket.project_id,
2733            "state": ticket.state,
2734            "run": active_run,
2735            "reason": reason,
2736        }));
2737    }
2738    Ok(json!({"tickets": rows}))
2739}
2740
2741fn spawn_daemon(project: &Project) -> Result<(), DaemonError> {
2742    let executable = std::env::current_exe().map_err(DaemonError::CurrentExecutable)?;
2743    Command::new(executable)
2744        .args(["daemon", "--foreground"])
2745        .current_dir(&project.root)
2746        .stdin(Stdio::null())
2747        .stdout(Stdio::null())
2748        .stderr(Stdio::null())
2749        .spawn()
2750        .map(|_| ())
2751        .map_err(DaemonError::Spawn)
2752}
2753
2754fn send_existing(project: &Project, request: Request) -> Result<ResponseEnvelope, DaemonError> {
2755    match send(&project.operator_socket, request.clone()) {
2756        Ok(response) => Ok(response),
2757        Err(current_error) => {
2758            let Some(identity) = read_lock_identity(&project.lock_path) else {
2759                return Err(current_error);
2760            };
2761            if !process_identity_matches(identity.pid, identity.started_at_ms) {
2762                return Err(current_error);
2763            }
2764            let Some(socket) = identity.socket else {
2765                return Err(current_error);
2766            };
2767            if socket == project.operator_socket {
2768                return Err(current_error);
2769            }
2770            send(&socket, request)
2771        }
2772    }
2773}
2774
2775fn send(socket: &Path, request: Request) -> Result<ResponseEnvelope, DaemonError> {
2776    let mut stream = StdUnixStream::connect(socket).map_err(DaemonError::Connect)?;
2777    stream
2778        .set_read_timeout(Some(CLIENT_TIMEOUT))
2779        .map_err(DaemonError::Connect)?;
2780    stream
2781        .set_write_timeout(Some(CLIENT_TIMEOUT))
2782        .map_err(DaemonError::Connect)?;
2783
2784    let sequence = NEXT_REQUEST_ID.fetch_add(1, Ordering::Relaxed);
2785    let envelope = RequestEnvelope::new(
2786        RequestId::new(format!("req-{}-{sequence}", std::process::id())),
2787        request,
2788        None,
2789    );
2790    serde_json::to_writer(&mut stream, &envelope).map_err(DaemonError::Encode)?;
2791    stream.write_all(b"\n").map_err(DaemonError::Write)?;
2792
2793    let mut line = String::new();
2794    let mut reader = BufReader::new(stream).take(MAX_ENVELOPE_BYTES + 1);
2795    reader.read_line(&mut line).map_err(DaemonError::Read)?;
2796    if line.len() as u64 > MAX_ENVELOPE_BYTES {
2797        return Err(DaemonError::InvalidResponse(
2798            "response envelope is too large".into(),
2799        ));
2800    }
2801    serde_json::from_str(line.trim_end()).map_err(DaemonError::Decode)
2802}
2803
2804/// Validates a `run` request and persists one queued activation. Acceptance
2805/// never implies a spawn; reconciliation decides that separately.
2806fn handle_run(
2807    state: &mut DispatcherState,
2808    args: &crate::protocol::RunArgs,
2809) -> Result<serde_json::Value, ErrorBody> {
2810    use crate::protocol::RunActivation;
2811
2812    if args.ticket.is_some() && args.project.is_some() {
2813        return Err(invalid_arguments(
2814            "a run may target a ticket or a project, not both",
2815        ));
2816    }
2817    if let Some(ticket_id) = &args.ticket {
2818        let Some(ticket) = lookup(state, |store| store.ticket(ticket_id))? else {
2819            return Err(not_found(&format!(
2820                "ticket `{ticket_id}` is not registered"
2821            )));
2822        };
2823        if ticket.state == TicketState::Held.as_str() {
2824            return Err(conflict(&format!(
2825                "ticket `{ticket_id}` is held; release it with `sloop ready {ticket_id}`"
2826            )));
2827        }
2828    }
2829    if let Some(project) = &args.project
2830        && !lookup(state, |store| store.project_exists(project))?
2831    {
2832        return Err(not_found(&format!("project `{project}` is not indexed")));
2833    }
2834    for only in &args.only {
2835        let Some(ticket) = lookup(state, |store| store.ticket(only))? else {
2836            return Err(not_found(&format!("ticket `{only}` is not registered")));
2837        };
2838        if let Some(project) = &args.project
2839            && &ticket.project_id != project
2840        {
2841            return Err(invalid_arguments(&format!(
2842                "ticket `{only}` belongs to project `{}`, not `{project}`",
2843                ticket.project_id
2844            )));
2845        }
2846    }
2847
2848    let (kind, echo_kind, interval_ms) = match &args.activation {
2849        RunActivation::Now => (ActivationKind::Immediate, "now", None),
2850        RunActivation::At { .. } => (ActivationKind::At, "at", None),
2851        RunActivation::Every { interval_ms } => {
2852            (ActivationKind::Every, "every", Some(*interval_ms as i64))
2853        }
2854        RunActivation::Overnight => (ActivationKind::Overnight, "overnight", None),
2855    };
2856
2857    let now_ms = state.clock.now_ms();
2858    let activation_id = format!(
2859        "A{}",
2860        lookup(state, |store| store.next_activation_ordinal())?
2861    );
2862    lookup(state, |store| {
2863        store.insert_activation(
2864            &NewActivation {
2865                id: &activation_id,
2866                kind,
2867                ticket_id: args.ticket.as_deref(),
2868                project_id: args.project.as_deref(),
2869                // Eligible times for scheduled kinds arrive with clock
2870                // support; queued recording preserves the request today.
2871                eligible_at_ms: None,
2872                interval_ms,
2873            },
2874            now_ms,
2875        )
2876    })?;
2877    for only in &args.only {
2878        lookup(state, |store| {
2879            store.insert_activation_filter(&activation_id, only)
2880        })?;
2881    }
2882
2883    let mut activation = json!({
2884        "id": activation_id,
2885        "kind": echo_kind,
2886        "state": "queued",
2887    });
2888    if let Some(ticket) = &args.ticket {
2889        activation["ticket"] = json!(ticket);
2890    }
2891    if let Some(project) = &args.project {
2892        activation["project"] = json!(project);
2893    }
2894    Ok(json!({"activation": activation}))
2895}
2896
2897fn handle_hold(
2898    state: &mut DispatcherState,
2899    args: &crate::protocol::TicketReferenceArgs,
2900) -> Result<serde_json::Value, ErrorBody> {
2901    let requested = TicketState::Held;
2902    let previous = state
2903        .store
2904        .set_ticket_hold(&args.ticket, requested, state.clock.now_ms())
2905        .map_err(|error| match error {
2906            StoreError::TicketNotFound { .. } => not_found(&error.to_string()),
2907            StoreError::TicketStateConflict { .. } => conflict(&error.to_string()),
2908            _ => internal(&error.to_string()),
2909        })?;
2910    Ok(json!({
2911        "ticket": args.ticket,
2912        "previous_state": previous,
2913        "state": requested.as_str(),
2914        "overridden": previous != requested.as_str(),
2915    }))
2916}
2917
2918fn handle_ready(
2919    state: &mut DispatcherState,
2920    args: &crate::protocol::TicketReferenceArgs,
2921) -> Result<serde_json::Value, ErrorBody> {
2922    let requested = TicketState::Ready;
2923    let previous = state
2924        .store
2925        .set_ticket_hold(&args.ticket, requested, state.clock.now_ms())
2926        .map_err(|error| match error {
2927            StoreError::TicketNotFound { .. } => not_found(&error.to_string()),
2928            StoreError::TicketStateConflict { .. } => conflict(&error.to_string()),
2929            _ => internal(&error.to_string()),
2930        })?;
2931    Ok(json!({
2932        "ticket": args.ticket,
2933        "previous_state": previous,
2934        "state": requested.as_str(),
2935        "overridden": previous != requested.as_str(),
2936    }))
2937}
2938
2939fn handle_retry(
2940    state: &mut DispatcherState,
2941    args: &crate::protocol::TicketReferenceArgs,
2942) -> Result<serde_json::Value, ErrorBody> {
2943    let previous = state
2944        .store
2945        .retry_ticket(&args.ticket, state.clock.now_ms())
2946        .map_err(|error| match error {
2947            StoreError::TicketNotFound { .. } => not_found(&error.to_string()),
2948            StoreError::TicketStateConflict { .. } => conflict(&error.to_string()),
2949            _ => internal(&error.to_string()),
2950        })?;
2951    Ok(json!({
2952        "ticket": args.ticket,
2953        "previous_state": previous,
2954        "state": TicketState::Ready.as_str(),
2955    }))
2956}
2957
2958/// One non-blocking snapshot of a run's state; the client loops. Launch and
2959/// recovery closures are terminal alongside ordinary derived outcomes.
2960fn handle_wait(
2961    state: &DispatcherState,
2962    args: &crate::protocol::RunReferenceArgs,
2963) -> Result<serde_json::Value, ErrorBody> {
2964    let Some(run) = lookup(state, |store| store.run(&args.run))? else {
2965        return Err(not_found(&format!("run `{}` does not exist", args.run)));
2966    };
2967    let terminal = matches!(
2968        run.state.as_str(),
2969        "merged" | "failed" | "needs_review" | "cancelled" | "orphaned" | "aborted"
2970    );
2971    Ok(json!({
2972        "run": run.id,
2973        "state": run.state,
2974        "terminal": terminal,
2975        "exit_code": run.exit_code,
2976    }))
2977}
2978
2979/// Returns one finite page of captured run output. Records are stored
2980/// escaped inside the response; raw agent bytes never reach Sloop's stdout.
2981fn handle_logs(
2982    state: &DispatcherState,
2983    args: &crate::protocol::RunReferenceArgs,
2984) -> Result<serde_json::Value, ErrorBody> {
2985    if lookup(state, |store| store.run(&args.run))?.is_none() {
2986        return Err(not_found(&format!("run `{}` does not exist", args.run)));
2987    }
2988    let page = crate::run_log::read_page(
2989        &run_output_path(&state.state_dir, &args.run),
2990        0,
2991        LOGS_PAGE_LIMIT,
2992    )
2993    .map_err(|error| internal(&format!("cannot read run log: {error}")))?;
2994    let entries = page
2995        .entries
2996        .iter()
2997        .map(serde_json::to_value)
2998        .collect::<Result<Vec<_>, _>>()
2999        .map_err(|error| internal(&format!("cannot encode run log: {error}")))?;
3000    Ok(json!({
3001        "run": args.run,
3002        "entries": entries,
3003        "next_cursor": page.next_cursor,
3004        "complete": page.complete,
3005    }))
3006}
3007
3008/// Records cancellation intent durably, then kills the run's whole process
3009/// group. Termination is confirmed by the exit event, which reads the intent
3010/// and settles the outcome as `Cancelled`; the worktree, branch, and captured
3011/// logs are preserved as evidence.
3012fn handle_cancel(
3013    state: &mut DispatcherState,
3014    args: &crate::protocol::RunReferenceArgs,
3015) -> Result<serde_json::Value, ErrorBody> {
3016    let Some(run) = lookup(state, |store| store.run(&args.run))? else {
3017        return Err(not_found(&format!("run `{}` does not exist", args.run)));
3018    };
3019    if !matches!(run.state.as_str(), "running" | "aftercare") || run.exited_at_ms.is_some() {
3020        return Err(conflict(&format!(
3021            "run `{}` is `{}` and cannot be cancelled",
3022            run.id, run.state
3023        )));
3024    }
3025
3026    // Intent must be durable before any signal: if the daemon dies between
3027    // the kill and the exit event, recovery still reads the cancellation.
3028    lookup(state, |store| {
3029        store.record_cancel_requested(&run.id, state.clock.now_ms())
3030    })?;
3031    state.cancelling.insert(run.id.clone());
3032
3033    if run.state == "aftercare" {
3034        let rows = lookup(state, |store| store.run_evidence(&run.id))?;
3035        for kind in ["merge_process", "test_process"] {
3036            if let Some((pid, start_time, group)) =
3037                aftercare_process_identity(&rows, kind).map_err(|error| internal(&error))?
3038                && process_identity_matches(pid, Some(start_time))
3039            {
3040                unsafe {
3041                    libc::kill(-(group as libc::pid_t), libc::SIGKILL);
3042                }
3043                break;
3044            }
3045        }
3046    } else {
3047        let process_matches = run
3048            .pid
3049            .and_then(|pid| u32::try_from(pid).ok())
3050            .is_some_and(|pid| process_identity_matches(pid, run.pid_start_time));
3051        if process_matches && let Some(group) = run.process_group_id {
3052            // A negative PID signals the whole group, so grandchildren die too.
3053            // ESRCH means the group already exited; the race resolves through
3054            // the recorded intent.
3055            unsafe {
3056                libc::kill(-(group as libc::pid_t), libc::SIGKILL);
3057            }
3058        }
3059    }
3060
3061    Ok(json!({
3062        "run": run.id,
3063        "state": "cancelling",
3064        "worktree": run.worktree_path,
3065        "preserved": true,
3066    }))
3067}
3068
3069/// Validates a stop request and, when forced, cancels every active run
3070/// through the same durable-intent path as `cancel`. The connection handler
3071/// owns the actual exit so the reply always reaches the caller first.
3072fn handle_stop(
3073    state: &mut DispatcherState,
3074    args: &crate::protocol::StopArgs,
3075) -> Result<serde_json::Value, ErrorBody> {
3076    let mut active: Vec<String> = state.active.iter().cloned().collect();
3077    active.sort();
3078    if !active.is_empty() && !args.force {
3079        return Err(conflict(&format!(
3080            "{} active run(s): {}; stop --force cancels them",
3081            active.len(),
3082            active.join(", "),
3083        )));
3084    }
3085    let mut cancelled = Vec::new();
3086    for run_id in active {
3087        if handle_cancel(
3088            state,
3089            &crate::protocol::RunReferenceArgs {
3090                run: run_id.clone(),
3091            },
3092        )
3093        .is_ok()
3094        {
3095            cancelled.push(run_id);
3096        }
3097    }
3098    Ok(json!({
3099        "stopping": true,
3100        "pid": state.pid,
3101        "cancelled_runs": cancelled,
3102    }))
3103}
3104
3105fn lookup<T>(
3106    state: &DispatcherState,
3107    query: impl FnOnce(&Store) -> Result<T, StoreError>,
3108) -> Result<T, ErrorBody> {
3109    query(&state.store).map_err(|error| internal(&error.to_string()))
3110}
3111
3112fn invalid_arguments(message: &str) -> ErrorBody {
3113    ErrorBody {
3114        code: ErrorCode::InvalidArguments,
3115        message: message.into(),
3116        details: json!({}),
3117    }
3118}
3119
3120fn not_found(message: &str) -> ErrorBody {
3121    ErrorBody {
3122        code: ErrorCode::NotFound,
3123        message: message.into(),
3124        details: json!({}),
3125    }
3126}
3127
3128fn conflict(message: &str) -> ErrorBody {
3129    ErrorBody {
3130        code: ErrorCode::Conflict,
3131        message: message.into(),
3132        details: json!({}),
3133    }
3134}
3135
3136fn post_error_body(error: &crate::post::PostError) -> ErrorBody {
3137    use crate::post::PostError;
3138    let code = match error {
3139        PostError::TicketFileNotFound(_)
3140        | PostError::UnknownProject(_)
3141        | PostError::UnknownFlow { .. }
3142        | PostError::UnknownBlockedBy { .. } => ErrorCode::NotFound,
3143        PostError::OutsideRepository(_)
3144        | PostError::OutsideTicketDirectory { .. }
3145        | PostError::InvalidTicket { .. }
3146        | PostError::MissingName { .. }
3147        | PostError::MissingBlockedBy { .. }
3148        | PostError::InvalidBlockedBy { .. }
3149        | PostError::EmptyBody { .. }
3150        | PostError::UnknownTarget(_)
3151        | PostError::MissingTargetValue { .. } => ErrorCode::InvalidArguments,
3152        PostError::ProjectConflict { .. }
3153        | PostError::FlowConflict { .. }
3154        | PostError::TicketIdTaken { .. }
3155        | PostError::DependencyCycle(_) => ErrorCode::Conflict,
3156        PostError::Io { .. } | PostError::Store(_) | PostError::IdAllocation(_) => {
3157            ErrorCode::Internal
3158        }
3159    };
3160    ErrorBody {
3161        code,
3162        message: error.to_string(),
3163        details: json!({}),
3164    }
3165}
3166
3167fn protocol_error(message: &str) -> ErrorBody {
3168    ErrorBody {
3169        code: ErrorCode::InvalidRequest,
3170        message: message.into(),
3171        details: json!({}),
3172    }
3173}
3174
3175fn unauthorized(message: &str) -> ErrorBody {
3176    ErrorBody {
3177        code: ErrorCode::Unauthorized,
3178        message: message.into(),
3179        details: json!({}),
3180    }
3181}
3182
3183fn internal(message: &str) -> ErrorBody {
3184    ErrorBody {
3185        code: ErrorCode::Internal,
3186        message: message.into(),
3187        details: json!({}),
3188    }
3189}
3190
3191#[derive(Debug)]
3192pub enum DaemonError {
3193    Config(ConfigError),
3194    Store(StoreError),
3195    CurrentDirectory(io::Error),
3196    CurrentExecutable(io::Error),
3197    Io {
3198        path: PathBuf,
3199        source: io::Error,
3200    },
3201    AlreadyRunning,
3202    Runtime(io::Error),
3203    Spawn(io::Error),
3204    Connect(io::Error),
3205    Write(io::Error),
3206    Read(io::Error),
3207    Encode(serde_json::Error),
3208    Decode(serde_json::Error),
3209    InvalidResponse(String),
3210    Frontmatter {
3211        path: PathBuf,
3212        error: FrontmatterError,
3213    },
3214    IdAllocation(IdError),
3215}
3216
3217impl DaemonError {
3218    pub fn error_body(&self) -> ErrorBody {
3219        let code = match self {
3220            Self::Config(_) => ErrorCode::InvalidArguments,
3221            _ => ErrorCode::DaemonUnavailable,
3222        };
3223        ErrorBody {
3224            code,
3225            message: self.to_string(),
3226            details: json!({}),
3227        }
3228    }
3229}
3230
3231impl From<ConfigError> for DaemonError {
3232    fn from(error: ConfigError) -> Self {
3233        Self::Config(error)
3234    }
3235}
3236
3237impl From<IdError> for DaemonError {
3238    fn from(error: IdError) -> Self {
3239        Self::IdAllocation(error)
3240    }
3241}
3242
3243impl std::fmt::Display for DaemonError {
3244    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
3245        match self {
3246            Self::Config(error) => error.fmt(formatter),
3247            Self::Store(error) => error.fmt(formatter),
3248            Self::CurrentDirectory(error) => {
3249                write!(formatter, "cannot read current directory: {error}")
3250            }
3251            Self::CurrentExecutable(error) => {
3252                write!(formatter, "cannot locate sloop executable: {error}")
3253            }
3254            Self::Io { path, source } => write!(formatter, "{}: {source}", path.display()),
3255            Self::AlreadyRunning => formatter.write_str("another sloop daemon holds the lock"),
3256            Self::Runtime(error) => write!(formatter, "cannot start async runtime: {error}"),
3257            Self::Spawn(error) => write!(formatter, "cannot spawn daemon: {error}"),
3258            Self::Connect(error) => write!(formatter, "cannot connect to daemon: {error}"),
3259            Self::Write(error) => write!(formatter, "cannot write daemon request: {error}"),
3260            Self::Read(error) => write!(formatter, "cannot read daemon response: {error}"),
3261            Self::Encode(error) => write!(formatter, "cannot encode daemon request: {error}"),
3262            Self::Decode(error) => write!(formatter, "cannot decode daemon response: {error}"),
3263            Self::InvalidResponse(message) => formatter.write_str(message),
3264            Self::Frontmatter { path, error } => write!(formatter, "{}: {error}", path.display()),
3265            Self::IdAllocation(error) => error.fmt(formatter),
3266        }
3267    }
3268}
3269
3270impl std::error::Error for DaemonError {}
3271
3272#[cfg(test)]
3273mod tests {
3274    use std::fs;
3275
3276    use tempfile::tempdir;
3277
3278    use super::{
3279        WORKER_BOOTSTRAP_PROMPT, compose_worker_prompt, index_projects, process_start_time,
3280        recoverable_process_matches,
3281    };
3282    use crate::config::expand_agent_cmd;
3283    use crate::store::{RecoverableRun, Store};
3284
3285    fn recoverable_current_process(start_time: Option<i64>) -> RecoverableRun {
3286        RecoverableRun {
3287            id: "R1".into(),
3288            ticket_id: "T1".into(),
3289            state: "running".into(),
3290            branch: None,
3291            worktree_path: None,
3292            pid: Some(i64::from(std::process::id())),
3293            pid_start_time: start_time,
3294            process_group_id: None,
3295            worker_token: None,
3296            worker_socket_path: None,
3297            exit_code: None,
3298            lease_expires_at_ms: 1,
3299        }
3300    }
3301
3302    #[test]
3303    fn recovery_requires_both_pid_and_start_time_to_match() {
3304        let Some(start_time) = process_start_time(std::process::id()) else {
3305            return;
3306        };
3307        assert!(recoverable_process_matches(&recoverable_current_process(
3308            Some(start_time)
3309        )));
3310        assert!(!recoverable_process_matches(&recoverable_current_process(
3311            Some(start_time + 1)
3312        )));
3313        assert!(!recoverable_process_matches(&recoverable_current_process(
3314            None
3315        )));
3316    }
3317
3318    #[test]
3319    fn agent_command_expands_ticket_model_and_effort() {
3320        let template = vec![
3321            "agent".to_owned(),
3322            "--model={model}".to_owned(),
3323            "--effort".to_owned(),
3324            "{effort}".to_owned(),
3325            "prompt={prompt}".to_owned(),
3326        ];
3327
3328        assert_eq!(
3329            expand_agent_cmd(&template, Some("sonnet"), Some("medium"), "assignment").unwrap(),
3330            [
3331                "agent",
3332                "--model=sonnet",
3333                "--effort",
3334                "medium",
3335                "prompt=assignment"
3336            ]
3337        );
3338    }
3339
3340    #[test]
3341    fn agent_command_rejects_a_missing_ticket_field() {
3342        let template = vec!["agent".to_owned(), "{model}".to_owned()];
3343
3344        assert_eq!(
3345            expand_agent_cmd(&template, None, Some("medium"), "assignment"),
3346            Err("does not specify `model`".to_owned())
3347        );
3348    }
3349
3350    #[test]
3351    fn worker_prompt_uses_the_builtin_when_instructions_are_absent() {
3352        let root = tempdir().unwrap();
3353
3354        assert_eq!(
3355            compose_worker_prompt(root.path()).unwrap(),
3356            WORKER_BOOTSTRAP_PROMPT
3357        );
3358    }
3359
3360    #[test]
3361    fn worker_prompt_appends_repository_instructions() {
3362        let root = tempdir().unwrap();
3363        fs::create_dir_all(root.path().join(".agents/sloop")).unwrap();
3364        fs::write(
3365            root.path().join(".agents/sloop/instructions.md"),
3366            "Use repository conventions.\n",
3367        )
3368        .unwrap();
3369
3370        assert_eq!(
3371            compose_worker_prompt(root.path()).unwrap(),
3372            format!("{WORKER_BOOTSTRAP_PROMPT}\n\nUse repository conventions.\n")
3373        );
3374    }
3375
3376    #[test]
3377    fn orphan_disposition_stamps_waits_then_deletes_unreferenced_rows() {
3378        use super::OrphanDisposition::{Delete, Keep, MarkMissing};
3379        use super::orphan_disposition;
3380
3381        assert_eq!(orphan_disposition(None, false, 1_000, 100), MarkMissing);
3382        assert_eq!(orphan_disposition(Some(950), false, 1_000, 100), Keep);
3383        assert_eq!(orphan_disposition(Some(900), false, 1_000, 100), Delete);
3384        assert_eq!(orphan_disposition(Some(900), true, 1_000, 100), Keep);
3385    }
3386
3387    #[test]
3388    fn reconcile_stamps_deletes_and_restores_tickets() {
3389        use crate::store::TicketState;
3390
3391        let root = tempdir().unwrap();
3392        let tickets = root.path().join(".agents/sloop/tickets");
3393        fs::create_dir_all(&tickets).unwrap();
3394        fs::write(tickets.join("present.md"), "# Present\n").unwrap();
3395        let store = Store::open(&root.path().join("sloop.db"), 1_000).unwrap();
3396        store
3397            .insert_local_project(
3398                "default",
3399                ".agents/sloop/projects/default.md",
3400                "Default",
3401                1_000,
3402            )
3403            .unwrap();
3404        let insert = |id: &str, file: &str, blocked_by: &[String]| {
3405            store
3406                .insert_local_ticket(
3407                    id,
3408                    "default",
3409                    &format!(".agents/sloop/tickets/{file}"),
3410                    id,
3411                    blocked_by,
3412                    &format!("sloop/{id}"),
3413                    None,
3414                    None,
3415                    None,
3416                    "default",
3417                    TicketState::Ready,
3418                    1_000,
3419                )
3420                .unwrap();
3421        };
3422        insert("T1", "present.md", &[]);
3423        insert("T2", "gone.md", &[]);
3424        insert("T3", "blocked-gone.md", &[]);
3425        insert("T4", "dependent.md", &["T3".into()]);
3426        fs::write(tickets.join("dependent.md"), "# Dependent\n").unwrap();
3427
3428        let window = 100;
3429        let stamps = |store: &Store| -> Vec<(String, Option<i64>)> {
3430            store
3431                .local_ticket_files()
3432                .unwrap()
3433                .into_iter()
3434                .map(|ticket| (ticket.id, ticket.missing_at_ms))
3435                .collect()
3436        };
3437
3438        // First pass stamps the two tickets whose files are gone.
3439        super::reconcile_tickets(root.path(), &store, 2_000, window).unwrap();
3440        assert_eq!(
3441            stamps(&store),
3442            vec![
3443                ("T1".into(), None),
3444                ("T2".into(), Some(2_000)),
3445                ("T3".into(), Some(2_000)),
3446                ("T4".into(), None),
3447            ]
3448        );
3449
3450        // Within the window nothing is deleted and stamps keep their origin.
3451        super::reconcile_tickets(root.path(), &store, 2_050, window).unwrap();
3452        assert_eq!(stamps(&store)[1], ("T2".into(), Some(2_000)));
3453
3454        // Past the window the unreferenced orphan is deleted; T3 survives
3455        // because T4 still names it as a blocker.
3456        super::reconcile_tickets(root.path(), &store, 2_100, window).unwrap();
3457        assert_eq!(
3458            stamps(&store),
3459            vec![
3460                ("T1".into(), None),
3461                ("T3".into(), Some(2_000)),
3462                ("T4".into(), None),
3463            ]
3464        );
3465
3466        // The file coming back clears the stamp even after the window.
3467        fs::write(tickets.join("blocked-gone.md"), "# Returned\n").unwrap();
3468        super::reconcile_tickets(root.path(), &store, 3_000, window).unwrap();
3469        assert_eq!(stamps(&store)[1], ("T3".into(), None));
3470    }
3471
3472    #[test]
3473    fn project_allocation_uses_sorted_paths_after_explicit_high_water_marks() {
3474        let root = tempdir().unwrap();
3475        let projects = root.path().join(".agents/sloop/projects");
3476        fs::create_dir_all(&projects).unwrap();
3477        fs::write(projects.join("zeta.md"), "# Zeta\n").unwrap();
3478        fs::write(projects.join("alpha.md"), "# Alpha\n").unwrap();
3479        fs::write(
3480            projects.join("middle.md"),
3481            "---\nid: PROJ-7\ntitle: Explicit\n---\n",
3482        )
3483        .unwrap();
3484        let store = Store::open(&root.path().join("sloop.db"), 1_000).unwrap();
3485
3486        index_projects(
3487            root.path(),
3488            std::path::Path::new(".agents/sloop/projects"),
3489            &store,
3490            1_000,
3491            "PROJ",
3492        )
3493        .unwrap();
3494
3495        assert!(
3496            fs::read_to_string(projects.join("alpha.md"))
3497                .unwrap()
3498                .contains("id: PROJ-8")
3499        );
3500        assert!(
3501            fs::read_to_string(projects.join("zeta.md"))
3502                .unwrap()
3503                .contains("id: PROJ-9")
3504        );
3505    }
3506}