Skip to main content

magi/
waiter.rs

1//! The waiter: something that keeps waiting on a question after the process
2//! that asked it is gone.
3//!
4//! `magi ask` blocks in the agent's own shell tool, and that tool kills a wait
5//! that runs long (see `ask::WAIT_SLICE`). An agent can also background the
6//! call and simply finish. Either way the question outlives the only process
7//! that would have read the owner's answer, and the owner's reply lands in a
8//! file nobody is polling - the phone then says "waiting for the agent" while
9//! there is no agent to wait for.
10//!
11//! This module is the guarantee that something is always on the other end. It
12//! runs inside `magi serve`, on its own cadence rather than the queue's, and
13//! for every question a `magi ask` filed it decides one of three things:
14//!
15//! - **the asker is still there** (its [`Lease`] is fresh): do nothing. A
16//!   second agent must never be started while the first is blocked;
17//! - **the asker is gone and the owner has said or answered something the agent
18//!   has not read**: resume the *same seat's* CLI session with the question,
19//!   the thread so far and the owner's words, so the agent that holds the
20//!   context carries on;
21//! - **nobody answered before `answer_timeout`**: abandon it, in the same words
22//!   the asker would have used.
23//!
24//! # Shape
25//!
26//! Same split as [`crate::queue`] and [`crate::ask`]: [`decide`] is pure - a
27//! question, a lease, a clock and a bool in, an [`Action`] out - and every
28//! filesystem or process effect is in [`Waiter::tick`]. State lives on disk (the
29//! question record and its lease), so a restarted daemon picks up exactly where
30//! the last one stopped, including an answer nobody has read yet.
31//!
32//! # What it will not do
33//!
34//! It never starts a fresh consultant. If the seat's session cannot be resumed
35//! (`agent::has_session` says no, the working directory is gone, the run is
36//! over, the agent left the roster) the question stays as it is, the owner
37//! gets a notification saying why, and nothing else runs. A new agent would
38//! have to be caught up on everything the first one knew, and would act on the
39//! owner's words without the context they were written against.
40//!
41//! It never assumes a resume worked either: a turn that fails or times out
42//! leaves the word undelivered, and a later tick tries again.
43
44use std::collections::HashMap;
45use std::path::PathBuf;
46use std::time::{Duration, Instant};
47
48use anyhow::Result;
49use jiff::Timestamp;
50
51use crate::agent::{self, Invocation, SeatState};
52use crate::ask::{Lease, Question, QuestionStatus, Questions, Waiter as Note, WaiterKind, Who};
53use crate::config::{AgentSpec, Config};
54use crate::notices::{self, Notice};
55use crate::prompt;
56use crate::run::RunState;
57
58/// How often the waiter looks at the store.
59pub const TICK: Duration = Duration::from_secs(5);
60
61/// Wall-clock limit for one resumed turn. A resumed agent may go on to do real
62/// work once it has the owner's word, but it is one conversation turn, not a
63/// whole node.
64const DELIVERY_TIMEOUT: Duration = Duration::from_secs(15 * 60);
65
66/// How long a failed or refused delivery is left alone before the same input
67/// is tried again. Long enough not to hammer a seat that is out of quota,
68/// short enough that a fixed problem does not wait a day.
69const RETRY_AFTER: Duration = Duration::from_secs(10 * 60);
70
71#[derive(serde::Serialize, serde::Deserialize)]
72struct Retry {
73    key: String,
74    until: i64,
75}
76
77fn retry_path(store: &Questions, id: &str) -> PathBuf {
78    store.root().join(format!("{id}.waiter-retry.json"))
79}
80
81fn retry_blocked(store: &Questions, id: &str, key: &str, now: Timestamp) -> bool {
82    std::fs::read(retry_path(store, id))
83        .ok()
84        .and_then(|bytes| serde_json::from_slice::<Retry>(&bytes).ok())
85        .is_some_and(|retry| retry.key == key && now.as_second() < retry.until)
86}
87
88/// What the agent has not read yet.
89#[derive(Debug, Clone, PartialEq, Eq)]
90pub enum Word {
91    /// The owner spoke back without deciding.
92    Said(String),
93    /// The owner decided.
94    Answered(String),
95}
96
97/// What to do about one question, this tick.
98#[derive(Debug, Clone, PartialEq, Eq)]
99pub enum Action {
100    /// Nothing: not ours, someone is waiting, or nothing new to hand over.
101    Idle,
102    /// The deadline passed with no answer and nobody waiting.
103    Expire,
104    /// Resume the asking seat with this.
105    Deliver(Word),
106}
107
108/// The whole decision, with no I/O.
109///
110/// `seat_busy` is whether the asking seat's CLI is still mid-turn - the agent
111/// that exited its `magi ask` may not have exited itself, and resuming a
112/// conversation that is running would fork it.
113///
114/// Order matters: a word the owner gave before the deadline is delivered even
115/// if the deadline has passed by the time the waiter looks, and only then does
116/// an unanswered question expire.
117pub fn decide(
118    q: &Question,
119    lease: Option<&Lease>,
120    seat_busy: bool,
121    default_timeout: u64,
122    now: Timestamp,
123) -> Action {
124    decide_owned(q, lease, seat_busy, default_timeout, now, false)
125}
126
127/// [`decide`], told whether the daemon owns the answer's action.
128///
129/// `daemon_owns_action` is true when the daemon will apply the action (the
130/// task exists and is in a state `daemon::decide_action` acts on). An answer
131/// that carries a [`crate::ask::ChoiceAction`] is then the daemon's alone
132/// (`daemon::apply_choice_actions`, idempotent through `Task::actions_applied`
133/// under the task's claim): resuming the dead seat here as well would run the
134/// same seat twice when the action is a resume or a requeue. With no task
135/// there is no one to apply it, so the words are delivered as before.
136pub fn decide_owned(
137    q: &Question,
138    lease: Option<&Lease>,
139    seat_busy: bool,
140    default_timeout: u64,
141    now: Timestamp,
142    daemon_owns_action: bool,
143) -> Action {
144    if q.routed_task.is_some() || q.cwd.is_none() || lease.is_some_and(|l| l.fresh(now)) {
145        return Action::Idle;
146    }
147    match q.status {
148        QuestionStatus::Abandoned => Action::Idle,
149        QuestionStatus::Answered => match q.resolution() {
150            _ if q.chosen_action().is_some_and(|a| a.maintenance())
151                || (daemon_owns_action && q.chosen_action().is_some()) =>
152            {
153                Action::Idle
154            }
155            Some(a) if !q.answer_delivered && !seat_busy => Action::Deliver(Word::Answered(a)),
156            _ => Action::Idle,
157        },
158        QuestionStatus::Open => {
159            if let Some(said) = q.unread_from_owner() {
160                if seat_busy {
161                    return Action::Idle;
162                }
163                return Action::Deliver(Word::Said(said));
164            }
165            let secs = if q.answer_timeout > 0 {
166                q.answer_timeout
167            } else {
168                default_timeout
169            };
170            let deadline = q.last_activity().saturating_add(secs as i64);
171            if now.as_second() > deadline {
172                Action::Expire
173            } else {
174                Action::Idle
175            }
176        }
177    }
178}
179
180/// Everything needed to resume one seat.
181struct Target {
182    coordination: Option<crate::config::Coordination>,
183    editing_network: crate::config::EditingNetwork,
184    fetched_base: Option<crate::run::FetchedBase>,
185    spec: AgentSpec,
186    seat: SeatState,
187    cwd: PathBuf,
188    sessions: bool,
189    allow_write: bool,
190    language: String,
191}
192
193/// The waiter's state: the store it watches and what it must remember between
194/// ticks (only which inputs it has already failed on).
195pub struct Waiter {
196    store: Questions,
197    home: PathBuf,
198    /// The config the conductor runs under, for a question the conductor asked
199    /// (it has no run whose snapshot could say).
200    conduct_cfg: Option<Config>,
201    /// `answer_timeout` for a question that did not record its own.
202    default_timeout: u64,
203    /// Inputs already tried and failed, by question id: `(key, until)`.
204    memo: HashMap<String, (String, Instant)>,
205}
206
207impl Waiter {
208    /// A waiter over `store`, with `home` as the magi home its runs live under.
209    pub fn new(store: Questions, home: PathBuf, conduct_cfg: Option<Config>) -> Self {
210        let default_timeout = conduct_cfg
211            .as_ref()
212            .map_or(Config::default().graph.answer_timeout, |c| {
213                c.graph.answer_timeout
214            });
215        Self {
216            store,
217            home,
218            conduct_cfg,
219            default_timeout,
220            memo: HashMap::new(),
221        }
222    }
223
224    /// Admission check without starting a process for an idle question store.
225    pub fn has_work(&self, now: Timestamp) -> bool {
226        self.store.list().iter().any(|q| {
227            let lease = self.store.read_lease(&q.id);
228            match decide_owned(
229                q,
230                lease.as_ref(),
231                false,
232                self.default_timeout,
233                now,
234                self.daemon_owns_action(q),
235            ) {
236                Action::Idle => false,
237                Action::Expire => true,
238                Action::Deliver(_) if q.deputy.is_some() => crate::deputy::exhausted_past_deadline(
239                    q,
240                    crate::deputy::can_start(self.conduct_cfg.as_ref(), crate::deputy::agent_of(q)),
241                    self.default_timeout,
242                    now,
243                ),
244                Action::Deliver(word) => {
245                    let key = format!("{}:{:?}", q.thread.len(), word);
246                    !self.seat_busy(q, now) && !retry_blocked(&self.store, &q.id, &key, now)
247                }
248            }
249        })
250    }
251
252    /// Is this question's asking seat still mid-turn?
253    fn seat_busy(&self, q: &Question, now: Timestamp) -> bool {
254        if q.run == crate::conduct::NODE {
255            return crate::conduct::busy(&self.home);
256        }
257        RunState::load_under(&q.run, &self.home).is_ok_and(|s| {
258            s.seats_active()
259                .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
260        })
261    }
262
263    /// Whether the daemon applies this question's chosen action, so the
264    /// waiter must not deliver the answer (see [`decide_owned`]).
265    fn daemon_owns_action(&self, q: &Question) -> bool {
266        q.chosen_action().is_some_and(|a| a.maintenance())
267            || crate::daemon::task_of_question(
268                &crate::queue::Queue::at(self.home.join("queue")).list(),
269                q,
270            )
271            .is_some_and(|t| crate::daemon::action_standing(t, q).daemon_owns())
272    }
273
274    /// Look at every question once and act on what needs acting on. `halt` is
275    /// asked between questions and during a delivery: true means the daemon is
276    /// parking, and whatever is in flight is dropped, undelivered and
277    /// resumable.
278    pub async fn tick(&mut self, now: Timestamp, halt: &(dyn Fn() -> bool + Sync)) {
279        for q in self.store.list() {
280            if halt() {
281                return;
282            }
283            if crate::task_chat::routes(&q, &self.home) {
284                continue;
285            }
286            let lease = self.store.read_lease(&q.id);
287            let owned = self.daemon_owns_action(&q);
288            match decide_owned(&q, lease.as_ref(), false, self.default_timeout, now, owned) {
289                Action::Idle => {}
290                Action::Expire => self.expire(&q),
291                Action::Deliver(word) => {
292                    // A question with a deputy is the deputy's to read
293                    // (`crate::deputy`); resuming the conductor's own seat
294                    // here would fork a conversation that is not its own.
295                    if q.deputy.is_some() {
296                        // Unless that deputy is spent: then nobody will read
297                        // the word, and the deadline still has to retire the
298                        // question.
299                        if crate::deputy::exhausted_past_deadline(
300                            &q,
301                            crate::deputy::can_start(
302                                self.conduct_cfg.as_ref(),
303                                crate::deputy::agent_of(&q),
304                            ),
305                            self.default_timeout,
306                            now,
307                        ) {
308                            self.expire(&q);
309                        }
310                        continue;
311                    }
312                    if self.seat_busy(&q, now) {
313                        continue;
314                    }
315                    let key = format!("{}:{:?}", q.thread.len(), word);
316                    if retry_blocked(&self.store, &q.id, &key, now)
317                        || matches!(self.memo.get(&q.id), Some((k, until)) if *k == key && Instant::now() < *until)
318                    {
319                        continue;
320                    }
321                    if let Err(e) = self.deliver(&q, &word, halt).await {
322                        tracing::warn!("question {}: {e:#}", q.short());
323                    }
324                }
325            }
326        }
327    }
328
329    /// Abandon an unanswered question whose deadline passed with nobody
330    /// waiting - the words the asker's own wait would have used.
331    fn expire(&self, q: &Question) {
332        let secs = if q.answer_timeout > 0 {
333            q.answer_timeout
334        } else {
335            self.default_timeout
336        };
337        let startable =
338            crate::deputy::can_start(self.conduct_cfg.as_ref(), crate::deputy::agent_of(q));
339        let why = format!("no answer within {}s of asking", secs.max(1));
340        let done = self.store.update(&q.id, |r| {
341            // Decided again on the record as it is now: the owner may have
342            // spoken since the tick read it, and a word given in time is
343            // delivered, not abandoned.
344            let now = Timestamp::now();
345            let lease = self.store.read_lease(&r.id);
346            if decide(r, lease.as_ref(), false, self.default_timeout, now) != Action::Expire
347                && !(lease.as_ref().is_none_or(|l| !l.fresh(now))
348                    && crate::deputy::exhausted_past_deadline(
349                        r,
350                        startable,
351                        self.default_timeout,
352                        now,
353                    ))
354            {
355                return Ok(false);
356            }
357            r.abandon(&why);
358            r.waiter = None;
359            Ok(true)
360        });
361        match done {
362            Ok((_, false)) => {}
363            Ok((_, true)) => {
364                self.store.drop_lease(&q.id);
365                tracing::warn!(
366                    "question {} went unanswered for {secs}s with nobody waiting; \
367                     it stays as the record of it",
368                    q.short()
369                );
370            }
371            Err(e) => tracing::warn!("could not expire question {}: {e:#}", q.short()),
372        }
373    }
374
375    /// Admission uses the very same target resolution as actual delivery.
376    pub fn can_resume(&self, q: &Question) -> bool {
377        self.target(q).is_ok()
378    }
379
380    /// Work out how to resume the seat, or why it cannot be.
381    fn target(&self, q: &Question) -> std::result::Result<Target, String> {
382        let cwd = q
383            .cwd
384            .as_deref()
385            .map(PathBuf::from)
386            .filter(|p| p.is_dir())
387            .ok_or("the directory the agent was working in is gone")?;
388        let (cfg, seat, allow_write, fetched_base) = if q.run == crate::conduct::NODE {
389            let cfg = self
390                .conduct_cfg
391                .clone()
392                .ok_or("the conductor's configuration is not available")?;
393            let seat = crate::conduct::load_seat(&self.home)
394                .ok_or("the conductor has no recorded session")?;
395            (cfg, seat, false, None)
396        } else {
397            let state = RunState::load_under(&q.run, &self.home)
398                .map_err(|_| "the run that asked is gone".to_owned())?;
399            if !state.status.resumable() {
400                return Err(format!(
401                    "the run that asked has already {}",
402                    state.status.as_str()
403                ));
404            }
405            let seat = state
406                .seats
407                .get(&q.seat)
408                .cloned()
409                .ok_or("the run has no record of the asking seat")?;
410            let write = matches!(q.node.as_str(), "implement" | "fix");
411            (state.config, seat, write, state.fetched_base)
412        };
413        let spec = cfg
414            .agent(&seat.agent)
415            .map_err(|_| format!("agent `{}` is no longer in the roster", seat.agent))?
416            .clone();
417        if !agent::has_session(spec.kind, &seat, cfg.graph.sessions) {
418            return Err("the seat has no session to resume".to_owned());
419        }
420        Ok(Target {
421            coordination: cfg.coordination.clone(),
422            editing_network: cfg.graph.editing_network,
423            fetched_base,
424            spec,
425            seat,
426            cwd,
427            sessions: cfg.graph.sessions,
428            allow_write,
429            language: cfg.graph.language.clone(),
430        })
431    }
432
433    /// Say why the owner's word cannot reach anybody, once per input.
434    fn refuse(&mut self, q: &Question, key: String, why: &str) {
435        tracing::warn!("question {} cannot be delivered: {why}", q.short());
436        notices::raise_in(
437            &self.home,
438            Notice::warn(
439                &format!("question:{}", q.id),
440                format!(
441                    "Question {} \"{}\": what you said cannot reach the agent \
442                     ({why}). It is recorded, but nothing will read it.",
443                    q.short(),
444                    q.summary
445                ),
446            )
447            .link(if q.run_names_task() {
448                notices::Link::Task { id: q.run.clone() }
449            } else {
450                notices::Link::Run { id: q.run.clone() }
451            }),
452        );
453        let _ = self.store.update(&q.id, |r| {
454            r.waiter = None;
455            Ok(())
456        });
457        let retry = Retry {
458            key: key.clone(),
459            until: Timestamp::now()
460                .as_second()
461                .saturating_add(RETRY_AFTER.as_secs() as i64),
462        };
463        let written = serde_json::to_vec(&retry)
464            .map_err(anyhow::Error::from)
465            .and_then(|bytes| {
466                std::fs::write(retry_path(&self.store, &q.id), bytes).map_err(Into::into)
467            });
468        if let Err(error) = written {
469            tracing::warn!(
470                "question {} retry cooldown could not be saved: {error:#}",
471                q.short()
472            );
473        }
474        self.memo
475            .insert(q.id.clone(), (key, Instant::now() + RETRY_AFTER));
476    }
477
478    /// Resume the asking seat with the owner's word.
479    async fn deliver(
480        &mut self,
481        q: &Question,
482        word: &Word,
483        halt: &(dyn Fn() -> bool + Sync),
484    ) -> Result<()> {
485        let key = format!("{}:{:?}", q.thread.len(), word);
486        let mut target = match self.target(q) {
487            Ok(t) => t,
488            Err(why) => {
489                self.refuse(q, key, &why);
490                return Ok(());
491            }
492        };
493
494        let claim = self.store.root().join(format!("{}.claim", q.id));
495        if !take_claim(&claim, DELIVERY_TIMEOUT + Duration::from_secs(60)) {
496            return Ok(());
497        }
498        let _release = Release(claim);
499
500        // Re-read under the claim: the asker may have come back, or the owner
501        // may have said more, since the tick's copy was taken.
502        let q = self.store.get(&q.id)?;
503        if crate::task_chat::routes(&q, &self.home) {
504            return Ok(());
505        }
506        let now = Timestamp::now();
507        let lease = self.store.read_lease(&q.id);
508        let owned = self.daemon_owns_action(&q);
509        let Action::Deliver(word) =
510            decide_owned(&q, lease.as_ref(), false, self.default_timeout, now, owned)
511        else {
512            return Ok(());
513        };
514        let snapshot = q.thread.len();
515
516        // Ownership of an action answer is settled against the daemon's task
517        // claim: hold it while the lease goes up, so the daemon either sees the
518        // fresh lease or has already applied the action (and we step aside).
519        // A claim held by a running competition is not the daemon applying an
520        // action; that case is delivered, with the re-check below.
521        let queue = crate::queue::Queue::at(self.home.join("queue"));
522        let task_id = crate::daemon::task_of_question(&queue.list(), &q).map(|t| t.id.clone());
523        // Never go on without the claim: a failed claim means an attempt (or
524        // the daemon) holds the task, and whatever it does next decides who
525        // owns this answer. Step aside; the next tick judges it afresh.
526        let _task_claim = match (&task_id, q.chosen_action()) {
527            (Some(id), Some(_)) => match queue.claim(id) {
528                Ok(c) => Some(c),
529                Err(e) => {
530                    tracing::debug!(
531                        "question {}: task claim not available ({e:#}), deferring delivery",
532                        q.short()
533                    );
534                    return Ok(());
535                }
536            },
537            _ => None,
538        };
539        // Applied, stale and pending are all stop conditions; only a busy task
540        // (or no action at all) is delivered to.
541        let task_still_ours = || {
542            if q.chosen_action().is_some_and(|a| a.maintenance()) {
543                false
544            } else {
545                match &task_id {
546                    Some(id) => queue
547                        .get(id)
548                        .map(|t| !crate::daemon::action_standing(&t, &q).daemon_owns())
549                        .unwrap_or(true),
550                    None => true,
551                }
552            }
553        };
554        if !task_still_ours() {
555            return Ok(());
556        }
557
558        self.store.beat(&q.id, WaiterKind::Daemon);
559        // Re-check after the lease is up: a daemon that decided before it
560        // existed has by now written its mark (or is blocked on our claim).
561        if !task_still_ours() {
562            return Ok(());
563        }
564        self.store.update(&q.id, |r| {
565            r.waiter = Some(Note {
566                kind: WaiterKind::Daemon,
567                since: now,
568            });
569            Ok(())
570        })?;
571        tracing::info!(
572            "question {}: the asker is gone, resuming seat {} with the owner's word",
573            q.short(),
574            q.seat
575        );
576
577        let thread: Vec<(&str, &str)> = q
578            .thread
579            .iter()
580            .map(|t| {
581                (
582                    if t.who == Who::Operator {
583                        "operator"
584                    } else {
585                        "agent"
586                    },
587                    t.body.as_str(),
588                )
589            })
590            .collect();
591        // What the owner said and the agent has not read has its own section in
592        // the prompt, so the recap stops at what it already read.
593        let shown = match word {
594            Word::Said(_) => &thread[..q.delivered_turns.min(thread.len())],
595            Word::Answered(_) => &thread[..],
596        };
597        let owner_word = match &word {
598            Word::Said(s) => prompt::OwnerWord::Said(s),
599            Word::Answered(a) => prompt::OwnerWord::Answered(a),
600        };
601        let body = prompt::question_resumed(
602            &q.id,
603            &q.summary,
604            &q.detail,
605            shown,
606            &owner_word,
607            &target.language,
608        );
609
610        let artifacts = self.store.root().join(format!("{}.delivery", q.id));
611        let stem = format!("deliver-{}", now.as_second());
612        let cache_dir = None;
613        let inv = Invocation {
614            coordination: target.coordination.as_ref(),
615            editing_network: target.editing_network,
616            fetched_base: target.fetched_base.as_ref(),
617            cwd: &target.cwd,
618            prompt: &body,
619            timeout: DELIVERY_TIMEOUT,
620            allow_write: target.allow_write,
621            unsandboxed: false,
622            sessions: target.sessions,
623            artifacts: &artifacts,
624            stem: &stem,
625            run: &q.run,
626            node: &q.node,
627            cache_dir,
628            attachments: &[],
629            writable: &[],
630        };
631
632        let store = self.store.clone();
633        let id = q.id.clone();
634        let out = {
635            let fut = agent::invoke(&target.spec, &mut target.seat, &inv);
636            tokio::pin!(fut);
637            let mut beat = tokio::time::interval(Duration::from_secs(1));
638            let mut beats = 0u32;
639            loop {
640                tokio::select! {
641                    r = &mut fut => break Some(r),
642                    _ = beat.tick() => {
643                        if halt() {
644                            break None;
645                        }
646                        beats += 1;
647                        if beats % 20 == 0 {
648                            store.beat(&id, WaiterKind::Daemon);
649                        }
650                    }
651                }
652            }
653        };
654
655        let Some(out) = out else {
656            // Parking: leave everything as it was, resumable.
657            let _ = self.store.update(&q.id, |r| {
658                r.waiter = None;
659                Ok(())
660            });
661            return Ok(());
662        };
663        let out = match out {
664            Ok(o) if o.usable() => o,
665            other => {
666                let why = match other {
667                    Ok(o) if o.timed_out => "the resumed turn timed out".to_owned(),
668                    Ok(o) if o.quota_exhausted() => "the agent is out of quota".to_owned(),
669                    Ok(o) => format!(
670                        "the resumed turn failed ({})",
671                        notices::exit_status(o.exit_code)
672                    ),
673                    Err(e) => format!("the agent could not be started: {e:#}"),
674                };
675                self.refuse(&q, key, &why);
676                return Ok(());
677            }
678        };
679
680        let text = out.text.trim().to_owned();
681        self.store.update(&q.id, |r| {
682            r.waiter = None;
683            match &word {
684                Word::Answered(_) => r.answer_delivered = true,
685                Word::Said(_) => {
686                    r.delivered_turns = r.delivered_turns.max(snapshot);
687                    // An agent that answered in prose instead of calling
688                    // `magi ask --thread` still spoke; keep it on the record so
689                    // the owner reads it. If it did use --thread, or the owner
690                    // said more meanwhile, the record already moved on.
691                    let agent_spoke = r.thread[snapshot.min(r.thread.len())..]
692                        .iter()
693                        .any(|t| t.who == Who::Agent);
694                    if !agent_spoke && r.thread.len() == snapshot && r.status.open() {
695                        let choices = r.choices.clone();
696                        r.reply(text, choices)?;
697                    }
698                }
699            }
700            Ok(())
701        })?;
702        self.memo.remove(&q.id);
703        let _ = std::fs::remove_file(retry_path(&self.store, &q.id));
704        Ok(())
705    }
706}
707
708/// Take the delivery claim, so two waiters (`magi serve` twice, or a stray
709/// second daemon) cannot resume the same seat at once. A claim older than a
710/// whole delivery plus slack was left by a process that died.
711pub(crate) fn take_claim(path: &std::path::Path, stale_after: Duration) -> bool {
712    let retry = crate::lock_file::Retry::new(None);
713    let attempt = || retry.create(path).is_ok();
714    if attempt() {
715        return true;
716    }
717    let stale = std::fs::metadata(path)
718        .and_then(|m| m.modified())
719        .ok()
720        .and_then(|t| t.elapsed().ok())
721        .is_some_and(|age| age > stale_after);
722    if stale {
723        let _ = std::fs::remove_file(path);
724        return attempt();
725    }
726    false
727}
728
729/// Removes the delivery claim when the delivery ends, however it ends.
730pub(crate) struct Release(pub(crate) PathBuf);
731
732impl Drop for Release {
733    fn drop(&mut self) {
734        let _ = std::fs::remove_file(&self.0);
735    }
736}
737
738#[cfg(test)]
739mod tests {
740    use super::*;
741
742    #[test]
743    fn editing_network_waiter_target_retains_saved_run_policy() {
744        use crate::config::EditingNetwork::{Enabled, Restricted};
745        for network in [Restricted, Enabled] {
746            let home = tempfile::tempdir().unwrap();
747            let mut cfg = Config::default();
748            cfg.graph.editing_network = network;
749            cfg.graph.sessions = true;
750            cfg.agents = vec![toml::from_str("id = 'fixture'\nkind = 'codex'\n").unwrap()];
751            let mut state = RunState::new(
752                home.path().into(),
753                "main".into(),
754                "0".repeat(40),
755                "repair".into(),
756                cfg,
757            );
758            state.status = crate::run::RunStatus::Implementing;
759            let mut seat = agent::SeatState::new("impl-A", "fixture", 1);
760            seat.turns = 1;
761            seat.captured_session = Some("saved-thread".into());
762            state.seats.insert(seat.key.clone(), seat);
763            state.save_under(home.path()).unwrap();
764            let mut q = asked(0);
765            q.run = state.id;
766            q.cwd = Some(home.path().to_string_lossy().into_owned());
767            let waiter = Waiter::new(
768                Questions::at(home.path().join("questions")),
769                home.path().into(),
770                None,
771            );
772            let target = waiter.target(&q).unwrap();
773            assert_eq!(target.editing_network, network);
774            assert!(target.allow_write);
775            assert_eq!(
776                target.seat.captured_session.as_deref(),
777                Some("saved-thread")
778            );
779        }
780    }
781
782    #[test]
783    fn retry_backoff_survives_a_new_finite_waiter() {
784        let home = tempfile::tempdir().expect("home");
785        let store = Questions::at(home.path().join("questions"));
786        std::fs::create_dir_all(store.root()).expect("questions");
787        let retry = Retry {
788            key: "input".to_owned(),
789            until: 700,
790        };
791        std::fs::write(
792            retry_path(&store, "question"),
793            serde_json::to_vec(&retry).expect("retry"),
794        )
795        .expect("save retry");
796        let reopened = Questions::at(home.path().join("questions"));
797        assert!(retry_blocked(&reopened, "question", "input", ts(699)));
798        assert!(!retry_blocked(
799            &reopened,
800            "question",
801            "different input",
802            ts(699)
803        ));
804        assert!(!retry_blocked(&reopened, "question", "input", ts(700)));
805    }
806
807    fn ts(secs: i64) -> Timestamp {
808        Timestamp::from_second(secs).unwrap()
809    }
810
811    fn asked(at: i64) -> Question {
812        let mut q = Question::new(
813            "run".into(),
814            "implement".into(),
815            "impl-A".into(),
816            "which?".into(),
817            String::new(),
818            vec![],
819        );
820        q.cwd = Some("/tmp".into());
821        q.asked_at = ts(at);
822        q.answer_timeout = 1000;
823        q
824    }
825
826    fn lease(at: i64) -> Lease {
827        Lease {
828            kind: WaiterKind::Asker,
829            pid: 1,
830            beat_at: ts(at),
831        }
832    }
833
834    #[test]
835    fn a_fresh_lease_means_nobody_else_acts() {
836        let mut q = asked(0);
837        q.say("why?").unwrap();
838        assert_eq!(
839            decide(&q, Some(&lease(95)), false, 86_400, ts(100)),
840            Action::Idle
841        );
842    }
843
844    #[test]
845    fn a_stale_lease_delivers_what_the_owner_said() {
846        let mut q = asked(0);
847        q.say("why?").unwrap();
848        assert_eq!(
849            decide(&q, Some(&lease(0)), false, 86_400, ts(500)),
850            Action::Deliver(Word::Said("why?".into()))
851        );
852        assert_eq!(
853            decide(&q, None, true, 86_400, ts(500)),
854            Action::Idle,
855            "a seat still mid-turn is not resumed"
856        );
857    }
858
859    #[test]
860    fn an_answer_nobody_read_is_delivered_once() {
861        let mut q = asked(0);
862        q.choices = vec!["A".into()];
863        q.answer(crate::ask::Answer::Choice("A".into())).unwrap();
864        assert_eq!(
865            decide(&q, None, false, 86_400, ts(10)),
866            Action::Deliver(Word::Answered("A".into()))
867        );
868        q.answer_delivered = true;
869        assert_eq!(decide(&q, None, false, 86_400, ts(10)), Action::Idle);
870    }
871
872    #[test]
873    fn only_asked_questions_and_only_before_the_deadline() {
874        let mut q = asked(0);
875        assert_eq!(decide(&q, None, false, 86_400, ts(999)), Action::Idle);
876        assert_eq!(decide(&q, None, false, 86_400, ts(1001)), Action::Expire);
877        q.cwd = None;
878        assert_eq!(decide(&q, None, false, 86_400, ts(1001)), Action::Idle);
879    }
880
881    #[test]
882    fn the_deadline_runs_from_the_last_turn_not_from_asking() {
883        let mut q = asked(0);
884        q.thread.push(crate::ask::Turn {
885            who: Who::Agent,
886            body: "context".into(),
887            at: ts(4000),
888            note: None,
889        });
890        q.delivered_turns = 1;
891        assert_eq!(decide(&q, None, false, 86_400, ts(4500)), Action::Idle);
892        assert_eq!(decide(&q, None, false, 86_400, ts(5001)), Action::Expire);
893    }
894
895    #[test]
896    fn a_word_given_in_time_is_delivered_after_the_deadline() {
897        let mut q = asked(0);
898        q.say("wait").unwrap();
899        assert!(matches!(
900            decide(&q, None, false, 86_400, ts(5000)),
901            Action::Deliver(_)
902        ));
903    }
904
905    #[test]
906    fn an_action_answer_is_left_to_the_daemon() {
907        let mut q = asked(0);
908        q.choices = vec!["A".into()];
909        q.actions
910            .insert("A".into(), crate::ask::ChoiceAction::Requeue);
911        q.answer(crate::ask::Answer::Choice("A".into())).unwrap();
912        assert_eq!(
913            decide_owned(&q, None, false, 86_400, ts(10), true),
914            Action::Idle,
915            "the daemon applies the action; the waiter must not also resume the seat"
916        );
917        // Delivered or not, a stale lease changes nothing.
918        q.answer_delivered = true;
919        assert_eq!(
920            decide_owned(&q, None, false, 86_400, ts(10), true),
921            Action::Idle
922        );
923        // No task to apply it: the words are delivered as before.
924        q.answer_delivered = false;
925        assert_eq!(
926            decide_owned(&q, None, false, 86_400, ts(10), false),
927            Action::Deliver(Word::Answered("A".into()))
928        );
929        // No action: delivered even when a task exists.
930        let mut plain = asked(0);
931        plain.choices = vec!["A".into()];
932        plain
933            .answer(crate::ask::Answer::Choice("A".into()))
934            .unwrap();
935        assert_eq!(
936            decide_owned(&plain, None, false, 86_400, ts(10), true),
937            Action::Deliver(Word::Answered("A".into()))
938        );
939    }
940}