Skip to main content

magi/
waiter.rs

1//! The waiter: something that keeps waiting on a question after the process
2//! that asked it is gone.
3//!
4//! `magi ask` blocks in the agent's own shell tool, and that tool kills a wait
5//! that runs long (see `ask::WAIT_SLICE`). An agent can also background the
6//! call and simply finish. Either way the question outlives the only process
7//! that would have read the owner's answer, and the owner's reply lands in a
8//! file nobody is polling - the phone then says "waiting for the agent" while
9//! there is no agent to wait for.
10//!
11//! This module is the guarantee that something is always on the other end. It
12//! runs inside `magi serve`, on its own cadence rather than the queue's, and
13//! for every question a `magi ask` filed it decides one of three things:
14//!
15//! - **the asker is still there** (its [`Lease`] is fresh): do nothing. A
16//!   second agent must never be started while the first is blocked;
17//! - **the asker is gone and the owner has said or answered something the agent
18//!   has not read**: resume the *same seat's* CLI session with the question,
19//!   the thread so far and the owner's words, so the agent that holds the
20//!   context carries on;
21//! - **nobody answered before `answer_timeout`**: abandon it, in the same words
22//!   the asker would have used.
23//!
24//! # Shape
25//!
26//! Same split as [`crate::queue`] and [`crate::ask`]: [`decide`] is pure - a
27//! question, a lease, a clock and a bool in, an [`Action`] out - and every
28//! filesystem or process effect is in [`Waiter::tick`]. State lives on disk (the
29//! question record and its lease), so a restarted daemon picks up exactly where
30//! the last one stopped, including an answer nobody has read yet.
31//!
32//! # What it will not do
33//!
34//! It never starts a fresh consultant. If the seat's session cannot be resumed
35//! (`agent::has_session` says no, the working directory is gone, the run is
36//! over, the agent left the roster) the question stays as it is, the owner
37//! gets a notification saying why, and nothing else runs. A new agent would
38//! have to be caught up on everything the first one knew, and would act on the
39//! owner's words without the context they were written against.
40//!
41//! It never assumes a resume worked either: a turn that fails or times out
42//! leaves the word undelivered, and a later tick tries again.
43
44use std::collections::HashMap;
45use std::path::PathBuf;
46use std::time::{Duration, Instant};
47
48use anyhow::Result;
49use jiff::Timestamp;
50
51use crate::agent::{self, Invocation, SeatState};
52use crate::ask::{Lease, Question, QuestionStatus, Questions, Waiter as Note, WaiterKind, Who};
53use crate::config::{AgentSpec, Config};
54use crate::notices::{self, Notice};
55use crate::prompt;
56use crate::run::RunState;
57
58/// How often the waiter looks at the store.
59pub const TICK: Duration = Duration::from_secs(5);
60
61/// Wall-clock limit for one resumed turn. A resumed agent may go on to do real
62/// work once it has the owner's word, but it is one conversation turn, not a
63/// whole node.
64const DELIVERY_TIMEOUT: Duration = Duration::from_secs(15 * 60);
65
66/// How long a failed or refused delivery is left alone before the same input
67/// is tried again. Long enough not to hammer a seat that is out of quota,
68/// short enough that a fixed problem does not wait a day.
69const RETRY_AFTER: Duration = Duration::from_secs(10 * 60);
70
71#[derive(serde::Serialize, serde::Deserialize)]
72struct Retry {
73    key: String,
74    until: i64,
75}
76
77fn retry_path(store: &Questions, id: &str) -> PathBuf {
78    store.root().join(format!("{id}.waiter-retry.json"))
79}
80
81fn retry_blocked(store: &Questions, id: &str, key: &str, now: Timestamp) -> bool {
82    std::fs::read(retry_path(store, id))
83        .ok()
84        .and_then(|bytes| serde_json::from_slice::<Retry>(&bytes).ok())
85        .is_some_and(|retry| retry.key == key && now.as_second() < retry.until)
86}
87
88/// What the agent has not read yet.
89#[derive(Debug, Clone, PartialEq, Eq)]
90pub enum Word {
91    /// The owner spoke back without deciding.
92    Said(String),
93    /// The owner decided.
94    Answered(String),
95}
96
97/// What to do about one question, this tick.
98#[derive(Debug, Clone, PartialEq, Eq)]
99pub enum Action {
100    /// Nothing: not ours, someone is waiting, or nothing new to hand over.
101    Idle,
102    /// The deadline passed with no answer and nobody waiting.
103    Expire,
104    /// Resume the asking seat with this.
105    Deliver(Word),
106}
107
108/// The whole decision, with no I/O.
109///
110/// `seat_busy` is whether the asking seat's CLI is still mid-turn - the agent
111/// that exited its `magi ask` may not have exited itself, and resuming a
112/// conversation that is running would fork it.
113///
114/// Order matters: a word the owner gave before the deadline is delivered even
115/// if the deadline has passed by the time the waiter looks, and only then does
116/// an unanswered question expire.
117pub fn decide(
118    q: &Question,
119    lease: Option<&Lease>,
120    seat_busy: bool,
121    default_timeout: u64,
122    now: Timestamp,
123) -> Action {
124    decide_owned(q, lease, seat_busy, default_timeout, now, false)
125}
126
127/// [`decide`], told whether the daemon owns the answer's action.
128///
129/// `daemon_owns_action` is true when the daemon will apply the action (the
130/// task exists and is in a state `daemon::decide_action` acts on). An answer
131/// that carries a [`crate::ask::ChoiceAction`] is then the daemon's alone
132/// (`daemon::apply_choice_actions`, idempotent through `Task::actions_applied`
133/// under the task's claim): resuming the dead seat here as well would run the
134/// same seat twice when the action is a resume or a requeue. With no task
135/// there is no one to apply it, so the words are delivered as before.
136pub fn decide_owned(
137    q: &Question,
138    lease: Option<&Lease>,
139    seat_busy: bool,
140    default_timeout: u64,
141    now: Timestamp,
142    daemon_owns_action: bool,
143) -> Action {
144    if q.routed_task.is_some() || q.cwd.is_none() || lease.is_some_and(|l| l.fresh(now)) {
145        return Action::Idle;
146    }
147    match q.status {
148        QuestionStatus::Abandoned => Action::Idle,
149        QuestionStatus::Answered => match q.resolution() {
150            _ if q.chosen_action().is_some_and(|a| a.maintenance())
151                || (daemon_owns_action && q.chosen_action().is_some()) =>
152            {
153                Action::Idle
154            }
155            Some(a) if !q.answer_delivered && !seat_busy => Action::Deliver(Word::Answered(a)),
156            _ => Action::Idle,
157        },
158        QuestionStatus::Open => {
159            if let Some(said) = q.unread_from_owner() {
160                if seat_busy {
161                    return Action::Idle;
162                }
163                return Action::Deliver(Word::Said(said));
164            }
165            let secs = if q.answer_timeout > 0 {
166                q.answer_timeout
167            } else {
168                default_timeout
169            };
170            let deadline = q.last_activity().saturating_add(secs as i64);
171            if now.as_second() > deadline {
172                Action::Expire
173            } else {
174                Action::Idle
175            }
176        }
177    }
178}
179
180/// Everything needed to resume one seat.
181struct Target {
182    coordination: Option<crate::config::Coordination>,
183    fetched_base: Option<crate::run::FetchedBase>,
184    spec: AgentSpec,
185    seat: SeatState,
186    cwd: PathBuf,
187    sessions: bool,
188    allow_write: bool,
189    language: String,
190}
191
192/// The waiter's state: the store it watches and what it must remember between
193/// ticks (only which inputs it has already failed on).
194pub struct Waiter {
195    store: Questions,
196    home: PathBuf,
197    /// The config the conductor runs under, for a question the conductor asked
198    /// (it has no run whose snapshot could say).
199    conduct_cfg: Option<Config>,
200    /// `answer_timeout` for a question that did not record its own.
201    default_timeout: u64,
202    /// Inputs already tried and failed, by question id: `(key, until)`.
203    memo: HashMap<String, (String, Instant)>,
204}
205
206impl Waiter {
207    /// A waiter over `store`, with `home` as the magi home its runs live under.
208    pub fn new(store: Questions, home: PathBuf, conduct_cfg: Option<Config>) -> Self {
209        let default_timeout = conduct_cfg
210            .as_ref()
211            .map_or(Config::default().graph.answer_timeout, |c| {
212                c.graph.answer_timeout
213            });
214        Self {
215            store,
216            home,
217            conduct_cfg,
218            default_timeout,
219            memo: HashMap::new(),
220        }
221    }
222
223    /// Admission check without starting a process for an idle question store.
224    pub fn has_work(&self, now: Timestamp) -> bool {
225        self.store.list_for_effects().iter().any(|q| {
226            if crate::task_chat::migrated_source(&self.home, &q.id) {
227                return false;
228            }
229            let lease = self.store.read_lease(&q.id);
230            match decide_owned(
231                q,
232                lease.as_ref(),
233                false,
234                self.default_timeout,
235                now,
236                self.daemon_owns_action(q),
237            ) {
238                Action::Idle => false,
239                Action::Expire => true,
240                Action::Deliver(_) if q.deputy.is_some() => crate::deputy::exhausted_past_deadline(
241                    q,
242                    crate::deputy::can_start(self.conduct_cfg.as_ref(), crate::deputy::agent_of(q)),
243                    self.default_timeout,
244                    now,
245                ),
246                Action::Deliver(word) => {
247                    let key = format!("{}:{:?}", q.thread.len(), word);
248                    !self.seat_busy(q, now) && !retry_blocked(&self.store, &q.id, &key, now)
249                }
250            }
251        })
252    }
253
254    /// Is this question's asking seat still mid-turn?
255    fn seat_busy(&self, q: &Question, now: Timestamp) -> bool {
256        if q.run == crate::conduct::NODE {
257            return crate::conduct::busy(&self.home);
258        }
259        RunState::load_under(&q.run, &self.home).is_ok_and(|s| {
260            s.seats_active()
261                .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
262        })
263    }
264
265    /// Whether the daemon applies this question's chosen action, so the
266    /// waiter must not deliver the answer (see [`decide_owned`]).
267    fn daemon_owns_action(&self, q: &Question) -> bool {
268        q.chosen_action().is_some_and(|a| a.maintenance())
269            || crate::daemon::task_of_question(
270                &crate::queue::Queue::at(self.home.join("queue")).list(),
271                q,
272            )
273            .is_some_and(|t| crate::daemon::action_standing(t, q).daemon_owns())
274    }
275
276    /// Look at every question once and act on what needs acting on. `halt` is
277    /// asked between questions and during a delivery: true means the daemon is
278    /// parking, and whatever is in flight is dropped, undelivered and
279    /// resumable.
280    pub async fn tick(&mut self, now: Timestamp, halt: &(dyn Fn() -> bool + Sync)) {
281        for q in self.store.list_for_effects() {
282            if halt() {
283                return;
284            }
285            if crate::task_chat::routes(&q, &self.home)
286                || crate::task_chat::migrated_source(&self.home, &q.id)
287            {
288                continue;
289            }
290            let lease = self.store.read_lease(&q.id);
291            let owned = self.daemon_owns_action(&q);
292            match decide_owned(&q, lease.as_ref(), false, self.default_timeout, now, owned) {
293                Action::Idle => {}
294                Action::Expire => self.expire(&q),
295                Action::Deliver(word) => {
296                    // A question with a deputy is the deputy's to read
297                    // (`crate::deputy`); resuming the conductor's own seat
298                    // here would fork a conversation that is not its own.
299                    if q.deputy.is_some() {
300                        // Unless that deputy is spent: then nobody will read
301                        // the word, and the deadline still has to retire the
302                        // question.
303                        if crate::deputy::exhausted_past_deadline(
304                            &q,
305                            crate::deputy::can_start(
306                                self.conduct_cfg.as_ref(),
307                                crate::deputy::agent_of(&q),
308                            ),
309                            self.default_timeout,
310                            now,
311                        ) {
312                            self.expire(&q);
313                        }
314                        continue;
315                    }
316                    if self.seat_busy(&q, now) {
317                        continue;
318                    }
319                    let key = format!("{}:{:?}", q.thread.len(), word);
320                    if retry_blocked(&self.store, &q.id, &key, now)
321                        || matches!(self.memo.get(&q.id), Some((k, until)) if *k == key && Instant::now() < *until)
322                    {
323                        continue;
324                    }
325                    if let Err(e) = self.deliver(&q, &word, halt).await {
326                        tracing::warn!("question {}: {e:#}", q.short());
327                    }
328                }
329            }
330        }
331    }
332
333    /// Abandon an unanswered question whose deadline passed with nobody
334    /// waiting - the words the asker's own wait would have used.
335    fn expire(&self, q: &Question) {
336        let secs = if q.answer_timeout > 0 {
337            q.answer_timeout
338        } else {
339            self.default_timeout
340        };
341        let startable =
342            crate::deputy::can_start(self.conduct_cfg.as_ref(), crate::deputy::agent_of(q));
343        let why = format!("no answer within {}s of asking", secs.max(1));
344        let done = self.store.update(&q.id, |r| {
345            // Decided again on the record as it is now: the owner may have
346            // spoken since the tick read it, and a word given in time is
347            // delivered, not abandoned.
348            let now = Timestamp::now();
349            let lease = self.store.read_lease(&r.id);
350            if decide(r, lease.as_ref(), false, self.default_timeout, now) != Action::Expire
351                && !(lease.as_ref().is_none_or(|l| !l.fresh(now))
352                    && crate::deputy::exhausted_past_deadline(
353                        r,
354                        startable,
355                        self.default_timeout,
356                        now,
357                    ))
358            {
359                return Ok(false);
360            }
361            r.abandon(&why);
362            r.waiter = None;
363            Ok(true)
364        });
365        match done {
366            Ok((_, false)) => {}
367            Ok((_, true)) => {
368                self.store.drop_lease(&q.id);
369                tracing::warn!(
370                    "question {} went unanswered for {secs}s with nobody waiting; \
371                     it stays as the record of it",
372                    q.short()
373                );
374            }
375            Err(e) => tracing::warn!("could not expire question {}: {e:#}", q.short()),
376        }
377    }
378
379    /// Admission uses the very same target resolution as actual delivery.
380    pub fn can_resume(&self, q: &Question) -> bool {
381        self.target(q).is_ok()
382    }
383
384    /// Work out how to resume the seat, or why it cannot be.
385    fn target(&self, q: &Question) -> std::result::Result<Target, String> {
386        let cwd = q
387            .cwd
388            .as_deref()
389            .map(PathBuf::from)
390            .filter(|p| p.is_dir())
391            .ok_or("the directory the agent was working in is gone")?;
392        let (cfg, seat, allow_write, fetched_base) = if q.run == crate::conduct::NODE {
393            let cfg = self
394                .conduct_cfg
395                .clone()
396                .ok_or("the conductor's configuration is not available")?;
397            let seat = crate::conduct::load_seat(&self.home)
398                .ok_or("the conductor has no recorded session")?;
399            (cfg, seat, false, None)
400        } else {
401            let state = RunState::load_under(&q.run, &self.home)
402                .map_err(|_| "the run that asked is gone".to_owned())?;
403            if !state.status.resumable() {
404                return Err(format!(
405                    "the run that asked has already {}",
406                    state.status.as_str()
407                ));
408            }
409            let seat = state
410                .seats
411                .get(&q.seat)
412                .cloned()
413                .ok_or("the run has no record of the asking seat")?;
414            let write = matches!(q.node.as_str(), "implement" | "fix");
415            (state.config, seat, write, state.fetched_base)
416        };
417        let spec = cfg
418            .agent(&seat.agent)
419            .map_err(|_| format!("agent `{}` is no longer in the roster", seat.agent))?
420            .clone();
421        if !agent::has_session(spec.kind, &seat, cfg.graph.sessions) {
422            return Err("the seat has no session to resume".to_owned());
423        }
424        Ok(Target {
425            coordination: cfg.coordination.clone(),
426            fetched_base,
427            spec,
428            seat,
429            cwd,
430            sessions: cfg.graph.sessions,
431            allow_write,
432            language: cfg.graph.language.clone(),
433        })
434    }
435
436    /// Say why the owner's word cannot reach anybody, once per input.
437    fn refuse(&mut self, q: &Question, key: String, why: &str) {
438        tracing::warn!("question {} cannot be delivered: {why}", q.short());
439        notices::raise_in(
440            &self.home,
441            Notice::warn(
442                &format!("question:{}", q.id),
443                format!(
444                    "Question {} \"{}\": what you said cannot reach the agent \
445                     ({why}). It is recorded, but nothing will read it.",
446                    q.short(),
447                    q.summary
448                ),
449            )
450            .link(if q.run_names_task() {
451                notices::Link::Task { id: q.run.clone() }
452            } else {
453                notices::Link::Run { id: q.run.clone() }
454            }),
455        );
456        let _ = self.store.update(&q.id, |r| {
457            r.waiter = None;
458            Ok(())
459        });
460        let retry = Retry {
461            key: key.clone(),
462            until: Timestamp::now()
463                .as_second()
464                .saturating_add(RETRY_AFTER.as_secs() as i64),
465        };
466        let written = serde_json::to_vec(&retry)
467            .map_err(anyhow::Error::from)
468            .and_then(|bytes| {
469                std::fs::write(retry_path(&self.store, &q.id), bytes).map_err(Into::into)
470            });
471        if let Err(error) = written {
472            tracing::warn!(
473                "question {} retry cooldown could not be saved: {error:#}",
474                q.short()
475            );
476        }
477        self.memo
478            .insert(q.id.clone(), (key, Instant::now() + RETRY_AFTER));
479    }
480
481    /// Resume the asking seat with the owner's word.
482    async fn deliver(
483        &mut self,
484        q: &Question,
485        word: &Word,
486        halt: &(dyn Fn() -> bool + Sync),
487    ) -> Result<()> {
488        let key = format!("{}:{:?}", q.thread.len(), word);
489        let mut target = match self.target(q) {
490            Ok(t) => t,
491            Err(why) => {
492                self.refuse(q, key, &why);
493                return Ok(());
494            }
495        };
496
497        let claim = self.store.root().join(format!("{}.claim", q.id));
498        if !take_claim(&claim, DELIVERY_TIMEOUT + Duration::from_secs(60)) {
499            return Ok(());
500        }
501        let _release = Release(claim);
502
503        // Re-read under the claim: the asker may have come back, or the owner
504        // may have said more, since the tick's copy was taken.
505        let q = self.store.get_for_effects(&q.id)?;
506        if crate::task_chat::routes(&q, &self.home)
507            || crate::task_chat::migrated_source(&self.home, &q.id)
508        {
509            return Ok(());
510        }
511        let now = Timestamp::now();
512        let lease = self.store.read_lease(&q.id);
513        let owned = self.daemon_owns_action(&q);
514        let Action::Deliver(word) =
515            decide_owned(&q, lease.as_ref(), false, self.default_timeout, now, owned)
516        else {
517            return Ok(());
518        };
519        let snapshot = q.thread.len();
520
521        // Ownership of an action answer is settled against the daemon's task
522        // claim: hold it while the lease goes up, so the daemon either sees the
523        // fresh lease or has already applied the action (and we step aside).
524        // A claim held by a running competition is not the daemon applying an
525        // action; that case is delivered, with the re-check below.
526        let queue = crate::queue::Queue::at(self.home.join("queue"));
527        let task_id = crate::daemon::task_of_question(&queue.list(), &q).map(|t| t.id.clone());
528        // Never go on without the claim: a failed claim means an attempt (or
529        // the daemon) holds the task, and whatever it does next decides who
530        // owns this answer. Step aside; the next tick judges it afresh.
531        let _task_claim = match (&task_id, q.chosen_action()) {
532            (Some(id), Some(_)) => match queue.claim(id) {
533                Ok(c) => Some(c),
534                Err(e) => {
535                    tracing::debug!(
536                        "question {}: task claim not available ({e:#}), deferring delivery",
537                        q.short()
538                    );
539                    return Ok(());
540                }
541            },
542            _ => None,
543        };
544        // Applied, stale and pending are all stop conditions; only a busy task
545        // (or no action at all) is delivered to.
546        let task_still_ours = || {
547            if q.chosen_action().is_some_and(|a| a.maintenance()) {
548                false
549            } else {
550                match &task_id {
551                    Some(id) => queue
552                        .get(id)
553                        .map(|t| !crate::daemon::action_standing(&t, &q).daemon_owns())
554                        .unwrap_or(true),
555                    None => true,
556                }
557            }
558        };
559        if !task_still_ours() {
560            return Ok(());
561        }
562
563        self.store.beat(&q.id, WaiterKind::Daemon);
564        // Re-check after the lease is up: a daemon that decided before it
565        // existed has by now written its mark (or is blocked on our claim).
566        if !task_still_ours() {
567            return Ok(());
568        }
569        self.store.update(&q.id, |r| {
570            r.waiter = Some(Note {
571                kind: WaiterKind::Daemon,
572                since: now,
573            });
574            Ok(())
575        })?;
576        tracing::info!(
577            "question {}: the asker is gone, resuming seat {} with the owner's word",
578            q.short(),
579            q.seat
580        );
581
582        let thread: Vec<(&str, &str)> = q
583            .thread
584            .iter()
585            .map(|t| {
586                (
587                    if t.who == Who::Operator {
588                        "operator"
589                    } else {
590                        "agent"
591                    },
592                    t.body.as_str(),
593                )
594            })
595            .collect();
596        // What the owner said and the agent has not read has its own section in
597        // the prompt, so the recap stops at what it already read.
598        let shown = match word {
599            Word::Said(_) => &thread[..q.delivered_turns.min(thread.len())],
600            Word::Answered(_) => &thread[..],
601        };
602        let owner_word = match &word {
603            Word::Said(s) => prompt::OwnerWord::Said(s),
604            Word::Answered(a) => prompt::OwnerWord::Answered(a),
605        };
606        let body = prompt::question_resumed(
607            &q.id,
608            &q.summary,
609            &q.detail,
610            shown,
611            &owner_word,
612            &target.language,
613        );
614
615        let artifacts = self.store.root().join(format!("{}.delivery", q.id));
616        let stem = format!("deliver-{}", now.as_second());
617        let cache_dir = None;
618        let inv = Invocation {
619            coordination: target.coordination.as_ref(),
620            fetched_base: target.fetched_base.as_ref(),
621            cwd: &target.cwd,
622            prompt: &body,
623            timeout: DELIVERY_TIMEOUT,
624            allow_write: target.allow_write,
625            sessions: target.sessions,
626            artifacts: &artifacts,
627            stem: &stem,
628            run: &q.run,
629            node: &q.node,
630            cache_dir,
631            attachments: &[],
632            writable: &[],
633        };
634
635        let store = self.store.clone();
636        let id = q.id.clone();
637        let out = {
638            let fut = agent::invoke(&target.spec, &mut target.seat, &inv);
639            tokio::pin!(fut);
640            let mut beat = tokio::time::interval(Duration::from_secs(1));
641            let mut beats = 0u32;
642            loop {
643                tokio::select! {
644                    r = &mut fut => break Some(r),
645                    _ = beat.tick() => {
646                        if halt() {
647                            break None;
648                        }
649                        beats += 1;
650                        if beats % 20 == 0 {
651                            store.beat(&id, WaiterKind::Daemon);
652                        }
653                    }
654                }
655            }
656        };
657
658        let Some(out) = out else {
659            // Parking: leave everything as it was, resumable.
660            let _ = self.store.update(&q.id, |r| {
661                r.waiter = None;
662                Ok(())
663            });
664            return Ok(());
665        };
666        let out = match out {
667            Ok(o) if o.usable() => o,
668            other => {
669                let why = match other {
670                    Ok(o) if o.timed_out => "the resumed turn timed out".to_owned(),
671                    Ok(o) if o.quota_exhausted() => "the agent is out of quota".to_owned(),
672                    Ok(o) => format!(
673                        "the resumed turn failed ({})",
674                        notices::exit_status(o.exit_code)
675                    ),
676                    Err(e) => format!("the agent could not be started: {e:#}"),
677                };
678                self.refuse(&q, key, &why);
679                return Ok(());
680            }
681        };
682
683        let text = out.text.trim().to_owned();
684        self.store.update(&q.id, |r| {
685            r.waiter = None;
686            match &word {
687                Word::Answered(_) => r.answer_delivered = true,
688                Word::Said(_) => {
689                    r.delivered_turns = r.delivered_turns.max(snapshot);
690                    // An agent that answered in prose instead of calling
691                    // `magi ask --thread` still spoke; keep it on the record so
692                    // the owner reads it. If it did use --thread, or the owner
693                    // said more meanwhile, the record already moved on.
694                    let agent_spoke = r.thread[snapshot.min(r.thread.len())..]
695                        .iter()
696                        .any(|t| t.who == Who::Agent);
697                    if !agent_spoke && r.thread.len() == snapshot && r.status.open() {
698                        let choices = r.choices.clone();
699                        r.reply(text, choices)?;
700                    }
701                }
702            }
703            Ok(())
704        })?;
705        self.memo.remove(&q.id);
706        let _ = std::fs::remove_file(retry_path(&self.store, &q.id));
707        Ok(())
708    }
709}
710
711/// Take the delivery claim, so two waiters (`magi serve` twice, or a stray
712/// second daemon) cannot resume the same seat at once. A claim older than a
713/// whole delivery plus slack was left by a process that died.
714pub(crate) fn take_claim(path: &std::path::Path, stale_after: Duration) -> bool {
715    let retry = crate::lock_file::Retry::new(None);
716    let attempt = || retry.create(path).is_ok();
717    if attempt() {
718        return true;
719    }
720    let stale = std::fs::metadata(path)
721        .and_then(|m| m.modified())
722        .ok()
723        .and_then(|t| t.elapsed().ok())
724        .is_some_and(|age| age > stale_after);
725    if stale {
726        let _ = std::fs::remove_file(path);
727        return attempt();
728    }
729    false
730}
731
732/// Removes the delivery claim when the delivery ends, however it ends.
733pub(crate) struct Release(pub(crate) PathBuf);
734
735impl Drop for Release {
736    fn drop(&mut self) {
737        let _ = std::fs::remove_file(&self.0);
738    }
739}
740
741#[cfg(test)]
742mod tests {
743    use super::*;
744
745    #[test]
746    fn waiter_target_retains_saved_run_seat_and_write_access() {
747        let home = tempfile::tempdir().unwrap();
748        let mut cfg = Config::default();
749        cfg.graph.sessions = true;
750        cfg.agents = vec![toml::from_str("id = 'fixture'\nkind = 'codex'\n").unwrap()];
751        let mut state = RunState::new(
752            home.path().into(),
753            "main".into(),
754            "0".repeat(40),
755            "repair".into(),
756            cfg,
757        );
758        state.status = crate::run::RunStatus::Implementing;
759        let mut seat = agent::SeatState::new("impl-A", "fixture", 1);
760        seat.turns = 1;
761        seat.captured_session = Some("saved-thread".into());
762        state.seats.insert(seat.key.clone(), seat);
763        state.save_under(home.path()).unwrap();
764        let mut q = asked(0);
765        q.run = state.id;
766        q.cwd = Some(home.path().to_string_lossy().into_owned());
767        let waiter = Waiter::new(
768            Questions::at(home.path().join("questions")),
769            home.path().into(),
770            None,
771        );
772        let target = waiter.target(&q).unwrap();
773        assert!(target.allow_write);
774        assert_eq!(
775            target.seat.captured_session.as_deref(),
776            Some("saved-thread")
777        );
778    }
779
780    #[test]
781    fn retry_backoff_survives_a_new_finite_waiter() {
782        let home = tempfile::tempdir().expect("home");
783        let store = Questions::at(home.path().join("questions"));
784        std::fs::create_dir_all(store.root()).expect("questions");
785        let retry = Retry {
786            key: "input".to_owned(),
787            until: 700,
788        };
789        std::fs::write(
790            retry_path(&store, "question"),
791            serde_json::to_vec(&retry).expect("retry"),
792        )
793        .expect("save retry");
794        let reopened = Questions::at(home.path().join("questions"));
795        assert!(retry_blocked(&reopened, "question", "input", ts(699)));
796        assert!(!retry_blocked(
797            &reopened,
798            "question",
799            "different input",
800            ts(699)
801        ));
802        assert!(!retry_blocked(&reopened, "question", "input", ts(700)));
803    }
804
805    fn ts(secs: i64) -> Timestamp {
806        Timestamp::from_second(secs).unwrap()
807    }
808
809    fn asked(at: i64) -> Question {
810        let mut q = Question::new(
811            "run".into(),
812            "implement".into(),
813            "impl-A".into(),
814            "which?".into(),
815            String::new(),
816            vec![],
817        );
818        q.cwd = Some("/tmp".into());
819        q.asked_at = ts(at);
820        q.answer_timeout = 1000;
821        q
822    }
823
824    fn lease(at: i64) -> Lease {
825        Lease {
826            kind: WaiterKind::Asker,
827            pid: 1,
828            beat_at: ts(at),
829        }
830    }
831
832    #[test]
833    fn a_fresh_lease_means_nobody_else_acts() {
834        let mut q = asked(0);
835        q.say("why?").unwrap();
836        assert_eq!(
837            decide(&q, Some(&lease(95)), false, 86_400, ts(100)),
838            Action::Idle
839        );
840    }
841
842    #[test]
843    fn a_stale_lease_delivers_what_the_owner_said() {
844        let mut q = asked(0);
845        q.say("why?").unwrap();
846        assert_eq!(
847            decide(&q, Some(&lease(0)), false, 86_400, ts(500)),
848            Action::Deliver(Word::Said("why?".into()))
849        );
850        assert_eq!(
851            decide(&q, None, true, 86_400, ts(500)),
852            Action::Idle,
853            "a seat still mid-turn is not resumed"
854        );
855    }
856
857    #[test]
858    fn an_answer_nobody_read_is_delivered_once() {
859        let mut q = asked(0);
860        q.choices = vec!["A".into()];
861        q.answer(crate::ask::Answer::Choice("A".into())).unwrap();
862        assert_eq!(
863            decide(&q, None, false, 86_400, ts(10)),
864            Action::Deliver(Word::Answered("A".into()))
865        );
866        q.answer_delivered = true;
867        assert_eq!(decide(&q, None, false, 86_400, ts(10)), Action::Idle);
868    }
869
870    #[test]
871    fn only_asked_questions_and_only_before_the_deadline() {
872        let mut q = asked(0);
873        assert_eq!(decide(&q, None, false, 86_400, ts(999)), Action::Idle);
874        assert_eq!(decide(&q, None, false, 86_400, ts(1001)), Action::Expire);
875        q.cwd = None;
876        assert_eq!(decide(&q, None, false, 86_400, ts(1001)), Action::Idle);
877    }
878
879    #[test]
880    fn the_deadline_runs_from_the_last_turn_not_from_asking() {
881        let mut q = asked(0);
882        q.thread.push(crate::ask::Turn {
883            who: Who::Agent,
884            body: "context".into(),
885            at: ts(4000),
886            note: None,
887        });
888        q.delivered_turns = 1;
889        assert_eq!(decide(&q, None, false, 86_400, ts(4500)), Action::Idle);
890        assert_eq!(decide(&q, None, false, 86_400, ts(5001)), Action::Expire);
891    }
892
893    #[test]
894    fn a_word_given_in_time_is_delivered_after_the_deadline() {
895        let mut q = asked(0);
896        q.say("wait").unwrap();
897        assert!(matches!(
898            decide(&q, None, false, 86_400, ts(5000)),
899            Action::Deliver(_)
900        ));
901    }
902
903    #[test]
904    fn an_action_answer_is_left_to_the_daemon() {
905        let mut q = asked(0);
906        q.choices = vec!["A".into()];
907        q.actions
908            .insert("A".into(), crate::ask::ChoiceAction::Requeue);
909        q.answer(crate::ask::Answer::Choice("A".into())).unwrap();
910        assert_eq!(
911            decide_owned(&q, None, false, 86_400, ts(10), true),
912            Action::Idle,
913            "the daemon applies the action; the waiter must not also resume the seat"
914        );
915        // Delivered or not, a stale lease changes nothing.
916        q.answer_delivered = true;
917        assert_eq!(
918            decide_owned(&q, None, false, 86_400, ts(10), true),
919            Action::Idle
920        );
921        // No task to apply it: the words are delivered as before.
922        q.answer_delivered = false;
923        assert_eq!(
924            decide_owned(&q, None, false, 86_400, ts(10), false),
925            Action::Deliver(Word::Answered("A".into()))
926        );
927        // No action: delivered even when a task exists.
928        let mut plain = asked(0);
929        plain.choices = vec!["A".into()];
930        plain
931            .answer(crate::ask::Answer::Choice("A".into()))
932            .unwrap();
933        assert_eq!(
934            decide_owned(&plain, None, false, 86_400, ts(10), true),
935            Action::Deliver(Word::Answered("A".into()))
936        );
937    }
938}