1use std::collections::HashMap;
45use std::path::PathBuf;
46use std::time::{Duration, Instant};
47
48use anyhow::Result;
49use jiff::Timestamp;
50
51use crate::agent::{self, Invocation, SeatState};
52use crate::ask::{Lease, Question, QuestionStatus, Questions, Waiter as Note, WaiterKind, Who};
53use crate::config::{AgentSpec, Config};
54use crate::notices::{self, Notice};
55use crate::prompt;
56use crate::run::RunState;
57
58pub const TICK: Duration = Duration::from_secs(5);
60
61const DELIVERY_TIMEOUT: Duration = Duration::from_secs(15 * 60);
65
66const RETRY_AFTER: Duration = Duration::from_secs(10 * 60);
70
71#[derive(Debug, Clone, PartialEq, Eq)]
73pub enum Word {
74 Said(String),
76 Answered(String),
78}
79
80#[derive(Debug, Clone, PartialEq, Eq)]
82pub enum Action {
83 Idle,
85 Expire,
87 Deliver(Word),
89}
90
91pub fn decide(
101 q: &Question,
102 lease: Option<&Lease>,
103 seat_busy: bool,
104 default_timeout: u64,
105 now: Timestamp,
106) -> Action {
107 decide_owned(q, lease, seat_busy, default_timeout, now, false)
108}
109
110pub fn decide_owned(
120 q: &Question,
121 lease: Option<&Lease>,
122 seat_busy: bool,
123 default_timeout: u64,
124 now: Timestamp,
125 daemon_owns_action: bool,
126) -> Action {
127 if q.routed_task.is_some() || q.cwd.is_none() || lease.is_some_and(|l| l.fresh(now)) {
128 return Action::Idle;
129 }
130 match q.status {
131 QuestionStatus::Abandoned => Action::Idle,
132 QuestionStatus::Answered => match q.resolution() {
133 _ if q.chosen_action().is_some_and(|a| a.maintenance())
134 || (daemon_owns_action && q.chosen_action().is_some()) =>
135 {
136 Action::Idle
137 }
138 Some(a) if !q.answer_delivered && !seat_busy => Action::Deliver(Word::Answered(a)),
139 _ => Action::Idle,
140 },
141 QuestionStatus::Open => {
142 if let Some(said) = q.unread_from_owner() {
143 if seat_busy {
144 return Action::Idle;
145 }
146 return Action::Deliver(Word::Said(said));
147 }
148 let secs = if q.answer_timeout > 0 {
149 q.answer_timeout
150 } else {
151 default_timeout
152 };
153 let deadline = q.last_activity().saturating_add(secs as i64);
154 if now.as_second() > deadline {
155 Action::Expire
156 } else {
157 Action::Idle
158 }
159 }
160 }
161}
162
163struct Target {
165 coordination: Option<crate::config::Coordination>,
166 fetched_base: Option<crate::run::FetchedBase>,
167 spec: AgentSpec,
168 seat: SeatState,
169 cwd: PathBuf,
170 sessions: bool,
171 allow_write: bool,
172 language: String,
173}
174
175pub struct Waiter {
178 store: Questions,
179 home: PathBuf,
180 conduct_cfg: Option<Config>,
183 default_timeout: u64,
185 memo: HashMap<String, (String, Instant)>,
187}
188
189impl Waiter {
190 pub fn new(store: Questions, home: PathBuf, conduct_cfg: Option<Config>) -> Self {
192 let default_timeout = conduct_cfg
193 .as_ref()
194 .map_or(Config::default().graph.answer_timeout, |c| {
195 c.graph.answer_timeout
196 });
197 Self {
198 store,
199 home,
200 conduct_cfg,
201 default_timeout,
202 memo: HashMap::new(),
203 }
204 }
205
206 fn seat_busy(&self, q: &Question, now: Timestamp) -> bool {
208 if q.run == crate::conduct::NODE {
209 return crate::conduct::busy(&self.home);
210 }
211 RunState::load_under(&q.run, &self.home).is_ok_and(|s| {
212 s.seats_active()
213 .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
214 })
215 }
216
217 fn daemon_owns_action(&self, q: &Question) -> bool {
220 q.chosen_action().is_some_and(|a| a.maintenance())
221 || crate::daemon::task_of_question(
222 &crate::queue::Queue::at(self.home.join("queue")).list(),
223 q,
224 )
225 .is_some_and(|t| crate::daemon::action_standing(t, q).daemon_owns())
226 }
227
228 pub async fn tick(&mut self, now: Timestamp, halt: &(dyn Fn() -> bool + Sync)) {
233 for q in self.store.list() {
234 if halt() {
235 return;
236 }
237 if crate::task_chat::routes(&q, &self.home) {
238 continue;
239 }
240 let lease = self.store.read_lease(&q.id);
241 let owned = self.daemon_owns_action(&q);
242 match decide_owned(&q, lease.as_ref(), false, self.default_timeout, now, owned) {
243 Action::Idle => {}
244 Action::Expire => self.expire(&q),
245 Action::Deliver(word) => {
246 if q.deputy.is_some() {
250 if crate::deputy::exhausted_past_deadline(
254 &q,
255 crate::deputy::can_start(
256 self.conduct_cfg.as_ref(),
257 crate::deputy::agent_of(&q),
258 ),
259 self.default_timeout,
260 now,
261 ) {
262 self.expire(&q);
263 }
264 continue;
265 }
266 if self.seat_busy(&q, now) {
267 continue;
268 }
269 let key = format!("{}:{:?}", q.thread.len(), word);
270 if matches!(self.memo.get(&q.id), Some((k, until)) if *k == key && Instant::now() < *until)
271 {
272 continue;
273 }
274 if let Err(e) = self.deliver(&q, &word, halt).await {
275 tracing::warn!("question {}: {e:#}", q.short());
276 }
277 }
278 }
279 }
280 }
281
282 fn expire(&self, q: &Question) {
285 let secs = if q.answer_timeout > 0 {
286 q.answer_timeout
287 } else {
288 self.default_timeout
289 };
290 let startable =
291 crate::deputy::can_start(self.conduct_cfg.as_ref(), crate::deputy::agent_of(q));
292 let why = format!("no answer within {}s of asking", secs.max(1));
293 let done = self.store.update(&q.id, |r| {
294 let now = Timestamp::now();
298 let lease = self.store.read_lease(&r.id);
299 if decide(r, lease.as_ref(), false, self.default_timeout, now) != Action::Expire
300 && !(lease.as_ref().is_none_or(|l| !l.fresh(now))
301 && crate::deputy::exhausted_past_deadline(
302 r,
303 startable,
304 self.default_timeout,
305 now,
306 ))
307 {
308 return Ok(false);
309 }
310 r.abandon(&why);
311 r.waiter = None;
312 Ok(true)
313 });
314 match done {
315 Ok((_, false)) => {}
316 Ok((_, true)) => {
317 self.store.drop_lease(&q.id);
318 tracing::warn!(
319 "question {} went unanswered for {secs}s with nobody waiting; \
320 it stays as the record of it",
321 q.short()
322 );
323 }
324 Err(e) => tracing::warn!("could not expire question {}: {e:#}", q.short()),
325 }
326 }
327
328 pub fn can_resume(&self, q: &Question) -> bool {
330 self.target(q).is_ok()
331 }
332
333 fn target(&self, q: &Question) -> std::result::Result<Target, String> {
335 let cwd = q
336 .cwd
337 .as_deref()
338 .map(PathBuf::from)
339 .filter(|p| p.is_dir())
340 .ok_or("the directory the agent was working in is gone")?;
341 let (cfg, seat, allow_write, fetched_base) = if q.run == crate::conduct::NODE {
342 let cfg = self
343 .conduct_cfg
344 .clone()
345 .ok_or("the conductor's configuration is not available")?;
346 let seat = crate::conduct::load_seat(&self.home)
347 .ok_or("the conductor has no recorded session")?;
348 (cfg, seat, false, None)
349 } else {
350 let state = RunState::load_under(&q.run, &self.home)
351 .map_err(|_| "the run that asked is gone".to_owned())?;
352 if !state.status.resumable() {
353 return Err(format!(
354 "the run that asked has already {}",
355 state.status.as_str()
356 ));
357 }
358 let seat = state
359 .seats
360 .get(&q.seat)
361 .cloned()
362 .ok_or("the run has no record of the asking seat")?;
363 let write = matches!(q.node.as_str(), "implement" | "fix");
364 (state.config, seat, write, state.fetched_base)
365 };
366 let spec = cfg
367 .agent(&seat.agent)
368 .map_err(|_| format!("agent `{}` is no longer in the roster", seat.agent))?
369 .clone();
370 if !agent::has_session(spec.kind, &seat, cfg.graph.sessions) {
371 return Err("the seat has no session to resume".to_owned());
372 }
373 Ok(Target {
374 coordination: cfg.coordination.clone(),
375 fetched_base,
376 spec,
377 seat,
378 cwd,
379 sessions: cfg.graph.sessions,
380 allow_write,
381 language: cfg.graph.language.clone(),
382 })
383 }
384
385 fn refuse(&mut self, q: &Question, key: String, why: &str) {
387 tracing::warn!("question {} cannot be delivered: {why}", q.short());
388 notices::raise_in(
389 &self.home,
390 Notice::warn(
391 &format!("question:{}", q.id),
392 format!(
393 "Question {} \"{}\": what you said cannot reach the agent \
394 ({why}). It is recorded, but nothing will read it.",
395 q.short(),
396 q.summary
397 ),
398 ),
399 );
400 let _ = self.store.update(&q.id, |r| {
401 r.waiter = None;
402 Ok(())
403 });
404 self.memo
405 .insert(q.id.clone(), (key, Instant::now() + RETRY_AFTER));
406 }
407
408 async fn deliver(
410 &mut self,
411 q: &Question,
412 word: &Word,
413 halt: &(dyn Fn() -> bool + Sync),
414 ) -> Result<()> {
415 let key = format!("{}:{:?}", q.thread.len(), word);
416 let mut target = match self.target(q) {
417 Ok(t) => t,
418 Err(why) => {
419 self.refuse(q, key, &why);
420 return Ok(());
421 }
422 };
423
424 let claim = self.store.root().join(format!("{}.claim", q.id));
425 if !take_claim(&claim, DELIVERY_TIMEOUT + Duration::from_secs(60)) {
426 return Ok(());
427 }
428 let _release = Release(claim);
429
430 let q = self.store.get(&q.id)?;
433 if crate::task_chat::routes(&q, &self.home) {
434 return Ok(());
435 }
436 let now = Timestamp::now();
437 let lease = self.store.read_lease(&q.id);
438 let owned = self.daemon_owns_action(&q);
439 let Action::Deliver(word) =
440 decide_owned(&q, lease.as_ref(), false, self.default_timeout, now, owned)
441 else {
442 return Ok(());
443 };
444 let snapshot = q.thread.len();
445
446 let queue = crate::queue::Queue::at(self.home.join("queue"));
452 let task_id = crate::daemon::task_of_question(&queue.list(), &q).map(|t| t.id.clone());
453 let _task_claim = match (&task_id, q.chosen_action()) {
457 (Some(id), Some(_)) => match queue.claim(id) {
458 Ok(c) => Some(c),
459 Err(e) => {
460 tracing::debug!(
461 "question {}: task claim not available ({e:#}), deferring delivery",
462 q.short()
463 );
464 return Ok(());
465 }
466 },
467 _ => None,
468 };
469 let task_still_ours = || {
472 if q.chosen_action().is_some_and(|a| a.maintenance()) {
473 false
474 } else {
475 match &task_id {
476 Some(id) => queue
477 .get(id)
478 .map(|t| !crate::daemon::action_standing(&t, &q).daemon_owns())
479 .unwrap_or(true),
480 None => true,
481 }
482 }
483 };
484 if !task_still_ours() {
485 return Ok(());
486 }
487
488 self.store.beat(&q.id, WaiterKind::Daemon);
489 if !task_still_ours() {
492 return Ok(());
493 }
494 self.store.update(&q.id, |r| {
495 r.waiter = Some(Note {
496 kind: WaiterKind::Daemon,
497 since: now,
498 });
499 Ok(())
500 })?;
501 tracing::info!(
502 "question {}: the asker is gone, resuming seat {} with the owner's word",
503 q.short(),
504 q.seat
505 );
506
507 let thread: Vec<(&str, &str)> = q
508 .thread
509 .iter()
510 .map(|t| {
511 (
512 if t.who == Who::Operator {
513 "operator"
514 } else {
515 "agent"
516 },
517 t.body.as_str(),
518 )
519 })
520 .collect();
521 let shown = match word {
524 Word::Said(_) => &thread[..q.delivered_turns.min(thread.len())],
525 Word::Answered(_) => &thread[..],
526 };
527 let owner_word = match &word {
528 Word::Said(s) => prompt::OwnerWord::Said(s),
529 Word::Answered(a) => prompt::OwnerWord::Answered(a),
530 };
531 let body = prompt::question_resumed(
532 &q.id,
533 &q.summary,
534 &q.detail,
535 shown,
536 &owner_word,
537 &target.language,
538 );
539
540 let artifacts = self.store.root().join(format!("{}.delivery", q.id));
541 let stem = format!("deliver-{}", now.as_second());
542 let cache_dir = None;
543 let inv = Invocation {
544 scope: crate::agent::InvocationScope::Repository,
545 coordination: target.coordination.as_ref(),
546 fetched_base: target.fetched_base.as_ref(),
547 cwd: &target.cwd,
548 prompt: &body,
549 timeout: DELIVERY_TIMEOUT,
550 allow_write: target.allow_write,
551 unsandboxed: false,
552 sessions: target.sessions,
553 artifacts: &artifacts,
554 stem: &stem,
555 run: &q.run,
556 node: &q.node,
557 cache_dir,
558 attachments: &[],
559 writable: &[],
560 };
561
562 let store = self.store.clone();
563 let id = q.id.clone();
564 let out = {
565 let fut = agent::invoke(&target.spec, &mut target.seat, &inv);
566 tokio::pin!(fut);
567 let mut beat = tokio::time::interval(Duration::from_secs(1));
568 let mut beats = 0u32;
569 loop {
570 tokio::select! {
571 r = &mut fut => break Some(r),
572 _ = beat.tick() => {
573 if halt() {
574 break None;
575 }
576 beats += 1;
577 if beats % 20 == 0 {
578 store.beat(&id, WaiterKind::Daemon);
579 }
580 }
581 }
582 }
583 };
584
585 let Some(out) = out else {
586 let _ = self.store.update(&q.id, |r| {
588 r.waiter = None;
589 Ok(())
590 });
591 return Ok(());
592 };
593 let out = match out {
594 Ok(o) if o.usable() => o,
595 other => {
596 let why = match other {
597 Ok(o) if o.timed_out => "the resumed turn timed out".to_owned(),
598 Ok(o) if o.quota_exhausted() => "the agent is out of quota".to_owned(),
599 Ok(o) => format!(
600 "the resumed turn failed ({})",
601 notices::exit_status(o.exit_code)
602 ),
603 Err(e) => format!("the agent could not be started: {e:#}"),
604 };
605 self.refuse(&q, key, &why);
606 return Ok(());
607 }
608 };
609
610 let text = out.text.trim().to_owned();
611 self.store.update(&q.id, |r| {
612 r.waiter = None;
613 match &word {
614 Word::Answered(_) => r.answer_delivered = true,
615 Word::Said(_) => {
616 r.delivered_turns = r.delivered_turns.max(snapshot);
617 let agent_spoke = r.thread[snapshot.min(r.thread.len())..]
622 .iter()
623 .any(|t| t.who == Who::Agent);
624 if !agent_spoke && r.thread.len() == snapshot && r.status.open() {
625 let choices = r.choices.clone();
626 r.reply(text, choices)?;
627 }
628 }
629 }
630 Ok(())
631 })?;
632 self.memo.remove(&q.id);
633 Ok(())
634 }
635}
636
637pub(crate) fn take_claim(path: &std::path::Path, stale_after: Duration) -> bool {
641 let retry = crate::lock_file::Retry::new(None);
642 let attempt = || retry.create(path).is_ok();
643 if attempt() {
644 return true;
645 }
646 let stale = std::fs::metadata(path)
647 .and_then(|m| m.modified())
648 .ok()
649 .and_then(|t| t.elapsed().ok())
650 .is_some_and(|age| age > stale_after);
651 if stale {
652 let _ = std::fs::remove_file(path);
653 return attempt();
654 }
655 false
656}
657
658pub(crate) struct Release(pub(crate) PathBuf);
660
661impl Drop for Release {
662 fn drop(&mut self) {
663 let _ = std::fs::remove_file(&self.0);
664 }
665}
666
667pub async fn run(mut waiter: Waiter, stop: crate::daemon::Stop) {
673 let halt = {
674 let stop = stop.clone();
675 move || stop.parking()
676 };
677 while !stop.stopped() {
678 waiter.tick(Timestamp::now(), &halt).await;
679 tokio::time::sleep(TICK).await;
680 }
681}
682
683#[cfg(test)]
684mod tests {
685 use super::*;
686
687 fn ts(secs: i64) -> Timestamp {
688 Timestamp::from_second(secs).unwrap()
689 }
690
691 fn asked(at: i64) -> Question {
692 let mut q = Question::new(
693 "run".into(),
694 "implement".into(),
695 "impl-A".into(),
696 "which?".into(),
697 String::new(),
698 vec![],
699 );
700 q.cwd = Some("/tmp".into());
701 q.asked_at = ts(at);
702 q.answer_timeout = 1000;
703 q
704 }
705
706 fn lease(at: i64) -> Lease {
707 Lease {
708 kind: WaiterKind::Asker,
709 pid: 1,
710 beat_at: ts(at),
711 }
712 }
713
714 #[test]
715 fn a_fresh_lease_means_nobody_else_acts() {
716 let mut q = asked(0);
717 q.say("why?").unwrap();
718 assert_eq!(
719 decide(&q, Some(&lease(95)), false, 86_400, ts(100)),
720 Action::Idle
721 );
722 }
723
724 #[test]
725 fn a_stale_lease_delivers_what_the_owner_said() {
726 let mut q = asked(0);
727 q.say("why?").unwrap();
728 assert_eq!(
729 decide(&q, Some(&lease(0)), false, 86_400, ts(500)),
730 Action::Deliver(Word::Said("why?".into()))
731 );
732 assert_eq!(
733 decide(&q, None, true, 86_400, ts(500)),
734 Action::Idle,
735 "a seat still mid-turn is not resumed"
736 );
737 }
738
739 #[test]
740 fn an_answer_nobody_read_is_delivered_once() {
741 let mut q = asked(0);
742 q.choices = vec!["A".into()];
743 q.answer(crate::ask::Answer::Choice("A".into())).unwrap();
744 assert_eq!(
745 decide(&q, None, false, 86_400, ts(10)),
746 Action::Deliver(Word::Answered("A".into()))
747 );
748 q.answer_delivered = true;
749 assert_eq!(decide(&q, None, false, 86_400, ts(10)), Action::Idle);
750 }
751
752 #[test]
753 fn only_asked_questions_and_only_before_the_deadline() {
754 let mut q = asked(0);
755 assert_eq!(decide(&q, None, false, 86_400, ts(999)), Action::Idle);
756 assert_eq!(decide(&q, None, false, 86_400, ts(1001)), Action::Expire);
757 q.cwd = None;
758 assert_eq!(decide(&q, None, false, 86_400, ts(1001)), Action::Idle);
759 }
760
761 #[test]
762 fn the_deadline_runs_from_the_last_turn_not_from_asking() {
763 let mut q = asked(0);
764 q.thread.push(crate::ask::Turn {
765 who: Who::Agent,
766 body: "context".into(),
767 at: ts(4000),
768 note: None,
769 });
770 q.delivered_turns = 1;
771 assert_eq!(decide(&q, None, false, 86_400, ts(4500)), Action::Idle);
772 assert_eq!(decide(&q, None, false, 86_400, ts(5001)), Action::Expire);
773 }
774
775 #[test]
776 fn a_word_given_in_time_is_delivered_after_the_deadline() {
777 let mut q = asked(0);
778 q.say("wait").unwrap();
779 assert!(matches!(
780 decide(&q, None, false, 86_400, ts(5000)),
781 Action::Deliver(_)
782 ));
783 }
784
785 #[test]
786 fn an_action_answer_is_left_to_the_daemon() {
787 let mut q = asked(0);
788 q.choices = vec!["A".into()];
789 q.actions
790 .insert("A".into(), crate::ask::ChoiceAction::Requeue);
791 q.answer(crate::ask::Answer::Choice("A".into())).unwrap();
792 assert_eq!(
793 decide_owned(&q, None, false, 86_400, ts(10), true),
794 Action::Idle,
795 "the daemon applies the action; the waiter must not also resume the seat"
796 );
797 q.answer_delivered = true;
799 assert_eq!(
800 decide_owned(&q, None, false, 86_400, ts(10), true),
801 Action::Idle
802 );
803 q.answer_delivered = false;
805 assert_eq!(
806 decide_owned(&q, None, false, 86_400, ts(10), false),
807 Action::Deliver(Word::Answered("A".into()))
808 );
809 let mut plain = asked(0);
811 plain.choices = vec!["A".into()];
812 plain
813 .answer(crate::ask::Answer::Choice("A".into()))
814 .unwrap();
815 assert_eq!(
816 decide_owned(&plain, None, false, 86_400, ts(10), true),
817 Action::Deliver(Word::Answered("A".into()))
818 );
819 }
820}