1use std::collections::HashMap;
45use std::path::PathBuf;
46use std::time::{Duration, Instant};
47
48use anyhow::Result;
49use jiff::Timestamp;
50
51use crate::agent::{self, Invocation, SeatState};
52use crate::ask::{Lease, Question, QuestionStatus, Questions, Waiter as Note, WaiterKind, Who};
53use crate::config::{AgentSpec, Config};
54use crate::notices::{self, Notice};
55use crate::prompt;
56use crate::run::RunState;
57
58pub const TICK: Duration = Duration::from_secs(5);
60
61const DELIVERY_TIMEOUT: Duration = Duration::from_secs(15 * 60);
65
66const RETRY_AFTER: Duration = Duration::from_secs(10 * 60);
70
71#[derive(Debug, Clone, PartialEq, Eq)]
73pub enum Word {
74 Said(String),
76 Answered(String),
78}
79
80#[derive(Debug, Clone, PartialEq, Eq)]
82pub enum Action {
83 Idle,
85 Expire,
87 Deliver(Word),
89}
90
91pub fn decide(
101 q: &Question,
102 lease: Option<&Lease>,
103 seat_busy: bool,
104 default_timeout: u64,
105 now: Timestamp,
106) -> Action {
107 decide_owned(q, lease, seat_busy, default_timeout, now, false)
108}
109
110pub fn decide_owned(
120 q: &Question,
121 lease: Option<&Lease>,
122 seat_busy: bool,
123 default_timeout: u64,
124 now: Timestamp,
125 daemon_owns_action: bool,
126) -> Action {
127 if q.cwd.is_none() || lease.is_some_and(|l| l.fresh(now)) {
128 return Action::Idle;
129 }
130 match q.status {
131 QuestionStatus::Abandoned => Action::Idle,
132 QuestionStatus::Answered => match q.resolution() {
133 _ if q.chosen_action().is_some_and(|a| a.maintenance())
134 || (daemon_owns_action && q.chosen_action().is_some()) =>
135 {
136 Action::Idle
137 }
138 Some(a) if !q.answer_delivered && !seat_busy => Action::Deliver(Word::Answered(a)),
139 _ => Action::Idle,
140 },
141 QuestionStatus::Open => {
142 if let Some(said) = q.unread_from_owner() {
143 if seat_busy {
144 return Action::Idle;
145 }
146 return Action::Deliver(Word::Said(said));
147 }
148 let secs = if q.answer_timeout > 0 {
149 q.answer_timeout
150 } else {
151 default_timeout
152 };
153 let deadline = q.last_activity().saturating_add(secs as i64);
154 if now.as_second() > deadline {
155 Action::Expire
156 } else {
157 Action::Idle
158 }
159 }
160 }
161}
162
163struct Target {
165 coordination: Option<crate::config::Coordination>,
166 fetched_base: Option<crate::run::FetchedBase>,
167 spec: AgentSpec,
168 seat: SeatState,
169 cwd: PathBuf,
170 sessions: bool,
171 allow_write: bool,
172 language: String,
173}
174
175pub struct Waiter {
178 store: Questions,
179 home: PathBuf,
180 conduct_cfg: Option<Config>,
183 default_timeout: u64,
185 memo: HashMap<String, (String, Instant)>,
187}
188
189impl Waiter {
190 pub fn new(store: Questions, home: PathBuf, conduct_cfg: Option<Config>) -> Self {
192 let default_timeout = conduct_cfg
193 .as_ref()
194 .map_or(Config::default().graph.answer_timeout, |c| {
195 c.graph.answer_timeout
196 });
197 Self {
198 store,
199 home,
200 conduct_cfg,
201 default_timeout,
202 memo: HashMap::new(),
203 }
204 }
205
206 fn seat_busy(&self, q: &Question, now: Timestamp) -> bool {
208 if q.run == crate::conduct::NODE {
209 return crate::conduct::busy(&self.home);
210 }
211 RunState::load_under(&q.run, &self.home).is_ok_and(|s| {
212 s.seats_active()
213 .any(|(k, a)| *k == q.seat && a.remaining_secs(now) > 0)
214 })
215 }
216
217 fn daemon_owns_action(&self, q: &Question) -> bool {
220 q.chosen_action().is_some_and(|a| a.maintenance())
221 || crate::daemon::task_of_question(
222 &crate::queue::Queue::at(self.home.join("queue")).list(),
223 q,
224 )
225 .is_some_and(|t| crate::daemon::action_standing(t, q).daemon_owns())
226 }
227
228 pub async fn tick(&mut self, now: Timestamp, halt: &(dyn Fn() -> bool + Sync)) {
233 for q in self.store.list() {
234 if halt() {
235 return;
236 }
237 let lease = self.store.read_lease(&q.id);
238 let owned = self.daemon_owns_action(&q);
239 match decide_owned(&q, lease.as_ref(), false, self.default_timeout, now, owned) {
240 Action::Idle => {}
241 Action::Expire => self.expire(&q),
242 Action::Deliver(word) => {
243 if q.deputy.is_some() {
247 if crate::deputy::exhausted_past_deadline(
251 &q,
252 crate::deputy::can_start(
253 self.conduct_cfg.as_ref(),
254 crate::deputy::agent_of(&q),
255 ),
256 self.default_timeout,
257 now,
258 ) {
259 self.expire(&q);
260 }
261 continue;
262 }
263 if self.seat_busy(&q, now) {
264 continue;
265 }
266 let key = format!("{}:{:?}", q.thread.len(), word);
267 if matches!(self.memo.get(&q.id), Some((k, until)) if *k == key && Instant::now() < *until)
268 {
269 continue;
270 }
271 if let Err(e) = self.deliver(&q, &word, halt).await {
272 tracing::warn!("question {}: {e:#}", q.short());
273 }
274 }
275 }
276 }
277 }
278
279 fn expire(&self, q: &Question) {
282 let secs = if q.answer_timeout > 0 {
283 q.answer_timeout
284 } else {
285 self.default_timeout
286 };
287 let startable =
288 crate::deputy::can_start(self.conduct_cfg.as_ref(), crate::deputy::agent_of(q));
289 let why = format!("no answer within {}s of asking", secs.max(1));
290 let done = self.store.update(&q.id, |r| {
291 let now = Timestamp::now();
295 let lease = self.store.read_lease(&r.id);
296 if decide(r, lease.as_ref(), false, self.default_timeout, now) != Action::Expire
297 && !(lease.as_ref().is_none_or(|l| !l.fresh(now))
298 && crate::deputy::exhausted_past_deadline(
299 r,
300 startable,
301 self.default_timeout,
302 now,
303 ))
304 {
305 return Ok(false);
306 }
307 r.abandon(&why);
308 r.waiter = None;
309 Ok(true)
310 });
311 match done {
312 Ok((_, false)) => {}
313 Ok((_, true)) => {
314 self.store.drop_lease(&q.id);
315 tracing::warn!(
316 "question {} went unanswered for {secs}s with nobody waiting; \
317 it stays as the record of it",
318 q.short()
319 );
320 }
321 Err(e) => tracing::warn!("could not expire question {}: {e:#}", q.short()),
322 }
323 }
324
325 pub fn can_resume(&self, q: &Question) -> bool {
327 self.target(q).is_ok()
328 }
329
330 fn target(&self, q: &Question) -> std::result::Result<Target, String> {
332 let cwd = q
333 .cwd
334 .as_deref()
335 .map(PathBuf::from)
336 .filter(|p| p.is_dir())
337 .ok_or("the directory the agent was working in is gone")?;
338 let (cfg, seat, allow_write, fetched_base) = if q.run == crate::conduct::NODE {
339 let cfg = self
340 .conduct_cfg
341 .clone()
342 .ok_or("the conductor's configuration is not available")?;
343 let seat = crate::conduct::load_seat(&self.home)
344 .ok_or("the conductor has no recorded session")?;
345 (cfg, seat, false, None)
346 } else {
347 let state = RunState::load_under(&q.run, &self.home)
348 .map_err(|_| "the run that asked is gone".to_owned())?;
349 if !state.status.resumable() {
350 return Err(format!(
351 "the run that asked has already {}",
352 state.status.as_str()
353 ));
354 }
355 let seat = state
356 .seats
357 .get(&q.seat)
358 .cloned()
359 .ok_or("the run has no record of the asking seat")?;
360 let write = matches!(q.node.as_str(), "implement" | "fix");
361 (state.config, seat, write, state.fetched_base)
362 };
363 let spec = cfg
364 .agent(&seat.agent)
365 .map_err(|_| format!("agent `{}` is no longer in the roster", seat.agent))?
366 .clone();
367 if !agent::has_session(spec.kind, &seat, cfg.graph.sessions) {
368 return Err("the seat has no session to resume".to_owned());
369 }
370 Ok(Target {
371 coordination: cfg.coordination.clone(),
372 fetched_base,
373 spec,
374 seat,
375 cwd,
376 sessions: cfg.graph.sessions,
377 allow_write,
378 language: cfg.graph.language.clone(),
379 })
380 }
381
382 fn refuse(&mut self, q: &Question, key: String, why: &str) {
384 tracing::warn!("question {} cannot be delivered: {why}", q.short());
385 notices::raise_in(
386 &self.home,
387 Notice::warn(
388 &format!("question:{}", q.id),
389 format!(
390 "Question {} \"{}\": what you said cannot reach the agent \
391 ({why}). It is recorded, but nothing will read it.",
392 q.short(),
393 q.summary
394 ),
395 ),
396 );
397 let _ = self.store.update(&q.id, |r| {
398 r.waiter = None;
399 Ok(())
400 });
401 self.memo
402 .insert(q.id.clone(), (key, Instant::now() + RETRY_AFTER));
403 }
404
405 async fn deliver(
407 &mut self,
408 q: &Question,
409 word: &Word,
410 halt: &(dyn Fn() -> bool + Sync),
411 ) -> Result<()> {
412 let key = format!("{}:{:?}", q.thread.len(), word);
413 let mut target = match self.target(q) {
414 Ok(t) => t,
415 Err(why) => {
416 self.refuse(q, key, &why);
417 return Ok(());
418 }
419 };
420
421 let claim = self.store.root().join(format!("{}.claim", q.id));
422 if !take_claim(&claim, DELIVERY_TIMEOUT + Duration::from_secs(60)) {
423 return Ok(());
424 }
425 let _release = Release(claim);
426
427 let q = self.store.get(&q.id)?;
430 let now = Timestamp::now();
431 let lease = self.store.read_lease(&q.id);
432 let owned = self.daemon_owns_action(&q);
433 let Action::Deliver(word) =
434 decide_owned(&q, lease.as_ref(), false, self.default_timeout, now, owned)
435 else {
436 return Ok(());
437 };
438 let snapshot = q.thread.len();
439
440 let queue = crate::queue::Queue::at(self.home.join("queue"));
446 let task_id = crate::daemon::task_of_question(&queue.list(), &q).map(|t| t.id.clone());
447 let _task_claim = match (&task_id, q.chosen_action()) {
451 (Some(id), Some(_)) => match queue.claim(id) {
452 Ok(c) => Some(c),
453 Err(e) => {
454 tracing::debug!(
455 "question {}: task claim not available ({e:#}), deferring delivery",
456 q.short()
457 );
458 return Ok(());
459 }
460 },
461 _ => None,
462 };
463 let task_still_ours = || {
466 if q.chosen_action().is_some_and(|a| a.maintenance()) {
467 false
468 } else {
469 match &task_id {
470 Some(id) => queue
471 .get(id)
472 .map(|t| !crate::daemon::action_standing(&t, &q).daemon_owns())
473 .unwrap_or(true),
474 None => true,
475 }
476 }
477 };
478 if !task_still_ours() {
479 return Ok(());
480 }
481
482 self.store.beat(&q.id, WaiterKind::Daemon);
483 if !task_still_ours() {
486 return Ok(());
487 }
488 self.store.update(&q.id, |r| {
489 r.waiter = Some(Note {
490 kind: WaiterKind::Daemon,
491 since: now,
492 });
493 Ok(())
494 })?;
495 tracing::info!(
496 "question {}: the asker is gone, resuming seat {} with the owner's word",
497 q.short(),
498 q.seat
499 );
500
501 let thread: Vec<(&str, &str)> = q
502 .thread
503 .iter()
504 .map(|t| {
505 (
506 if t.who == Who::Operator {
507 "operator"
508 } else {
509 "agent"
510 },
511 t.body.as_str(),
512 )
513 })
514 .collect();
515 let shown = match word {
518 Word::Said(_) => &thread[..q.delivered_turns.min(thread.len())],
519 Word::Answered(_) => &thread[..],
520 };
521 let owner_word = match &word {
522 Word::Said(s) => prompt::OwnerWord::Said(s),
523 Word::Answered(a) => prompt::OwnerWord::Answered(a),
524 };
525 let body = prompt::question_resumed(
526 &q.id,
527 &q.summary,
528 &q.detail,
529 shown,
530 &owner_word,
531 &target.language,
532 );
533
534 let artifacts = self.store.root().join(format!("{}.delivery", q.id));
535 let stem = format!("deliver-{}", now.as_second());
536 let cache_dir = None;
537 let inv = Invocation {
538 scope: crate::agent::InvocationScope::Repository,
539 coordination: target.coordination.as_ref(),
540 fetched_base: target.fetched_base.as_ref(),
541 cwd: &target.cwd,
542 prompt: &body,
543 timeout: DELIVERY_TIMEOUT,
544 allow_write: target.allow_write,
545 unsandboxed: false,
546 sessions: target.sessions,
547 artifacts: &artifacts,
548 stem: &stem,
549 run: &q.run,
550 node: &q.node,
551 cache_dir,
552 attachments: &[],
553 writable: &[],
554 };
555
556 let store = self.store.clone();
557 let id = q.id.clone();
558 let out = {
559 let fut = agent::invoke(&target.spec, &mut target.seat, &inv);
560 tokio::pin!(fut);
561 let mut beat = tokio::time::interval(Duration::from_secs(1));
562 let mut beats = 0u32;
563 loop {
564 tokio::select! {
565 r = &mut fut => break Some(r),
566 _ = beat.tick() => {
567 if halt() {
568 break None;
569 }
570 beats += 1;
571 if beats % 20 == 0 {
572 store.beat(&id, WaiterKind::Daemon);
573 }
574 }
575 }
576 }
577 };
578
579 let Some(out) = out else {
580 let _ = self.store.update(&q.id, |r| {
582 r.waiter = None;
583 Ok(())
584 });
585 return Ok(());
586 };
587 let out = match out {
588 Ok(o) if o.usable() => o,
589 other => {
590 let why = match other {
591 Ok(o) if o.timed_out => "the resumed turn timed out".to_owned(),
592 Ok(o) if o.quota_exhausted() => "the agent is out of quota".to_owned(),
593 Ok(o) => format!(
594 "the resumed turn failed ({})",
595 notices::exit_status(o.exit_code)
596 ),
597 Err(e) => format!("the agent could not be started: {e:#}"),
598 };
599 self.refuse(&q, key, &why);
600 return Ok(());
601 }
602 };
603
604 let text = out.text.trim().to_owned();
605 self.store.update(&q.id, |r| {
606 r.waiter = None;
607 match &word {
608 Word::Answered(_) => r.answer_delivered = true,
609 Word::Said(_) => {
610 r.delivered_turns = r.delivered_turns.max(snapshot);
611 let agent_spoke = r.thread[snapshot.min(r.thread.len())..]
616 .iter()
617 .any(|t| t.who == Who::Agent);
618 if !agent_spoke && r.thread.len() == snapshot && r.status.open() {
619 let choices = r.choices.clone();
620 r.reply(text, choices)?;
621 }
622 }
623 }
624 Ok(())
625 })?;
626 self.memo.remove(&q.id);
627 Ok(())
628 }
629}
630
631pub(crate) fn take_claim(path: &std::path::Path, stale_after: Duration) -> bool {
635 let attempt = || {
636 std::fs::OpenOptions::new()
637 .write(true)
638 .create_new(true)
639 .open(path)
640 .is_ok()
641 };
642 if attempt() {
643 return true;
644 }
645 let stale = std::fs::metadata(path)
646 .and_then(|m| m.modified())
647 .ok()
648 .and_then(|t| t.elapsed().ok())
649 .is_some_and(|age| age > stale_after);
650 if stale {
651 let _ = std::fs::remove_file(path);
652 return attempt();
653 }
654 false
655}
656
657pub(crate) struct Release(pub(crate) PathBuf);
659
660impl Drop for Release {
661 fn drop(&mut self) {
662 let _ = std::fs::remove_file(&self.0);
663 }
664}
665
666pub async fn run(mut waiter: Waiter, stop: crate::daemon::Stop) {
672 let halt = {
673 let stop = stop.clone();
674 move || stop.parking()
675 };
676 while !stop.stopped() {
677 waiter.tick(Timestamp::now(), &halt).await;
678 tokio::time::sleep(TICK).await;
679 }
680}
681
682#[cfg(test)]
683mod tests {
684 use super::*;
685
686 fn ts(secs: i64) -> Timestamp {
687 Timestamp::from_second(secs).unwrap()
688 }
689
690 fn asked(at: i64) -> Question {
691 let mut q = Question::new(
692 "run".into(),
693 "implement".into(),
694 "impl-A".into(),
695 "which?".into(),
696 String::new(),
697 vec![],
698 );
699 q.cwd = Some("/tmp".into());
700 q.asked_at = ts(at);
701 q.answer_timeout = 1000;
702 q
703 }
704
705 fn lease(at: i64) -> Lease {
706 Lease {
707 kind: WaiterKind::Asker,
708 pid: 1,
709 beat_at: ts(at),
710 }
711 }
712
713 #[test]
714 fn a_fresh_lease_means_nobody_else_acts() {
715 let mut q = asked(0);
716 q.say("why?").unwrap();
717 assert_eq!(
718 decide(&q, Some(&lease(95)), false, 86_400, ts(100)),
719 Action::Idle
720 );
721 }
722
723 #[test]
724 fn a_stale_lease_delivers_what_the_owner_said() {
725 let mut q = asked(0);
726 q.say("why?").unwrap();
727 assert_eq!(
728 decide(&q, Some(&lease(0)), false, 86_400, ts(500)),
729 Action::Deliver(Word::Said("why?".into()))
730 );
731 assert_eq!(
732 decide(&q, None, true, 86_400, ts(500)),
733 Action::Idle,
734 "a seat still mid-turn is not resumed"
735 );
736 }
737
738 #[test]
739 fn an_answer_nobody_read_is_delivered_once() {
740 let mut q = asked(0);
741 q.choices = vec!["A".into()];
742 q.answer(crate::ask::Answer::Choice("A".into())).unwrap();
743 assert_eq!(
744 decide(&q, None, false, 86_400, ts(10)),
745 Action::Deliver(Word::Answered("A".into()))
746 );
747 q.answer_delivered = true;
748 assert_eq!(decide(&q, None, false, 86_400, ts(10)), Action::Idle);
749 }
750
751 #[test]
752 fn only_asked_questions_and_only_before_the_deadline() {
753 let mut q = asked(0);
754 assert_eq!(decide(&q, None, false, 86_400, ts(999)), Action::Idle);
755 assert_eq!(decide(&q, None, false, 86_400, ts(1001)), Action::Expire);
756 q.cwd = None;
757 assert_eq!(decide(&q, None, false, 86_400, ts(1001)), Action::Idle);
758 }
759
760 #[test]
761 fn the_deadline_runs_from_the_last_turn_not_from_asking() {
762 let mut q = asked(0);
763 q.thread.push(crate::ask::Turn {
764 who: Who::Agent,
765 body: "context".into(),
766 at: ts(4000),
767 note: None,
768 });
769 q.delivered_turns = 1;
770 assert_eq!(decide(&q, None, false, 86_400, ts(4500)), Action::Idle);
771 assert_eq!(decide(&q, None, false, 86_400, ts(5001)), Action::Expire);
772 }
773
774 #[test]
775 fn a_word_given_in_time_is_delivered_after_the_deadline() {
776 let mut q = asked(0);
777 q.say("wait").unwrap();
778 assert!(matches!(
779 decide(&q, None, false, 86_400, ts(5000)),
780 Action::Deliver(_)
781 ));
782 }
783
784 #[test]
785 fn an_action_answer_is_left_to_the_daemon() {
786 let mut q = asked(0);
787 q.choices = vec!["A".into()];
788 q.actions
789 .insert("A".into(), crate::ask::ChoiceAction::Requeue);
790 q.answer(crate::ask::Answer::Choice("A".into())).unwrap();
791 assert_eq!(
792 decide_owned(&q, None, false, 86_400, ts(10), true),
793 Action::Idle,
794 "the daemon applies the action; the waiter must not also resume the seat"
795 );
796 q.answer_delivered = true;
798 assert_eq!(
799 decide_owned(&q, None, false, 86_400, ts(10), true),
800 Action::Idle
801 );
802 q.answer_delivered = false;
804 assert_eq!(
805 decide_owned(&q, None, false, 86_400, ts(10), false),
806 Action::Deliver(Word::Answered("A".into()))
807 );
808 let mut plain = asked(0);
810 plain.choices = vec!["A".into()];
811 plain
812 .answer(crate::ask::Answer::Choice("A".into()))
813 .unwrap();
814 assert_eq!(
815 decide_owned(&plain, None, false, 86_400, ts(10), true),
816 Action::Deliver(Word::Answered("A".into()))
817 );
818 }
819}