1use std::collections::{BTreeMap, HashMap, HashSet};
67use std::path::{Path, PathBuf};
68use std::sync::Arc;
69use std::time::{Duration, Instant};
70
71use anyhow::{Context as _, Result};
72use jiff::Timestamp;
73use tokio::task::JoinSet;
74
75use crate::agent::{self, Invocation, SeatState};
76use crate::ask::{ChoiceAction, Deputy, Question, Questions, Waiter as Note, WaiterKind, Who};
77use crate::config::Config;
78use crate::notices::{self, Notice};
79use crate::prompt;
80
81pub const NODE: &str = "deputy";
84
85pub const TICK: Duration = Duration::from_secs(5);
87
88pub const MAX_STARTS: u32 = 3;
92
93const RESTART_AFTER: Duration = Duration::from_secs(30);
97
98fn next_start_path(store: &Questions, id: &str) -> PathBuf {
99 store.root().join(format!("{id}.deputy")).join("next-start")
100}
101
102fn start_due(store: &Questions, id: &str, now: Timestamp) -> bool {
103 std::fs::read_to_string(next_start_path(store, id))
104 .ok()
105 .and_then(|text| text.parse::<i64>().ok())
106 .is_none_or(|next| now.as_second() >= next)
107}
108
109const SLACK_SECS: u64 = 120;
112
113const CLAIM_STALE: Duration = Duration::from_secs(60);
115
116const HANDOVER_TIMEOUT: Duration = Duration::from_secs(10 * 60);
118
119pub type Halt = Arc<dyn Fn() -> bool + Send + Sync>;
121
122pub fn brief(
129 task_id: &str,
130 reason: &str,
131 choices: &[String],
132 actions: &BTreeMap<String, ChoiceAction>,
133) -> String {
134 let reason = reason.trim();
135 let mut s = format!(
136 "Task {task_id} (`magi task show {task_id}`) is blocked on this question. \
137 The conductor asked because: {}\n\n\
138 When the owner answers, the daemon applies the exact attached action \
139 once, after rechecking its preconditions. You only record the choice; \
140 you never execute maintenance or edit repository or run records.",
141 if reason.is_empty() {
142 "(it recorded no reasoning)"
143 } else {
144 reason
145 }
146 );
147 if !choices.is_empty() {
148 s.push_str("\n\nWhat each option does:");
149 for c in choices {
150 match actions.get(c) {
151 Some(a) => s.push_str(&format!("\n- `{c}`: also {}", a.describe())),
152 None => s.push_str(&format!(
153 "\n- `{c}`: no executable operation; keep held and ask for a supported choice"
154 )),
155 }
156 }
157 }
158 s
159}
160
161#[derive(Debug, Clone, Copy, PartialEq, Eq)]
163pub enum Kind {
164 Conduct,
166 Land,
168 Release,
170 Triage,
173 Generic,
176}
177
178pub fn kind_of(q: &Question) -> Option<Kind> {
186 if q.routed_task.is_some() {
187 return None;
188 }
189 match q.node.as_str() {
190 crate::conduct::NODE => Some(Kind::Conduct),
191 crate::land::APPROVAL_NODE => Some(Kind::Land),
192 crate::bump::NOTICE_NODE if q.seat == "release-watch" => Some(Kind::Release),
195 crate::bump::NOTICE_NODE => None,
196 crate::triage::NODE | crate::triage::DEPS_NODE => Some(Kind::Triage),
197 _ if q.cwd.is_none() && !q.choices.is_empty() => Some(Kind::Generic),
198 _ => None,
199 }
200}
201
202pub fn generic_brief(
205 q: &Question,
206 actions: &BTreeMap<String, ChoiceAction>,
207 home: &Path,
208) -> String {
209 let run = crate::run::is_run_id(&q.run).then_some(q.run.as_str());
211 let knows = if run.is_some() {
212 "You know what the question itself says and what you can learn from \
213 its run by the read-only investigation described below."
214 } else {
215 "You know only what the question itself says."
216 };
217 let mut s = format!(
218 "This question was filed by magi (node `{}`, seat `{}`) with no agent \
219 waiting on it. Magi records the owner's answer and the component that \
220 asked applies it; you apply nothing. {knows} When the owner's words \
221 pick an option, settle it; when you cannot tell which option they \
222 mean, ask them with `--thread`.",
223 q.node, q.seat
224 );
225 if let Some(run) = run {
226 let artifacts = home.join("runs").join(run).join("artifacts");
227 s.push_str(&format!(
228 "\n\nThis question concerns run `{run}`. You may investigate it on \
229 your own, read-only: run `magi show {run}` and read files under \
230 `{}`. When the owner asks about details (for example which part \
231 was wrong), look it up yourself and answer; do not reply that you \
232 cannot see the details, and do not tell the owner to run `magi show` \
233 themselves. Investigating never changes anything: edit, commit, push \
234 and apply nothing. If the run cannot be read, say exactly that and do \
235 not guess at the cause or at what an option does.",
236 artifacts.display()
237 ));
238 }
239 if q.node == crate::github_text::ASK_NODE {
240 s.push_str("\n\n");
241 s.push_str(&crate::github_text::deputy_brief());
242 }
243 if !q.choices.is_empty() {
244 s.push_str("\n\nWhat each option does:");
245 for c in &q.choices {
246 match actions.get(c) {
247 Some(a) => s.push_str(&format!("\n- `{c}`: also {}", a.describe())),
248 None => s.push_str(&format!(
249 "\n- `{c}`: {}",
250 crate::choice_effect::describe(q, c)
251 .unwrap_or_else(|| "unsupported legacy choice; nothing applies it".into())
252 )),
253 }
254 }
255 }
256 s
257}
258
259pub fn fixed_clock(q: &Question) -> bool {
263 q.node == crate::land::APPROVAL_NODE
264 || (q.node == crate::bump::NOTICE_NODE && q.seat == "release-watch")
265 || q.node == crate::github_text::ASK_NODE
266}
267
268pub fn deadline(q: &Question, default_timeout: u64) -> i64 {
276 let secs = if q.answer_timeout > 0 {
277 q.answer_timeout
278 } else {
279 default_timeout
280 };
281 let from = if fixed_clock(q) {
282 q.asked_at.as_second()
283 } else {
284 q.last_activity()
285 };
286 from.saturating_add(secs as i64)
287}
288
289pub fn restart_in_time(q: &Question, default_timeout: u64, now: Timestamp) -> bool {
291 now.as_second() <= deadline(q, default_timeout) || q.unread_from_owner().is_some()
292}
293
294pub fn can_start(cfg: Option<&Config>, agent: &str) -> bool {
302 cfg.is_some_and(|c| {
303 c.daemon.max_deputies > 0
304 && ((!agent.is_empty() && c.agent(agent).is_ok()) || c.resolve_roles().is_ok())
305 })
306}
307
308pub fn agent_of(q: &Question) -> &str {
310 q.deputy.as_ref().map_or("", |d| d.agent.as_str())
311}
312
313pub fn exhausted_past_deadline(
320 q: &Question,
321 startable: bool,
322 default_timeout: u64,
323 now: Timestamp,
324) -> bool {
325 q.status.open()
326 && q.deputy
327 .as_ref()
328 .is_some_and(|d| d.starts >= MAX_STARTS || !startable)
329 && now.as_second() > deadline(q, default_timeout)
330}
331
332pub struct Deputies {
334 store: Questions,
335 home: PathBuf,
336 cfg: Option<Config>,
337 fallback_repo: PathBuf,
339 max: usize,
340 halt: Halt,
341 tasks: JoinSet<String>,
342 inflight: HashSet<String>,
343 memo: HashMap<String, Instant>,
345}
346
347impl Deputies {
348 pub fn new(
350 store: Questions,
351 home: PathBuf,
352 cfg: Option<Config>,
353 fallback_repo: PathBuf,
354 max: usize,
355 halt: Halt,
356 ) -> Self {
357 Self {
358 store,
359 home,
360 cfg,
361 fallback_repo,
362 max,
363 halt,
364 tasks: JoinSet::new(),
365 inflight: HashSet::new(),
366 memo: HashMap::new(),
367 }
368 }
369
370 pub fn has_work(&self, now: Timestamp) -> bool {
372 self.max > 0
373 && self.store.list().iter().any(|q| {
374 kind_of(q).is_some()
375 && !crate::task_chat::routes(q, &self.home)
376 && q.status.open()
377 && q.deputy
378 .as_ref()
379 .is_none_or(|deputy| deputy.starts < MAX_STARTS)
380 && self
381 .store
382 .read_lease(&q.id)
383 .is_none_or(|lease| !lease.fresh(now))
384 && can_start(self.cfg.as_ref(), agent_of(q))
385 && restart_in_time(q, self.default_timeout(), now)
386 && start_due(&self.store, &q.id, now)
387 })
388 }
389
390 fn default_timeout(&self) -> u64 {
391 self.cfg
392 .as_ref()
393 .map_or(Config::default().graph.answer_timeout, |c| {
394 c.graph.answer_timeout
395 })
396 }
397
398 fn reap(&mut self) {
399 while let Some(done) = self.tasks.try_join_next() {
400 if let Ok(id) = done {
401 self.inflight.remove(&id);
402 }
403 }
404 if self.tasks.is_empty() {
405 self.inflight.clear();
406 }
407 }
408
409 fn attach(&self, q: &Question, kind: Kind) -> Option<Question> {
419 let default_timeout = self.default_timeout();
420 let repo = self.fallback_repo.to_string_lossy().into_owned();
421 let state = match kind {
422 Kind::Land => crate::run::RunState::load(&q.run).ok(),
423 Kind::Conduct | Kind::Release | Kind::Triage | Kind::Generic => None,
424 };
425 let timeout = state
427 .as_ref()
428 .map_or(default_timeout, |s| s.config.graph.answer_timeout);
429 self.store
430 .update(&q.id, |r| {
431 if r.deputy.is_none() {
432 r.deputy = Some(Deputy::new(brief_for(&self.home, r, kind, state.as_ref())));
433 }
434 if kind == Kind::Conduct && r.cwd.is_none() {
435 r.cwd = Some(repo.clone());
436 }
437 if r.answer_timeout == 0 {
438 r.answer_timeout = match kind {
439 Kind::Conduct => default_timeout,
440 Kind::Land => timeout,
441 Kind::Release | Kind::Triage | Kind::Generic => default_timeout,
442 };
443 }
444 Ok(())
445 })
446 .map(|(r, ())| r)
447 .map_err(|e| tracing::warn!("question {}: cannot attach a deputy: {e:#}", q.short()))
448 .ok()
449 }
450
451 fn ready(&self, q: Question, now: Timestamp) -> Option<Question> {
452 if (self.halt)() || !q.status.open() || crate::task_chat::routes(&q, &self.home) {
453 return None;
454 }
455 let kind = kind_of(&q)?;
456 let needs_cwd = kind == Kind::Conduct && q.cwd.is_none();
457 let q = if q.deputy.is_none() || needs_cwd || q.answer_timeout == 0 {
458 self.attach(&q, kind)?
459 } else {
460 q
461 };
462 let deputy = q.deputy.as_ref()?;
463 if self
464 .store
465 .read_lease(&q.id)
466 .is_some_and(|lease| lease.fresh(now))
467 || deputy.starts >= MAX_STARTS
468 || !restart_in_time(&q, self.default_timeout(), now)
469 || !can_start(self.cfg.as_ref(), &deputy.agent)
470 || !start_due(&self.store, &q.id, now)
471 {
472 return None;
473 }
474 Some(q)
475 }
476
477 pub async fn dispatch(&self, now: Timestamp) -> Result<()> {
480 let Some(_dispatch) = crate::worker::try_lease(&self.home, "deputy-dispatch")? else {
481 return Ok(());
482 };
483 let jobs = crate::worker::active(&self.home)?;
484 let mut occupied: HashSet<String> = jobs
485 .iter()
486 .filter(|job| job.kind == "deputy")
487 .map(|job| {
488 job.key
489 .strip_prefix("deputy:")
490 .unwrap_or(&job.id)
491 .to_owned()
492 })
493 .collect();
494 let questions = self.store.list();
495 occupied.extend(
498 questions
499 .iter()
500 .filter(|q| {
501 self.store
502 .read_lease(&q.id)
503 .is_some_and(|lease| lease.kind == WaiterKind::Deputy && lease.fresh(now))
504 })
505 .map(|q| q.id.clone()),
506 );
507 for q in questions {
508 if occupied.len() >= self.max || (self.halt)() {
509 break;
510 }
511 if occupied.contains(&q.id) {
512 continue;
513 }
514 let Some(q) = self.ready(q, now) else {
515 continue;
516 };
517 let request = WorkerRequest {
518 question: q.id.clone(),
519 cfg: self.cfg.clone(),
520 fallback_repo: self.fallback_repo.clone(),
521 };
522 match crate::worker::spawn(
523 &self.home,
524 "deputy",
525 &format!("deputy:{}", q.id),
526 serde_json::to_value(request)?,
527 )
528 .await
529 {
530 Ok(_) => {
531 occupied.insert(q.id);
532 }
533 Err(error) => tracing::warn!("question {} deputy worker: {error:#}", q.short()),
534 }
535 }
536 Ok(())
537 }
538
539 pub fn tick(&mut self, now: Timestamp) {
541 self.reap();
542 for q in self.store.list() {
543 let Some(q) = self.ready(q, now) else {
544 continue;
545 };
546 if self.inflight.contains(&q.id)
547 || self.inflight.len() >= self.max
548 || matches!(self.memo.get(&q.id), Some(until) if Instant::now() < *until)
549 {
550 continue;
551 }
552 self.memo
553 .insert(q.id.clone(), Instant::now() + RESTART_AFTER);
554 self.inflight.insert(q.id.clone());
555 let job = Job {
556 store: self.store.clone(),
557 home: self.home.clone(),
558 cfg: self.cfg.clone(),
559 fallback_repo: self.fallback_repo.clone(),
560 halt: Arc::clone(&self.halt),
561 };
562 let id = q.id.clone();
563 self.tasks.spawn(async move {
564 if let Err(error) = job.turn(&id).await {
565 tracing::warn!(
566 "deputy for question {}: {error:#}",
567 crate::ask::short_id(&id)
568 );
569 }
570 id
571 });
572 }
573 }
574
575 pub async fn drain(&mut self) {
578 while let Some(done) = self.tasks.join_next().await {
579 if let Ok(id) = done {
580 self.inflight.remove(&id);
581 }
582 }
583 self.inflight.clear();
584 }
585}
586
587fn brief_for(
590 home: &Path,
591 q: &Question,
592 kind: Kind,
593 state: Option<&crate::run::RunState>,
594) -> String {
595 match kind {
596 Kind::Conduct => brief(&q.run, &q.detail, &q.choices, &q.actions),
597 Kind::Land => crate::land::deputy_brief(q, state),
598 Kind::Release => crate::release_watch::deputy_brief(q, home),
599 Kind::Triage => {
600 crate::triage::deputy_brief(q, &crate::queue::Queue::at(home.join("queue")))
601 }
602 Kind::Generic => generic_brief(q, &q.actions, home),
603 }
604}
605
606#[derive(serde::Serialize, serde::Deserialize)]
607struct WorkerRequest {
608 question: String,
609 cfg: Option<Config>,
610 fallback_repo: PathBuf,
611}
612
613pub async fn run_worker(home: &Path, payload: serde_json::Value) -> Result<()> {
615 let request: WorkerRequest = serde_json::from_value(payload)?;
616 Job {
617 store: Questions::at(home.join("questions")),
618 home: home.to_path_buf(),
619 cfg: request.cfg,
620 fallback_repo: request.fallback_repo,
621 halt: Arc::new(|| false),
622 }
623 .turn(&request.question)
624 .await
625}
626
627struct Job {
629 store: Questions,
630 home: PathBuf,
631 cfg: Option<Config>,
632 fallback_repo: PathBuf,
633 halt: Halt,
634}
635
636impl Job {
637 async fn drive(
640 &self,
641 spec: &crate::config::AgentSpec,
642 seat: &mut SeatState,
643 inv: &Invocation<'_>,
644 id: &str,
645 ) -> Option<Result<agent::AgentOutput>> {
646 let fut = agent::invoke(spec, seat, inv);
647 tokio::pin!(fut);
648 let mut beat = tokio::time::interval(Duration::from_secs(1));
649 let mut beats = 0u32;
650 loop {
651 tokio::select! {
652 r = &mut fut => break Some(r),
653 _ = beat.tick() => {
654 if (self.halt)() {
655 break None;
656 }
657 beats += 1;
658 if beats % 20 == 0 {
659 self.store.beat(id, WaiterKind::Deputy);
660 }
661 }
662 }
663 }
664 }
665
666 fn park_quietly(&self, id: &str) {
669 let _ = self.store.update(id, |r| {
670 r.waiter = None;
671 Ok(())
672 });
673 self.store.drop_lease(id);
674 }
675
676 fn park(&self, id: &str) {
679 let _ = self.store.update(id, |r| {
680 if let Some(d) = r.deputy.as_mut() {
681 d.starts = d.starts.saturating_sub(1);
682 }
683 r.waiter = None;
684 Ok(())
685 });
686 self.store.drop_lease(id);
687 }
688
689 async fn turn(&self, id: &str) -> Result<()> {
690 let claim = self.store.root().join(format!("{id}.deputy-claim"));
691 if !crate::waiter::take_claim(&claim, CLAIM_STALE) {
695 return Ok(());
696 }
697 let release = crate::waiter::Release(claim);
698
699 let q = self.store.get(id)?;
701 let now = Timestamp::now();
702 let Some(dep) = q.deputy.clone() else {
703 return Ok(());
704 };
705 if !q.status.open()
706 || crate::task_chat::routes(&q, &self.home)
707 || dep.starts >= MAX_STARTS
708 || self.store.read_lease(&q.id).is_some_and(|l| l.fresh(now))
709 || !start_due(&self.store, &q.id, now)
710 {
711 return Ok(());
712 }
713 let cfg = self
714 .cfg
715 .as_ref()
716 .context("the deputy's configuration is not available")?;
717 let spec = match cfg.agent(&dep.agent) {
718 Ok(s) if !dep.agent.is_empty() => s.clone(),
719 _ => {
720 cfg.resolve_roles()
721 .context("resolving the deputy's agent")?
722 .conductor
723 }
724 };
725 let key = crate::ask::deputy_seat_key(&q.id);
728 let (mut seat, resumed) = match dep.seat.clone() {
729 Some(s)
730 if s.agent == spec.id && agent::has_session(spec.kind, &s, cfg.graph.sessions) =>
731 {
732 (s, true)
733 }
734 _ => (SeatState::new(&key, &spec.id, crate::rng::entropy()), false),
735 };
736 let recorded = q.cwd.clone().or_else(|| match kind_of(&q) {
739 Some(Kind::Release) => {
740 crate::release_watch::state_for_question(&self.home, &q.id).map(|st| st.repo)
741 }
742 Some(Kind::Triage | Kind::Generic) => crate::queue::Queue::at(self.home.join("queue"))
745 .get(&q.run)
746 .ok()
747 .map(|t| t.repo.to_string_lossy().into_owned())
748 .filter(|r| !r.is_empty()),
749 _ => None,
750 });
751 let cwd = recorded
752 .as_deref()
753 .map(PathBuf::from)
754 .filter(|p| p.is_dir())
755 .unwrap_or_else(|| self.fallback_repo.clone());
756
757 let bodies: Vec<String> = q
760 .thread
761 .iter()
762 .map(|t| match &t.note {
763 Some(n) => format!("{}\n(note: {n})", t.body),
764 None => t.body.clone(),
765 })
766 .collect();
767 let thread: Vec<(&str, &str)> = q
768 .thread
769 .iter()
770 .zip(&bodies)
771 .map(|(t, body)| {
772 (
773 if t.who == Who::Operator {
774 "operator"
775 } else {
776 "agent"
777 },
778 body.as_str(),
779 )
780 })
781 .collect();
782 let read = &thread[..q.delivered_turns.min(thread.len())];
783 let unread = q.unread_from_owner();
784 let snapshot = q.thread.len();
785 let brief = match kind_of(&q) {
792 Some(kind @ (Kind::Land | Kind::Release | Kind::Triage | Kind::Generic)) => {
793 let state = (kind == Kind::Land)
794 .then(|| crate::run::RunState::load(&q.run).ok())
795 .flatten();
796 brief_for(&self.home, &q, kind, state.as_ref())
797 }
798 Some(Kind::Conduct) | None => dep.brief.clone(),
799 };
800 let current_brief = format!(
801 "{}\n\n{}\n\n{}",
802 brief,
803 crate::choice_effect::brief(&q),
804 crate::queue::dependency_facts(
805 &crate::queue::Queue::at(self.home.join("queue")),
806 &self.store
807 )
808 );
809 let body = prompt::deputy(&prompt::DeputyPrompt {
810 id: &q.id,
811 summary: &q.summary,
812 detail: &q.detail,
813 brief: ¤t_brief,
814 choices: &q.choices,
815 thread: read,
816 unread: unread.as_deref(),
817 resumed,
818 handover: false,
819 kind: kind_of(&q).unwrap_or(Kind::Conduct),
820 language: &cfg.graph.language,
821 });
822
823 let next_start = next_start_path(&self.store, &q.id);
826 std::fs::create_dir_all(next_start.parent().expect("deputy artifacts parent"))?;
827 std::fs::write(
828 next_start,
829 now.as_second()
830 .saturating_add(RESTART_AFTER.as_secs() as i64)
831 .to_string(),
832 )?;
833 self.store.beat(&q.id, WaiterKind::Deputy);
834 let starts = dep.starts + 1;
835 let first = seat.clone();
836 self.store.update(&q.id, |r| {
837 if let Some(d) = r.deputy.as_mut() {
838 d.agent = spec.id.clone();
839 d.seat = Some(first);
840 d.starts = starts;
841 }
842 r.waiter = Some(Note {
843 kind: WaiterKind::Deputy,
844 since: now,
845 });
846 Ok(())
847 })?;
848 drop(release);
849 tracing::info!(
850 "question {}: deputy seat {} {} (start {starts}/{MAX_STARTS})",
851 q.short(),
852 seat.key,
853 if resumed { "resuming" } else { "starting" }
854 );
855
856 let left = (deadline(&q, cfg.graph.answer_timeout) - now.as_second()).max(0) as u64;
857 let artifacts = self.store.root().join(format!("{}.deputy", q.id));
858 let cache_dir = cfg.cache_dir();
859 let allow_write = true;
863 let writable = [self.store.root().to_path_buf(), self.home.join("queue")];
868 macro_rules! invocation {
869 ($prompt:expr, $stem:expr, $timeout:expr) => {
870 Invocation {
871 coordination: cfg.coordination.as_ref(),
872 fetched_base: None,
873 cwd: &cwd,
874 prompt: $prompt,
875 timeout: $timeout,
876 allow_write,
877 unsandboxed: false,
878 sessions: cfg.graph.sessions,
879 artifacts: &artifacts,
880 stem: $stem,
881 run: &q.run,
885 node: NODE,
886 cache_dir: cache_dir.as_deref(),
887 attachments: &[],
888 writable: &writable,
889 }
890 };
891 }
892
893 let mut early = None;
898 if !resumed && cfg.graph.sessions {
899 let hbody = prompt::deputy(&prompt::DeputyPrompt {
900 id: &q.id,
901 summary: &q.summary,
902 detail: &q.detail,
903 brief: ¤t_brief,
904 choices: &q.choices,
905 thread: read,
906 unread: None,
907 resumed: false,
908 handover: true,
909 kind: kind_of(&q).unwrap_or(Kind::Conduct),
910 language: &cfg.graph.language,
911 });
912 let hstem = format!("handover-{starts}");
913 let hlimit = HANDOVER_TIMEOUT.min(Duration::from_secs(left.max(1)));
916 let hinv = invocation!(&hbody, &hstem, hlimit);
917 let Some(done) = self.drive(&spec, &mut seat, &hinv, &q.id).await else {
918 self.park(&q.id);
919 return Ok(());
920 };
921 let kept = seat.clone();
922 self.store.update(&q.id, |r| {
923 if let Some(d) = r.deputy.as_mut() {
924 d.seat = Some(kept);
925 }
926 Ok(())
927 })?;
928 if !matches!(&done, Ok(o) if o.usable()) {
929 early = Some(done);
930 }
931 }
932
933 let left = if early.is_none() && !resumed && cfg.graph.sessions {
936 let now = Timestamp::now();
937 let again = self.store.get(&q.id)?;
938 let left = (deadline(&again, cfg.graph.answer_timeout) - now.as_second()).max(0) as u64;
939 if !again.status.open() || (left == 0 && again.unread_from_owner().is_none()) {
940 self.park_quietly(&q.id);
941 return Ok(());
942 }
943 left
944 } else {
945 left
946 };
947 let out = match early {
948 Some(done) => done,
949 None => {
950 let stem = format!("turn-{starts}");
951 let timeout = Duration::from_secs(left.max(60) + SLACK_SECS);
952 let inv = invocation!(&body, &stem, timeout);
953 match self.drive(&spec, &mut seat, &inv, &q.id).await {
954 Some(out) => out,
955 None => {
956 self.park(&q.id);
957 return Ok(());
958 }
959 }
960 }
961 };
962
963 let (text, why) = match out {
964 Ok(o) if o.usable() => (Some(o.text.trim().to_owned()), None),
965 Ok(o) if o.timed_out => (None, Some("its turn timed out".to_owned())),
966 Ok(o) if o.quota_exhausted() => (None, Some("the agent is out of quota".to_owned())),
967 Ok(o) => (
968 None,
969 Some(format!(
970 "its turn failed ({})",
971 notices::exit_status(o.exit_code)
972 )),
973 ),
974 Err(e) => (None, Some(format!("the agent could not be started: {e:#}"))),
975 };
976 let kept = seat.clone();
977 self.store.update(&q.id, |r| {
978 if let Some(d) = r.deputy.as_mut() {
979 d.seat = Some(kept);
980 }
981 r.waiter = None;
982 self.store.drop_lease(&q.id);
984 if let Some(text) = text
988 && unread.is_some()
989 && !text.is_empty()
990 && r.status.open()
991 && r.thread.len() == snapshot
992 {
993 r.delivered_turns = r.delivered_turns.max(snapshot);
994 let choices = r.choices.clone();
995 r.reply(text, choices)?;
996 }
997 Ok(())
998 })?;
999 if let Some(why) = why {
1000 tracing::warn!("question {}: the deputy ended: {why}", q.short());
1001 notices::raise_in(
1002 &self.home,
1003 Notice::warn(
1004 &format!("deputy-turn:{}", q.id),
1005 format!(
1006 "Question {} \"{}\": its follow-up agent stopped ({why}).",
1007 q.short(),
1008 q.summary
1009 ),
1010 )
1011 .link(if q.run_names_task() {
1012 notices::Link::Task { id: q.run.clone() }
1013 } else {
1014 notices::Link::Run { id: q.run.clone() }
1015 }),
1016 );
1017 }
1018 Ok(())
1019 }
1020}
1021
1022#[cfg(test)]
1023mod worker_tests {
1024 use super::*;
1025
1026 #[test]
1027 fn restart_cooldown_survives_a_new_runner() {
1028 let home = tempfile::tempdir().expect("home");
1029 let questions = Questions::at(home.path().join("questions"));
1030 let path = next_start_path(&questions, "question");
1031 std::fs::create_dir_all(path.parent().expect("parent")).expect("artifacts");
1032 std::fs::write(&path, "130").expect("next start");
1033 let reopened = Questions::at(home.path().join("questions"));
1034 assert!(!start_due(
1035 &reopened,
1036 "question",
1037 Timestamp::from_second(129).unwrap()
1038 ));
1039 assert!(start_due(
1040 &reopened,
1041 "question",
1042 Timestamp::from_second(130).unwrap()
1043 ));
1044 }
1045}