Skip to main content

scv_tools/delegate/
background.rs

1//! Background delegations: an `agent` call with `background: true` returns a
2//! job handle at once while the agent keeps working; `agent_status` and
3//! `agent_wait` observe the job, `agent_cancel` stops it, and the session is
4//! told when it finishes so the server can report it in a turn of its own.
5//!
6//! Each call that starts a job, or shows the model a job's result, leaves a
7//! [`JobChange`] under its call ID, which the server hands to the session's
8//! clients with the call's `tool.completed` ([`BackgroundJobs::take_changes`]).
9//!
10//! A finished job stays unreported until a report turn about it succeeds. A
11//! failed one makes it due again after a delay, a bounded number of times;
12//! after that, or when another turn must not be tried, the server reports it
13//! to the client directly ([`BackgroundJobs::report_failed`]).
14//!
15//! A call that continues a conversation takes its place in that
16//! conversation's lane as it arrives. When a turn or another call is
17//! ahead, the call's busy policy decides: it steers the running turn, waits
18//! in the foreground, fails, or becomes a queued job that runs once its
19//! place comes up.
20//!
21//! A call with `review` runs as a reviewed job ([`review`](super::review)):
22//! the same job, whose one call runs the whole builder and reviewer loop and
23//! whose changes and report carry SCV's outcome.
24
25pub(in crate::delegate) mod lane;
26mod owed;
27
28use std::{
29    collections::HashSet,
30    path::PathBuf,
31    sync::{Arc, Mutex, Weak},
32    time::{Duration, Instant},
33};
34
35use async_trait::async_trait;
36use scv_core::{
37    ApprovalGate, ProgressSink, Tool, ToolApprovals, ToolContext, ToolError, ToolOutput, ToolRisk,
38    ToolSpec,
39};
40use scv_protocol::{JobChange, JobOutcome, JobReport, JobStatus, describe_reports};
41use serde::Deserialize;
42use serde_json::{Map, Value, json};
43use tokio::sync::{mpsc, watch};
44use tokio_util::sync::CancellationToken;
45
46use self::lane::{Lanes, Place};
47use crate::{
48    BusyBehavior,
49    args::{Timeouts, bounded, parse_args, timeout_schema},
50    delegate::{
51        agent::{AGENT_TOOL, AgentTool, Offered},
52        conversation::{self, ConversationStore},
53        output::AgentReply,
54        request::AgentArgs,
55        review::{self, Record, ReviewedTool},
56    },
57    sync::lock,
58};
59
60/// Finished jobs a session keeps for `agent_status` beyond the running ones.
61const MAX_FINISHED: usize = 16;
62/// Characters of a finished job's reply quoted in a server-started report turn.
63const REPORT_REPLY_CHARS: usize = 6000;
64/// Jobs one report turn covers; any more wait for the next.
65const REPORT_MAX_JOBS: usize = 4;
66/// How long after each failed report turn the next one is due: after the
67/// first failure, then after the second. Once they run out, the job is
68/// reported directly.
69const REPORT_RETRY_DELAYS: [Duration; 2] = [Duration::from_secs(30), Duration::from_secs(120)];
70/// How long `agent_cancel` waits for a stopped job to settle.
71const CANCEL_SETTLE: Duration = Duration::from_secs(10);
72/// Job changes kept for calls whose `tool.completed` has not taken them,
73/// such as a call whose turn was aborted; the oldest go first.
74const MAX_PENDING_CHANGES: usize = 64;
75/// Characters of a delegated prompt's first line kept as its job's task.
76const TASK_CHARS: usize = 80;
77
78/// One session's background jobs. Dropping the store (with the session's
79/// tools) cancels every job still running.
80pub struct BackgroundJobs {
81    limit: usize,
82    state: Mutex<JobsState>,
83    cancellation: CancellationToken,
84    /// Woken whenever a job finishes, so the session can report it.
85    finished: Option<mpsc::UnboundedSender<()>>,
86    /// Decides a running job's nested approval requests, since no turn is
87    /// left to carry them to a person. Without it they are denied.
88    approvals: Option<Arc<dyn ApprovalGate>>,
89    /// The `agent` calls continuing each conversation, in arrival order.
90    lanes: Arc<Lanes>,
91}
92
93impl std::fmt::Debug for BackgroundJobs {
94    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
95        formatter
96            .debug_struct("BackgroundJobs")
97            .field("limit", &self.limit)
98            .finish_non_exhaustive()
99    }
100}
101
102#[derive(Default)]
103struct JobsState {
104    next: u64,
105    jobs: Vec<Job>,
106    /// Changes by the ID of the call that made them, oldest first.
107    changes: Vec<(String, JobChange)>,
108    /// The final outcomes of cancelled reviewed jobs the session is owed.
109    owed: owed::Owed,
110}
111
112impl JobsState {
113    /// Bring the outcome in the change the call `call_id` made for
114    /// `outcome`'s job up to date, if no client has taken it yet.
115    fn refresh(&mut self, call_id: &str, outcome: JobOutcome) {
116        if let Some((_, change)) = self
117            .changes
118            .iter_mut()
119            .rev()
120            .find(|(call, change)| call == call_id && change.job == outcome.job)
121        {
122            change.outcome = Some(outcome);
123        }
124    }
125
126    /// Keep `change` for the call `call_id`, which a client learns of with
127    /// that call's `tool.completed`. A call without an ID has no event.
128    fn record(&mut self, call_id: &str, change: JobChange) {
129        if call_id.is_empty() {
130            return;
131        }
132        if self.changes.len() >= MAX_PENDING_CHANGES {
133            let (call, dropped) = self.changes.remove(0);
134            self.owed.evicted(&call, &dropped.job);
135        }
136        self.changes.push((call_id.to_owned(), change));
137    }
138}
139
140struct Job {
141    id: String,
142    /// The tool that started it, `agent`.
143    tool: String,
144    /// The agent that runs it, such as `codex`.
145    agent: String,
146    /// The first line of the delegated prompt, shortened.
147    task: String,
148    started: Instant,
149    progress: ProgressSink,
150    last_progress: Option<String>,
151    /// Stops this job alone.
152    cancel: CancellationToken,
153    /// `agent_cancel` stopped it.
154    cancelled: bool,
155    /// A busy conversation's queued prompt, which `agent.max_queued_turns`
156    /// bounds instead of `agent.max_background`.
157    queued: bool,
158    outcome: Option<Outcome>,
159    /// The model has seen the result (through `agent_wait`, `agent_status`,
160    /// or a report turn that completed), the user stopped its report turn,
161    /// the server reported it directly, or the model asked for the stop with
162    /// `agent_cancel`, so it needs no report turn.
163    reported: bool,
164    /// A report turn about it runs.
165    reporting: bool,
166    /// Report turns about it that failed.
167    report_attempts: u32,
168    /// After a failed report turn: when the next one is due.
169    report_due: Option<tokio::time::Instant>,
170    done: watch::Receiver<bool>,
171    /// A reviewed job's record, whose outcome its changes and report state.
172    review: Option<Arc<Record>>,
173    /// `agent_cancel` waits for it, and describes it after: never pruned
174    /// meanwhile.
175    cancelling: bool,
176}
177
178/// `agent_cancel` waiting for a job to stop. Settled after the wait, or,
179/// if the call is dropped first, as when its turn is aborted, its change
180/// withdrawn and any outcome it owed left to an update.
181struct Waiting<'a> {
182    jobs: &'a BackgroundJobs,
183    job: String,
184    call: String,
185    /// The job is reviewed: the session is owed its final outcome.
186    owed: bool,
187    settled: bool,
188}
189
190impl Waiting<'_> {
191    /// The wait is over: the change the call made says the final outcome
192    /// if the job stopped, else the outcome as decided, its journal still
193    /// open; [`BackgroundJobs::take_changes`] says the final one if the job
194    /// stops before the change is sent.
195    fn settle(mut self) {
196        self.settled = true;
197        let mut state = self.jobs.state();
198        let Some(entry) = state.jobs.iter_mut().find(|entry| entry.id == self.job) else {
199            return;
200        };
201        entry.cancelling = false;
202        if !self.owed {
203            return;
204        }
205        let pending = entry.outcome.is_none().then(|| entry.outcome()).flatten();
206        let pending = pending.map(|mut outcome| {
207            outcome.review.journal_pending = true;
208            outcome
209        });
210        if let Some(outcome) = state.owed.waited(&self.job, &self.call, pending) {
211            state.refresh(&self.call, outcome);
212        }
213    }
214}
215
216impl Drop for Waiting<'_> {
217    fn drop(&mut self) {
218        if self.settled {
219            return;
220        }
221        let mut state = self.jobs.state();
222        if let Some(entry) = state.jobs.iter_mut().find(|entry| entry.id == self.job) {
223            entry.cancelling = false;
224        }
225        if !self.owed {
226            return;
227        }
228        // Its call is gone, so its change is never sent.
229        let (call, job) = (&self.call, &self.job);
230        state
231            .changes
232            .retain(|(recorded, change)| !(recorded == call && &change.job == job));
233        let due = state.owed.abandoned(job, call);
234        drop(state);
235        if due {
236            self.jobs.wake();
237        }
238    }
239}
240
241struct Outcome {
242    output: ToolOutput,
243    elapsed: Duration,
244}
245
246/// What a failed report turn leaves its jobs to.
247#[derive(Debug, PartialEq, Eq)]
248pub enum ReportFailure {
249    /// Another report turn about them is due after this long.
250    Retry(Duration),
251    /// No more report turns: the server reports them directly. `attempts`
252    /// counts the report turns that failed.
253    GiveUp {
254        attempts: u32,
255        reports: Vec<JobReport>,
256    },
257    /// The model saw each of them through a call during the turn.
258    Settled,
259}
260
261impl Drop for BackgroundJobs {
262    fn drop(&mut self) {
263        self.cancellation.cancel();
264    }
265}
266
267impl BackgroundJobs {
268    /// At most `limit` jobs run at once. `finished` is woken as jobs finish.
269    pub fn new(limit: usize, finished: Option<mpsc::UnboundedSender<()>>) -> Self {
270        Self {
271            limit,
272            state: Mutex::default(),
273            cancellation: CancellationToken::new(),
274            finished,
275            approvals: None,
276            lanes: Arc::default(),
277        }
278    }
279
280    /// Decide running jobs' nested approval requests with `gate`, which must
281    /// never grant more than the session's foreground would.
282    #[must_use]
283    pub fn with_approvals(mut self, gate: Arc<dyn ApprovalGate>) -> Self {
284        self.approvals = Some(gate);
285        self
286    }
287
288    pub(crate) fn limit(&self) -> usize {
289        self.limit
290    }
291
292    fn state(&self) -> std::sync::MutexGuard<'_, JobsState> {
293        lock(&self.state)
294    }
295
296    /// Start `tool` with `arguments` in the background for the call in
297    /// `context` (whose workspace and call ID it takes; the job has its own
298    /// cancellation), on `agent`, and return its job's
299    /// `{"job","agent","status":"running","background":true}` description.
300    /// A reviewed job's `review` records its start in its journal first, and
301    /// the call fails if it cannot. That write happens outside the job
302    /// table's lock, so a slow disk never holds up the session's other job
303    /// calls: the job's number is taken first, and a failed write leaves it
304    /// unused.
305    #[allow(
306        clippy::too_many_arguments,
307        reason = "a job's call, its context, its turn, and its review, each its own"
308    )]
309    pub(in crate::delegate) fn start(
310        self: &Arc<Self>,
311        tool: Arc<dyn Tool>,
312        name: &str,
313        agent: &str,
314        arguments: Value,
315        context: &ToolContext,
316        turn: Turn,
317        review: Option<&Arc<ReviewedTool>>,
318    ) -> Result<Value, ToolError> {
319        let task = task_line(
320            arguments
321                .get("prompt")
322                .and_then(Value::as_str)
323                .unwrap_or_default(),
324        );
325        let queued = matches!(turn, Turn::Queued(_));
326        let reserved = match review {
327            Some(review) => {
328                let id = {
329                    let mut state = self.state();
330                    self.room(&state, queued)?;
331                    state.next += 1;
332                    format!("job-{}", state.next)
333                };
334                review.begin(&id)?;
335                Some(id)
336            }
337            None => None,
338        };
339        let (id, progress, done_tx, cancellation) = {
340            let mut state = self.state();
341            // Checked again: another call may have started a job meanwhile.
342            self.room(&state, queued)?;
343            let id = reserved.unwrap_or_else(|| {
344                state.next += 1;
345                format!("job-{}", state.next)
346            });
347            let progress = ProgressSink::buffered();
348            let (done_tx, done) = watch::channel(false);
349            let cancel = self.cancellation.child_token();
350            state.record(
351                &context.call_id,
352                JobChange {
353                    job: id.clone(),
354                    tool: name.to_owned(),
355                    agent: agent.to_owned(),
356                    status: JobStatus::Running,
357                    task: task.clone(),
358                    journal: review.map(|review| review.journal_id()),
359                    outcome: None,
360                },
361            );
362            state.jobs.push(Job {
363                id: id.clone(),
364                tool: name.to_owned(),
365                agent: agent.to_owned(),
366                task,
367                started: Instant::now(),
368                progress: progress.clone(),
369                last_progress: None,
370                cancel: cancel.clone(),
371                cancelled: false,
372                queued,
373                outcome: None,
374                reported: false,
375                reporting: false,
376                report_attempts: 0,
377                report_due: None,
378                done,
379                review: review.map(|review| Arc::clone(review.record())),
380                cancelling: false,
381            });
382            (id, progress, done_tx, cancel)
383        };
384        let jobs = Arc::downgrade(self);
385        let job = id.clone();
386        let approvals = self.approvals.clone();
387        let workspace = context.workspace.clone();
388        let reviewed = review.map(Arc::clone);
389        tokio::spawn(async move {
390            let started = Instant::now();
391            let mut context = ToolContext::new(workspace, cancellation);
392            context.background_job = true;
393            // No turn is left to ask a person, so nested approval requests
394            // get the session's unattended answer, or are denied.
395            if let Some(gate) = &approvals {
396                context.approvals = ToolApprovals::new(Arc::clone(gate), job.clone());
397            }
398            context.progress = progress;
399            let place = turn.place();
400            let output = match &place {
401                Some(place) if queued => {
402                    context.progress.report(&format!(
403                        "queued: waits for conversation {}'s running turn",
404                        place.handle()
405                    ));
406                    match place.wait_first(&context.cancellation).await {
407                        Ok(()) => {
408                            context.progress.take();
409                            context.progress.report(&format!(
410                                "started its turn in conversation {}",
411                                place.handle()
412                            ));
413                            tool.execute(arguments, context).await
414                        }
415                        // A reviewed job ends its review even when its loop
416                        // never ran, so its result says how it ended.
417                        Err(error) => match &reviewed {
418                            Some(review) => Ok(review.unstarted(&error)),
419                            None => Err(error),
420                        },
421                    }
422                }
423                _ => tool.execute(arguments, context).await,
424            }
425            .unwrap_or_else(ToolOutput::from);
426            // Free the conversation before the result is visible, so a call
427            // continuing it after `agent_wait` finds it idle.
428            drop(place);
429            finish(&jobs, &job, output, started.elapsed());
430            let _ = done_tx.send(true);
431        });
432        let mut started = json!({
433            "job": id,
434            "agent": agent,
435            "status": "running",
436            "background": true,
437            "note": "The agent is working in the background. SCV reports the result in a new \
438                     turn when it finishes. agent_status shows its progress and agent_cancel \
439                     stops it."
440        });
441        if queued {
442            started["queued"] = true.into();
443            started["note"] = "The conversation is busy, so this prompt is queued: it runs in \
444                the background after the turns ahead of it, and SCV reports the result in a new \
445                turn when it finishes. agent_status shows where it stands and agent_cancel \
446                withdraws it."
447                .into();
448        }
449        Ok(started)
450    }
451
452    /// Refuse a job beyond `agent.max_background`; a queued prompt counts
453    /// toward `agent.max_queued_turns` instead.
454    fn room(&self, state: &JobsState, queued: bool) -> Result<(), ToolError> {
455        let running = state
456            .jobs
457            .iter()
458            .filter(|job| job.outcome.is_none() && !job.queued)
459            .count();
460        if !queued && running >= self.limit {
461            return Err(ToolError::limit(format!(
462                "{running} background jobs are already running, the limit \
463                 (agent.max_background). Start this one after a job finishes, or \
464                 stop one with agent_cancel if the user no longer needs it."
465            )));
466        }
467        Ok(())
468    }
469
470    /// Stop `job` for the call `call_id` and wait briefly for it to settle,
471    /// then describe it. A stopped job needs no report turn: the model asked
472    /// for the stop.
473    ///
474    /// A reviewed job's decision is fixed at once, but its journal goes on
475    /// until it really stops, so the session is owed its final outcome
476    /// ([`owed`]): the call's change says it if the job stops before the
477    /// change is sent, and otherwise says the journal is still open, and a
478    /// `background.updated` follows ([`Self::take_updates`]). The wait
479    /// reads only the job's live state, never its journal.
480    async fn cancel(&self, job: &str, call_id: &str) -> Result<Value, ToolError> {
481        let (mut done, owed) = {
482            let mut state = self.state();
483            let index = state
484                .jobs
485                .iter()
486                .position(|candidate| candidate.id == job)
487                .ok_or_else(|| unknown_job(job))?;
488            let entry = &mut state.jobs[index];
489            if entry.outcome.is_some() {
490                let (mut value, change) = entry.describe();
491                value["note"] = "The job had already finished.".into();
492                if let Some(change) = change {
493                    state.record(call_id, change);
494                }
495                return Ok(value);
496            }
497            entry.cancelled = true;
498            entry.cancel.cancel();
499            entry.cancelling = true;
500            let done = entry.done.clone();
501            let mut owed = false;
502            if !entry.reported {
503                entry.reported = true;
504                let change = entry.change(JobStatus::Cancelled);
505                owed = entry.review.is_some();
506                state.record(call_id, change);
507                if owed {
508                    state.owed.cancelling(job, call_id);
509                }
510            }
511            (done, owed)
512        };
513        let waiting = Waiting {
514            jobs: self,
515            job: job.to_owned(),
516            call: call_id.to_owned(),
517            owed,
518            settled: false,
519        };
520        let _ = tokio::time::timeout(CANCEL_SETTLE, done.wait_for(|finished| *finished)).await;
521        waiting.settle();
522        self.describe(Some(job), call_id)
523    }
524
525    /// The final outcomes of cancelled reviewed jobs the session is owed
526    /// and has not been sent, for the server's `background.updated`. Each
527    /// is being sent until [`Self::updated`] says how that went.
528    pub fn take_updates(&self) -> Vec<JobOutcome> {
529        self.state().owed.take_updates()
530    }
531
532    /// The update naming `jobs` reached the client, when `delivered`, which
533    /// ends what was owed; or it could not be sent, which leaves them due.
534    pub fn updated(&self, jobs: &[String], delivered: bool) {
535        self.state().owed.updated(jobs, delivered);
536    }
537
538    /// The `tool.completed` of the call `call_id`, carrying the changes
539    /// [`Self::take_changes`] gave, reached the client, when `delivered`, or
540    /// could not be sent, as when its turn was cancelled first. A change
541    /// that did not deliver a reviewed job's final outcome leaves it to a
542    /// `background.updated`.
543    pub fn published(&self, call_id: &str, delivered: bool) {
544        let due = self.state().owed.sent(call_id, delivered);
545        if due {
546            self.wake();
547        }
548    }
549
550    /// A turn ended: no change of its calls can be sent any more, so what
551    /// such a change owed goes to a `background.updated`.
552    pub fn turn_ended(&self) {
553        let due = self.state().owed.turn_ended();
554        if due {
555            self.wake();
556        }
557    }
558
559    /// For tests of how a client of this store delivers reviewed jobs'
560    /// outcomes, such as the server's, with the `test-support` feature: owe
561    /// the session the final outcome of `change.job`, through `change`,
562    /// recorded for the call `call_id` once `agent_cancel` waited, which says
563    /// the journal is still open; `last` is the final outcome if the job has
564    /// stopped.
565    #[cfg(any(test, feature = "test-support"))]
566    pub fn owe_for_tests(&self, call_id: &str, change: JobChange, last: Option<JobOutcome>) {
567        let mut state = self.state();
568        state.owed.owe(call_id, &change, last);
569        state.record(call_id, change);
570    }
571
572    /// For the same tests: the job of `outcome` stops with it.
573    #[cfg(any(test, feature = "test-support"))]
574    pub fn stop_for_tests(&self, outcome: &JobOutcome) {
575        let due = self.state().owed.stopped(&outcome.job, outcome);
576        if due {
577            self.wake();
578        }
579    }
580
581    /// For tests of what waits for a session's jobs, with the
582    /// `test-support` feature: start a job on `agent` that runs until the
583    /// returned sender is used or dropped.
584    #[cfg(any(test, feature = "test-support"))]
585    pub fn start_for_tests(self: &Arc<Self>, agent: &str) -> tokio::sync::oneshot::Sender<()> {
586        struct Waits(Mutex<Option<tokio::sync::oneshot::Receiver<()>>>);
587        #[async_trait]
588        impl Tool for Waits {
589            fn spec(&self) -> ToolSpec {
590                ToolSpec {
591                    name: AGENT_TOOL.into(),
592                    description: String::new(),
593                    parameters: json!({"type":"object"}),
594                }
595            }
596            fn risk(&self, _arguments: &Value) -> Result<ToolRisk, ToolError> {
597                Ok(ToolRisk::Delegate)
598            }
599            fn approval_summary(&self, _arguments: &Value) -> Result<String, ToolError> {
600                Ok(String::new())
601            }
602            async fn execute(
603                &self,
604                _arguments: Value,
605                _context: ToolContext,
606            ) -> Result<ToolOutput, ToolError> {
607                let release = lock(&self.0).take();
608                if let Some(release) = release {
609                    let _ = release.await;
610                }
611                Ok(ToolOutput::success(
612                    r#"{"status":"completed","reply":"done"}"#,
613                ))
614            }
615        }
616        let (release, waiting) = tokio::sync::oneshot::channel();
617        let context = ToolContext::new(std::env::temp_dir(), CancellationToken::new());
618        self.start(
619            Arc::new(Waits(Mutex::new(Some(waiting)))),
620            AGENT_TOOL,
621            agent,
622            json!({"prompt": "wait"}),
623            &context,
624            Turn::New,
625            None,
626        )
627        .expect("a test job starts");
628        release
629    }
630
631    /// Wake the session, as when a job finishes, for what is now due.
632    fn wake(&self) {
633        if let Some(finished) = &self.finished {
634            let _ = finished.send(());
635        }
636    }
637
638    /// Wait up to `limit` for `job` and describe it for the call `call_id`;
639    /// a finished job is then marked seen.
640    async fn wait(
641        &self,
642        job: &str,
643        limit: Duration,
644        cancellation: &CancellationToken,
645        call_id: &str,
646    ) -> Result<Value, ToolError> {
647        let mut done = self
648            .state()
649            .jobs
650            .iter()
651            .find(|candidate| candidate.id == job)
652            .map(|candidate| candidate.done.clone())
653            .ok_or_else(|| unknown_job(job))?;
654        tokio::select! {
655            () = cancellation.cancelled() => return Err(ToolError::cancelled("wait cancelled")),
656            _ = tokio::time::timeout(limit, done.wait_for(|finished| *finished)) => {}
657        }
658        self.describe(Some(job), call_id)
659    }
660
661    /// Describe one job, or every job this session remembers, for the call
662    /// `call_id`; finished jobs described are marked seen.
663    fn describe(&self, job: Option<&str>, call_id: &str) -> Result<Value, ToolError> {
664        let mut state = self.state();
665        let mut changes = Vec::new();
666        let value = if let Some(job) = job {
667            let entry = state
668                .jobs
669                .iter_mut()
670                .find(|candidate| candidate.id == job)
671                .ok_or_else(|| unknown_job(job))?;
672            let (value, change) = entry.describe();
673            changes.extend(change);
674            value
675        } else {
676            let jobs: Vec<Value> = state
677                .jobs
678                .iter_mut()
679                .map(|entry| {
680                    let (value, change) = entry.describe();
681                    changes.extend(change);
682                    value
683                })
684                .collect();
685            json!({ "jobs": jobs })
686        };
687        for change in changes {
688            state.record(call_id, change);
689        }
690        Ok(value)
691    }
692
693    /// The jobs the call `call_id` started or showed the model the result of,
694    /// for its `tool.completed`, which then reports how its sending went
695    /// ([`Self::published`]). A cancelled reviewed job's change says its
696    /// final outcome if the job has stopped by now, superseding the
697    /// snapshot taken when the cancel's wait ended.
698    pub fn take_changes(&self, call_id: &str) -> Vec<JobChange> {
699        let mut state = self.state();
700        let mut taken = Vec::new();
701        state.changes.retain(|(call, change)| {
702            if call == call_id {
703                taken.push(change.clone());
704                false
705            } else {
706                true
707            }
708        });
709        for change in &mut taken {
710            state.owed.take(call_id, change);
711        }
712        taken
713    }
714
715    /// Finished jobs the model has not seen yet whose report is due, at most
716    /// a few, for a report turn. Each stays unreported, and is not taken
717    /// again, until the turn ends: [`report_settled`](Self::report_settled)
718    /// or [`report_failed`](Self::report_failed).
719    pub fn take_unreported(&self) -> Vec<JobReport> {
720        let now = tokio::time::Instant::now();
721        let mut state = self.state();
722        state
723            .jobs
724            .iter_mut()
725            .filter(|job| job.awaits_report() && job.report_due.is_none_or(|due| due <= now))
726            .take(REPORT_MAX_JOBS)
727            .filter_map(|job| {
728                job.reporting = true;
729                job.report()
730            })
731            .collect()
732    }
733
734    /// The report turn about `jobs` ended without failing: it completed, so
735    /// the model has seen them, or the user cancelled it. Either settles them.
736    pub fn report_settled(&self, jobs: &[String]) {
737        let mut state = self.state();
738        for job in state.jobs.iter_mut().filter(|job| jobs.contains(&job.id)) {
739            job.reporting = false;
740            job.reported = true;
741            job.report_due = None;
742        }
743    }
744
745    /// The report turn about `jobs` failed, so the model has not seen them
746    /// (a failed turn leaves no history). With `retry`, another turn is due
747    /// after a delay, up to three turns in all; otherwise, or once they are
748    /// used up, the jobs are settled and returned for the server to report
749    /// directly. A turn that reported several jobs decides for all of them,
750    /// by the one that failed most.
751    pub fn report_failed(&self, jobs: &[String], retry: bool) -> ReportFailure {
752        let mut state = self.state();
753        let mut failed: Vec<&mut Job> = state
754            .jobs
755            .iter_mut()
756            .filter(|job| job.reporting && jobs.contains(&job.id))
757            .collect();
758        for job in &mut failed {
759            job.reporting = false;
760            job.report_attempts += 1;
761        }
762        // One the turn's model looked up through a call is settled already.
763        failed.retain(|job| !job.reported);
764        let Some(attempts) = failed.iter().map(|job| job.report_attempts).max() else {
765            return ReportFailure::Settled;
766        };
767        let delay = usize::try_from(attempts - 1)
768            .ok()
769            .and_then(|earlier| REPORT_RETRY_DELAYS.get(earlier));
770        if retry && let Some(&delay) = delay {
771            let due = tokio::time::Instant::now() + delay;
772            for job in failed {
773                job.report_due = Some(due);
774            }
775            return ReportFailure::Retry(delay);
776        }
777        let reports = failed
778            .into_iter()
779            .filter_map(|job| {
780                job.reported = true;
781                job.report_due = None;
782                job.report()
783            })
784            .collect();
785        ReportFailure::GiveUp { attempts, reports }
786    }
787
788    /// Make every report waiting out a failed turn due now, as once a turn
789    /// succeeds and the model is evidently reachable again. Whether any was
790    /// waiting.
791    pub fn retry_now(&self) -> bool {
792        let mut state = self.state();
793        let mut waiting = false;
794        for job in state.jobs.iter_mut().filter(|job| job.awaits_report()) {
795            waiting |= job.report_due.take().is_some();
796        }
797        waiting
798    }
799
800    /// When the earliest report waiting out a failed turn is due; it may
801    /// already have passed.
802    pub fn next_retry(&self) -> Option<tokio::time::Instant> {
803        self.state()
804            .jobs
805            .iter()
806            .filter(|job| job.awaits_report())
807            .filter_map(|job| job.report_due)
808            .min()
809    }
810
811    /// Jobs still running or whose result is not reported yet, including
812    /// those a report turn is reporting now, and cancelled reviewed jobs
813    /// whose final outcome the session is still owed, until it was sent;
814    /// each job once.
815    pub fn pending(&self) -> usize {
816        let state = self.state();
817        let mut pending: HashSet<&str> = state
818            .jobs
819            .iter()
820            .filter(|job| job.outcome.is_none() || !job.reported)
821            .map(|job| job.id.as_str())
822            .collect();
823        pending.extend(state.owed.jobs());
824        pending.len()
825    }
826
827    /// Whether any job is still running.
828    pub fn running(&self) -> usize {
829        self.state()
830            .jobs
831            .iter()
832            .filter(|job| job.outcome.is_none())
833            .count()
834    }
835}
836
837/// How a job's call takes its conversation's turn.
838pub(in crate::delegate) enum Turn {
839    /// A new conversation, which nothing else can reach yet.
840    New,
841    /// The conversation's next turn, which nothing was ahead of.
842    Next(Place),
843    /// A prompt queued behind the turns ahead of it.
844    Queued(Place),
845}
846
847impl Turn {
848    fn place(self) -> Option<Place> {
849        match self {
850            Self::New => None,
851            Self::Next(place) | Self::Queued(place) => Some(place),
852        }
853    }
854}
855
856fn finish(jobs: &Weak<BackgroundJobs>, id: &str, output: ToolOutput, elapsed: Duration) {
857    // The session ended first: its jobs were cancelled with it.
858    let Some(jobs) = jobs.upgrade() else {
859        return;
860    };
861    {
862        let mut state = jobs.state();
863        let mut last = None;
864        if let Some(job) = state.jobs.iter_mut().find(|job| job.id == id) {
865            job.last_progress = job.progress.take().or(job.last_progress.take());
866            job.outcome = Some(Outcome { output, elapsed });
867            // Stopped on request: the model already knows.
868            job.reported |= job.cancelled;
869            last = job.outcome();
870        }
871        // What a cancel owed of it is now final.
872        if let Some(last) = last {
873            state.owed.stopped(id, &last);
874        }
875        // Keep every running job and the newest finished ones.
876        let finished = state
877            .jobs
878            .iter()
879            .filter(|job| job.outcome.is_some())
880            .count();
881        let mut excess = finished.saturating_sub(MAX_FINISHED);
882        state.jobs.retain(|job| {
883            if excess > 0 && job.outcome.is_some() && job.reported && !job.cancelling {
884                excess -= 1;
885                false
886            } else {
887                true
888            }
889        });
890    }
891    if let Some(finished) = &jobs.finished {
892        let _ = finished.send(());
893    }
894}
895
896impl Job {
897    /// Finished, unseen by the model, and not in a report turn.
898    fn awaits_report(&self) -> bool {
899        self.outcome.is_some() && !self.reported && !self.reporting
900    }
901
902    /// The finished job's result, for a report.
903    fn report(&self) -> Option<JobReport> {
904        let outcome = self.outcome.as_ref()?;
905        let reply = AgentReply::read(&result_value(&outcome.output)).unwrap_or_default();
906        Some(JobReport {
907            job: self.id.clone(),
908            agent: self.agent.clone(),
909            task: self.task.clone(),
910            status: job_status(&outcome.output, &reply),
911            session: reply.session,
912            reply: bounded(
913                reply
914                    .reply
915                    .as_deref()
916                    .unwrap_or(outcome.output.content.as_str()),
917                REPORT_REPLY_CHARS,
918            ),
919            outcome: self.outcome(),
920        })
921    }
922
923    /// A reviewed job's outcome as it stands. A cancelled job's is what it
924    /// was when the cancel arrived.
925    fn outcome(&self) -> Option<JobOutcome> {
926        self.review.as_ref().map(|record| {
927            let (review, landing) = record
928                .live
929                .snapshot(self.cancelled || self.cancel.is_cancelled());
930            JobOutcome {
931                job: self.id.clone(),
932                review,
933                landing,
934            }
935        })
936    }
937
938    /// This job with `status`, as its clients learn of it: a reviewed job's
939    /// settling change carries its outcome.
940    fn change(&self, status: JobStatus) -> JobChange {
941        JobChange {
942            job: self.id.clone(),
943            tool: self.tool.clone(),
944            agent: self.agent.clone(),
945            status,
946            task: self.task.clone(),
947            journal: None,
948            outcome: if status == JobStatus::Running {
949                None
950            } else {
951                self.outcome()
952            },
953        }
954    }
955
956    /// The job as the model reads it, and its change when this is the first
957    /// time the model sees its result.
958    fn describe(&mut self) -> (Value, Option<JobChange>) {
959        if let Some(line) = self.progress.take() {
960            self.last_progress = Some(line);
961        }
962        let mut change = None;
963        let mut value = Map::new();
964        value.insert("job".into(), self.id.clone().into());
965        value.insert("agent".into(), self.agent.clone().into());
966        match &self.outcome {
967            None => {
968                value.insert("status".into(), "running".into());
969                value.insert(
970                    "elapsed_seconds".into(),
971                    self.started.elapsed().as_secs().into(),
972                );
973                if let Some(progress) = &self.last_progress {
974                    value.insert("progress".into(), progress.clone().into());
975                }
976                if let Some(record) = &self.review {
977                    value.insert("review".into(), record.live.status());
978                }
979            }
980            Some(outcome) => {
981                let result = result_value(&outcome.output);
982                let status = if self.cancelled {
983                    JobStatus::Cancelled
984                } else {
985                    job_status(
986                        &outcome.output,
987                        &AgentReply::read(&result).unwrap_or_default(),
988                    )
989                };
990                value.insert("status".into(), status.as_str().into());
991                value.insert("elapsed_seconds".into(), outcome.elapsed.as_secs().into());
992                value.insert("result".into(), result);
993                if !self.reported {
994                    self.reported = true;
995                    change = Some(self.change(status));
996                }
997            }
998        }
999        (Value::Object(value), change)
1000    }
1001}
1002
1003/// The agent tool's structured result, or its text when it was not JSON.
1004fn result_value(output: &ToolOutput) -> Value {
1005    match serde_json::from_str::<Value>(&output.content) {
1006        Ok(value @ Value::Object(_)) => value,
1007        _ => json!({ "reply": output.content, "is_error": output.is_error() }),
1008    }
1009}
1010
1011/// How a finished job ended: its agent's reported status, else whether its
1012/// call failed.
1013fn job_status(output: &ToolOutput, reply: &AgentReply) -> JobStatus {
1014    reply.status.unwrap_or(if output.is_error() {
1015        JobStatus::Failed
1016    } else {
1017        JobStatus::Completed
1018    })
1019}
1020
1021/// The first non-empty line of `prompt`, at most [`TASK_CHARS`] characters,
1022/// which names a job to people.
1023fn task_line(prompt: &str) -> String {
1024    let line = prompt
1025        .lines()
1026        .map(str::trim)
1027        .find(|line| !line.is_empty())
1028        .unwrap_or_default();
1029    let mut task: String = line
1030        .chars()
1031        .filter(|character| !character.is_control())
1032        .take(TASK_CHARS)
1033        .collect();
1034    if line.chars().count() > TASK_CHARS {
1035        task.push('…');
1036    }
1037    task
1038}
1039
1040fn unknown_job(job: &str) -> ToolError {
1041    ToolError::invalid_arguments(format!(
1042        "unknown background job {:?}; agent_status lists this session's jobs",
1043        bounded(job, 64)
1044    ))
1045}
1046
1047/// What the model is told when a report includes reviewed jobs, whose
1048/// outcome SCV states itself.
1049const REVIEWED_NOTE: &str = " SCV shows the user its own Review and Landing lines for \
1050     reviewed jobs. If you mention the outcome, repeat them as given; never call unapproved \
1051     work approved or done, and never call landed work approved. If a reviewer declined, tell \
1052     the user what it said. If a landing is NOT confirmed, or the reviewer could NOT confirm \
1053     it, say so plainly, and don't fix, revert, or land anything on your own.";
1054
1055fn reviewed_note(reports: &[JobReport]) -> &'static str {
1056    if reports.iter().any(|report| report.outcome.is_some()) {
1057        REVIEWED_NOTE
1058    } else {
1059        ""
1060    }
1061}
1062
1063/// The server-started turn that reports finished jobs to the model.
1064pub fn report_prompt(reports: &[JobReport]) -> String {
1065    format!(
1066        "[SCV background report] Delegated work you started in the background has \
1067         finished. The user did not send this message: tell them briefly what \
1068         happened and the key result.{}\n\n{}",
1069        reviewed_note(reports),
1070        describe_reports(reports)
1071    )
1072}
1073
1074/// What the model is told of jobs the server reported directly, because
1075/// their report turns failed with `error`: the user has their results.
1076pub fn delivered_note(reports: &[JobReport], error: &str) -> String {
1077    format!(
1078        "[SCV background report, already delivered] Delegated work you started in \
1079         the background has finished. Your report of it failed ({error}), so SCV \
1080         sent the user the results below directly. The user did not send this \
1081         message; repeat the results only if they ask.{}\n\n{}",
1082        reviewed_note(reports),
1083        describe_reports(reports)
1084    )
1085}
1086
1087/// The `agent` tool, able to run a call in the background as well, and to
1088/// run one as a reviewed job.
1089pub(crate) struct BackgroundCapable {
1090    pub(crate) inner: Arc<AgentTool>,
1091    pub(crate) jobs: Arc<BackgroundJobs>,
1092    /// The session's conversations, which a reviewed job pins and releases.
1093    pub(crate) conversations: Arc<ConversationStore>,
1094    /// Where reviewed jobs keep their journals; without it, `review` is not
1095    /// offered.
1096    pub(crate) reviews: Option<PathBuf>,
1097    /// The SCV session, which a review's journal names.
1098    pub(crate) session: Option<String>,
1099}
1100
1101/// The `agent` tool for a queued prompt, whose turn has come by the time it
1102/// runs, so no busy policy applies to it again.
1103struct QueuedTool(Arc<AgentTool>);
1104
1105#[async_trait]
1106impl Tool for QueuedTool {
1107    fn spec(&self) -> ToolSpec {
1108        self.0.spec()
1109    }
1110    fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
1111        self.0.risk(arguments)
1112    }
1113    fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
1114        self.0.approval_summary(arguments)
1115    }
1116    async fn execute(
1117        &self,
1118        arguments: Value,
1119        context: ToolContext,
1120    ) -> Result<ToolOutput, ToolError> {
1121        self.0.execute_when_idle(arguments, context).await
1122    }
1123}
1124
1125/// An agent call's arguments with `background` and `review` split off.
1126struct Split {
1127    arguments: Value,
1128    /// `background` as the call gave it.
1129    background: Option<bool>,
1130    /// The `review` object, when the call gave one.
1131    review: Option<Value>,
1132}
1133
1134impl Split {
1135    fn new(arguments: &Value) -> Self {
1136        let mut arguments = arguments.clone();
1137        let object = arguments.as_object_mut();
1138        let (background, review) = match object {
1139            Some(object) => (
1140                object
1141                    .remove("background")
1142                    .map(|value| value.as_bool() == Some(true)),
1143                object.remove("review").filter(|review| !review.is_null()),
1144            ),
1145            None => (None, None),
1146        };
1147        Self {
1148            arguments,
1149            background,
1150            review,
1151        }
1152    }
1153
1154    fn background(&self) -> bool {
1155        self.background == Some(true)
1156    }
1157}
1158
1159#[async_trait]
1160impl Tool for BackgroundCapable {
1161    fn spec(&self) -> ToolSpec {
1162        let mut spec = self.inner.spec();
1163        let reviewed = self.offers_review();
1164        if let Some(properties) = spec
1165            .parameters
1166            .get_mut("properties")
1167            .and_then(Value::as_object_mut)
1168        {
1169            properties.insert(
1170                "background".into(),
1171                json!({
1172                    "type":"boolean",
1173                    "description":"Run in the background: the call returns a job handle at once, \
1174                        the user can keep talking to you while the agent works, and SCV reports \
1175                        the result in a new turn when it finishes. Use it for any substantial \
1176                        task; run in the foreground only for quick work whose result you need \
1177                        within this turn."
1178                }),
1179            );
1180            if reviewed {
1181                properties.insert("review".into(), review::schema(&self.inner));
1182            }
1183        }
1184        spec.description.push_str(&format!(
1185            " Set background to true for anything beyond a quick task: the call returns a job \
1186             handle at once (at most {} running per session), SCV reports the result when the \
1187             job finishes, agent_status shows progress, and agent_cancel stops it. A background \
1188             job's own approval requests get only the answer this session would give without \
1189             asking a person.",
1190            self.jobs.limit()
1191        ));
1192        if reviewed {
1193            spec.description.push_str(
1194                " A call with review runs as a reviewed background job: an independent \
1195                 reviewer checks the work in up to review.rounds rounds.",
1196            );
1197        }
1198        spec
1199    }
1200
1201    fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
1202        let split = Split::new(arguments);
1203        if let Some(review) = &split.review {
1204            self.plan(&split, review)?;
1205        }
1206        self.inner.risk_for(
1207            &split.arguments,
1208            split.background() || split.review.is_some(),
1209        )
1210    }
1211
1212    fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
1213        let split = Split::new(arguments);
1214        let background = split.background() || split.review.is_some();
1215        let mut summary = self
1216            .inner
1217            .approval_summary_for(&split.arguments, background)?;
1218        if let Some(review) = &split.review {
1219            let plan = self.plan(&split, review)?;
1220            summary.push_str(
1221                " Runs in the background: the call returns at once and the result is \
1222                 reported when the review ends.",
1223            );
1224            summary.push_str(&review::approval_summary(&plan, &self.inner)?);
1225        } else if split.background() {
1226            summary.push_str(
1227                " Runs in the background: the call returns at once and the result is \
1228                 reported when the agent finishes.",
1229            );
1230        }
1231        Ok(summary)
1232    }
1233
1234    async fn execute(
1235        &self,
1236        arguments: Value,
1237        context: ToolContext,
1238    ) -> Result<ToolOutput, ToolError> {
1239        let split = Split::new(&arguments);
1240        if let Some(review) = &split.review {
1241            return self.start_reviewed(&split, review, &context);
1242        }
1243        let background = split.background();
1244        let arguments = split.arguments;
1245        let (agent, routed) = self.inner.route(&arguments)?;
1246        // Validate before returning a job handle, so a bad call fails now,
1247        // and as the call was made: a foreground call queued behind a busy
1248        // conversation runs as a job but may not set `no_timeout`.
1249        self.inner.risk_for(&arguments, background)?;
1250        let Some(handle) = parse_args::<AgentArgs>(&routed)?
1251            .session
1252            .filter(|session| conversation::is_handle(session))
1253        else {
1254            return if background {
1255                self.start(agent, arguments, &context, Turn::New)
1256            } else {
1257                self.inner.execute(arguments, context).await
1258            };
1259        };
1260        // Taken before anything runs or waits, so calls keep their order.
1261        let place = self.jobs.lanes.join(&handle);
1262        if place.ahead == 0 && !agent.backend.busy(&routed)? {
1263            return if background {
1264                self.start(agent, arguments, &context, Turn::Next(place))
1265            } else {
1266                // Held until the turn ends, so a later call finds it busy.
1267                let output = self.inner.execute(arguments, context).await;
1268                drop(place);
1269                output
1270            };
1271        }
1272        let behavior = match agent.on_busy(&routed)? {
1273            BusyBehavior::Steer => {
1274                if let Some(output) = agent.backend.steer(routed, context.clone()).await? {
1275                    return Ok(output);
1276                }
1277                agent.busy.fallback()
1278            }
1279            behavior => behavior,
1280        };
1281        match behavior {
1282            // A background call cannot hold its caller, so it queues instead.
1283            BusyBehavior::Wait if !background => {
1284                place.wait_first(&context.cancellation).await?;
1285                let output = self.inner.execute_when_idle(arguments, context).await;
1286                drop(place);
1287                output
1288            }
1289            BusyBehavior::Fail => Err(busy(&handle, &place)),
1290            _ => {
1291                queue_room(agent, &handle, &place)?;
1292                self.start(agent, arguments, &context, Turn::Queued(place))
1293            }
1294        }
1295    }
1296}
1297
1298/// The `session busy` error for a call to conversation `handle` that found
1299/// a turn running, or calls waiting, at `place`.
1300fn busy(handle: &str, place: &Place) -> ToolError {
1301    // Calls ahead of this one other than the running turn.
1302    let waiting = place.ahead.saturating_sub(1);
1303    ToolError::failed(if waiting == 0 {
1304        format!("session busy: conversation {handle} is still running a turn")
1305    } else {
1306        format!(
1307            "session busy: conversation {handle} is still running a turn, and {waiting} more \
1308             prompts wait for it"
1309        )
1310    })
1311}
1312
1313/// Refuse a prompt queued at `place` beyond `agent.max_queued_turns`.
1314fn queue_room(agent: &Offered, handle: &str, place: &Place) -> Result<(), ToolError> {
1315    let waiting = place.ahead.saturating_sub(1);
1316    if waiting >= agent.busy.max_queued_turns {
1317        return Err(ToolError::limit(format!(
1318            "session busy: conversation {handle} is running a turn and {waiting} prompts \
1319             already wait for it, of the {} agent.max_queued_turns allows. Send this one \
1320             once they have run, or stop one with agent_cancel.",
1321            agent.busy.max_queued_turns
1322        )));
1323    }
1324    Ok(())
1325}
1326
1327impl BackgroundCapable {
1328    /// `inner` with `jobs` and a store of its own, without reviews: for
1329    /// tests of plain background calls.
1330    #[cfg(test)]
1331    pub(crate) fn plain(inner: Arc<AgentTool>, jobs: Arc<BackgroundJobs>) -> Self {
1332        Self {
1333            inner,
1334            jobs,
1335            conversations: Arc::new(ConversationStore::new(
1336                conversation::ConversationLimits {
1337                    max: 8,
1338                    idle: Duration::from_secs(86400),
1339                },
1340                None,
1341            )),
1342            reviews: None,
1343            session: None,
1344        }
1345    }
1346
1347    /// Whether calls may set `review`: background jobs exist (this tool),
1348    /// the session keeps journals, and an offered agent can continue a
1349    /// conversation.
1350    fn offers_review(&self) -> bool {
1351        self.reviews.is_some() && !self.inner.continuing().is_empty()
1352    }
1353
1354    /// Check a reviewed call, before approval and again before it starts.
1355    fn plan(&self, split: &Split, review: &Value) -> Result<review::Plan, ToolError> {
1356        if !self.offers_review() {
1357            return Err(ToolError::invalid_arguments(
1358                "review is not available in this session",
1359            ));
1360        }
1361        if split.background == Some(false) {
1362            return Err(ToolError::invalid_arguments(
1363                "a reviewed call runs as a background job; omit background or set it to true",
1364            ));
1365        }
1366        let (builder, routed) = self.inner.route(&split.arguments)?;
1367        builder.backend.risk(&routed)?;
1368        review::plan(
1369            &self.inner,
1370            builder,
1371            &routed,
1372            review,
1373            self.conversations.limits().max,
1374        )
1375    }
1376
1377    /// Start a reviewed call as a background job. A busy builder
1378    /// conversation queues it, or fails it with `on_busy: fail`; it never
1379    /// waits, since it is a background job, and never steers, since its
1380    /// prompt opens a review of its own.
1381    fn start_reviewed(
1382        &self,
1383        split: &Split,
1384        review: &Value,
1385        context: &ToolContext,
1386    ) -> Result<ToolOutput, ToolError> {
1387        let plan = self.plan(split, review)?;
1388        let arguments = split.arguments.clone();
1389        self.inner.risk_for(&arguments, true)?;
1390        let (agent, routed) = self.inner.route(&arguments)?;
1391        let turn = match parse_args::<AgentArgs>(&routed)?
1392            .session
1393            .filter(|session| conversation::is_handle(session))
1394        {
1395            None => Turn::New,
1396            Some(handle) => {
1397                let place = self.jobs.lanes.join(&handle);
1398                if place.ahead == 0 && !agent.backend.busy(&routed)? {
1399                    Turn::Next(place)
1400                } else if agent.on_busy(&routed)? == BusyBehavior::Fail {
1401                    return Err(busy(&handle, &place));
1402                } else {
1403                    queue_room(agent, &handle, &place)?;
1404                    Turn::Queued(place)
1405                }
1406            }
1407        };
1408        let dir = self.reviews.as_deref().ok_or_else(|| {
1409            ToolError::invalid_arguments("review is not available in this session")
1410        })?;
1411        let reviewers: Vec<String> = plan
1412            .reviewer_names()
1413            .into_iter()
1414            .map(str::to_owned)
1415            .collect();
1416        let (rounds, land) = (plan.rounds, plan.land);
1417        let tool = Arc::new(ReviewedTool::new(
1418            Arc::clone(&self.inner),
1419            Arc::clone(&self.jobs.lanes),
1420            Arc::clone(&self.conversations),
1421            plan,
1422            arguments.clone(),
1423            self.session.clone(),
1424            dir,
1425        )?);
1426        let queued = matches!(turn, Turn::Queued(_));
1427        let mut started = self.jobs.start(
1428            Arc::clone(&tool) as Arc<dyn Tool>,
1429            AGENT_TOOL,
1430            &agent.name,
1431            arguments,
1432            context,
1433            turn,
1434            Some(&tool),
1435        )?;
1436        let first = reviewers.first().cloned().unwrap_or_default();
1437        started["review"] = json!({
1438            "reviewers": reviewers,
1439            "rounds": rounds,
1440            "land": review::land_name(land),
1441            "journal": tool.journal_id(),
1442        });
1443        started["note"] = format!(
1444            "{}The agent works on it, then a fresh {first} reviews it (or the next reviewer \
1445             available), for up to {rounds} rounds. SCV reports the result, with its own Review \
1446             and Landing lines, in a new turn when the review ends. agent_status shows the \
1447             round and phase; agent_cancel stops the whole review.",
1448            if queued {
1449                "The conversation is busy, so this reviewed call is queued behind the turns \
1450                 ahead of it. "
1451            } else {
1452                ""
1453            }
1454        )
1455        .into();
1456        Ok(ToolOutput::success(started.to_string()))
1457    }
1458
1459    /// Start the call on `agent` as a background job of this session.
1460    fn start(
1461        &self,
1462        agent: &Offered,
1463        arguments: Value,
1464        context: &ToolContext,
1465        turn: Turn,
1466    ) -> Result<ToolOutput, ToolError> {
1467        let tool: Arc<dyn Tool> = if matches!(turn, Turn::Queued(_)) {
1468            Arc::new(QueuedTool(Arc::clone(&self.inner)))
1469        } else {
1470            Arc::clone(&self.inner) as Arc<dyn Tool>
1471        };
1472        let started = self.jobs.start(
1473            tool,
1474            AGENT_TOOL,
1475            &agent.name,
1476            arguments,
1477            context,
1478            turn,
1479            None,
1480        )?;
1481        Ok(ToolOutput::success(started.to_string()))
1482    }
1483}
1484
1485#[derive(Deserialize)]
1486#[serde(deny_unknown_fields)]
1487struct WaitArgs {
1488    job: String,
1489    timeout_seconds: Option<u64>,
1490}
1491
1492#[derive(Deserialize)]
1493#[serde(deny_unknown_fields)]
1494struct StatusArgs {
1495    job: Option<String>,
1496}
1497
1498/// `agent_wait`: block until a background job finishes, or the timeout.
1499pub(crate) struct WaitTool {
1500    pub(crate) jobs: Arc<BackgroundJobs>,
1501    pub(crate) timeouts: Timeouts,
1502}
1503
1504#[async_trait]
1505impl Tool for WaitTool {
1506    fn spec(&self) -> ToolSpec {
1507        let mut timeout = timeout_schema(self.timeouts);
1508        timeout["description"] = format!(
1509            "Seconds to wait before returning the job still running. Defaults to {}; at most {}.",
1510            self.timeouts.default.min(self.timeouts.max).as_secs(),
1511            self.timeouts.max.as_secs()
1512        )
1513        .into();
1514        ToolSpec {
1515            name: "agent_wait".into(),
1516            description: "Wait for a background agent job (the `job` handle an agent call with \
1517                background: true returned) to finish, and return its result. Returns early with \
1518                status running when the timeout passes. Waiting holds your turn open, so the \
1519                user cannot reach you meanwhile; usually let SCV report the result instead."
1520                .into(),
1521            parameters: json!({
1522                "type":"object",
1523                "properties":{"job":{"type":"string"},"timeout_seconds":timeout},
1524                "required":["job"],
1525                "additionalProperties":false
1526            }),
1527        }
1528    }
1529
1530    fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
1531        let args: WaitArgs = parse_args(arguments)?;
1532        self.timeouts.resolve(args.timeout_seconds)?;
1533        Ok(ToolRisk::ReadOnly)
1534    }
1535
1536    fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
1537        let args: WaitArgs = parse_args(arguments)?;
1538        Ok(format!(
1539            "Wait for background job {}",
1540            bounded(&args.job, 64)
1541        ))
1542    }
1543
1544    async fn execute(
1545        &self,
1546        arguments: Value,
1547        context: ToolContext,
1548    ) -> Result<ToolOutput, ToolError> {
1549        let args: WaitArgs = parse_args(&arguments)?;
1550        let limit = self.timeouts.resolve(args.timeout_seconds)?;
1551        let value = self
1552            .jobs
1553            .wait(&args.job, limit, &context.cancellation, &context.call_id)
1554            .await?;
1555        Ok(ToolOutput::success(value.to_string()))
1556    }
1557}
1558
1559/// `agent_status`: describe one background job or all of them.
1560pub(crate) struct StatusTool {
1561    pub(crate) jobs: Arc<BackgroundJobs>,
1562}
1563
1564#[async_trait]
1565impl Tool for StatusTool {
1566    fn spec(&self) -> ToolSpec {
1567        ToolSpec {
1568            name: "agent_status".into(),
1569            description: "Show this session's background agent jobs: running ones with their \
1570                latest progress, finished ones with their result. Pass job for one job."
1571                .into(),
1572            parameters: json!({
1573                "type":"object",
1574                "properties":{"job":{"type":"string"}},
1575                "additionalProperties":false
1576            }),
1577        }
1578    }
1579
1580    fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
1581        let _: StatusArgs = parse_args(arguments)?;
1582        Ok(ToolRisk::ReadOnly)
1583    }
1584
1585    fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
1586        let args: StatusArgs = parse_args(arguments)?;
1587        Ok(args.job.map_or_else(
1588            || "List background jobs".into(),
1589            |job| format!("Show background job {}", bounded(&job, 64)),
1590        ))
1591    }
1592
1593    async fn execute(
1594        &self,
1595        arguments: Value,
1596        context: ToolContext,
1597    ) -> Result<ToolOutput, ToolError> {
1598        let args: StatusArgs = parse_args(&arguments)?;
1599        let value = self.jobs.describe(args.job.as_deref(), &context.call_id)?;
1600        Ok(ToolOutput::success(value.to_string()))
1601    }
1602}
1603
1604#[derive(Deserialize)]
1605#[serde(deny_unknown_fields)]
1606struct CancelArgs {
1607    job: String,
1608}
1609
1610/// `agent_cancel`: stop one running background job.
1611pub(crate) struct CancelTool {
1612    pub(crate) jobs: Arc<BackgroundJobs>,
1613}
1614
1615#[async_trait]
1616impl Tool for CancelTool {
1617    fn spec(&self) -> ToolSpec {
1618        ToolSpec {
1619            name: "agent_cancel".into(),
1620            description: "Stop a running background agent job (the `job` handle an agent call \
1621                with background: true returned), for example when the user no longer wants \
1622                it. The agent and every process it started are stopped; work it already wrote \
1623                stays. Returns the job with status cancelled, and no report turn follows."
1624                .into(),
1625            parameters: json!({
1626                "type":"object",
1627                "properties":{"job":{"type":"string"}},
1628                "required":["job"],
1629                "additionalProperties":false
1630            }),
1631        }
1632    }
1633
1634    fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
1635        let _: CancelArgs = parse_args(arguments)?;
1636        Ok(ToolRisk::Process)
1637    }
1638
1639    fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
1640        let args: CancelArgs = parse_args(arguments)?;
1641        Ok(format!("Stop background job {}", bounded(&args.job, 64)))
1642    }
1643
1644    async fn execute(
1645        &self,
1646        arguments: Value,
1647        context: ToolContext,
1648    ) -> Result<ToolOutput, ToolError> {
1649        let args: CancelArgs = parse_args(&arguments)?;
1650        let value = self.jobs.cancel(&args.job, &context.call_id).await?;
1651        Ok(ToolOutput::success(value.to_string()))
1652    }
1653}
1654
1655#[cfg(test)]
1656mod tests;