Skip to main content

scv_tools/delegate/
background.rs

1//! Background delegations: an `agent` call with `background: true` returns a
2//! job handle at once while the agent keeps working; `agent_status` and
3//! `agent_wait` observe the job, `agent_cancel` stops it, and the session is
4//! told when it finishes so the server can report it in a turn of its own.
5//!
6//! Each call that starts a job, or shows the model a job's result, leaves a
7//! [`JobChange`] under its call ID, which the server hands to the session's
8//! clients with the call's `tool.completed` ([`BackgroundJobs::take_changes`]).
9//!
10//! A finished job stays unreported until a report turn about it succeeds. A
11//! failed one makes it due again after a delay, a bounded number of times;
12//! after that, or when another turn must not be tried, the server reports it
13//! to the client directly ([`BackgroundJobs::report_failed`]).
14//!
15//! A call that continues a conversation takes its place in that
16//! conversation's lane as it arrives. When a turn or another call is
17//! ahead, the call's busy policy decides: it steers the running turn, waits
18//! in the foreground, fails, or becomes a queued job that runs once its
19//! place comes up.
20//!
21//! A call with `review` runs as a reviewed job ([`review`](super::review)):
22//! the same job, whose one call runs the whole builder and reviewer loop and
23//! whose changes and report carry SCV's outcome.
24
25pub(in crate::delegate) mod lane;
26pub mod ledger;
27mod owed;
28
29use std::{
30    collections::HashSet,
31    path::PathBuf,
32    sync::{Arc, Mutex, Weak},
33    time::{Duration, Instant},
34};
35
36use async_trait::async_trait;
37use scv_core::{
38    ApprovalGate, ProgressSink, Tool, ToolApprovals, ToolContext, ToolError, ToolOutput, ToolRisk,
39    ToolSpec,
40};
41use scv_protocol::{
42    AgentState, AttachOffer, AttachState, JobChange, JobOrigin, JobOutcome, JobReport, JobStatus,
43    describe_reports,
44};
45use serde::Deserialize;
46use serde_json::{Map, Value, json};
47use tokio::sync::{mpsc, watch};
48use tokio_util::sync::CancellationToken;
49
50use self::{
51    lane::{Front, Lanes, Place, Unreserved},
52    ledger::{JobLedger, JobRecord, NewJob},
53};
54use crate::{
55    BusyBehavior,
56    args::{Timeouts, bounded, parse_args, timeout_schema},
57    delegate::{
58        agent::{AGENT_TOOL, AgentTool, Offered},
59        conversation::{self, ConversationStore, Obstacle},
60        output::AgentReply,
61        request::AgentArgs,
62        review::{self, Record, ReviewedTool},
63        takeover,
64    },
65    sync::lock,
66};
67
68/// Finished jobs a session keeps for `agent_status` beyond the running ones.
69const MAX_FINISHED: usize = 16;
70/// Characters of a finished job's reply quoted in a server-started report turn.
71const REPORT_REPLY_CHARS: usize = 6000;
72/// Jobs one report turn covers; any more wait for the next.
73const REPORT_MAX_JOBS: usize = 4;
74/// How long after each failed report turn the next one is due: after the
75/// first failure, then after the second. Once they run out, the job is
76/// reported directly.
77const REPORT_RETRY_DELAYS: [Duration; 2] = [Duration::from_secs(30), Duration::from_secs(120)];
78/// How long `agent_cancel` waits for a stopped job to settle.
79const CANCEL_SETTLE: Duration = Duration::from_secs(10);
80/// Job changes kept for calls whose `tool.completed` has not taken them,
81/// such as a call whose turn was aborted; the oldest go first.
82const MAX_PENDING_CHANGES: usize = 64;
83/// Characters of a delegated prompt's first line kept as its job's task.
84const TASK_CHARS: usize = 80;
85
86/// One session's background jobs. Dropping the store (with the session's
87/// tools) cancels every job still running.
88pub struct BackgroundJobs {
89    limit: usize,
90    state: Mutex<JobsState>,
91    cancellation: CancellationToken,
92    /// Woken whenever a job finishes, so the session can report it.
93    finished: Option<mpsc::UnboundedSender<()>>,
94    /// Decides a running job's nested approval requests, since no turn is
95    /// left to carry them to a person. Without it they are denied.
96    approvals: Option<Arc<dyn ApprovalGate>>,
97    /// The `agent` calls continuing each conversation, in arrival order.
98    lanes: Arc<Lanes>,
99    /// Where the session's jobs are recorded durably, when it keeps them.
100    ledger: Option<JobLedger>,
101}
102
103impl std::fmt::Debug for BackgroundJobs {
104    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
105        formatter
106            .debug_struct("BackgroundJobs")
107            .field("limit", &self.limit)
108            .finish_non_exhaustive()
109    }
110}
111
112#[derive(Default)]
113struct JobsState {
114    next: u64,
115    jobs: Vec<Job>,
116    /// Changes by the ID of the call that made them, oldest first.
117    changes: Vec<(String, JobChange)>,
118    /// The final outcomes of cancelled reviewed jobs the session is owed.
119    owed: owed::Owed,
120}
121
122impl JobsState {
123    /// Bring the outcome in the change the call `call_id` made for
124    /// `outcome`'s job up to date, if no client has taken it yet.
125    fn refresh(&mut self, call_id: &str, outcome: JobOutcome) {
126        if let Some((_, change)) = self
127            .changes
128            .iter_mut()
129            .rev()
130            .find(|(call, change)| call == call_id && change.job == outcome.job)
131        {
132            change.outcome = Some(outcome);
133        }
134    }
135
136    /// Keep `change` for the call `call_id`, which a client learns of with
137    /// that call's `tool.completed`. A call without an ID has no event.
138    fn record(&mut self, call_id: &str, change: JobChange) {
139        if call_id.is_empty() {
140            return;
141        }
142        if self.changes.len() >= MAX_PENDING_CHANGES {
143            let (call, dropped) = self.changes.remove(0);
144            self.owed.evicted(&call, &dropped.job);
145        }
146        self.changes.push((call_id.to_owned(), change));
147    }
148}
149
150struct Job {
151    id: String,
152    /// The tool that started it, `agent`.
153    tool: String,
154    /// The agent that runs it, such as `codex`.
155    agent: String,
156    /// The first line of the delegated prompt, shortened.
157    task: String,
158    started: Instant,
159    progress: ProgressSink,
160    last_progress: Option<String>,
161    /// Stops this job alone.
162    cancel: CancellationToken,
163    /// `agent_cancel` stopped it.
164    cancelled: bool,
165    /// A busy conversation's queued prompt, which `agent.max_queued_turns`
166    /// bounds instead of `agent.max_background`.
167    queued: bool,
168    outcome: Option<Outcome>,
169    /// The model has seen the result (through `agent_wait`, `agent_status`,
170    /// or a report turn that completed), the user stopped its report turn,
171    /// the server reported it directly, or the model asked for the stop with
172    /// `agent_cancel`, so it needs no report turn.
173    reported: bool,
174    /// A report turn about it runs.
175    reporting: bool,
176    /// Report turns about it that failed.
177    report_attempts: u32,
178    /// After a failed report turn: when the next one is due.
179    report_due: Option<tokio::time::Instant>,
180    done: watch::Receiver<bool>,
181    /// A reviewed job's record, whose outcome its changes and report state.
182    review: Option<Arc<Record>>,
183    /// `agent_cancel` waits for it, and describes it after: never pruned
184    /// meanwhile.
185    cancelling: bool,
186    /// Who started it. The owner's own jobs are never reported to the
187    /// model: the owner watches them in the workbench.
188    origin: JobOrigin,
189    /// A queued prompt whose turn has not come yet.
190    queued_waiting: bool,
191    /// The conversation whose lane a queued prompt waits in.
192    lane: Option<String>,
193    /// The owner stopped it from the workbench, so the model was not told.
194    owner_cancelled: bool,
195    /// Its durable record, when the session keeps a ledger, and that
196    /// record's revision: each change bumps it, and it is written after the
197    /// job table's lock is let go ([`BackgroundJobs::persist`]).
198    record: Option<JobRecord>,
199    record_revision: u64,
200}
201
202/// A revision of a job's record to write once the job table is let go.
203type RecordWrite = (JobRecord, u64);
204
205impl Job {
206    /// The record as it is now, under a new revision, to write.
207    fn record_changed(&mut self) -> Option<RecordWrite> {
208        let record = self.record.as_ref()?;
209        self.record_revision += 1;
210        Some((record.clone(), self.record_revision))
211    }
212}
213
214/// `agent_cancel` waiting for a job to stop. Settled after the wait, or,
215/// if the call is dropped first, as when its turn is aborted, its change
216/// withdrawn and any outcome it owed left to an update.
217struct Waiting<'a> {
218    jobs: &'a BackgroundJobs,
219    job: String,
220    call: String,
221    /// The job is reviewed: the session is owed its final outcome.
222    owed: bool,
223    settled: bool,
224}
225
226impl Waiting<'_> {
227    /// The wait is over: the change the call made says the final outcome
228    /// if the job stopped, else the outcome as decided, its journal still
229    /// open; [`BackgroundJobs::take_changes`] says the final one if the job
230    /// stops before the change is sent.
231    fn settle(mut self) {
232        self.settled = true;
233        let mut state = self.jobs.state();
234        let Some(entry) = state.jobs.iter_mut().find(|entry| entry.id == self.job) else {
235            return;
236        };
237        entry.cancelling = false;
238        if !self.owed {
239            return;
240        }
241        let pending = entry.outcome.is_none().then(|| entry.outcome()).flatten();
242        let pending = pending.map(|mut outcome| {
243            outcome.review.journal_pending = true;
244            outcome
245        });
246        if let Some(outcome) = state.owed.waited(&self.job, &self.call, pending) {
247            state.refresh(&self.call, outcome);
248        }
249    }
250}
251
252impl Drop for Waiting<'_> {
253    fn drop(&mut self) {
254        if self.settled {
255            return;
256        }
257        let mut state = self.jobs.state();
258        if let Some(entry) = state.jobs.iter_mut().find(|entry| entry.id == self.job) {
259            entry.cancelling = false;
260        }
261        if !self.owed {
262            return;
263        }
264        // Its call is gone, so its change is never sent.
265        let (call, job) = (&self.call, &self.job);
266        state
267            .changes
268            .retain(|(recorded, change)| !(recorded == call && &change.job == job));
269        let due = state.owed.abandoned(job, call);
270        drop(state);
271        if due {
272            self.jobs.wake();
273        }
274    }
275}
276
277struct Outcome {
278    output: ToolOutput,
279    elapsed: Duration,
280}
281
282/// What a failed report turn leaves its jobs to.
283#[derive(Debug, PartialEq, Eq)]
284pub enum ReportFailure {
285    /// Another report turn about them is due after this long.
286    Retry(Duration),
287    /// No more report turns: the server reports them directly. `attempts`
288    /// counts the report turns that failed.
289    GiveUp {
290        attempts: u32,
291        reports: Vec<JobReport>,
292    },
293    /// The model saw each of them through a call during the turn.
294    Settled,
295}
296
297impl Drop for BackgroundJobs {
298    fn drop(&mut self) {
299        self.cancellation.cancel();
300        // Jobs still going stop with their session: say so durably, since
301        // their own end is never recorded once the store is gone.
302        if let Some(ledger) = &self.ledger {
303            let state = self
304                .state
305                .get_mut()
306                .unwrap_or_else(std::sync::PoisonError::into_inner);
307            for job in state.jobs.iter_mut().filter(|job| job.outcome.is_none()) {
308                if let Some(record) = job.record.as_mut() {
309                    ledger::interrupt(record, "its session ended before it finished");
310                    ledger.save(record);
311                }
312            }
313        }
314    }
315}
316
317impl BackgroundJobs {
318    /// At most `limit` jobs run at once. `finished` is woken as jobs finish.
319    pub fn new(limit: usize, finished: Option<mpsc::UnboundedSender<()>>) -> Self {
320        Self {
321            limit,
322            state: Mutex::default(),
323            cancellation: CancellationToken::new(),
324            finished,
325            approvals: None,
326            lanes: Arc::default(),
327            ledger: None,
328        }
329    }
330
331    /// Record the session's jobs durably in `ledger`. Job IDs continue
332    /// after the ones it already holds.
333    #[must_use]
334    pub fn with_ledger(mut self, ledger: JobLedger) -> Self {
335        self.state().next = ledger.last_number();
336        self.ledger = Some(ledger);
337        self
338    }
339
340    /// Number new jobs after the `job-<n>` IDs `jobs` names too: the
341    /// session's runs name jobs whose records may be gone.
342    pub(crate) fn continue_after<'a>(&self, jobs: impl IntoIterator<Item = &'a str>) {
343        let mut state = self.state();
344        for job in jobs {
345            if let Some(number) = job
346                .strip_prefix("job-")
347                .and_then(|number| number.parse::<u64>().ok())
348            {
349                state.next = state.next.max(number);
350            }
351        }
352    }
353
354    /// Decide running jobs' nested approval requests with `gate`, which must
355    /// never grant more than the session's foreground would.
356    #[must_use]
357    pub fn with_approvals(mut self, gate: Arc<dyn ApprovalGate>) -> Self {
358        self.approvals = Some(gate);
359        self
360    }
361
362    pub(crate) fn limit(&self) -> usize {
363        self.limit
364    }
365
366    fn state(&self) -> std::sync::MutexGuard<'_, JobsState> {
367        lock(&self.state)
368    }
369
370    /// Start `tool` with `arguments` in the background for the call in
371    /// `context` (whose workspace and call ID it takes; the job has its own
372    /// cancellation), on `agent`, and return its job's
373    /// `{"job","agent","status":"running","background":true}` description.
374    /// A reviewed job's `review` records its start in its journal first, and
375    /// the call fails if it cannot. That write happens outside the job
376    /// table's lock, so a slow disk never holds up the session's other job
377    /// calls: the job's number is taken first, and a failed write leaves it
378    /// unused.
379    #[allow(
380        clippy::too_many_arguments,
381        reason = "a job's call, its context, its turn, and its review, each its own"
382    )]
383    pub(in crate::delegate) fn start(
384        self: &Arc<Self>,
385        tool: Arc<dyn Tool>,
386        name: &str,
387        agent: &str,
388        arguments: Value,
389        context: &ToolContext,
390        turn: Turn,
391        review: Option<&Arc<ReviewedTool>>,
392        meta: JobMeta,
393    ) -> Result<Value, ToolError> {
394        let JobMeta {
395            origin,
396            worktree,
397            resumed_from,
398        } = meta;
399        let task = task_line(
400            arguments
401                .get("prompt")
402                .and_then(Value::as_str)
403                .unwrap_or_default(),
404        );
405        let queued = matches!(turn, Turn::Queued(_));
406        let lane = match &turn {
407            Turn::Queued(place) => Some(place.handle().to_owned()),
408            Turn::New | Turn::Next(_) => None,
409        };
410        let reserved = match review {
411            Some(review) => {
412                let id = {
413                    let mut state = self.state();
414                    self.room(&state, queued)?;
415                    state.next += 1;
416                    format!("job-{}", state.next)
417                };
418                review.begin(&id)?;
419                Some(id)
420            }
421            None => None,
422        };
423        let (id, progress, done_tx, cancellation, first) = {
424            let mut state = self.state();
425            // Checked again: another call may have started a job meanwhile.
426            self.room(&state, queued)?;
427            let id = reserved.unwrap_or_else(|| {
428                state.next += 1;
429                format!("job-{}", state.next)
430            });
431            let progress = ProgressSink::buffered();
432            let (done_tx, done) = watch::channel(false);
433            let cancel = self.cancellation.child_token();
434            let record = self.ledger.as_ref().and_then(|ledger| {
435                let mut record = ledger.record(&NewJob {
436                    job: &id,
437                    agent,
438                    task: &task,
439                    arguments: &arguments,
440                    started_by: origin,
441                    reviewed: review.is_some(),
442                    queued,
443                })?;
444                record.worktree.clone_from(&worktree);
445                record.resumed_from = resumed_from;
446                Some(record)
447            });
448            let first = record.clone().map(|record| (record, 1));
449            state.record(
450                &context.call_id,
451                JobChange {
452                    job: id.clone(),
453                    tool: name.to_owned(),
454                    agent: agent.to_owned(),
455                    status: JobStatus::Running,
456                    task: task.clone(),
457                    journal: review.map(|review| review.journal_id()),
458                    outcome: None,
459                },
460            );
461            state.jobs.push(Job {
462                id: id.clone(),
463                tool: name.to_owned(),
464                agent: agent.to_owned(),
465                task,
466                started: Instant::now(),
467                progress: progress.clone(),
468                last_progress: None,
469                cancel: cancel.clone(),
470                cancelled: false,
471                queued,
472                outcome: None,
473                reported: false,
474                reporting: false,
475                report_attempts: 0,
476                report_due: None,
477                done,
478                review: review.map(|review| Arc::clone(review.record())),
479                cancelling: false,
480                origin,
481                queued_waiting: queued,
482                lane,
483                owner_cancelled: false,
484                record,
485                record_revision: 1,
486            });
487            (id, progress, done_tx, cancel, first)
488        };
489        self.persist(first);
490        // The owner can stop it to take its conversation over. A reviewed
491        // job's own place is not offered: stopping it ends a whole review.
492        if review.is_none()
493            && let Turn::Next(place) | Turn::Queued(place) = &turn
494        {
495            place.run_as(&id);
496        }
497        let jobs = Arc::downgrade(self);
498        let job = id.clone();
499        let approvals = self.approvals.clone();
500        let workspace = context.workspace.clone();
501        let reviewed = review.map(Arc::clone);
502        tokio::spawn(async move {
503            let started = Instant::now();
504            let mut context = ToolContext::new(workspace, cancellation);
505            context.background_job = true;
506            context.job = Some(job.clone());
507            // No turn is left to ask a person, so nested approval requests
508            // get the session's unattended answer, or are denied.
509            if let Some(gate) = &approvals {
510                context.approvals = ToolApprovals::new(Arc::clone(gate), job.clone());
511            }
512            context.progress = progress;
513            let place = turn.place();
514            let output = match &place {
515                Some(place) if queued => {
516                    context.progress.report(&format!(
517                        "queued: waits for conversation {}'s running turn",
518                        place.handle()
519                    ));
520                    match place.wait_first(&context.cancellation).await {
521                        Ok(()) => {
522                            if let Some(jobs) = jobs.upgrade() {
523                                jobs.began(&job);
524                            }
525                            context.progress.take();
526                            context.progress.report(&format!(
527                                "started its turn in conversation {}",
528                                place.handle()
529                            ));
530                            tool.execute(arguments, context).await
531                        }
532                        // A reviewed job ends its review even when its loop
533                        // never ran, so its result says how it ended.
534                        Err(error) => match &reviewed {
535                            Some(review) => Ok(review.unstarted(&error)),
536                            None => Err(error),
537                        },
538                    }
539                }
540                _ => tool.execute(arguments, context).await,
541            }
542            .unwrap_or_else(ToolOutput::from);
543            // Free the conversation before the result is visible, so a call
544            // continuing it after `agent_wait` finds it idle.
545            drop(place);
546            finish(&jobs, &job, output, started.elapsed());
547            let _ = done_tx.send(true);
548        });
549        let mut started = json!({
550            "job": id,
551            "agent": agent,
552            "status": "running",
553            "background": true,
554            "note": "The agent is working in the background. SCV reports the result in a new \
555                     turn when it finishes. agent_status shows its progress and agent_cancel \
556                     stops it."
557        });
558        if let Some(worktree) = &worktree {
559            started["worktree"] = serde_json::to_value(worktree).unwrap_or(Value::Null);
560        }
561        if queued {
562            started["queued"] = true.into();
563            started["note"] = "The conversation is busy, so this prompt is queued: it runs in \
564                the background after the turns ahead of it, and SCV reports the result in a new \
565                turn when it finishes. agent_status shows where it stands and agent_cancel \
566                withdraws it."
567                .into();
568        }
569        Ok(started)
570    }
571
572    /// A queued job's turn has come: it runs now.
573    fn began(&self, id: &str) {
574        let write = {
575            let mut state = self.state();
576            state
577                .jobs
578                .iter_mut()
579                .find(|job| job.id == id)
580                .and_then(|job| {
581                    job.queued_waiting = false;
582                    let record = job.record.as_mut()?;
583                    record.state = AgentState::Working;
584                    record.updated_unix = ledger::now();
585                    job.record_changed()
586                })
587        };
588        self.persist(write);
589    }
590
591    /// The conversation handles the session's job records name.
592    pub(crate) fn recorded_conversations(&self) -> Vec<String> {
593        self.ledger
594            .as_ref()
595            .and_then(JobLedger::conversations)
596            .unwrap_or_default()
597    }
598
599    /// Whether the session still owes its client `job`'s result: dropping
600    /// its record now would lose it.
601    pub(crate) fn unreported(&self, job: &str) -> bool {
602        self.state()
603            .jobs
604            .iter()
605            .any(|entry| entry.id == job && !entry.reported)
606    }
607
608    /// Mark `job`'s result reported in its durable record; the change to
609    /// write, when there is one.
610    fn mark_reported(job: &mut Job) -> Option<RecordWrite> {
611        let record = job.record.as_mut().filter(|record| !record.reported)?;
612        record.reported = true;
613        record.updated_unix = ledger::now();
614        job.record_changed()
615    }
616
617    /// Write the record changes `writes`, outside the job table's lock, so
618    /// a slow disk never holds up the session's job calls. The write syncs
619    /// to disk, so on a runtime's worker thread the runtime is told it
620    /// blocks, and moves its other tasks elsewhere meanwhile.
621    fn persist(&self, writes: impl IntoIterator<Item = RecordWrite>) {
622        let Some(ledger) = &self.ledger else {
623            return;
624        };
625        let write = || {
626            for (record, revision) in writes {
627                ledger.save_revision(&record, revision);
628            }
629        };
630        match tokio::runtime::Handle::try_current() {
631            Ok(runtime)
632                if runtime.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread =>
633            {
634                tokio::task::block_in_place(write);
635            }
636            _ => write(),
637        }
638    }
639
640    /// The session's jobs as they stand, for the workbench.
641    pub(crate) fn live(&self) -> Vec<LiveJob> {
642        let mut state = self.state();
643        state
644            .jobs
645            .iter_mut()
646            .map(|job| {
647                if let Some(line) = job.progress.take() {
648                    job.last_progress = Some(line);
649                }
650                LiveJob {
651                    id: job.id.clone(),
652                    waiting: job.queued_waiting,
653                    running: job.outcome.is_none(),
654                    progress: job.last_progress.clone(),
655                    elapsed: job.started.elapsed(),
656                }
657            })
658            .collect()
659    }
660
661    /// Reviewed job `job`'s outcome as it stands, while this session still
662    /// keeps the job; `None` for any other job.
663    pub(crate) fn outcome(&self, job: &str) -> Option<JobOutcome> {
664        self.state()
665            .jobs
666            .iter()
667            .find(|entry| entry.id == job)
668            .and_then(Job::outcome)
669    }
670
671    /// Stop `job` for the owner, from the workbench: as `agent_cancel` does,
672    /// but the model did not ask, so its report turn tells it the owner
673    /// stopped the job. A reviewed job's report carries its outcome as the
674    /// cancel left it.
675    pub(crate) fn cancel_for_owner(&self, job: &str) -> Result<(), String> {
676        let mut state = self.state();
677        let entry = state
678            .jobs
679            .iter_mut()
680            .find(|candidate| candidate.id == job)
681            .ok_or_else(|| format!("this session has no job {job:?}"))?;
682        if entry.outcome.is_some() {
683            return Err(format!("{job} has already finished"));
684        }
685        entry.cancelled = true;
686        entry.owner_cancelled = true;
687        entry.cancel.cancel();
688        Ok(())
689    }
690
691    /// Refuse a new running job beyond `agent.max_background`, before
692    /// anything is prepared for it.
693    fn has_room(&self) -> Result<(), ToolError> {
694        let state = self.state();
695        self.room(&state, false)
696    }
697
698    /// Refuse a job beyond `agent.max_background`; a queued prompt counts
699    /// toward `agent.max_queued_turns` instead.
700    fn room(&self, state: &JobsState, queued: bool) -> Result<(), ToolError> {
701        let running = state
702            .jobs
703            .iter()
704            .filter(|job| job.outcome.is_none() && !job.queued)
705            .count();
706        if !queued && running >= self.limit {
707            return Err(ToolError::limit(format!(
708                "{running} background jobs are already running, the limit \
709                 (agent.max_background). Start this one after a job finishes, or \
710                 stop one with agent_cancel if the user no longer needs it."
711            )));
712        }
713        Ok(())
714    }
715
716    /// Stop `job` for the call `call_id` and wait briefly for it to settle,
717    /// then describe it. A stopped job needs no report turn: the model asked
718    /// for the stop.
719    ///
720    /// A reviewed job's decision is fixed at once, but its journal goes on
721    /// until it really stops, so the session is owed its final outcome
722    /// ([`owed`]): the call's change says it if the job stops before the
723    /// change is sent, and otherwise says the journal is still open, and a
724    /// `background.updated` follows ([`Self::take_updates`]). The wait
725    /// reads only the job's live state, never its journal.
726    async fn cancel(&self, job: &str, call_id: &str) -> Result<Value, ToolError> {
727        let (mut done, owed, write) = {
728            let mut state = self.state();
729            let index = state
730                .jobs
731                .iter()
732                .position(|candidate| candidate.id == job)
733                .ok_or_else(|| unknown_job(job))?;
734            let entry = &mut state.jobs[index];
735            if entry.origin == JobOrigin::Owner && entry.outcome.is_none() {
736                return Err(ToolError::invalid_arguments(format!(
737                    "{job} is the user's own job, started from the SCV workbench; only the \
738                     user stops it"
739                )));
740            }
741            if entry.outcome.is_some() {
742                let (mut value, change) = entry.describe();
743                value["note"] = "The job had already finished.".into();
744                // The model has its result now, as `agent_status` gives it.
745                let write = change
746                    .is_some()
747                    .then(|| Self::mark_reported(entry))
748                    .flatten();
749                if let Some(change) = change {
750                    state.record(call_id, change);
751                }
752                drop(state);
753                self.persist(write);
754                return Ok(value);
755            }
756            entry.cancelled = true;
757            entry.cancel.cancel();
758            entry.cancelling = true;
759            let done = entry.done.clone();
760            let mut owed = false;
761            let mut write = None;
762            if !entry.reported {
763                entry.reported = true;
764                write = Self::mark_reported(entry);
765                let change = entry.change(JobStatus::Cancelled);
766                owed = entry.review.is_some();
767                state.record(call_id, change);
768                if owed {
769                    state.owed.cancelling(job, call_id);
770                }
771            }
772            (done, owed, write)
773        };
774        self.persist(write);
775        let waiting = Waiting {
776            jobs: self,
777            job: job.to_owned(),
778            call: call_id.to_owned(),
779            owed,
780            settled: false,
781        };
782        let _ = tokio::time::timeout(CANCEL_SETTLE, done.wait_for(|finished| *finished)).await;
783        waiting.settle();
784        self.describe(Some(job), call_id)
785    }
786
787    /// The final outcomes of cancelled reviewed jobs the session is owed
788    /// and has not been sent, for the server's `background.updated`. Each
789    /// is being sent until [`Self::updated`] says how that went.
790    pub fn take_updates(&self) -> Vec<JobOutcome> {
791        self.state().owed.take_updates()
792    }
793
794    /// The update naming `jobs` reached the client, when `delivered`, which
795    /// ends what was owed; or it could not be sent, which leaves them due.
796    pub fn updated(&self, jobs: &[String], delivered: bool) {
797        self.state().owed.updated(jobs, delivered);
798    }
799
800    /// The `tool.completed` of the call `call_id`, carrying the changes
801    /// [`Self::take_changes`] gave, reached the client, when `delivered`, or
802    /// could not be sent, as when its turn was cancelled first. A change
803    /// that did not deliver a reviewed job's final outcome leaves it to a
804    /// `background.updated`.
805    pub fn published(&self, call_id: &str, delivered: bool) {
806        let due = self.state().owed.sent(call_id, delivered);
807        if due {
808            self.wake();
809        }
810    }
811
812    /// A turn ended: no change of its calls can be sent any more, so what
813    /// such a change owed goes to a `background.updated`.
814    pub fn turn_ended(&self) {
815        let due = self.state().owed.turn_ended();
816        if due {
817            self.wake();
818        }
819    }
820
821    /// For tests of how a client of this store delivers reviewed jobs'
822    /// outcomes, such as the server's, with the `test-support` feature: owe
823    /// the session the final outcome of `change.job`, through `change`,
824    /// recorded for the call `call_id` once `agent_cancel` waited, which says
825    /// the journal is still open; `last` is the final outcome if the job has
826    /// stopped.
827    #[cfg(any(test, feature = "test-support"))]
828    pub fn owe_for_tests(&self, call_id: &str, change: JobChange, last: Option<JobOutcome>) {
829        let mut state = self.state();
830        state.owed.owe(call_id, &change, last);
831        state.record(call_id, change);
832    }
833
834    /// For the same tests: the job of `outcome` stops with it.
835    #[cfg(any(test, feature = "test-support"))]
836    pub fn stop_for_tests(&self, outcome: &JobOutcome) {
837        let due = self.state().owed.stopped(&outcome.job, outcome);
838        if due {
839            self.wake();
840        }
841    }
842
843    /// For tests of what waits for a session's jobs, with the
844    /// `test-support` feature: start a job on `agent` that runs until the
845    /// returned sender is used or dropped.
846    #[cfg(any(test, feature = "test-support"))]
847    pub fn start_for_tests(self: &Arc<Self>, agent: &str) -> tokio::sync::oneshot::Sender<()> {
848        struct Waits(Mutex<Option<tokio::sync::oneshot::Receiver<()>>>);
849        #[async_trait]
850        impl Tool for Waits {
851            fn spec(&self) -> ToolSpec {
852                ToolSpec {
853                    name: AGENT_TOOL.into(),
854                    description: String::new(),
855                    parameters: json!({"type":"object"}),
856                }
857            }
858            fn risk(&self, _arguments: &Value) -> Result<ToolRisk, ToolError> {
859                Ok(ToolRisk::Delegate)
860            }
861            fn approval_summary(&self, _arguments: &Value) -> Result<String, ToolError> {
862                Ok(String::new())
863            }
864            async fn execute(
865                &self,
866                _arguments: Value,
867                _context: ToolContext,
868            ) -> Result<ToolOutput, ToolError> {
869                let release = lock(&self.0).take();
870                if let Some(release) = release {
871                    let _ = release.await;
872                }
873                Ok(ToolOutput::success(
874                    r#"{"status":"completed","reply":"done"}"#,
875                ))
876            }
877        }
878        let (release, waiting) = tokio::sync::oneshot::channel();
879        let context = ToolContext::new(std::env::temp_dir(), CancellationToken::new());
880        self.start(
881            Arc::new(Waits(Mutex::new(Some(waiting)))),
882            AGENT_TOOL,
883            agent,
884            json!({"prompt": "wait"}),
885            &context,
886            Turn::New,
887            None,
888            JobMeta::default(),
889        )
890        .expect("a test job starts");
891        release
892    }
893
894    /// Wake the session, as when a job finishes, for what is now due.
895    fn wake(&self) {
896        if let Some(finished) = &self.finished {
897            let _ = finished.send(());
898        }
899    }
900
901    /// Wait up to `limit` for `job` and describe it for the call `call_id`;
902    /// a finished job is then marked seen.
903    async fn wait(
904        &self,
905        job: &str,
906        limit: Duration,
907        cancellation: &CancellationToken,
908        call_id: &str,
909    ) -> Result<Value, ToolError> {
910        let mut done = self
911            .state()
912            .jobs
913            .iter()
914            .find(|candidate| candidate.id == job)
915            .map(|candidate| candidate.done.clone())
916            .ok_or_else(|| unknown_job(job))?;
917        tokio::select! {
918            () = cancellation.cancelled() => return Err(ToolError::cancelled("wait cancelled")),
919            _ = tokio::time::timeout(limit, done.wait_for(|finished| *finished)) => {}
920        }
921        self.describe(Some(job), call_id)
922    }
923
924    /// Describe one job, or every job this session remembers, for the call
925    /// `call_id`; finished jobs described are marked seen.
926    fn describe(&self, job: Option<&str>, call_id: &str) -> Result<Value, ToolError> {
927        let mut state = self.state();
928        let mut changes = Vec::new();
929        let mut writes = Vec::new();
930        let value = if let Some(job) = job {
931            let entry = state
932                .jobs
933                .iter_mut()
934                .find(|candidate| candidate.id == job)
935                .ok_or_else(|| unknown_job(job))?;
936            let (value, change) = entry.describe();
937            if change.is_some() {
938                writes.extend(Self::mark_reported(entry));
939            }
940            changes.extend(change);
941            value
942        } else {
943            let jobs: Vec<Value> = state
944                .jobs
945                .iter_mut()
946                .map(|entry| {
947                    let (value, change) = entry.describe();
948                    if change.is_some() {
949                        writes.extend(Self::mark_reported(entry));
950                    }
951                    changes.extend(change);
952                    value
953                })
954                .collect();
955            json!({ "jobs": jobs })
956        };
957        for change in changes {
958            state.record(call_id, change);
959        }
960        drop(state);
961        self.persist(writes);
962        Ok(value)
963    }
964
965    /// The jobs the call `call_id` started or showed the model the result of,
966    /// for its `tool.completed`, which then reports how its sending went
967    /// ([`Self::published`]). A cancelled reviewed job's change says its
968    /// final outcome if the job has stopped by now, superseding the
969    /// snapshot taken when the cancel's wait ended.
970    pub fn take_changes(&self, call_id: &str) -> Vec<JobChange> {
971        let mut state = self.state();
972        let mut taken = Vec::new();
973        state.changes.retain(|(call, change)| {
974            if call == call_id {
975                taken.push(change.clone());
976                false
977            } else {
978                true
979            }
980        });
981        for change in &mut taken {
982            state.owed.take(call_id, change);
983        }
984        taken
985    }
986
987    /// Finished jobs the model has not seen yet whose report is due, at most
988    /// a few, for a report turn. Each stays unreported, and is not taken
989    /// again, until the turn ends: [`report_settled`](Self::report_settled)
990    /// or [`report_failed`](Self::report_failed).
991    pub fn take_unreported(&self) -> Vec<JobReport> {
992        let now = tokio::time::Instant::now();
993        let mut state = self.state();
994        state
995            .jobs
996            .iter_mut()
997            .filter(|job| job.awaits_report() && job.report_due.is_none_or(|due| due <= now))
998            .take(REPORT_MAX_JOBS)
999            .filter_map(|job| {
1000                job.reporting = true;
1001                job.report()
1002            })
1003            .collect()
1004    }
1005
1006    /// The report turn about `jobs` ended without failing: it completed, so
1007    /// the model has seen them, or the user cancelled it. Either settles them.
1008    pub fn report_settled(&self, jobs: &[String]) {
1009        let mut writes = Vec::new();
1010        {
1011            let mut state = self.state();
1012            for job in state.jobs.iter_mut().filter(|job| jobs.contains(&job.id)) {
1013                job.reporting = false;
1014                job.reported = true;
1015                job.report_due = None;
1016                writes.extend(Self::mark_reported(job));
1017            }
1018        }
1019        self.persist(writes);
1020    }
1021
1022    /// The report turn about `jobs` failed, so the model has not seen them
1023    /// (a failed turn leaves no history). With `retry`, another turn is due
1024    /// after a delay, up to three turns in all; otherwise, or once they are
1025    /// used up, the jobs are settled and returned for the server to report
1026    /// directly. A turn that reported several jobs decides for all of them,
1027    /// by the one that failed most.
1028    pub fn report_failed(&self, jobs: &[String], retry: bool) -> ReportFailure {
1029        let mut writes = Vec::new();
1030        let failure = self.report_failed_locked(jobs, retry, &mut writes);
1031        self.persist(writes);
1032        failure
1033    }
1034
1035    /// [`Self::report_failed`] under the job table's lock, gathering the
1036    /// record changes to write in `writes`.
1037    fn report_failed_locked(
1038        &self,
1039        jobs: &[String],
1040        retry: bool,
1041        writes: &mut Vec<RecordWrite>,
1042    ) -> ReportFailure {
1043        let mut state = self.state();
1044        let mut failed: Vec<&mut Job> = state
1045            .jobs
1046            .iter_mut()
1047            .filter(|job| job.reporting && jobs.contains(&job.id))
1048            .collect();
1049        for job in &mut failed {
1050            job.reporting = false;
1051            job.report_attempts += 1;
1052        }
1053        // One the turn's model looked up through a call is settled already.
1054        failed.retain(|job| !job.reported);
1055        let Some(attempts) = failed.iter().map(|job| job.report_attempts).max() else {
1056            return ReportFailure::Settled;
1057        };
1058        let delay = usize::try_from(attempts - 1)
1059            .ok()
1060            .and_then(|earlier| REPORT_RETRY_DELAYS.get(earlier));
1061        if retry && let Some(&delay) = delay {
1062            let due = tokio::time::Instant::now() + delay;
1063            for job in failed {
1064                job.report_due = Some(due);
1065            }
1066            return ReportFailure::Retry(delay);
1067        }
1068        let reports = failed
1069            .into_iter()
1070            .filter_map(|job| {
1071                job.reported = true;
1072                job.report_due = None;
1073                writes.extend(Self::mark_reported(job));
1074                job.report()
1075            })
1076            .collect();
1077        ReportFailure::GiveUp { attempts, reports }
1078    }
1079
1080    /// Make every report waiting out a failed turn due now, as once a turn
1081    /// succeeds and the model is evidently reachable again. Whether any was
1082    /// waiting.
1083    pub fn retry_now(&self) -> bool {
1084        let mut state = self.state();
1085        let mut waiting = false;
1086        for job in state.jobs.iter_mut().filter(|job| job.awaits_report()) {
1087            waiting |= job.report_due.take().is_some();
1088        }
1089        waiting
1090    }
1091
1092    /// When the earliest report waiting out a failed turn is due; it may
1093    /// already have passed.
1094    pub fn next_retry(&self) -> Option<tokio::time::Instant> {
1095        self.state()
1096            .jobs
1097            .iter()
1098            .filter(|job| job.awaits_report())
1099            .filter_map(|job| job.report_due)
1100            .min()
1101    }
1102
1103    /// Jobs still running or whose result is not reported yet, including
1104    /// those a report turn is reporting now, and cancelled reviewed jobs
1105    /// whose final outcome the session is still owed, until it was sent;
1106    /// each job once.
1107    pub fn pending(&self) -> usize {
1108        let state = self.state();
1109        let mut pending: HashSet<&str> = state
1110            .jobs
1111            .iter()
1112            .filter(|job| job.outcome.is_none() || !job.reported)
1113            .map(|job| job.id.as_str())
1114            .collect();
1115        pending.extend(state.owed.jobs());
1116        pending.len()
1117    }
1118
1119    /// Of the [`pending`](Self::pending) jobs, the queued prompts that wait
1120    /// only for the owner's takeover of their conversation to end. A planned
1121    /// restart does not wait for them: the restart ends the takeover, and
1122    /// they are kept for the next session like any queued prompt.
1123    pub fn behind_takeovers(&self) -> usize {
1124        self.state()
1125            .jobs
1126            .iter()
1127            .filter(|job| job.outcome.is_none() && job.queued_waiting)
1128            .filter(|job| {
1129                job.lane
1130                    .as_deref()
1131                    .is_some_and(|lane| self.lanes.held_by_takeover(lane))
1132            })
1133            .count()
1134    }
1135
1136    /// Whether any job is still running.
1137    pub fn running(&self) -> usize {
1138        self.state()
1139            .jobs
1140            .iter()
1141            .filter(|job| job.outcome.is_none())
1142            .count()
1143    }
1144}
1145
1146/// Who started a job, the worktree it runs in, and the interrupted job it
1147/// resumes: what its record holds from the start.
1148#[derive(Debug, Clone, Default)]
1149pub(crate) struct JobMeta {
1150    pub(crate) origin: JobOrigin,
1151    pub(crate) worktree: Option<scv_protocol::WorktreeRef>,
1152    /// `<session>/<job>` of the interrupted job it runs again.
1153    pub(crate) resumed_from: Option<String>,
1154}
1155
1156/// What an owner's prompt to the workbench became.
1157#[derive(Debug, Clone, PartialEq)]
1158pub(crate) enum OwnerCall {
1159    /// A background job: its start, as an `agent` call returns it.
1160    Job(Value),
1161    /// The prompt went into the conversation's running turn.
1162    Steered,
1163}
1164
1165/// A job as the workbench sees it while its session lives.
1166#[derive(Debug, Clone)]
1167pub(crate) struct LiveJob {
1168    pub(crate) id: String,
1169    /// A queued prompt whose turn has not come yet.
1170    pub(crate) waiting: bool,
1171    pub(crate) running: bool,
1172    pub(crate) progress: Option<String>,
1173    pub(crate) elapsed: Duration,
1174}
1175
1176/// The workbench state of a job that ended with `status`.
1177pub(crate) fn agent_state(status: JobStatus) -> AgentState {
1178    match status {
1179        JobStatus::Completed => AgentState::Completed,
1180        JobStatus::Declined => AgentState::Declined,
1181        JobStatus::Timeout => AgentState::Timeout,
1182        JobStatus::Cancelled => AgentState::Cancelled,
1183        JobStatus::Running => AgentState::Working,
1184        _ => AgentState::Failed,
1185    }
1186}
1187
1188/// How a job's call takes its conversation's turn.
1189pub(in crate::delegate) enum Turn {
1190    /// A new conversation, which nothing else can reach yet.
1191    New,
1192    /// The conversation's next turn, which nothing was ahead of.
1193    Next(Place),
1194    /// A prompt queued behind the turns ahead of it.
1195    Queued(Place),
1196}
1197
1198impl Turn {
1199    fn place(self) -> Option<Place> {
1200        match self {
1201            Self::New => None,
1202            Self::Next(place) | Self::Queued(place) => Some(place),
1203        }
1204    }
1205}
1206
1207fn finish(jobs: &Weak<BackgroundJobs>, id: &str, output: ToolOutput, elapsed: Duration) {
1208    // The session ended first: its jobs were cancelled with it.
1209    let Some(jobs) = jobs.upgrade() else {
1210        return;
1211    };
1212    let mut write = None;
1213    {
1214        let mut state = jobs.state();
1215        let mut last = None;
1216        if let Some(job) = state.jobs.iter_mut().find(|job| job.id == id) {
1217            job.last_progress = job.progress.take().or(job.last_progress.take());
1218            job.queued_waiting = false;
1219            // Stopped on the model's request: the model already knows. The
1220            // owner's own jobs are the owner's to watch, never the model's.
1221            job.reported |=
1222                (job.cancelled && !job.owner_cancelled) || job.origin == JobOrigin::Owner;
1223            let result = result_value(&output);
1224            let status = if job.cancelled {
1225                JobStatus::Cancelled
1226            } else {
1227                job_status(&output, &AgentReply::read(&result).unwrap_or_default())
1228            };
1229            job.outcome = Some(Outcome { output, elapsed });
1230            if let Some(record) = job.record.as_mut() {
1231                ledger::finish(record, agent_state(status), &result, job.reported);
1232                write = job.record_changed();
1233            }
1234            last = job.outcome();
1235        }
1236        // What a cancel owed of it is now final.
1237        if let Some(last) = last {
1238            state.owed.stopped(id, &last);
1239        }
1240        // Keep every running job and the newest finished ones.
1241        let finished = state
1242            .jobs
1243            .iter()
1244            .filter(|job| job.outcome.is_some())
1245            .count();
1246        let mut excess = finished.saturating_sub(MAX_FINISHED);
1247        state.jobs.retain(|job| {
1248            if excess > 0 && job.outcome.is_some() && job.reported && !job.cancelling {
1249                excess -= 1;
1250                false
1251            } else {
1252                true
1253            }
1254        });
1255    }
1256    // Recorded before anyone hears the job finished.
1257    jobs.persist(write);
1258    if let Some(finished) = &jobs.finished {
1259        let _ = finished.send(());
1260    }
1261}
1262
1263impl Job {
1264    /// Finished, unseen by the model, and not in a report turn.
1265    fn awaits_report(&self) -> bool {
1266        self.outcome.is_some() && !self.reported && !self.reporting
1267    }
1268
1269    /// The finished job's result, for a report.
1270    fn report(&self) -> Option<JobReport> {
1271        let outcome = self.outcome.as_ref()?;
1272        let reply = AgentReply::read(&result_value(&outcome.output)).unwrap_or_default();
1273        Some(JobReport {
1274            job: self.id.clone(),
1275            agent: self.agent.clone(),
1276            task: self.task.clone(),
1277            // Stopped by the owner: the model asked for no stop of its own.
1278            status: if self.cancelled {
1279                JobStatus::Cancelled
1280            } else {
1281                job_status(&outcome.output, &reply)
1282            },
1283            session: reply.session,
1284            reply: bounded(
1285                reply
1286                    .reply
1287                    .as_deref()
1288                    .unwrap_or(outcome.output.content.as_str()),
1289                REPORT_REPLY_CHARS,
1290            ),
1291            outcome: self.outcome(),
1292        })
1293    }
1294
1295    /// A reviewed job's outcome as it stands. A cancelled job's is what it
1296    /// was when the cancel arrived.
1297    fn outcome(&self) -> Option<JobOutcome> {
1298        self.review.as_ref().map(|record| {
1299            let (review, landing) = record
1300                .live
1301                .snapshot(self.cancelled || self.cancel.is_cancelled());
1302            JobOutcome {
1303                job: self.id.clone(),
1304                review,
1305                landing,
1306            }
1307        })
1308    }
1309
1310    /// This job with `status`, as its clients learn of it: a reviewed job's
1311    /// settling change carries its outcome.
1312    fn change(&self, status: JobStatus) -> JobChange {
1313        JobChange {
1314            job: self.id.clone(),
1315            tool: self.tool.clone(),
1316            agent: self.agent.clone(),
1317            status,
1318            task: self.task.clone(),
1319            journal: None,
1320            outcome: if status == JobStatus::Running {
1321                None
1322            } else {
1323                self.outcome()
1324            },
1325        }
1326    }
1327
1328    /// The job as the model reads it, and its change when this is the first
1329    /// time the model sees its result.
1330    fn describe(&mut self) -> (Value, Option<JobChange>) {
1331        if let Some(line) = self.progress.take() {
1332            self.last_progress = Some(line);
1333        }
1334        let mut change = None;
1335        let mut value = Map::new();
1336        value.insert("job".into(), self.id.clone().into());
1337        value.insert("agent".into(), self.agent.clone().into());
1338        if self.origin == JobOrigin::Owner {
1339            value.insert("started_by".into(), "owner".into());
1340        }
1341        match &self.outcome {
1342            None => {
1343                value.insert("status".into(), "running".into());
1344                value.insert(
1345                    "elapsed_seconds".into(),
1346                    self.started.elapsed().as_secs().into(),
1347                );
1348                if let Some(progress) = &self.last_progress {
1349                    value.insert("progress".into(), progress.clone().into());
1350                }
1351                if let Some(record) = &self.review {
1352                    value.insert("review".into(), record.live.status());
1353                }
1354            }
1355            Some(outcome) => {
1356                let result = result_value(&outcome.output);
1357                let status = if self.cancelled {
1358                    JobStatus::Cancelled
1359                } else {
1360                    job_status(
1361                        &outcome.output,
1362                        &AgentReply::read(&result).unwrap_or_default(),
1363                    )
1364                };
1365                value.insert("status".into(), status.as_str().into());
1366                value.insert("elapsed_seconds".into(), outcome.elapsed.as_secs().into());
1367                value.insert("result".into(), result);
1368                if !self.reported {
1369                    self.reported = true;
1370                    change = Some(self.change(status));
1371                }
1372            }
1373        }
1374        (Value::Object(value), change)
1375    }
1376}
1377
1378/// The agent tool's structured result, or its text when it was not JSON.
1379fn result_value(output: &ToolOutput) -> Value {
1380    match serde_json::from_str::<Value>(&output.content) {
1381        Ok(value @ Value::Object(_)) => value,
1382        _ => json!({ "reply": output.content, "is_error": output.is_error() }),
1383    }
1384}
1385
1386/// How a finished job ended: its agent's reported status, else whether its
1387/// call failed.
1388fn job_status(output: &ToolOutput, reply: &AgentReply) -> JobStatus {
1389    reply.status.unwrap_or(if output.is_error() {
1390        JobStatus::Failed
1391    } else {
1392        JobStatus::Completed
1393    })
1394}
1395
1396/// The first non-empty line of `prompt`, at most [`TASK_CHARS`] characters,
1397/// which names a job to people.
1398fn task_line(prompt: &str) -> String {
1399    let line = prompt
1400        .lines()
1401        .map(str::trim)
1402        .find(|line| !line.is_empty())
1403        .unwrap_or_default();
1404    let mut task: String = line
1405        .chars()
1406        .filter(|character| !character.is_control())
1407        .take(TASK_CHARS)
1408        .collect();
1409    if line.chars().count() > TASK_CHARS {
1410        task.push('…');
1411    }
1412    task
1413}
1414
1415fn unknown_job(job: &str) -> ToolError {
1416    ToolError::invalid_arguments(format!(
1417        "unknown background job {:?}; agent_status lists this session's jobs",
1418        bounded(job, 64)
1419    ))
1420}
1421
1422/// What the model is told when a report includes reviewed jobs, whose
1423/// outcome SCV states itself.
1424const REVIEWED_NOTE: &str = " SCV shows the user its own Review and Landing lines for \
1425     reviewed jobs. If you mention the outcome, repeat them as given; never call unapproved \
1426     work approved or done, and never call landed work approved. If a reviewer declined, tell \
1427     the user what it said. If a landing is NOT confirmed, or the reviewer could NOT confirm \
1428     it, say so plainly, and don't fix, revert, or land anything on your own.";
1429
1430fn reviewed_note(reports: &[JobReport]) -> &'static str {
1431    if reports.iter().any(|report| report.outcome.is_some()) {
1432        REVIEWED_NOTE
1433    } else {
1434        ""
1435    }
1436}
1437
1438/// The server-started turn that reports finished jobs to the model.
1439pub fn report_prompt(reports: &[JobReport]) -> String {
1440    format!(
1441        "[SCV background report] Delegated work you started in the background has \
1442         finished. The user did not send this message: tell them briefly what \
1443         happened and the key result.{}{}\n\n{}",
1444        cancelled_note(reports),
1445        reviewed_note(reports),
1446        describe_reports(reports)
1447    )
1448}
1449
1450/// What the model is told when a report includes a job someone else
1451/// stopped: a job the model stops itself is never reported.
1452fn cancelled_note(reports: &[JobReport]) -> &'static str {
1453    if reports
1454        .iter()
1455        .any(|report| report.status == JobStatus::Cancelled)
1456    {
1457        " A job reported as cancelled was stopped by the user (from the SCV workbench or \
1458         `scv agents kill`), not by you; don't restart it unless they ask."
1459    } else {
1460        ""
1461    }
1462}
1463
1464/// What the model is told of jobs the server reported directly, because
1465/// their report turns failed with `error`: the user has their results.
1466pub fn delivered_note(reports: &[JobReport], error: &str) -> String {
1467    format!(
1468        "[SCV background report, already delivered] Delegated work you started in \
1469         the background has finished. Your report of it failed ({error}), so SCV \
1470         sent the user the results below directly. The user did not send this \
1471         message; repeat the results only if they ask.{}\n\n{}",
1472        reviewed_note(reports),
1473        describe_reports(reports)
1474    )
1475}
1476
1477/// The `agent` tool, able to run a call in the background as well, and to
1478/// run one as a reviewed job.
1479pub(crate) struct BackgroundCapable {
1480    pub(crate) inner: Arc<AgentTool>,
1481    pub(crate) jobs: Arc<BackgroundJobs>,
1482    /// The session's conversations, which a reviewed job pins and releases.
1483    pub(crate) conversations: Arc<ConversationStore>,
1484    /// Where reviewed jobs keep their journals; without it, `review` is not
1485    /// offered.
1486    pub(crate) reviews: Option<PathBuf>,
1487    /// The SCV session, which a review's journal names.
1488    pub(crate) session: Option<String>,
1489}
1490
1491/// The `agent` tool for a queued prompt, whose turn has come by the time it
1492/// runs, so no busy policy applies to it again.
1493struct QueuedTool(Arc<AgentTool>);
1494
1495#[async_trait]
1496impl Tool for QueuedTool {
1497    fn spec(&self) -> ToolSpec {
1498        self.0.spec()
1499    }
1500    fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
1501        self.0.risk(arguments)
1502    }
1503    fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
1504        self.0.approval_summary(arguments)
1505    }
1506    async fn execute(
1507        &self,
1508        arguments: Value,
1509        context: ToolContext,
1510    ) -> Result<ToolOutput, ToolError> {
1511        self.0.execute_when_idle(arguments, context).await
1512    }
1513}
1514
1515/// An agent call's arguments with `background` and `review` split off.
1516struct Split {
1517    arguments: Value,
1518    /// `background` as the call gave it.
1519    background: Option<bool>,
1520    /// The `review` object, when the call gave one.
1521    review: Option<Value>,
1522}
1523
1524impl Split {
1525    fn new(arguments: &Value) -> Self {
1526        let mut arguments = arguments.clone();
1527        let object = arguments.as_object_mut();
1528        let (background, review) = match object {
1529            Some(object) => (
1530                object
1531                    .remove("background")
1532                    .map(|value| value.as_bool() == Some(true)),
1533                object.remove("review").filter(|review| !review.is_null()),
1534            ),
1535            None => (None, None),
1536        };
1537        Self {
1538            arguments,
1539            background,
1540            review,
1541        }
1542    }
1543
1544    fn background(&self) -> bool {
1545        self.background == Some(true)
1546    }
1547}
1548
1549#[async_trait]
1550impl Tool for BackgroundCapable {
1551    fn spec(&self) -> ToolSpec {
1552        let mut spec = self.inner.spec();
1553        let reviewed = self.offers_review();
1554        if let Some(properties) = spec
1555            .parameters
1556            .get_mut("properties")
1557            .and_then(Value::as_object_mut)
1558        {
1559            properties.insert(
1560                "background".into(),
1561                json!({
1562                    "type":"boolean",
1563                    "description":"Run in the background: the call returns a job handle at once, \
1564                        the user can keep talking to you while the agent works, and SCV reports \
1565                        the result in a new turn when it finishes. Use it for any substantial \
1566                        task; run in the foreground only for quick work whose result you need \
1567                        within this turn."
1568                }),
1569            );
1570            if reviewed {
1571                properties.insert("review".into(), review::schema(&self.inner));
1572            }
1573        }
1574        spec.description.push_str(&format!(
1575            " Set background to true for anything beyond a quick task: the call returns a job \
1576             handle at once (at most {} running per session), SCV reports the result when the \
1577             job finishes, agent_status shows progress, and agent_cancel stops it. A background \
1578             job's own approval requests get only the answer this session would give without \
1579             asking a person.",
1580            self.jobs.limit()
1581        ));
1582        if reviewed {
1583            spec.description.push_str(
1584                " A call with review runs as a reviewed background job: an independent \
1585                 reviewer checks the work in up to review.rounds rounds.",
1586            );
1587        }
1588        spec
1589    }
1590
1591    fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
1592        let split = Split::new(arguments);
1593        if let Some(review) = &split.review {
1594            self.plan(&split, review)?;
1595        }
1596        self.inner.risk_for(
1597            &split.arguments,
1598            split.background() || split.review.is_some(),
1599        )
1600    }
1601
1602    fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
1603        let split = Split::new(arguments);
1604        let background = split.background() || split.review.is_some();
1605        let mut summary = self
1606            .inner
1607            .approval_summary_for(&split.arguments, background)?;
1608        if let Some(review) = &split.review {
1609            let plan = self.plan(&split, review)?;
1610            summary.push_str(
1611                " Runs in the background: the call returns at once and the result is \
1612                 reported when the review ends.",
1613            );
1614            summary.push_str(&review::approval_summary(&plan, &self.inner)?);
1615        } else if split.background() {
1616            summary.push_str(
1617                " Runs in the background: the call returns at once and the result is \
1618                 reported when the agent finishes.",
1619            );
1620        }
1621        Ok(summary)
1622    }
1623
1624    async fn execute(
1625        &self,
1626        arguments: Value,
1627        context: ToolContext,
1628    ) -> Result<ToolOutput, ToolError> {
1629        let split = Split::new(&arguments);
1630        if let Some(review) = &split.review {
1631            return self.start_reviewed(&split, review, &context).await;
1632        }
1633        let background = split.background();
1634        let mut arguments = split.arguments;
1635        self.inner.route(&arguments)?;
1636        // Validate before returning a job handle, so a bad call fails now,
1637        // and as the call was made: a foreground call queued behind a busy
1638        // conversation runs as a job but may not set `no_timeout`.
1639        self.inner.risk_for(&arguments, background)?;
1640        // A background call's worktree is created before its job starts,
1641        // so the job's record and start name it. A foreground call's is
1642        // the agent tool's.
1643        let worktree = if background && parse_args::<AgentArgs>(&arguments)?.worktree {
1644            self.jobs.has_room()?;
1645            self.inner
1646                .prepare_worktree(&mut arguments, &context)
1647                .await?
1648        } else {
1649            None
1650        };
1651        let (agent, routed) = self.inner.route(&arguments)?;
1652        let Some(handle) = parse_args::<AgentArgs>(&routed)?
1653            .session
1654            .filter(|session| conversation::is_handle(session))
1655        else {
1656            return if background {
1657                let started = self.start_as(
1658                    agent,
1659                    arguments,
1660                    &context,
1661                    Turn::New,
1662                    JobMeta {
1663                        origin: JobOrigin::Model,
1664                        worktree,
1665                        resumed_from: None,
1666                    },
1667                )?;
1668                Ok(ToolOutput::success(started.to_string()))
1669            } else {
1670                self.inner.execute(arguments, context).await
1671            };
1672        };
1673        // Taken before anything runs or waits, so calls keep their order.
1674        let place = self.jobs.lanes.join(&handle);
1675        if place.ahead == 0 && !agent.backend.busy(&routed)? {
1676            return if background {
1677                self.start(agent, arguments, &context, Turn::Next(place))
1678            } else {
1679                // Held until the turn ends, so a later call finds it busy.
1680                let output = self.inner.execute(arguments, context).await;
1681                drop(place);
1682                output
1683            };
1684        }
1685        let behavior = match agent.on_busy(&routed)? {
1686            // A turn the owner's takeover is about to stop takes nothing.
1687            BusyBehavior::Steer => {
1688                if !self.jobs.lanes.has_takeover(&handle)
1689                    && let Some(output) = agent.backend.steer(routed, context.clone()).await?
1690                {
1691                    return Ok(output);
1692                }
1693                agent.busy.fallback()
1694            }
1695            behavior => behavior,
1696        };
1697        match behavior {
1698            // A background call cannot hold its caller, so it queues instead.
1699            BusyBehavior::Wait if !background => {
1700                place.wait_first(&context.cancellation).await?;
1701                let output = self.inner.execute_when_idle(arguments, context).await;
1702                drop(place);
1703                output
1704            }
1705            BusyBehavior::Fail => Err(busy(&handle, &place)),
1706            _ => {
1707                queue_room(agent, &handle, &place)?;
1708                self.start(agent, arguments, &context, Turn::Queued(place))
1709            }
1710        }
1711    }
1712}
1713
1714/// The `session busy` error for a call to conversation `handle` that found
1715/// a turn running, or calls waiting, at `place`.
1716fn busy(handle: &str, place: &Place) -> ToolError {
1717    // Calls ahead of this one other than the running turn.
1718    let waiting = place.ahead.saturating_sub(1);
1719    ToolError::failed(if waiting == 0 {
1720        format!("session busy: conversation {handle} is still running a turn")
1721    } else {
1722        format!(
1723            "session busy: conversation {handle} is still running a turn, and {waiting} more \
1724             prompts wait for it"
1725        )
1726    })
1727}
1728
1729/// Refuse a prompt queued at `place` beyond `agent.max_queued_turns`.
1730fn queue_room(agent: &Offered, handle: &str, place: &Place) -> Result<(), ToolError> {
1731    let waiting = place.ahead.saturating_sub(1);
1732    if waiting >= agent.busy.max_queued_turns {
1733        return Err(ToolError::limit(format!(
1734            "session busy: conversation {handle} is running a turn and {waiting} prompts \
1735             already wait for it, of the {} agent.max_queued_turns allows. Send this one \
1736             once they have run, or stop one with agent_cancel.",
1737            agent.busy.max_queued_turns
1738        )));
1739    }
1740    Ok(())
1741}
1742
1743impl BackgroundCapable {
1744    /// `inner` with `jobs` and a store of its own, without reviews: for
1745    /// tests of plain background calls.
1746    #[cfg(test)]
1747    pub(crate) fn plain(inner: Arc<AgentTool>, jobs: Arc<BackgroundJobs>) -> Self {
1748        Self {
1749            inner,
1750            jobs,
1751            conversations: Arc::new(ConversationStore::new(
1752                conversation::ConversationLimits {
1753                    max: 8,
1754                    idle: Duration::from_secs(86400),
1755                },
1756                None,
1757            )),
1758            reviews: None,
1759            session: None,
1760        }
1761    }
1762
1763    /// Whether calls may set `review`: background jobs exist (this tool),
1764    /// the session keeps journals, and an offered agent can continue a
1765    /// conversation.
1766    fn offers_review(&self) -> bool {
1767        self.reviews.is_some() && !self.inner.continuing().is_empty()
1768    }
1769
1770    /// Check a reviewed call, before approval and again before it starts.
1771    fn plan(&self, split: &Split, review: &Value) -> Result<review::Plan, ToolError> {
1772        if !self.offers_review() {
1773            return Err(ToolError::invalid_arguments(
1774                "review is not available in this session",
1775            ));
1776        }
1777        if split.background == Some(false) {
1778            return Err(ToolError::invalid_arguments(
1779                "a reviewed call runs as a background job; omit background or set it to true",
1780            ));
1781        }
1782        let (builder, routed) = self.inner.route(&split.arguments)?;
1783        builder.backend.risk(&routed)?;
1784        review::plan(
1785            &self.inner,
1786            builder,
1787            &routed,
1788            review,
1789            self.conversations.limits().max,
1790        )
1791    }
1792
1793    /// Start a reviewed call as a background job. A busy builder
1794    /// conversation queues it, or fails it with `on_busy: fail`; it never
1795    /// waits, since it is a background job, and never steers, since its
1796    /// prompt opens a review of its own.
1797    async fn start_reviewed(
1798        &self,
1799        split: &Split,
1800        review: &Value,
1801        context: &ToolContext,
1802    ) -> Result<ToolOutput, ToolError> {
1803        let mut plan = self.plan(split, review)?;
1804        let mut arguments = split.arguments.clone();
1805        self.inner.risk_for(&arguments, true)?;
1806        // The builder and every reviewer work in the same worktree.
1807        let worktree = if parse_args::<AgentArgs>(&arguments)?.worktree {
1808            self.jobs.has_room()?;
1809            let worktree = self.inner.prepare_worktree(&mut arguments, context).await?;
1810            plan.cwd = arguments
1811                .get("cwd")
1812                .and_then(Value::as_str)
1813                .map(str::to_owned);
1814            worktree
1815        } else {
1816            None
1817        };
1818        let (agent, routed) = self.inner.route(&arguments)?;
1819        let turn = match parse_args::<AgentArgs>(&routed)?
1820            .session
1821            .filter(|session| conversation::is_handle(session))
1822        {
1823            None => Turn::New,
1824            Some(handle) => {
1825                let place = self.jobs.lanes.join(&handle);
1826                if place.ahead == 0 && !agent.backend.busy(&routed)? {
1827                    Turn::Next(place)
1828                } else if agent.on_busy(&routed)? == BusyBehavior::Fail {
1829                    return Err(busy(&handle, &place));
1830                } else {
1831                    queue_room(agent, &handle, &place)?;
1832                    Turn::Queued(place)
1833                }
1834            }
1835        };
1836        let dir = self.reviews.as_deref().ok_or_else(|| {
1837            ToolError::invalid_arguments("review is not available in this session")
1838        })?;
1839        let reviewers: Vec<String> = plan
1840            .reviewer_names()
1841            .into_iter()
1842            .map(str::to_owned)
1843            .collect();
1844        let (rounds, land) = (plan.rounds, plan.land);
1845        let tool = Arc::new(ReviewedTool::new(
1846            Arc::clone(&self.inner),
1847            Arc::clone(&self.jobs.lanes),
1848            Arc::clone(&self.conversations),
1849            plan,
1850            arguments.clone(),
1851            self.session.clone(),
1852            dir,
1853        )?);
1854        let queued = matches!(turn, Turn::Queued(_));
1855        let mut started = self.jobs.start(
1856            Arc::clone(&tool) as Arc<dyn Tool>,
1857            AGENT_TOOL,
1858            &agent.name,
1859            arguments,
1860            context,
1861            turn,
1862            Some(&tool),
1863            JobMeta {
1864                origin: JobOrigin::Model,
1865                worktree,
1866                resumed_from: None,
1867            },
1868        )?;
1869        let first = reviewers.first().cloned().unwrap_or_default();
1870        started["review"] = json!({
1871            "reviewers": reviewers,
1872            "rounds": rounds,
1873            "land": review::land_name(land),
1874            "journal": tool.journal_id(),
1875        });
1876        started["note"] = format!(
1877            "{}The agent works on it, then a fresh {first} reviews it (or the next reviewer \
1878             available), for up to {rounds} rounds. SCV reports the result, with its own Review \
1879             and Landing lines, in a new turn when the review ends. agent_status shows the \
1880             round and phase; agent_cancel stops the whole review.",
1881            if queued {
1882                "The conversation is busy, so this reviewed call is queued behind the turns \
1883                 ahead of it. "
1884            } else {
1885                ""
1886            }
1887        )
1888        .into();
1889        Ok(ToolOutput::success(started.to_string()))
1890    }
1891
1892    /// Start the call on `agent` as a background job of this session.
1893    fn start(
1894        &self,
1895        agent: &Offered,
1896        mut arguments: Value,
1897        context: &ToolContext,
1898        turn: Turn,
1899    ) -> Result<ToolOutput, ToolError> {
1900        let meta = self.queued_meta(&mut arguments, &turn);
1901        let started = self.start_as(agent, arguments, context, turn, meta)?;
1902        Ok(ToolOutput::success(started.to_string()))
1903    }
1904
1905    /// A queued follow-up has no run of its own. Remember the conversation's
1906    /// directory and SCV worktree on the job, so a later resume still has
1907    /// them after the job that run served was dismissed and the worktree
1908    /// removed.
1909    fn queued_meta(&self, arguments: &mut Value, turn: &Turn) -> JobMeta {
1910        if !matches!(turn, Turn::Queued(_)) {
1911            return JobMeta::default();
1912        }
1913        let Some(handle) = parse_args::<AgentArgs>(arguments)
1914            .ok()
1915            .and_then(|args| args.session)
1916            .filter(|session| conversation::is_handle(session))
1917        else {
1918            return JobMeta::default();
1919        };
1920        let Some(cwd) = self.conversations.cwd(&handle) else {
1921            return JobMeta::default();
1922        };
1923        if arguments
1924            .get("cwd")
1925            .and_then(Value::as_str)
1926            .is_none_or(str::is_empty)
1927            && let Some(object) = arguments.as_object_mut()
1928        {
1929            object.insert("cwd".into(), cwd.display().to_string().into());
1930        }
1931        JobMeta {
1932            worktree: self.inner.worktree_at(&cwd),
1933            ..JobMeta::default()
1934        }
1935    }
1936
1937    /// Start the call on `agent` as a background job `meta` describes.
1938    fn start_as(
1939        &self,
1940        agent: &Offered,
1941        arguments: Value,
1942        context: &ToolContext,
1943        turn: Turn,
1944        meta: JobMeta,
1945    ) -> Result<Value, ToolError> {
1946        let tool: Arc<dyn Tool> = if matches!(turn, Turn::Queued(_)) {
1947            Arc::new(QueuedTool(Arc::clone(&self.inner)))
1948        } else {
1949            Arc::clone(&self.inner) as Arc<dyn Tool>
1950        };
1951        self.jobs.start(
1952            tool,
1953            AGENT_TOOL,
1954            &agent.name,
1955            arguments,
1956            context,
1957            turn,
1958            None,
1959            meta,
1960        )
1961    }
1962
1963    /// Run the owner's call from the workbench as a background job of this
1964    /// session: no approval request, since it is the owner's own action,
1965    /// and no report turn, since the owner watches it. A call continuing a
1966    /// conversation that runs a turn steers it when `mode` allows and the
1967    /// agent can take the prompt, and otherwise queues behind it. `meta`'s
1968    /// worktree is recorded when the call makes none of its own, and its
1969    /// `resumed_from` as given; it is always the owner's.
1970    pub(crate) async fn owner_call(
1971        &self,
1972        mut arguments: Value,
1973        workspace: std::path::PathBuf,
1974        mode: scv_protocol::SendMode,
1975        meta: JobMeta,
1976    ) -> Result<OwnerCall, ToolError> {
1977        use scv_protocol::SendMode;
1978        let context = ToolContext::new(workspace, CancellationToken::new());
1979        self.inner.route(&arguments)?;
1980        self.inner.risk_for(&arguments, true)?;
1981        let worktree = if parse_args::<AgentArgs>(&arguments)?.worktree {
1982            self.jobs.has_room()?;
1983            self.inner
1984                .prepare_worktree(&mut arguments, &context)
1985                .await?
1986        } else {
1987            None
1988        };
1989        let (agent, routed) = self.inner.route(&arguments)?;
1990        let meta = JobMeta {
1991            origin: JobOrigin::Owner,
1992            worktree: worktree.or(meta.worktree),
1993            resumed_from: meta.resumed_from,
1994        };
1995        let Some(handle) = parse_args::<AgentArgs>(&routed)?
1996            .session
1997            .filter(|session| conversation::is_handle(session))
1998        else {
1999            return self
2000                .start_as(agent, arguments, &context, Turn::New, meta)
2001                .map(OwnerCall::Job);
2002        };
2003        let mut meta = meta;
2004        if let Some(cwd) = self.conversations.cwd(&handle) {
2005            if arguments.get("cwd").and_then(Value::as_str).is_none()
2006                && let Some(object) = arguments.as_object_mut()
2007            {
2008                object.insert("cwd".into(), cwd.display().to_string().into());
2009            }
2010            if meta.worktree.is_none() {
2011                meta.worktree = self.inner.worktree_at(&cwd);
2012            }
2013        }
2014        let place = self.jobs.lanes.join(&handle);
2015        let busy = agent.backend.busy(&routed)?;
2016        if !busy && !self.conversations.knows(&handle) {
2017            return Err(ToolError::invalid_arguments(format!(
2018                "no conversation {handle} in this session; it ended, or it was forgotten after it sat idle"
2019            )));
2020        }
2021        if place.ahead == 0 && !busy {
2022            return self
2023                .start_as(agent, arguments, &context, Turn::Next(place), meta)
2024                .map(OwnerCall::Job);
2025        }
2026        if mode != SendMode::Queue
2027            && !self.jobs.lanes.has_takeover(&handle)
2028            && agent
2029                .backend
2030                .steer(routed.clone(), context.clone())
2031                .await?
2032                .is_some()
2033        {
2034            return Ok(OwnerCall::Steered);
2035        }
2036        if mode == SendMode::Steer {
2037            return Err(ToolError::failed(format!(
2038                "conversation {handle} cannot take a prompt into its running turn (its agent \
2039                 does not steer, or no turn is running yet); send it queued instead"
2040            )));
2041        }
2042        queue_room(agent, &handle, &place)?;
2043        self.start_as(agent, arguments, &context, Turn::Queued(place), meta)
2044            .map(OwnerCall::Job)
2045    }
2046
2047    /// Whether `agent` is offered and takes `option` (`model`, `effort`,
2048    /// or `session`).
2049    pub(crate) fn takes(&self, agent: &str, option: &str) -> bool {
2050        self.inner.find(agent).is_some_and(|offered| match option {
2051            "model" => offered.accepts.model,
2052            "effort" => offered.accepts.effort,
2053            "session" => offered.accepts.session,
2054            _ => false,
2055        })
2056    }
2057
2058    /// Whether the session has conversation `handle`, so the owner can send
2059    /// it a prompt.
2060    pub(crate) fn knows(&self, handle: &str) -> bool {
2061        self.conversations.knows(handle)
2062    }
2063
2064    /// Take in conversation `vendor` that `agent` saved, so a call can
2065    /// continue it in `cwd`, when that agent can continue a conversation
2066    /// from its own session ID alone. Returns the handle, which is the one
2067    /// this session already holds for that agent, session, and directory.
2068    pub(crate) fn adopt(&self, agent: &str, cwd: &std::path::Path, vendor: &str) -> Option<String> {
2069        let offered = self.inner.find(agent)?;
2070        if !offered.accepts.session || !offered.backend.adoptable() {
2071            return None;
2072        }
2073        self.conversations.adopt(agent, cwd, vendor).ok()
2074    }
2075
2076    /// Take conversation `handle` over in its agent's own terminal for the
2077    /// owner, sized `size`, as SCV session `session`: refused, with why,
2078    /// when the agent's terminal may not be offered, or the conversation
2079    /// runs a turn, has turns waiting, or has no agent session to open.
2080    /// Calls that arrive meanwhile queue behind the takeover in its lane.
2081    pub(crate) async fn take_over(
2082        &self,
2083        handle: &str,
2084        session: &str,
2085        size: (u16, u16),
2086    ) -> Result<Arc<takeover::Takeover>, String> {
2087        let (agent, launch) = self.launch_for(handle)?;
2088        let place = self.jobs.lanes.join_takeover(handle);
2089        if place.ahead > 0 {
2090            return Err(waiting(handle));
2091        }
2092        self.open_terminal(&agent, &launch, handle, place, session, size)
2093            .await
2094    }
2095
2096    /// Hold conversation `handle`'s lane for the owner's takeover right
2097    /// behind its running turn, ahead of any prompt queued after it, and
2098    /// stop nothing yet: [`Self::take_over_reserved`] stops the background
2099    /// job running that turn. Refused, with why and with nothing held, as
2100    /// [`Self::take_over`] refuses but for the running turn, when that turn
2101    /// is no background job (a model's call, or a reviewed job's), and when
2102    /// no turn runs but a job's call is first in the lane: one about to
2103    /// start its turn (a queued prompt), or one whose turn just ended.
2104    pub(crate) fn reserve_takeover(&self, handle: &str) -> Result<Reserved, String> {
2105        let (agent, launch) = self.launch_for(handle)?;
2106        let obstacles = self
2107            .conversations
2108            .obstacles(handle)
2109            .ok_or_else(|| conversation::unknown_conversation(handle))?;
2110        if let Some(obstacle) = obstacles
2111            .into_iter()
2112            .find(|obstacle| *obstacle != Obstacle::Running)
2113        {
2114            return Err(obstacle.refusal(handle));
2115        }
2116        let (place, job) = self
2117            .jobs
2118            .lanes
2119            .reserve_takeover(handle)
2120            .map_err(|why| match why {
2121                Unreserved::TakenOver => Obstacle::TakenOver.refusal(handle),
2122                Unreserved::NotAJob => not_a_job(handle),
2123            })?;
2124        // Seen with the place held, so the turn is that of the call ahead
2125        // of it: no prompt queued behind can start one in between.
2126        let running = self.conversations.is_busy(handle);
2127        match &job {
2128            // A turn that holds no place in the lane is no job's to stop.
2129            None if running => return Err(not_a_job(handle)),
2130            Some(job) if !running => return Err(between_turns(handle, job)),
2131            _ => {}
2132        }
2133        Ok(Reserved {
2134            handle: handle.to_owned(),
2135            agent,
2136            launch,
2137            place,
2138            job,
2139        })
2140    }
2141
2142    /// Take the conversation `reserved` holds the lane of over for the
2143    /// owner: stop the background job running its turn, wait until that
2144    /// turn has ended and the takeover is first in the lane, and open the
2145    /// terminal. However it fails, the lane is let go and the conversation
2146    /// runs its queued prompts as before.
2147    pub(crate) async fn take_over_reserved(
2148        &self,
2149        reserved: Reserved,
2150        session: &str,
2151        size: (u16, u16),
2152    ) -> Result<Arc<takeover::Takeover>, String> {
2153        let Reserved {
2154            handle,
2155            agent,
2156            launch,
2157            place,
2158            job,
2159        } = reserved;
2160        let Some(job) = job else {
2161            return self
2162                .open_terminal(&agent, &launch, &handle, place, session, size)
2163                .await;
2164        };
2165        // One that finished meanwhile has let its turn go already.
2166        let _ = self.jobs.cancel_for_owner(&job);
2167        let never = CancellationToken::new();
2168        let ended = tokio::time::timeout(INTERRUPT_WAIT, async {
2169            place.wait_first(&never).await?;
2170            self.conversations.wait_idle(&handle, &never).await
2171        })
2172        .await;
2173        if ended.is_err() {
2174            return Err(format!(
2175                "Stopped {job}, but its turn had not ended after {} seconds, so {handle} was not \
2176                 taken over; attach once it is idle",
2177                INTERRUPT_WAIT.as_secs()
2178            ));
2179        }
2180        self.open_terminal(&agent, &launch, &handle, place, session, size)
2181            .await
2182            .map_err(|error| {
2183                format!(
2184                    "Stopped {job}, but {handle} could not be taken over: {error}. Its next \
2185                     prompt runs as before"
2186                )
2187            })
2188    }
2189
2190    /// Whether the owner could take conversation `handle` over in its
2191    /// agent's own terminal now, as [`Self::take_over`] and
2192    /// [`Self::reserve_takeover`] would decide; `None` when the session does
2193    /// not have it. The workbench adds what only it knows: how many
2194    /// takeovers run, and approvals waiting for the owner.
2195    pub(crate) fn attach_offer(&self, handle: &str) -> Option<AttachOffer> {
2196        let obstacles = self.conversations.obstacles(handle)?;
2197        let offer = |state, detail: Option<String>| AttachOffer {
2198            state,
2199            detail,
2200            interrupt: false,
2201            job: None,
2202        };
2203        if let Err(why) = self.launch_for(handle) {
2204            return Some(offer(AttachState::Ineligible, Some(why)));
2205        }
2206        if obstacles.contains(&Obstacle::TakenOver) || self.jobs.lanes.has_takeover(handle) {
2207            return Some(offer(AttachState::TakenOver, None));
2208        }
2209        if let Some(obstacle) = obstacles
2210            .iter()
2211            .find(|obstacle| **obstacle != Obstacle::Running)
2212        {
2213            return Some(offer(obstacle.state(), Some(obstacle.refusal(handle))));
2214        }
2215        let running = obstacles.contains(&Obstacle::Running);
2216        Some(match self.jobs.lanes.front(handle) {
2217            None if !running => offer(AttachState::Ready, None),
2218            None => offer(AttachState::Busy, Some(Obstacle::Running.refusal(handle))),
2219            Some(Front::Job(job)) if running => AttachOffer {
2220                interrupt: true,
2221                job: Some(job),
2222                ..offer(AttachState::Busy, Some(waiting(handle)))
2223            },
2224            Some(Front::Job(job)) => offer(AttachState::Busy, Some(between_turns(handle, &job))),
2225            Some(Front::Call | Front::Takeover) => {
2226                offer(AttachState::Busy, Some(not_a_job(handle)))
2227            }
2228        })
2229    }
2230
2231    /// The agent of conversation `handle` and how its terminal opens, or
2232    /// why it is never offered.
2233    fn launch_for(&self, handle: &str) -> Result<(String, takeover::TakeoverLaunch), String> {
2234        let agent = self
2235            .conversations
2236            .agent_of(handle)
2237            .ok_or_else(|| conversation::unknown_conversation(handle))?;
2238        let offered = self
2239            .inner
2240            .find(&agent)
2241            .ok_or_else(|| format!("{agent} is not offered in this session"))?;
2242        let launch = offered.backend.takeover()?;
2243        Ok((agent, launch))
2244    }
2245
2246    /// Hold conversation `handle`, whose lane `place` holds, stop what it
2247    /// kept between turns, and open `launch`'s terminal on it.
2248    async fn open_terminal(
2249        &self,
2250        agent: &str,
2251        launch: &takeover::TakeoverLaunch,
2252        handle: &str,
2253        place: Place,
2254        session: &str,
2255        size: (u16, u16),
2256    ) -> Result<Arc<takeover::Takeover>, String> {
2257        let mut hold = self.conversations.take_over(handle)?;
2258        if let Some(attachment) = hold.attachment.take() {
2259            match self.inner.find(agent) {
2260                Some(offered) => offered.backend.release(attachment).await,
2261                None => drop(attachment),
2262            }
2263        }
2264        takeover::start(launch, hold, place, session, size).await
2265    }
2266}
2267
2268/// The owner's takeover of a conversation whose lane is held right behind
2269/// its running turn, which is not stopped yet.
2270pub(crate) struct Reserved {
2271    handle: String,
2272    agent: String,
2273    launch: takeover::TakeoverLaunch,
2274    place: Place,
2275    /// The background job running the conversation's turn, which taking
2276    /// it over stops; `None` when it runs none.
2277    pub(crate) job: Option<String>,
2278}
2279
2280impl std::fmt::Debug for Reserved {
2281    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2282        formatter
2283            .debug_struct("Reserved")
2284            .field("handle", &self.handle)
2285            .field("job", &self.job)
2286            .finish_non_exhaustive()
2287    }
2288}
2289
2290/// How long the owner's interrupting takeover waits for the turn it
2291/// stopped to end before it gives the conversation's lane back.
2292const INTERRUPT_WAIT: Duration = Duration::from_secs(20);
2293
2294/// The refusal to take conversation `handle` over while calls are in its
2295/// lane.
2296fn waiting(handle: &str) -> String {
2297    format!(
2298        "{handle} has a turn running or waiting; attach once it is idle, or stop its jobs first"
2299    )
2300}
2301
2302/// The refusal to stop conversation `handle`'s running turn for a takeover
2303/// when that turn is no background job.
2304fn not_a_job(handle: &str) -> String {
2305    format!(
2306        "{handle} is running a turn that is no background job SCV can stop for you (a model's \
2307         call in its session, or a reviewed job's round); attach once it is idle"
2308    )
2309}
2310
2311/// The refusal to stop `job`, first in conversation `handle`'s lane, for a
2312/// takeover while no turn runs: its turn is about to start or just ended,
2313/// and a prompt that waits is never stopped.
2314fn between_turns(handle: &str, job: &str) -> String {
2315    format!(
2316        "{handle} is between turns: {job} is about to start the next one, or has just ended \
2317         its own, and only a running turn is stopped for a takeover; attach again in a moment"
2318    )
2319}
2320
2321#[derive(Deserialize)]
2322#[serde(deny_unknown_fields)]
2323struct WaitArgs {
2324    job: String,
2325    timeout_seconds: Option<u64>,
2326}
2327
2328#[derive(Deserialize)]
2329#[serde(deny_unknown_fields)]
2330struct StatusArgs {
2331    job: Option<String>,
2332}
2333
2334/// `agent_wait`: block until a background job finishes, or the timeout.
2335pub(crate) struct WaitTool {
2336    pub(crate) jobs: Arc<BackgroundJobs>,
2337    pub(crate) timeouts: Timeouts,
2338}
2339
2340#[async_trait]
2341impl Tool for WaitTool {
2342    fn spec(&self) -> ToolSpec {
2343        let mut timeout = timeout_schema(self.timeouts);
2344        timeout["description"] = format!(
2345            "Seconds to wait before returning the job still running. Defaults to {}; at most {}.",
2346            self.timeouts.default.min(self.timeouts.max).as_secs(),
2347            self.timeouts.max.as_secs()
2348        )
2349        .into();
2350        ToolSpec {
2351            name: "agent_wait".into(),
2352            description: "Wait for a background agent job (the `job` handle an agent call with \
2353                background: true returned) to finish, and return its result. Returns early with \
2354                status running when the timeout passes. Waiting holds your turn open, so the \
2355                user cannot reach you meanwhile; usually let SCV report the result instead."
2356                .into(),
2357            parameters: json!({
2358                "type":"object",
2359                "properties":{"job":{"type":"string"},"timeout_seconds":timeout},
2360                "required":["job"],
2361                "additionalProperties":false
2362            }),
2363        }
2364    }
2365
2366    fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
2367        let args: WaitArgs = parse_args(arguments)?;
2368        self.timeouts.resolve(args.timeout_seconds)?;
2369        Ok(ToolRisk::ReadOnly)
2370    }
2371
2372    fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
2373        let args: WaitArgs = parse_args(arguments)?;
2374        Ok(format!(
2375            "Wait for background job {}",
2376            bounded(&args.job, 64)
2377        ))
2378    }
2379
2380    async fn execute(
2381        &self,
2382        arguments: Value,
2383        context: ToolContext,
2384    ) -> Result<ToolOutput, ToolError> {
2385        let args: WaitArgs = parse_args(&arguments)?;
2386        let limit = self.timeouts.resolve(args.timeout_seconds)?;
2387        let value = self
2388            .jobs
2389            .wait(&args.job, limit, &context.cancellation, &context.call_id)
2390            .await?;
2391        Ok(ToolOutput::success(value.to_string()))
2392    }
2393}
2394
2395/// `agent_status`: describe one background job or all of them.
2396pub(crate) struct StatusTool {
2397    pub(crate) jobs: Arc<BackgroundJobs>,
2398}
2399
2400#[async_trait]
2401impl Tool for StatusTool {
2402    fn spec(&self) -> ToolSpec {
2403        ToolSpec {
2404            name: "agent_status".into(),
2405            description: "Show this session's background agent jobs: running ones with their \
2406                latest progress, finished ones with their result. Pass job for one job."
2407                .into(),
2408            parameters: json!({
2409                "type":"object",
2410                "properties":{"job":{"type":"string"}},
2411                "additionalProperties":false
2412            }),
2413        }
2414    }
2415
2416    fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
2417        let _: StatusArgs = parse_args(arguments)?;
2418        Ok(ToolRisk::ReadOnly)
2419    }
2420
2421    fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
2422        let args: StatusArgs = parse_args(arguments)?;
2423        Ok(args.job.map_or_else(
2424            || "List background jobs".into(),
2425            |job| format!("Show background job {}", bounded(&job, 64)),
2426        ))
2427    }
2428
2429    async fn execute(
2430        &self,
2431        arguments: Value,
2432        context: ToolContext,
2433    ) -> Result<ToolOutput, ToolError> {
2434        let args: StatusArgs = parse_args(&arguments)?;
2435        let value = self.jobs.describe(args.job.as_deref(), &context.call_id)?;
2436        Ok(ToolOutput::success(value.to_string()))
2437    }
2438}
2439
2440#[derive(Deserialize)]
2441#[serde(deny_unknown_fields)]
2442struct CancelArgs {
2443    job: String,
2444}
2445
2446/// `agent_cancel`: stop one running background job.
2447pub(crate) struct CancelTool {
2448    pub(crate) jobs: Arc<BackgroundJobs>,
2449}
2450
2451#[async_trait]
2452impl Tool for CancelTool {
2453    fn spec(&self) -> ToolSpec {
2454        ToolSpec {
2455            name: "agent_cancel".into(),
2456            description: "Stop a running background agent job (the `job` handle an agent call \
2457                with background: true returned), for example when the user no longer wants \
2458                it. The agent and every process it started are stopped; work it already wrote \
2459                stays. Returns the job with status cancelled, and no report turn follows."
2460                .into(),
2461            parameters: json!({
2462                "type":"object",
2463                "properties":{"job":{"type":"string"}},
2464                "required":["job"],
2465                "additionalProperties":false
2466            }),
2467        }
2468    }
2469
2470    fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
2471        let _: CancelArgs = parse_args(arguments)?;
2472        Ok(ToolRisk::Process)
2473    }
2474
2475    fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
2476        let args: CancelArgs = parse_args(arguments)?;
2477        Ok(format!("Stop background job {}", bounded(&args.job, 64)))
2478    }
2479
2480    async fn execute(
2481        &self,
2482        arguments: Value,
2483        context: ToolContext,
2484    ) -> Result<ToolOutput, ToolError> {
2485        let args: CancelArgs = parse_args(&arguments)?;
2486        let value = self.jobs.cancel(&args.job, &context.call_id).await?;
2487        Ok(ToolOutput::success(value.to_string()))
2488    }
2489}
2490
2491#[cfg(test)]
2492mod tests;