Skip to main content

scv_tools/delegate/
background.rs

1//! Background delegations: an `agent` call with `background: true` returns a
2//! job handle at once while the agent keeps working; `agent_status` and
3//! `agent_wait` observe the job, `agent_cancel` stops it, and the session is
4//! told when it finishes so the server can report it in a turn of its own.
5//!
6//! Each call that starts a job, or shows the model a job's result, leaves a
7//! [`JobChange`] under its call ID, which the server hands to the session's
8//! clients with the call's `tool.completed` ([`BackgroundJobs::take_changes`]).
9//!
10//! A finished job stays unreported until a report turn about it succeeds. A
11//! failed one makes it due again after a delay, a bounded number of times;
12//! after that, or when another turn must not be tried, the server reports it
13//! to the client directly ([`BackgroundJobs::report_failed`]).
14//!
15//! A call that continues a conversation takes its place in that
16//! conversation's lane as it arrives. When a turn or another call is
17//! ahead, the call's busy policy decides: it steers the running turn, waits
18//! in the foreground, fails, or becomes a queued job that runs once its
19//! place comes up.
20//!
21//! A call with `review` runs as a reviewed job ([`review`](super::review)):
22//! the same job, whose one call runs the whole builder and reviewer loop and
23//! whose changes and report carry SCV's outcome.
24//!
25//! A chat thread can keep a [`subscription`] to one job that already exists.
26//! Later owner messages on that thread are input to that job. The record sits
27//! beside the ledger, and a restart reloads it without resuming the job.
28
29pub(in crate::delegate) mod lane;
30pub mod ledger;
31mod owed;
32pub mod subscription;
33
34use std::{
35    collections::HashSet,
36    path::PathBuf,
37    sync::{Arc, Mutex, Weak},
38    time::{Duration, Instant},
39};
40
41use async_trait::async_trait;
42use scv_core::{
43    ApprovalGate, ProgressSink, Tool, ToolApprovals, ToolContext, ToolError, ToolOutput, ToolRisk,
44    ToolSpec,
45};
46use scv_protocol::{
47    AgentState, AttachOffer, AttachState, JobChange, JobOrigin, JobOutcome, JobReport, JobStatus,
48    describe_reports,
49};
50use serde::Deserialize;
51use serde_json::{Map, Value, json};
52use tokio::sync::{mpsc, watch};
53use tokio_util::sync::CancellationToken;
54
55use self::{
56    lane::{Front, Lanes, Place, Unreserved},
57    ledger::{JobLedger, JobRecord, NewJob},
58};
59use crate::{
60    BusyBehavior,
61    args::{Timeouts, bounded, parse_args, timeout_schema},
62    delegate::{
63        agent::{AGENT_TOOL, AgentTool, Offered},
64        conversation::{self, ConversationStore, Obstacle},
65        output::AgentReply,
66        request::AgentArgs,
67        review::{self, Record, ReviewedTool},
68        takeover,
69    },
70    sync::lock,
71};
72
73/// Finished jobs a session keeps for `agent_status` beyond the running ones.
74const MAX_FINISHED: usize = 16;
75/// Characters of a finished job's reply quoted in a server-started report turn.
76const REPORT_REPLY_CHARS: usize = 6000;
77/// Jobs one report turn covers; any more wait for the next.
78const REPORT_MAX_JOBS: usize = 4;
79/// How long after each failed report turn the next one is due: after the
80/// first failure, then after the second. Once they run out, the job is
81/// reported directly.
82const REPORT_RETRY_DELAYS: [Duration; 2] = [Duration::from_secs(30), Duration::from_secs(120)];
83/// How long `agent_cancel` waits for a stopped job to settle.
84const CANCEL_SETTLE: Duration = Duration::from_secs(10);
85/// Job changes kept for calls whose `tool.completed` has not taken them,
86/// such as a call whose turn was aborted; the oldest go first.
87const MAX_PENDING_CHANGES: usize = 64;
88/// Characters of a delegated prompt's first line kept as its job's task.
89const TASK_CHARS: usize = 80;
90
91/// One session's background jobs. Dropping the store (with the session's
92/// tools) cancels every job still running.
93pub struct BackgroundJobs {
94    limit: usize,
95    state: Mutex<JobsState>,
96    cancellation: CancellationToken,
97    /// Woken whenever a job finishes, so the session can report it. Absent
98    /// on a workbench session, which runs no report turn.
99    finished: Option<mpsc::UnboundedSender<()>>,
100    /// Decides a running job's nested approval requests, since no turn is
101    /// left to carry them to a person. Without it they are denied.
102    approvals: Option<Arc<dyn ApprovalGate>>,
103    /// The `agent` calls continuing each conversation, in arrival order.
104    lanes: Arc<Lanes>,
105    /// Where the session's jobs are recorded durably, when it keeps them.
106    ledger: Option<JobLedger>,
107}
108
109impl std::fmt::Debug for BackgroundJobs {
110    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
111        formatter
112            .debug_struct("BackgroundJobs")
113            .field("limit", &self.limit)
114            .finish_non_exhaustive()
115    }
116}
117
118#[derive(Default)]
119struct JobsState {
120    next: u64,
121    jobs: Vec<Job>,
122    /// Changes by the ID of the call that made them, oldest first.
123    changes: Vec<(String, JobChange)>,
124    /// The final outcomes of cancelled reviewed jobs the session is owed.
125    owed: owed::Owed,
126}
127
128impl JobsState {
129    /// Bring the outcome in the change the call `call_id` made for
130    /// `outcome`'s job up to date, if no client has taken it yet.
131    fn refresh(&mut self, call_id: &str, outcome: JobOutcome) {
132        if let Some((_, change)) = self
133            .changes
134            .iter_mut()
135            .rev()
136            .find(|(call, change)| call == call_id && change.job == outcome.job)
137        {
138            change.outcome = Some(outcome);
139        }
140    }
141
142    /// Keep `change` for the call `call_id`, which a client learns of with
143    /// that call's `tool.completed`. A call without an ID has no event.
144    fn record(&mut self, call_id: &str, change: JobChange) {
145        if call_id.is_empty() {
146            return;
147        }
148        if self.changes.len() >= MAX_PENDING_CHANGES {
149            let (call, dropped) = self.changes.remove(0);
150            self.owed.evicted(&call, &dropped.job);
151        }
152        self.changes.push((call_id.to_owned(), change));
153    }
154}
155
156struct Job {
157    id: String,
158    /// The tool that started it, `agent`.
159    tool: String,
160    /// The agent that runs it, such as `codex`.
161    agent: String,
162    /// The first line of the delegated prompt, shortened.
163    task: String,
164    started: Instant,
165    progress: ProgressSink,
166    last_progress: Option<String>,
167    /// Stops this job alone.
168    cancel: CancellationToken,
169    /// `agent_cancel` stopped it.
170    cancelled: bool,
171    /// A busy conversation's queued prompt, which `agent.max_queued_turns`
172    /// bounds instead of `agent.max_background`.
173    queued: bool,
174    outcome: Option<Outcome>,
175    /// The model has seen the result (through `agent_wait`, `agent_status`,
176    /// or a report turn that completed), the user stopped its report turn,
177    /// the server reported it directly, or the model asked for the stop with
178    /// `agent_cancel`, so it needs no report turn.
179    reported: bool,
180    /// A report turn about it runs.
181    reporting: bool,
182    /// Report turns about it that failed.
183    report_attempts: u32,
184    /// After a failed report turn: when the next one is due.
185    report_due: Option<tokio::time::Instant>,
186    done: watch::Receiver<bool>,
187    /// A reviewed job's record, whose outcome its changes and report state.
188    review: Option<Arc<Record>>,
189    /// `agent_cancel` waits for it, and describes it after: never pruned
190    /// meanwhile.
191    cancelling: bool,
192    /// Who started it. The owner's own jobs are never reported to the
193    /// model: the owner watches them in the workbench.
194    origin: JobOrigin,
195    /// A queued prompt whose turn has not come yet.
196    queued_waiting: bool,
197    /// The conversation whose lane a queued prompt waits in.
198    lane: Option<String>,
199    /// The owner stopped it from the workbench, so the model was not told.
200    owner_cancelled: bool,
201    /// Its durable record, when the session keeps a ledger, and that
202    /// record's revision: each change bumps it, and it is written after the
203    /// job table's lock is let go ([`BackgroundJobs::persist`]).
204    record: Option<JobRecord>,
205    record_revision: u64,
206}
207
208/// A revision of a job's record to write once the job table is let go.
209type RecordWrite = (JobRecord, u64);
210
211impl Job {
212    /// The record as it is now, under a new revision, to write.
213    fn record_changed(&mut self) -> Option<RecordWrite> {
214        let record = self.record.as_ref()?;
215        self.record_revision += 1;
216        Some((record.clone(), self.record_revision))
217    }
218}
219
220/// `agent_cancel` waiting for a job to stop. Settled after the wait, or,
221/// if the call is dropped first, as when its turn is aborted, its change
222/// withdrawn and any outcome it owed left to an update.
223struct Waiting<'a> {
224    jobs: &'a BackgroundJobs,
225    job: String,
226    call: String,
227    /// The job is reviewed: the session is owed its final outcome.
228    owed: bool,
229    settled: bool,
230}
231
232impl Waiting<'_> {
233    /// The wait is over: the change the call made says the final outcome
234    /// if the job stopped, else the outcome as decided, its journal still
235    /// open; [`BackgroundJobs::take_changes`] says the final one if the job
236    /// stops before the change is sent.
237    fn settle(mut self) {
238        self.settled = true;
239        let mut state = self.jobs.state();
240        let Some(entry) = state.jobs.iter_mut().find(|entry| entry.id == self.job) else {
241            return;
242        };
243        entry.cancelling = false;
244        if !self.owed {
245            return;
246        }
247        let pending = entry.outcome.is_none().then(|| entry.outcome()).flatten();
248        let pending = pending.map(|mut outcome| {
249            outcome.review.journal_pending = true;
250            outcome
251        });
252        if let Some(outcome) = state.owed.waited(&self.job, &self.call, pending) {
253            state.refresh(&self.call, outcome);
254        }
255    }
256}
257
258impl Drop for Waiting<'_> {
259    fn drop(&mut self) {
260        if self.settled {
261            return;
262        }
263        let mut state = self.jobs.state();
264        if let Some(entry) = state.jobs.iter_mut().find(|entry| entry.id == self.job) {
265            entry.cancelling = false;
266        }
267        if !self.owed {
268            return;
269        }
270        // Its call is gone, so its change is never sent.
271        let (call, job) = (&self.call, &self.job);
272        state
273            .changes
274            .retain(|(recorded, change)| !(recorded == call && &change.job == job));
275        let due = state.owed.abandoned(job, call);
276        drop(state);
277        if due {
278            self.jobs.wake();
279        }
280    }
281}
282
283struct Outcome {
284    output: ToolOutput,
285    elapsed: Duration,
286}
287
288/// What a failed report turn leaves its jobs to.
289#[derive(Debug, PartialEq, Eq)]
290pub enum ReportFailure {
291    /// Another report turn about them is due after this long.
292    Retry(Duration),
293    /// No more report turns: the server reports them directly. `attempts`
294    /// counts the report turns that failed.
295    GiveUp {
296        attempts: u32,
297        reports: Vec<JobReport>,
298    },
299    /// The model saw each of them through a call during the turn.
300    Settled,
301}
302
303impl Drop for BackgroundJobs {
304    fn drop(&mut self) {
305        self.cancellation.cancel();
306        // Jobs still going stop with their session: say so durably, since
307        // their own end is never recorded once the store is gone.
308        if let Some(ledger) = &self.ledger {
309            let state = self
310                .state
311                .get_mut()
312                .unwrap_or_else(std::sync::PoisonError::into_inner);
313            for job in state.jobs.iter_mut().filter(|job| job.outcome.is_none()) {
314                if let Some(record) = job.record.as_mut() {
315                    ledger::interrupt(record, "its session ended before it finished");
316                    ledger.save(record);
317                }
318            }
319        }
320    }
321}
322
323impl BackgroundJobs {
324    /// At most `limit` jobs run at once. `finished` is woken as jobs finish.
325    /// A workbench session passes none: nothing takes its unreported jobs.
326    pub fn new(limit: usize, finished: Option<mpsc::UnboundedSender<()>>) -> Self {
327        Self {
328            limit,
329            state: Mutex::default(),
330            cancellation: CancellationToken::new(),
331            finished,
332            approvals: None,
333            lanes: Arc::default(),
334            ledger: None,
335        }
336    }
337
338    /// Record the session's jobs durably in `ledger`. Job IDs continue
339    /// after the ones it already holds.
340    #[must_use]
341    pub fn with_ledger(mut self, ledger: JobLedger) -> Self {
342        self.state().next = ledger.last_number();
343        self.ledger = Some(ledger);
344        self
345    }
346
347    /// Number new jobs after the `job-<n>` IDs `jobs` names too: the
348    /// session's runs name jobs whose records may be gone.
349    pub(crate) fn continue_after<'a>(&self, jobs: impl IntoIterator<Item = &'a str>) {
350        let mut state = self.state();
351        for job in jobs {
352            if let Some(number) = job
353                .strip_prefix("job-")
354                .and_then(|number| number.parse::<u64>().ok())
355            {
356                state.next = state.next.max(number);
357            }
358        }
359    }
360
361    /// Decide running jobs' nested approval requests with `gate`, which must
362    /// never grant more than the session's foreground would.
363    #[must_use]
364    pub fn with_approvals(mut self, gate: Arc<dyn ApprovalGate>) -> Self {
365        self.approvals = Some(gate);
366        self
367    }
368
369    pub(crate) fn limit(&self) -> usize {
370        self.limit
371    }
372
373    /// Whether a finished model job is taken by a report turn. A workbench
374    /// session has none, so such a job would stay in [`Self::pending`].
375    pub(crate) fn has_report_turn(&self) -> bool {
376        self.finished.is_some()
377    }
378
379    fn state(&self) -> std::sync::MutexGuard<'_, JobsState> {
380        lock(&self.state)
381    }
382
383    /// Start `tool` with `arguments` in the background for the call in
384    /// `context` (whose workspace and call ID it takes; the job has its own
385    /// cancellation), on `agent`, and return its job's
386    /// `{"job","agent","status":"running","background":true}` description.
387    /// A reviewed job's `review` records its start in its journal first, and
388    /// the call fails if it cannot. That write happens outside the job
389    /// table's lock, so a slow disk never holds up the session's other job
390    /// calls: the job's number is taken first, and a failed write leaves it
391    /// unused. A job that takes a subscription follow-up
392    /// ([`JobMeta::accepted_follow_up`]) writes its first record under that
393    /// lock instead, and the call fails, starting nothing, if the write
394    /// does. Every other job's record writes stay best effort.
395    #[allow(
396        clippy::too_many_arguments,
397        reason = "a job's call, its context, its turn, and its review, each its own"
398    )]
399    pub(in crate::delegate) fn start(
400        self: &Arc<Self>,
401        tool: Arc<dyn Tool>,
402        name: &str,
403        agent: &str,
404        arguments: Value,
405        context: &ToolContext,
406        turn: Turn,
407        review: Option<&Arc<ReviewedTool>>,
408        meta: JobMeta,
409    ) -> Result<Value, ToolError> {
410        let JobMeta {
411            origin,
412            worktree,
413            resumed_from,
414            accepted_follow_up,
415            keep_origin: _,
416        } = meta;
417        let task = task_line(
418            arguments
419                .get("prompt")
420                .and_then(Value::as_str)
421                .unwrap_or_default(),
422        );
423        let queued = matches!(turn, Turn::Queued(_));
424        let lane = match &turn {
425            Turn::Queued(place) => Some(place.handle().to_owned()),
426            Turn::New | Turn::Next(_) => None,
427        };
428        let reserved = match review {
429            Some(review) => {
430                let id = {
431                    let mut state = self.state();
432                    self.room(&state, queued)?;
433                    state.next += 1;
434                    format!("job-{}", state.next)
435                };
436                review.begin(&id)?;
437                Some(id)
438            }
439            None => None,
440        };
441        let (id, progress, done_tx, cancellation, first) = {
442            let mut state = self.state();
443            // Checked again: another call may have started a job meanwhile.
444            self.room(&state, queued)?;
445            let id = reserved.unwrap_or_else(|| {
446                state.next += 1;
447                format!("job-{}", state.next)
448            });
449            let progress = ProgressSink::buffered();
450            let (done_tx, done) = watch::channel(false);
451            let cancel = self.cancellation.child_token();
452            let accepting = accepted_follow_up.is_some();
453            let record = self.ledger.as_ref().and_then(|ledger| {
454                let mut record = ledger.record(&NewJob {
455                    job: &id,
456                    agent,
457                    task: &task,
458                    arguments: &arguments,
459                    started_by: origin,
460                    reviewed: review.is_some(),
461                    queued,
462                })?;
463                record.worktree.clone_from(&worktree);
464                record.resumed_from = resumed_from;
465                record.accepted_follow_up = accepted_follow_up;
466                Some(record)
467            });
468            // A subscription follow-up's acceptance is on disk before the
469            // job is in the table, or there is no job: a restart retries a
470            // follow-up no record names, so it must not have started.
471            let first = if accepting {
472                let ledger = self.ledger.as_ref().zip(record.as_ref());
473                let saved = ledger
474                    .map(|(ledger, record)| blocking(|| ledger.save_revision_required(record, 1)));
475                match saved {
476                    Some(Ok(())) => None,
477                    Some(Err(error)) => {
478                        return Err(ToolError::failed(format!(
479                            "the job's record could not keep the follow-up it would take \
480                             ({error}), so the job did not start"
481                        )));
482                    }
483                    None => {
484                        return Err(ToolError::failed(
485                            "this session keeps no job records, so a follow-up job cannot start",
486                        ));
487                    }
488                }
489            } else {
490                record.clone().map(|record| (record, 1))
491            };
492            state.record(
493                &context.call_id,
494                JobChange {
495                    job: id.clone(),
496                    tool: name.to_owned(),
497                    agent: agent.to_owned(),
498                    status: JobStatus::Running,
499                    task: task.clone(),
500                    journal: review.map(|review| review.journal_id()),
501                    outcome: None,
502                },
503            );
504            state.jobs.push(Job {
505                id: id.clone(),
506                tool: name.to_owned(),
507                agent: agent.to_owned(),
508                task,
509                started: Instant::now(),
510                progress: progress.clone(),
511                last_progress: None,
512                cancel: cancel.clone(),
513                cancelled: false,
514                queued,
515                outcome: None,
516                reported: false,
517                reporting: false,
518                report_attempts: 0,
519                report_due: None,
520                done,
521                review: review.map(|review| Arc::clone(review.record())),
522                cancelling: false,
523                origin,
524                queued_waiting: queued,
525                lane,
526                owner_cancelled: false,
527                record,
528                record_revision: 1,
529            });
530            (id, progress, done_tx, cancel, first)
531        };
532        self.persist(first);
533        // The owner can stop it to take its conversation over. A reviewed
534        // job's own place is not offered: stopping it ends a whole review.
535        if review.is_none()
536            && let Turn::Next(place) | Turn::Queued(place) = &turn
537        {
538            place.run_as(&id);
539        }
540        let jobs = Arc::downgrade(self);
541        let job = id.clone();
542        let approvals = self.approvals.clone();
543        let workspace = context.workspace.clone();
544        let reviewed = review.map(Arc::clone);
545        tokio::spawn(async move {
546            let started = Instant::now();
547            let mut context = ToolContext::new(workspace, cancellation);
548            context.background_job = true;
549            context.job = Some(job.clone());
550            // No turn is left to ask a person, so nested approval requests
551            // get the session's unattended answer, or are denied.
552            if let Some(gate) = &approvals {
553                context.approvals = ToolApprovals::new(Arc::clone(gate), job.clone());
554            }
555            context.progress = progress;
556            let place = turn.place();
557            let output = match &place {
558                Some(place) if queued => {
559                    context.progress.report(&format!(
560                        "queued: waits for conversation {}'s running turn",
561                        place.handle()
562                    ));
563                    match place.wait_first(&context.cancellation).await {
564                        Ok(()) => {
565                            if let Some(jobs) = jobs.upgrade() {
566                                jobs.began(&job);
567                            }
568                            context.progress.take();
569                            context.progress.report(&format!(
570                                "started its turn in conversation {}",
571                                place.handle()
572                            ));
573                            tool.execute(arguments, context).await
574                        }
575                        // A reviewed job ends its review even when its loop
576                        // never ran, so its result says how it ended.
577                        Err(error) => match &reviewed {
578                            Some(review) => Ok(review.unstarted(&error)),
579                            None => Err(error),
580                        },
581                    }
582                }
583                _ => tool.execute(arguments, context).await,
584            }
585            .unwrap_or_else(ToolOutput::from);
586            // Free the conversation before the result is visible, so a call
587            // continuing it after `agent_wait` finds it idle.
588            drop(place);
589            finish(&jobs, &job, output, started.elapsed());
590            let _ = done_tx.send(true);
591        });
592        let mut started = json!({
593            "job": id,
594            "agent": agent,
595            "status": "running",
596            "background": true,
597            "note": "The agent is working in the background. SCV reports the result in a new \
598                     turn when it finishes. agent_status shows its progress and agent_cancel \
599                     stops it."
600        });
601        if let Some(worktree) = &worktree {
602            started["worktree"] = serde_json::to_value(worktree).unwrap_or(Value::Null);
603        }
604        if queued {
605            started["queued"] = true.into();
606            started["note"] = "The conversation is busy, so this prompt is queued: it runs in \
607                the background after the turns ahead of it, and SCV reports the result in a new \
608                turn when it finishes. agent_status shows where it stands and agent_cancel \
609                withdraws it."
610                .into();
611        }
612        Ok(started)
613    }
614
615    /// A queued job's turn has come: it runs now.
616    fn began(&self, id: &str) {
617        let write = {
618            let mut state = self.state();
619            state
620                .jobs
621                .iter_mut()
622                .find(|job| job.id == id)
623                .and_then(|job| {
624                    job.queued_waiting = false;
625                    let record = job.record.as_mut()?;
626                    record.state = AgentState::Working;
627                    record.updated_unix = ledger::now();
628                    job.record_changed()
629                })
630        };
631        self.persist(write);
632    }
633
634    /// The conversation handles the session's job records name.
635    pub(crate) fn recorded_conversations(&self) -> Vec<String> {
636        self.ledger
637            .as_ref()
638            .and_then(JobLedger::conversations)
639            .unwrap_or_default()
640    }
641
642    /// Whether the session still owes its client `job`'s result: dropping
643    /// its record now would lose it.
644    pub(crate) fn unreported(&self, job: &str) -> bool {
645        self.state()
646            .jobs
647            .iter()
648            .any(|entry| entry.id == job && !entry.reported)
649    }
650
651    /// Mark `job`'s result reported in its durable record; the change to
652    /// write, when there is one.
653    fn mark_reported(job: &mut Job) -> Option<RecordWrite> {
654        let record = job.record.as_mut().filter(|record| !record.reported)?;
655        record.reported = true;
656        record.updated_unix = ledger::now();
657        job.record_changed()
658    }
659
660    /// Write the record changes `writes`, outside the job table's lock, so
661    /// a slow disk never holds up the session's job calls. A failed write is
662    /// logged and skipped.
663    fn persist(&self, writes: impl IntoIterator<Item = RecordWrite>) {
664        let Some(ledger) = &self.ledger else {
665            return;
666        };
667        blocking(|| {
668            for (record, revision) in writes {
669                ledger.save_revision(&record, revision);
670            }
671        });
672    }
673
674    /// The session's jobs as they stand, for the workbench.
675    pub(crate) fn live(&self) -> Vec<LiveJob> {
676        let mut state = self.state();
677        state
678            .jobs
679            .iter_mut()
680            .map(|job| {
681                if let Some(line) = job.progress.take() {
682                    job.last_progress = Some(line);
683                }
684                LiveJob {
685                    id: job.id.clone(),
686                    waiting: job.queued_waiting,
687                    running: job.outcome.is_none(),
688                    progress: job.last_progress.clone(),
689                    elapsed: job.started.elapsed(),
690                }
691            })
692            .collect()
693    }
694
695    /// Reviewed job `job`'s outcome as it stands, while this session still
696    /// keeps the job; `None` for any other job.
697    pub(crate) fn outcome(&self, job: &str) -> Option<JobOutcome> {
698        self.state()
699            .jobs
700            .iter()
701            .find(|entry| entry.id == job)
702            .and_then(Job::outcome)
703    }
704
705    /// Stop `job` for the owner, from the workbench: as `agent_cancel` does,
706    /// but the model did not ask, so its report turn tells it the owner
707    /// stopped the job. A reviewed job's report carries its outcome as the
708    /// cancel left it.
709    pub(crate) fn cancel_for_owner(&self, job: &str) -> Result<(), String> {
710        let mut state = self.state();
711        let entry = state
712            .jobs
713            .iter_mut()
714            .find(|candidate| candidate.id == job)
715            .ok_or_else(|| format!("this session has no job {job:?}"))?;
716        if entry.outcome.is_some() {
717            return Err(format!("{job} has already finished"));
718        }
719        entry.cancelled = true;
720        entry.owner_cancelled = true;
721        entry.cancel.cancel();
722        Ok(())
723    }
724
725    /// Refuse a new running job beyond `agent.max_background`, before
726    /// anything is prepared for it.
727    fn has_room(&self) -> Result<(), ToolError> {
728        let state = self.state();
729        self.room(&state, false)
730    }
731
732    /// Refuse a job beyond `agent.max_background`; a queued prompt counts
733    /// toward `agent.max_queued_turns` instead.
734    fn room(&self, state: &JobsState, queued: bool) -> Result<(), ToolError> {
735        let running = state
736            .jobs
737            .iter()
738            .filter(|job| job.outcome.is_none() && !job.queued)
739            .count();
740        if !queued && running >= self.limit {
741            return Err(ToolError::limit(format!(
742                "{running} background jobs are already running, the limit \
743                 (agent.max_background). Start this one after a job finishes, or \
744                 stop one with agent_cancel if the user no longer needs it."
745            )));
746        }
747        Ok(())
748    }
749
750    /// Stop `job` for the call `call_id` and wait briefly for it to settle,
751    /// then describe it. A stopped job needs no report turn: the model asked
752    /// for the stop.
753    ///
754    /// A reviewed job's decision is fixed at once, but its journal goes on
755    /// until it really stops, so the session is owed its final outcome
756    /// ([`owed`]): the call's change says it if the job stops before the
757    /// change is sent, and otherwise says the journal is still open, and a
758    /// `background.updated` follows ([`Self::take_updates`]). The wait
759    /// reads only the job's live state, never its journal.
760    async fn cancel(&self, job: &str, call_id: &str) -> Result<Value, ToolError> {
761        let (mut done, owed, write) = {
762            let mut state = self.state();
763            let index = state
764                .jobs
765                .iter()
766                .position(|candidate| candidate.id == job)
767                .ok_or_else(|| unknown_job(job))?;
768            let entry = &mut state.jobs[index];
769            if entry.origin == JobOrigin::Owner && entry.outcome.is_none() {
770                return Err(ToolError::invalid_arguments(format!(
771                    "{job} is the user's own job, started from the SCV workbench; only the \
772                     user stops it"
773                )));
774            }
775            if entry.outcome.is_some() {
776                let (mut value, change) = entry.describe();
777                value["note"] = "The job had already finished.".into();
778                // The model has its result now, as `agent_status` gives it.
779                let write = change
780                    .is_some()
781                    .then(|| Self::mark_reported(entry))
782                    .flatten();
783                if let Some(change) = change {
784                    state.record(call_id, change);
785                }
786                drop(state);
787                self.persist(write);
788                return Ok(value);
789            }
790            entry.cancelled = true;
791            entry.cancel.cancel();
792            entry.cancelling = true;
793            let done = entry.done.clone();
794            let mut owed = false;
795            let mut write = None;
796            if !entry.reported {
797                entry.reported = true;
798                write = Self::mark_reported(entry);
799                let change = entry.change(JobStatus::Cancelled);
800                owed = entry.review.is_some();
801                state.record(call_id, change);
802                if owed {
803                    state.owed.cancelling(job, call_id);
804                }
805            }
806            (done, owed, write)
807        };
808        self.persist(write);
809        let waiting = Waiting {
810            jobs: self,
811            job: job.to_owned(),
812            call: call_id.to_owned(),
813            owed,
814            settled: false,
815        };
816        let _ = tokio::time::timeout(CANCEL_SETTLE, done.wait_for(|finished| *finished)).await;
817        waiting.settle();
818        self.describe(Some(job), call_id)
819    }
820
821    /// The final outcomes of cancelled reviewed jobs the session is owed
822    /// and has not been sent, for the server's `background.updated`. Each
823    /// is being sent until [`Self::updated`] says how that went.
824    pub fn take_updates(&self) -> Vec<JobOutcome> {
825        self.state().owed.take_updates()
826    }
827
828    /// The update naming `jobs` reached the client, when `delivered`, which
829    /// ends what was owed; or it could not be sent, which leaves them due.
830    pub fn updated(&self, jobs: &[String], delivered: bool) {
831        self.state().owed.updated(jobs, delivered);
832    }
833
834    /// The `tool.completed` of the call `call_id`, carrying the changes
835    /// [`Self::take_changes`] gave, reached the client, when `delivered`, or
836    /// could not be sent, as when its turn was cancelled first. A change
837    /// that did not deliver a reviewed job's final outcome leaves it to a
838    /// `background.updated`.
839    pub fn published(&self, call_id: &str, delivered: bool) {
840        let due = self.state().owed.sent(call_id, delivered);
841        if due {
842            self.wake();
843        }
844    }
845
846    /// A turn ended: no change of its calls can be sent any more, so what
847    /// such a change owed goes to a `background.updated`.
848    pub fn turn_ended(&self) {
849        let due = self.state().owed.turn_ended();
850        if due {
851            self.wake();
852        }
853    }
854
855    /// For tests of how a client of this store delivers reviewed jobs'
856    /// outcomes, such as the server's, with the `test-support` feature: owe
857    /// the session the final outcome of `change.job`, through `change`,
858    /// recorded for the call `call_id` once `agent_cancel` waited, which says
859    /// the journal is still open; `last` is the final outcome if the job has
860    /// stopped.
861    #[cfg(any(test, feature = "test-support"))]
862    pub fn owe_for_tests(&self, call_id: &str, change: JobChange, last: Option<JobOutcome>) {
863        let mut state = self.state();
864        state.owed.owe(call_id, &change, last);
865        state.record(call_id, change);
866    }
867
868    /// For the same tests: the job of `outcome` stops with it.
869    #[cfg(any(test, feature = "test-support"))]
870    pub fn stop_for_tests(&self, outcome: &JobOutcome) {
871        let due = self.state().owed.stopped(&outcome.job, outcome);
872        if due {
873            self.wake();
874        }
875    }
876
877    /// For tests of what waits for a session's jobs, with the
878    /// `test-support` feature: start a job on `agent` that runs until the
879    /// returned sender is used or dropped.
880    #[cfg(any(test, feature = "test-support"))]
881    pub fn start_for_tests(self: &Arc<Self>, agent: &str) -> tokio::sync::oneshot::Sender<()> {
882        struct Waits(Mutex<Option<tokio::sync::oneshot::Receiver<()>>>);
883        #[async_trait]
884        impl Tool for Waits {
885            fn spec(&self) -> ToolSpec {
886                ToolSpec {
887                    name: AGENT_TOOL.into(),
888                    description: String::new(),
889                    parameters: json!({"type":"object"}),
890                }
891            }
892            fn risk(&self, _arguments: &Value) -> Result<ToolRisk, ToolError> {
893                Ok(ToolRisk::Delegate)
894            }
895            fn approval_summary(&self, _arguments: &Value) -> Result<String, ToolError> {
896                Ok(String::new())
897            }
898            async fn execute(
899                &self,
900                _arguments: Value,
901                _context: ToolContext,
902            ) -> Result<ToolOutput, ToolError> {
903                let release = lock(&self.0).take();
904                if let Some(release) = release {
905                    let _ = release.await;
906                }
907                Ok(ToolOutput::success(
908                    r#"{"status":"completed","reply":"done"}"#,
909                ))
910            }
911        }
912        let (release, waiting) = tokio::sync::oneshot::channel();
913        let context = ToolContext::new(std::env::temp_dir(), CancellationToken::new());
914        self.start(
915            Arc::new(Waits(Mutex::new(Some(waiting)))),
916            AGENT_TOOL,
917            agent,
918            json!({"prompt": "wait"}),
919            &context,
920            Turn::New,
921            None,
922            JobMeta::default(),
923        )
924        .expect("a test job starts");
925        release
926    }
927
928    /// Wake the session, as when a job finishes, for what is now due.
929    fn wake(&self) {
930        if let Some(finished) = &self.finished {
931            let _ = finished.send(());
932        }
933    }
934
935    /// Wait up to `limit` for `job` and describe it for the call `call_id`;
936    /// a finished job is then marked seen.
937    async fn wait(
938        &self,
939        job: &str,
940        limit: Duration,
941        cancellation: &CancellationToken,
942        call_id: &str,
943    ) -> Result<Value, ToolError> {
944        let mut done = self
945            .state()
946            .jobs
947            .iter()
948            .find(|candidate| candidate.id == job)
949            .map(|candidate| candidate.done.clone())
950            .ok_or_else(|| unknown_job(job))?;
951        tokio::select! {
952            () = cancellation.cancelled() => return Err(ToolError::cancelled("wait cancelled")),
953            _ = tokio::time::timeout(limit, done.wait_for(|finished| *finished)) => {}
954        }
955        self.describe(Some(job), call_id)
956    }
957
958    /// Describe one job, or every job this session remembers, for the call
959    /// `call_id`; finished jobs described are marked seen.
960    fn describe(&self, job: Option<&str>, call_id: &str) -> Result<Value, ToolError> {
961        let mut state = self.state();
962        let mut changes = Vec::new();
963        let mut writes = Vec::new();
964        let value = if let Some(job) = job {
965            let entry = state
966                .jobs
967                .iter_mut()
968                .find(|candidate| candidate.id == job)
969                .ok_or_else(|| unknown_job(job))?;
970            let (value, change) = entry.describe();
971            if change.is_some() {
972                writes.extend(Self::mark_reported(entry));
973            }
974            changes.extend(change);
975            value
976        } else {
977            let jobs: Vec<Value> = state
978                .jobs
979                .iter_mut()
980                .map(|entry| {
981                    let (value, change) = entry.describe();
982                    if change.is_some() {
983                        writes.extend(Self::mark_reported(entry));
984                    }
985                    changes.extend(change);
986                    value
987                })
988                .collect();
989            json!({ "jobs": jobs })
990        };
991        for change in changes {
992            state.record(call_id, change);
993        }
994        drop(state);
995        self.persist(writes);
996        Ok(value)
997    }
998
999    /// The jobs the call `call_id` started or showed the model the result of,
1000    /// for its `tool.completed`, which then reports how its sending went
1001    /// ([`Self::published`]). A cancelled reviewed job's change says its
1002    /// final outcome if the job has stopped by now, superseding the
1003    /// snapshot taken when the cancel's wait ended.
1004    pub fn take_changes(&self, call_id: &str) -> Vec<JobChange> {
1005        let mut state = self.state();
1006        let mut taken = Vec::new();
1007        state.changes.retain(|(call, change)| {
1008            if call == call_id {
1009                taken.push(change.clone());
1010                false
1011            } else {
1012                true
1013            }
1014        });
1015        for change in &mut taken {
1016            state.owed.take(call_id, change);
1017        }
1018        taken
1019    }
1020
1021    /// Finished jobs the model has not seen yet whose report is due, at most
1022    /// a few, for a report turn. Each stays unreported, and is not taken
1023    /// again, until the turn ends: [`report_settled`](Self::report_settled)
1024    /// or [`report_failed`](Self::report_failed).
1025    pub fn take_unreported(&self) -> Vec<JobReport> {
1026        let now = tokio::time::Instant::now();
1027        let mut state = self.state();
1028        state
1029            .jobs
1030            .iter_mut()
1031            .filter(|job| job.awaits_report() && job.report_due.is_none_or(|due| due <= now))
1032            .take(REPORT_MAX_JOBS)
1033            .filter_map(|job| {
1034                job.reporting = true;
1035                job.report()
1036            })
1037            .collect()
1038    }
1039
1040    /// The report turn about `jobs` ended without failing: it completed, so
1041    /// the model has seen them, or the user cancelled it. Either settles them.
1042    pub fn report_settled(&self, jobs: &[String]) {
1043        let mut writes = Vec::new();
1044        {
1045            let mut state = self.state();
1046            for job in state.jobs.iter_mut().filter(|job| jobs.contains(&job.id)) {
1047                job.reporting = false;
1048                job.reported = true;
1049                job.report_due = None;
1050                writes.extend(Self::mark_reported(job));
1051            }
1052        }
1053        self.persist(writes);
1054    }
1055
1056    /// The report turn about `jobs` failed, so the model has not seen them
1057    /// (a failed turn leaves no history). With `retry`, another turn is due
1058    /// after a delay, up to three turns in all; otherwise, or once they are
1059    /// used up, the jobs are settled and returned for the server to report
1060    /// directly. A turn that reported several jobs decides for all of them,
1061    /// by the one that failed most.
1062    pub fn report_failed(&self, jobs: &[String], retry: bool) -> ReportFailure {
1063        let mut writes = Vec::new();
1064        let failure = self.report_failed_locked(jobs, retry, &mut writes);
1065        self.persist(writes);
1066        failure
1067    }
1068
1069    /// [`Self::report_failed`] under the job table's lock, gathering the
1070    /// record changes to write in `writes`.
1071    fn report_failed_locked(
1072        &self,
1073        jobs: &[String],
1074        retry: bool,
1075        writes: &mut Vec<RecordWrite>,
1076    ) -> ReportFailure {
1077        let mut state = self.state();
1078        let mut failed: Vec<&mut Job> = state
1079            .jobs
1080            .iter_mut()
1081            .filter(|job| job.reporting && jobs.contains(&job.id))
1082            .collect();
1083        for job in &mut failed {
1084            job.reporting = false;
1085            job.report_attempts += 1;
1086        }
1087        // One the turn's model looked up through a call is settled already.
1088        failed.retain(|job| !job.reported);
1089        let Some(attempts) = failed.iter().map(|job| job.report_attempts).max() else {
1090            return ReportFailure::Settled;
1091        };
1092        let delay = usize::try_from(attempts - 1)
1093            .ok()
1094            .and_then(|earlier| REPORT_RETRY_DELAYS.get(earlier));
1095        if retry && let Some(&delay) = delay {
1096            let due = tokio::time::Instant::now() + delay;
1097            for job in failed {
1098                job.report_due = Some(due);
1099            }
1100            return ReportFailure::Retry(delay);
1101        }
1102        let reports = failed
1103            .into_iter()
1104            .filter_map(|job| {
1105                job.reported = true;
1106                job.report_due = None;
1107                writes.extend(Self::mark_reported(job));
1108                job.report()
1109            })
1110            .collect();
1111        ReportFailure::GiveUp { attempts, reports }
1112    }
1113
1114    /// Make every report waiting out a failed turn due now, as once a turn
1115    /// succeeds and the model is evidently reachable again. Whether any was
1116    /// waiting.
1117    pub fn retry_now(&self) -> bool {
1118        let mut state = self.state();
1119        let mut waiting = false;
1120        for job in state.jobs.iter_mut().filter(|job| job.awaits_report()) {
1121            waiting |= job.report_due.take().is_some();
1122        }
1123        waiting
1124    }
1125
1126    /// When the earliest report waiting out a failed turn is due; it may
1127    /// already have passed.
1128    pub fn next_retry(&self) -> Option<tokio::time::Instant> {
1129        self.state()
1130            .jobs
1131            .iter()
1132            .filter(|job| job.awaits_report())
1133            .filter_map(|job| job.report_due)
1134            .min()
1135    }
1136
1137    /// Jobs still running or whose result is not reported yet, including
1138    /// those a report turn is reporting now, and cancelled reviewed jobs
1139    /// whose final outcome the session is still owed, until it was sent;
1140    /// each job once.
1141    pub fn pending(&self) -> usize {
1142        let state = self.state();
1143        let mut pending: HashSet<&str> = state
1144            .jobs
1145            .iter()
1146            .filter(|job| job.outcome.is_none() || !job.reported)
1147            .map(|job| job.id.as_str())
1148            .collect();
1149        pending.extend(state.owed.jobs());
1150        pending.len()
1151    }
1152
1153    /// Of the [`pending`](Self::pending) jobs, the queued prompts that wait
1154    /// only for the owner's takeover of their conversation to end. A planned
1155    /// restart does not wait for them: the restart ends the takeover, and
1156    /// they are kept for the next session like any queued prompt.
1157    pub fn behind_takeovers(&self) -> usize {
1158        self.state()
1159            .jobs
1160            .iter()
1161            .filter(|job| job.outcome.is_none() && job.queued_waiting)
1162            .filter(|job| {
1163                job.lane
1164                    .as_deref()
1165                    .is_some_and(|lane| self.lanes.held_by_takeover(lane))
1166            })
1167            .count()
1168    }
1169
1170    /// Whether any job is still running.
1171    pub fn running(&self) -> usize {
1172        self.state()
1173            .jobs
1174            .iter()
1175            .filter(|job| job.outcome.is_none())
1176            .count()
1177    }
1178}
1179
1180/// Who started a job, the worktree it runs in, and the interrupted job it
1181/// resumes: what its record holds from the start.
1182#[derive(Debug, Clone, Default)]
1183pub(crate) struct JobMeta {
1184    pub(crate) origin: JobOrigin,
1185    pub(crate) worktree: Option<scv_protocol::WorktreeRef>,
1186    /// `<session>/<job>` of the interrupted job it runs again.
1187    pub(crate) resumed_from: Option<String>,
1188    /// When set, [`BackgroundCapable::owner_call`] keeps [`Self::origin`].
1189    /// A subscription follow-up sets it on a session that reports jobs, so
1190    /// that session's report turn still sees the job. A session with no
1191    /// report turn leaves it unset: the job is the owner's to watch and is
1192    /// marked reported when it finishes.
1193    pub(crate) keep_origin: bool,
1194    /// The subscription follow-up this job is accepting. On disk in the
1195    /// job's record before the job starts; when that write fails,
1196    /// [`BackgroundJobs::start`] fails and starts nothing.
1197    pub(crate) accepted_follow_up: Option<String>,
1198}
1199
1200/// What an owner's prompt to the workbench became.
1201#[derive(Debug, Clone, PartialEq)]
1202pub(crate) enum OwnerCall {
1203    /// A background job: its start, as an `agent` call returns it.
1204    Job(Value),
1205    /// The prompt went into the conversation's running turn.
1206    Steered,
1207}
1208
1209/// A job as the workbench sees it while its session lives.
1210#[derive(Debug, Clone)]
1211pub(crate) struct LiveJob {
1212    pub(crate) id: String,
1213    /// A queued prompt whose turn has not come yet.
1214    pub(crate) waiting: bool,
1215    pub(crate) running: bool,
1216    pub(crate) progress: Option<String>,
1217    pub(crate) elapsed: Duration,
1218}
1219
1220/// The workbench state of a job that ended with `status`.
1221pub(crate) fn agent_state(status: JobStatus) -> AgentState {
1222    match status {
1223        JobStatus::Completed => AgentState::Completed,
1224        JobStatus::Declined => AgentState::Declined,
1225        JobStatus::Timeout => AgentState::Timeout,
1226        JobStatus::Cancelled => AgentState::Cancelled,
1227        JobStatus::Running => AgentState::Working,
1228        _ => AgentState::Failed,
1229    }
1230}
1231
1232/// How a job's call takes its conversation's turn.
1233pub(in crate::delegate) enum Turn {
1234    /// A new conversation, which nothing else can reach yet.
1235    New,
1236    /// The conversation's next turn, which nothing was ahead of.
1237    Next(Place),
1238    /// A prompt queued behind the turns ahead of it.
1239    Queued(Place),
1240}
1241
1242impl Turn {
1243    fn place(self) -> Option<Place> {
1244        match self {
1245            Self::New => None,
1246            Self::Next(place) | Self::Queued(place) => Some(place),
1247        }
1248    }
1249}
1250
1251fn finish(jobs: &Weak<BackgroundJobs>, id: &str, output: ToolOutput, elapsed: Duration) {
1252    // The session ended first: its jobs were cancelled with it.
1253    let Some(jobs) = jobs.upgrade() else {
1254        return;
1255    };
1256    let mut write = None;
1257    {
1258        let mut state = jobs.state();
1259        let mut last = None;
1260        if let Some(job) = state.jobs.iter_mut().find(|job| job.id == id) {
1261            job.last_progress = job.progress.take().or(job.last_progress.take());
1262            job.queued_waiting = false;
1263            // Stopped on the model's request: the model already knows. The
1264            // owner's own jobs are the owner's to watch, never the model's.
1265            job.reported |=
1266                (job.cancelled && !job.owner_cancelled) || job.origin == JobOrigin::Owner;
1267            let result = result_value(&output);
1268            let status = if job.cancelled {
1269                JobStatus::Cancelled
1270            } else {
1271                job_status(&output, &AgentReply::read(&result).unwrap_or_default())
1272            };
1273            job.outcome = Some(Outcome { output, elapsed });
1274            if let Some(record) = job.record.as_mut() {
1275                ledger::finish(record, agent_state(status), &result, job.reported);
1276                write = job.record_changed();
1277            }
1278            last = job.outcome();
1279        }
1280        // What a cancel owed of it is now final.
1281        if let Some(last) = last {
1282            state.owed.stopped(id, &last);
1283        }
1284        // Keep every running job and the newest finished ones.
1285        let finished = state
1286            .jobs
1287            .iter()
1288            .filter(|job| job.outcome.is_some())
1289            .count();
1290        let mut excess = finished.saturating_sub(MAX_FINISHED);
1291        state.jobs.retain(|job| {
1292            if excess > 0 && job.outcome.is_some() && job.reported && !job.cancelling {
1293                excess -= 1;
1294                false
1295            } else {
1296                true
1297            }
1298        });
1299    }
1300    // Recorded before anyone hears the job finished.
1301    jobs.persist(write);
1302    if let Some(finished) = &jobs.finished {
1303        let _ = finished.send(());
1304    }
1305}
1306
1307impl Job {
1308    /// Finished, unseen by the model, and not in a report turn.
1309    fn awaits_report(&self) -> bool {
1310        self.outcome.is_some() && !self.reported && !self.reporting
1311    }
1312
1313    /// The finished job's result, for a report.
1314    fn report(&self) -> Option<JobReport> {
1315        let outcome = self.outcome.as_ref()?;
1316        let reply = AgentReply::read(&result_value(&outcome.output)).unwrap_or_default();
1317        Some(JobReport {
1318            job: self.id.clone(),
1319            agent: self.agent.clone(),
1320            task: self.task.clone(),
1321            // Stopped by the owner: the model asked for no stop of its own.
1322            status: if self.cancelled {
1323                JobStatus::Cancelled
1324            } else {
1325                job_status(&outcome.output, &reply)
1326            },
1327            session: reply.session,
1328            reply: bounded(
1329                reply
1330                    .reply
1331                    .as_deref()
1332                    .unwrap_or(outcome.output.content.as_str()),
1333                REPORT_REPLY_CHARS,
1334            ),
1335            outcome: self.outcome(),
1336        })
1337    }
1338
1339    /// A reviewed job's outcome as it stands. A cancelled job's is what it
1340    /// was when the cancel arrived.
1341    fn outcome(&self) -> Option<JobOutcome> {
1342        self.review.as_ref().map(|record| {
1343            let (review, landing) = record
1344                .live
1345                .snapshot(self.cancelled || self.cancel.is_cancelled());
1346            JobOutcome {
1347                job: self.id.clone(),
1348                review,
1349                landing,
1350            }
1351        })
1352    }
1353
1354    /// This job with `status`, as its clients learn of it: a reviewed job's
1355    /// settling change carries its outcome.
1356    fn change(&self, status: JobStatus) -> JobChange {
1357        JobChange {
1358            job: self.id.clone(),
1359            tool: self.tool.clone(),
1360            agent: self.agent.clone(),
1361            status,
1362            task: self.task.clone(),
1363            journal: None,
1364            outcome: if status == JobStatus::Running {
1365                None
1366            } else {
1367                self.outcome()
1368            },
1369        }
1370    }
1371
1372    /// The job as the model reads it, and its change when this is the first
1373    /// time the model sees its result.
1374    fn describe(&mut self) -> (Value, Option<JobChange>) {
1375        if let Some(line) = self.progress.take() {
1376            self.last_progress = Some(line);
1377        }
1378        let mut change = None;
1379        let mut value = Map::new();
1380        value.insert("job".into(), self.id.clone().into());
1381        value.insert("agent".into(), self.agent.clone().into());
1382        if self.origin == JobOrigin::Owner {
1383            value.insert("started_by".into(), "owner".into());
1384        }
1385        match &self.outcome {
1386            None => {
1387                value.insert("status".into(), "running".into());
1388                value.insert(
1389                    "elapsed_seconds".into(),
1390                    self.started.elapsed().as_secs().into(),
1391                );
1392                if let Some(progress) = &self.last_progress {
1393                    value.insert("progress".into(), progress.clone().into());
1394                }
1395                if let Some(record) = &self.review {
1396                    value.insert("review".into(), record.live.status());
1397                }
1398            }
1399            Some(outcome) => {
1400                let result = result_value(&outcome.output);
1401                let status = if self.cancelled {
1402                    JobStatus::Cancelled
1403                } else {
1404                    job_status(
1405                        &outcome.output,
1406                        &AgentReply::read(&result).unwrap_or_default(),
1407                    )
1408                };
1409                value.insert("status".into(), status.as_str().into());
1410                value.insert("elapsed_seconds".into(), outcome.elapsed.as_secs().into());
1411                value.insert("result".into(), result);
1412                if !self.reported {
1413                    self.reported = true;
1414                    change = Some(self.change(status));
1415                }
1416            }
1417        }
1418        (Value::Object(value), change)
1419    }
1420}
1421
1422/// The agent tool's structured result, or its text when it was not JSON.
1423fn result_value(output: &ToolOutput) -> Value {
1424    match serde_json::from_str::<Value>(&output.content) {
1425        Ok(value @ Value::Object(_)) => value,
1426        _ => json!({ "reply": output.content, "is_error": output.is_error() }),
1427    }
1428}
1429
1430/// How a finished job ended: its agent's reported status, else whether its
1431/// call failed.
1432fn job_status(output: &ToolOutput, reply: &AgentReply) -> JobStatus {
1433    reply.status.unwrap_or(if output.is_error() {
1434        JobStatus::Failed
1435    } else {
1436        JobStatus::Completed
1437    })
1438}
1439
1440/// The first non-empty line of `prompt`, at most [`TASK_CHARS`] characters,
1441/// which names a job to people.
1442fn task_line(prompt: &str) -> String {
1443    let line = prompt
1444        .lines()
1445        .map(str::trim)
1446        .find(|line| !line.is_empty())
1447        .unwrap_or_default();
1448    let mut task: String = line
1449        .chars()
1450        .filter(|character| !character.is_control())
1451        .take(TASK_CHARS)
1452        .collect();
1453    if line.chars().count() > TASK_CHARS {
1454        task.push('…');
1455    }
1456    task
1457}
1458
1459/// Run the disk write `write`. The write syncs to disk, so on a runtime's
1460/// worker thread the runtime is told it blocks, and moves its other tasks
1461/// elsewhere meanwhile.
1462fn blocking<T>(write: impl FnOnce() -> T) -> T {
1463    match tokio::runtime::Handle::try_current() {
1464        Ok(runtime) if runtime.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread => {
1465            tokio::task::block_in_place(write)
1466        }
1467        _ => write(),
1468    }
1469}
1470
1471fn unknown_job(job: &str) -> ToolError {
1472    ToolError::invalid_arguments(format!(
1473        "unknown background job {:?}; agent_status lists this session's jobs",
1474        bounded(job, 64)
1475    ))
1476}
1477
1478/// What the model is told when a report includes reviewed jobs, whose
1479/// outcome SCV states itself.
1480const REVIEWED_NOTE: &str = " SCV shows the user its own Review and Landing lines for \
1481     reviewed jobs. If you mention the outcome, repeat them as given; never call unapproved \
1482     work approved or done, and never call landed work approved. If a reviewer declined, tell \
1483     the user what it said. If a landing is NOT confirmed, or the reviewer could NOT confirm \
1484     it, say so plainly, and don't fix, revert, or land anything on your own.";
1485
1486fn reviewed_note(reports: &[JobReport]) -> &'static str {
1487    if reports.iter().any(|report| report.outcome.is_some()) {
1488        REVIEWED_NOTE
1489    } else {
1490        ""
1491    }
1492}
1493
1494/// The server-started turn that reports finished jobs to the model.
1495pub fn report_prompt(reports: &[JobReport]) -> String {
1496    format!(
1497        "[SCV background report] Delegated work you started in the background has \
1498         finished. The user did not send this message: tell them briefly what \
1499         happened and the key result.{}{}\n\n{}",
1500        cancelled_note(reports),
1501        reviewed_note(reports),
1502        describe_reports(reports)
1503    )
1504}
1505
1506/// What the model is told when a report includes a job someone else
1507/// stopped: a job the model stops itself is never reported.
1508fn cancelled_note(reports: &[JobReport]) -> &'static str {
1509    if reports
1510        .iter()
1511        .any(|report| report.status == JobStatus::Cancelled)
1512    {
1513        " A job reported as cancelled was stopped by the user (from the SCV workbench or \
1514         `scv agents kill`), not by you; don't restart it unless they ask."
1515    } else {
1516        ""
1517    }
1518}
1519
1520/// What the model is told of jobs the server reported directly, because
1521/// their report turns failed with `error`: the user has their results.
1522pub fn delivered_note(reports: &[JobReport], error: &str) -> String {
1523    format!(
1524        "[SCV background report, already delivered] Delegated work you started in \
1525         the background has finished. Your report of it failed ({error}), so SCV \
1526         sent the user the results below directly. The user did not send this \
1527         message; repeat the results only if they ask.{}\n\n{}",
1528        reviewed_note(reports),
1529        describe_reports(reports)
1530    )
1531}
1532
1533/// The `agent` tool, able to run a call in the background as well, and to
1534/// run one as a reviewed job.
1535pub(crate) struct BackgroundCapable {
1536    pub(crate) inner: Arc<AgentTool>,
1537    pub(crate) jobs: Arc<BackgroundJobs>,
1538    /// The session's conversations, which a reviewed job pins and releases.
1539    pub(crate) conversations: Arc<ConversationStore>,
1540    /// Where reviewed jobs keep their journals; without it, `review` is not
1541    /// offered.
1542    pub(crate) reviews: Option<PathBuf>,
1543    /// The SCV session, which a review's journal names.
1544    pub(crate) session: Option<String>,
1545}
1546
1547/// The `agent` tool for a queued prompt, whose turn has come by the time it
1548/// runs, so no busy policy applies to it again.
1549struct QueuedTool(Arc<AgentTool>);
1550
1551#[async_trait]
1552impl Tool for QueuedTool {
1553    fn spec(&self) -> ToolSpec {
1554        self.0.spec()
1555    }
1556    fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
1557        self.0.risk(arguments)
1558    }
1559    fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
1560        self.0.approval_summary(arguments)
1561    }
1562    async fn execute(
1563        &self,
1564        arguments: Value,
1565        context: ToolContext,
1566    ) -> Result<ToolOutput, ToolError> {
1567        self.0.execute_when_idle(arguments, context).await
1568    }
1569}
1570
1571/// An agent call's arguments with `background` and `review` split off.
1572struct Split {
1573    arguments: Value,
1574    /// `background` as the call gave it.
1575    background: Option<bool>,
1576    /// The `review` object, when the call gave one.
1577    review: Option<Value>,
1578}
1579
1580impl Split {
1581    fn new(arguments: &Value) -> Self {
1582        let mut arguments = arguments.clone();
1583        let object = arguments.as_object_mut();
1584        let (background, review) = match object {
1585            Some(object) => (
1586                object
1587                    .remove("background")
1588                    .map(|value| value.as_bool() == Some(true)),
1589                object.remove("review").filter(|review| !review.is_null()),
1590            ),
1591            None => (None, None),
1592        };
1593        Self {
1594            arguments,
1595            background,
1596            review,
1597        }
1598    }
1599
1600    fn background(&self) -> bool {
1601        self.background == Some(true)
1602    }
1603}
1604
1605#[async_trait]
1606impl Tool for BackgroundCapable {
1607    fn spec(&self) -> ToolSpec {
1608        let mut spec = self.inner.spec();
1609        let reviewed = self.offers_review();
1610        if let Some(properties) = spec
1611            .parameters
1612            .get_mut("properties")
1613            .and_then(Value::as_object_mut)
1614        {
1615            properties.insert(
1616                "background".into(),
1617                json!({
1618                    "type":"boolean",
1619                    "description":"Run in the background: the call returns a job handle at once, \
1620                        the user can keep talking to you while the agent works, and SCV reports \
1621                        the result in a new turn when it finishes. Use it for any substantial \
1622                        task; run in the foreground only for quick work whose result you need \
1623                        within this turn."
1624                }),
1625            );
1626            if reviewed {
1627                properties.insert("review".into(), review::schema(&self.inner));
1628            }
1629        }
1630        spec.description.push_str(&format!(
1631            " Set background to true for anything beyond a quick task: the call returns a job \
1632             handle at once (at most {} running per session), SCV reports the result when the \
1633             job finishes, agent_status shows progress, and agent_cancel stops it. A background \
1634             job's own approval requests get only the answer this session would give without \
1635             asking a person.",
1636            self.jobs.limit()
1637        ));
1638        if reviewed {
1639            spec.description.push_str(
1640                " A call with review runs as a reviewed background job: an independent \
1641                 reviewer checks the work in up to review.rounds rounds.",
1642            );
1643        }
1644        spec
1645    }
1646
1647    fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
1648        let split = Split::new(arguments);
1649        if let Some(review) = &split.review {
1650            self.plan(&split, review)?;
1651        }
1652        self.inner.risk_for(
1653            &split.arguments,
1654            split.background() || split.review.is_some(),
1655        )
1656    }
1657
1658    fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
1659        let split = Split::new(arguments);
1660        let background = split.background() || split.review.is_some();
1661        let mut summary = self
1662            .inner
1663            .approval_summary_for(&split.arguments, background)?;
1664        if let Some(review) = &split.review {
1665            let plan = self.plan(&split, review)?;
1666            summary.push_str(
1667                " Runs in the background: the call returns at once and the result is \
1668                 reported when the review ends.",
1669            );
1670            summary.push_str(&review::approval_summary(&plan, &self.inner)?);
1671        } else if split.background() {
1672            summary.push_str(
1673                " Runs in the background: the call returns at once and the result is \
1674                 reported when the agent finishes.",
1675            );
1676        }
1677        Ok(summary)
1678    }
1679
1680    async fn execute(
1681        &self,
1682        arguments: Value,
1683        context: ToolContext,
1684    ) -> Result<ToolOutput, ToolError> {
1685        let split = Split::new(&arguments);
1686        if let Some(review) = &split.review {
1687            return self.start_reviewed(&split, review, &context).await;
1688        }
1689        let background = split.background();
1690        let mut arguments = split.arguments;
1691        self.inner.route(&arguments)?;
1692        // Validate before returning a job handle, so a bad call fails now,
1693        // and as the call was made: a foreground call queued behind a busy
1694        // conversation runs as a job but may not set `no_timeout`.
1695        self.inner.risk_for(&arguments, background)?;
1696        // A background call's worktree is created before its job starts,
1697        // so the job's record and start name it. A foreground call's is
1698        // the agent tool's.
1699        let worktree = if background && parse_args::<AgentArgs>(&arguments)?.worktree {
1700            self.jobs.has_room()?;
1701            self.inner
1702                .prepare_worktree(&mut arguments, &context)
1703                .await?
1704        } else {
1705            None
1706        };
1707        let (agent, routed) = self.inner.route(&arguments)?;
1708        let Some(handle) = parse_args::<AgentArgs>(&routed)?
1709            .session
1710            .filter(|session| conversation::is_handle(session))
1711        else {
1712            return if background {
1713                let started = self.start_as(
1714                    agent,
1715                    arguments,
1716                    &context,
1717                    Turn::New,
1718                    JobMeta {
1719                        origin: JobOrigin::Model,
1720                        worktree,
1721                        resumed_from: None,
1722                        keep_origin: false,
1723                        accepted_follow_up: None,
1724                    },
1725                )?;
1726                Ok(ToolOutput::success(started.to_string()))
1727            } else {
1728                self.inner.execute(arguments, context).await
1729            };
1730        };
1731        // Taken before anything runs or waits, so calls keep their order.
1732        let place = self.jobs.lanes.join(&handle);
1733        if place.ahead == 0 && !agent.backend.busy(&routed)? {
1734            return if background {
1735                self.start(agent, arguments, &context, Turn::Next(place))
1736            } else {
1737                // Held until the turn ends, so a later call finds it busy.
1738                let output = self.inner.execute(arguments, context).await;
1739                drop(place);
1740                output
1741            };
1742        }
1743        let behavior = match agent.on_busy(&routed)? {
1744            // A turn the owner's takeover is about to stop takes nothing.
1745            BusyBehavior::Steer => {
1746                if !self.jobs.lanes.has_takeover(&handle)
1747                    && let Some(output) = agent.backend.steer(routed, context.clone()).await?
1748                {
1749                    return Ok(output);
1750                }
1751                agent.busy.fallback()
1752            }
1753            behavior => behavior,
1754        };
1755        match behavior {
1756            // A background call cannot hold its caller, so it queues instead.
1757            BusyBehavior::Wait if !background => {
1758                place.wait_first(&context.cancellation).await?;
1759                let output = self.inner.execute_when_idle(arguments, context).await;
1760                drop(place);
1761                output
1762            }
1763            BusyBehavior::Fail => Err(busy(&handle, &place)),
1764            _ => {
1765                queue_room(agent, &handle, &place)?;
1766                self.start(agent, arguments, &context, Turn::Queued(place))
1767            }
1768        }
1769    }
1770}
1771
1772/// The `session busy` error for a call to conversation `handle` that found
1773/// a turn running, or calls waiting, at `place`.
1774fn busy(handle: &str, place: &Place) -> ToolError {
1775    // Calls ahead of this one other than the running turn.
1776    let waiting = place.ahead.saturating_sub(1);
1777    ToolError::failed(if waiting == 0 {
1778        format!("session busy: conversation {handle} is still running a turn")
1779    } else {
1780        format!(
1781            "session busy: conversation {handle} is still running a turn, and {waiting} more \
1782             prompts wait for it"
1783        )
1784    })
1785}
1786
1787/// Refuse a prompt queued at `place` beyond `agent.max_queued_turns`.
1788fn queue_room(agent: &Offered, handle: &str, place: &Place) -> Result<(), ToolError> {
1789    let waiting = place.ahead.saturating_sub(1);
1790    if waiting >= agent.busy.max_queued_turns {
1791        return Err(ToolError::limit(format!(
1792            "session busy: conversation {handle} is running a turn and {waiting} prompts \
1793             already wait for it, of the {} agent.max_queued_turns allows. Send this one \
1794             once they have run, or stop one with agent_cancel.",
1795            agent.busy.max_queued_turns
1796        )));
1797    }
1798    Ok(())
1799}
1800
1801impl BackgroundCapable {
1802    /// `inner` with `jobs` and a store of its own, without reviews: for
1803    /// tests of plain background calls.
1804    #[cfg(test)]
1805    pub(crate) fn plain(inner: Arc<AgentTool>, jobs: Arc<BackgroundJobs>) -> Self {
1806        Self {
1807            inner,
1808            jobs,
1809            conversations: Arc::new(ConversationStore::new(
1810                conversation::ConversationLimits {
1811                    max: 8,
1812                    idle: Duration::from_secs(86400),
1813                },
1814                None,
1815            )),
1816            reviews: None,
1817            session: None,
1818        }
1819    }
1820
1821    /// Whether calls may set `review`: background jobs exist (this tool),
1822    /// the session keeps journals, and an offered agent can continue a
1823    /// conversation.
1824    fn offers_review(&self) -> bool {
1825        self.reviews.is_some() && !self.inner.continuing().is_empty()
1826    }
1827
1828    /// Check a reviewed call, before approval and again before it starts.
1829    fn plan(&self, split: &Split, review: &Value) -> Result<review::Plan, ToolError> {
1830        if !self.offers_review() {
1831            return Err(ToolError::invalid_arguments(
1832                "review is not available in this session",
1833            ));
1834        }
1835        if split.background == Some(false) {
1836            return Err(ToolError::invalid_arguments(
1837                "a reviewed call runs as a background job; omit background or set it to true",
1838            ));
1839        }
1840        let (builder, routed) = self.inner.route(&split.arguments)?;
1841        builder.backend.risk(&routed)?;
1842        review::plan(
1843            &self.inner,
1844            builder,
1845            &routed,
1846            review,
1847            self.conversations.limits().max,
1848        )
1849    }
1850
1851    /// Start a reviewed call as a background job. A busy builder
1852    /// conversation queues it, or fails it with `on_busy: fail`; it never
1853    /// waits, since it is a background job, and never steers, since its
1854    /// prompt opens a review of its own.
1855    async fn start_reviewed(
1856        &self,
1857        split: &Split,
1858        review: &Value,
1859        context: &ToolContext,
1860    ) -> Result<ToolOutput, ToolError> {
1861        let mut plan = self.plan(split, review)?;
1862        let mut arguments = split.arguments.clone();
1863        self.inner.risk_for(&arguments, true)?;
1864        // The builder and every reviewer work in the same worktree.
1865        let worktree = if parse_args::<AgentArgs>(&arguments)?.worktree {
1866            self.jobs.has_room()?;
1867            let worktree = self.inner.prepare_worktree(&mut arguments, context).await?;
1868            plan.cwd = arguments
1869                .get("cwd")
1870                .and_then(Value::as_str)
1871                .map(str::to_owned);
1872            worktree
1873        } else {
1874            None
1875        };
1876        let (agent, routed) = self.inner.route(&arguments)?;
1877        let turn = match parse_args::<AgentArgs>(&routed)?
1878            .session
1879            .filter(|session| conversation::is_handle(session))
1880        {
1881            None => Turn::New,
1882            Some(handle) => {
1883                let place = self.jobs.lanes.join(&handle);
1884                if place.ahead == 0 && !agent.backend.busy(&routed)? {
1885                    Turn::Next(place)
1886                } else if agent.on_busy(&routed)? == BusyBehavior::Fail {
1887                    return Err(busy(&handle, &place));
1888                } else {
1889                    queue_room(agent, &handle, &place)?;
1890                    Turn::Queued(place)
1891                }
1892            }
1893        };
1894        let dir = self.reviews.as_deref().ok_or_else(|| {
1895            ToolError::invalid_arguments("review is not available in this session")
1896        })?;
1897        let reviewers: Vec<String> = plan
1898            .reviewer_names()
1899            .into_iter()
1900            .map(str::to_owned)
1901            .collect();
1902        let (rounds, land) = (plan.rounds, plan.land);
1903        let tool = Arc::new(ReviewedTool::new(
1904            Arc::clone(&self.inner),
1905            Arc::clone(&self.jobs.lanes),
1906            Arc::clone(&self.conversations),
1907            plan,
1908            arguments.clone(),
1909            self.session.clone(),
1910            dir,
1911        )?);
1912        let queued = matches!(turn, Turn::Queued(_));
1913        let mut started = self.jobs.start(
1914            Arc::clone(&tool) as Arc<dyn Tool>,
1915            AGENT_TOOL,
1916            &agent.name,
1917            arguments,
1918            context,
1919            turn,
1920            Some(&tool),
1921            JobMeta {
1922                origin: JobOrigin::Model,
1923                worktree,
1924                resumed_from: None,
1925                keep_origin: false,
1926                accepted_follow_up: None,
1927            },
1928        )?;
1929        let first = reviewers.first().cloned().unwrap_or_default();
1930        started["review"] = json!({
1931            "reviewers": reviewers,
1932            "rounds": rounds,
1933            "land": review::land_name(land),
1934            "journal": tool.journal_id(),
1935        });
1936        started["note"] = format!(
1937            "{}The agent works on it, then a fresh {first} reviews it (or the next reviewer \
1938             available), for up to {rounds} rounds. SCV reports the result, with its own Review \
1939             and Landing lines, in a new turn when the review ends. agent_status shows the \
1940             round and phase; agent_cancel stops the whole review.",
1941            if queued {
1942                "The conversation is busy, so this reviewed call is queued behind the turns \
1943                 ahead of it. "
1944            } else {
1945                ""
1946            }
1947        )
1948        .into();
1949        Ok(ToolOutput::success(started.to_string()))
1950    }
1951
1952    /// Start the call on `agent` as a background job of this session.
1953    fn start(
1954        &self,
1955        agent: &Offered,
1956        mut arguments: Value,
1957        context: &ToolContext,
1958        turn: Turn,
1959    ) -> Result<ToolOutput, ToolError> {
1960        let meta = self.queued_meta(&mut arguments, &turn);
1961        let started = self.start_as(agent, arguments, context, turn, meta)?;
1962        Ok(ToolOutput::success(started.to_string()))
1963    }
1964
1965    /// A queued follow-up has no run of its own. Remember the conversation's
1966    /// directory and SCV worktree on the job, so a later resume still has
1967    /// them after the job that run served was dismissed and the worktree
1968    /// removed.
1969    fn queued_meta(&self, arguments: &mut Value, turn: &Turn) -> JobMeta {
1970        if !matches!(turn, Turn::Queued(_)) {
1971            return JobMeta::default();
1972        }
1973        let Some(handle) = parse_args::<AgentArgs>(arguments)
1974            .ok()
1975            .and_then(|args| args.session)
1976            .filter(|session| conversation::is_handle(session))
1977        else {
1978            return JobMeta::default();
1979        };
1980        let Some(cwd) = self.conversations.cwd(&handle) else {
1981            return JobMeta::default();
1982        };
1983        if arguments
1984            .get("cwd")
1985            .and_then(Value::as_str)
1986            .is_none_or(str::is_empty)
1987            && let Some(object) = arguments.as_object_mut()
1988        {
1989            object.insert("cwd".into(), cwd.display().to_string().into());
1990        }
1991        JobMeta {
1992            worktree: self.inner.worktree_at(&cwd),
1993            ..JobMeta::default()
1994        }
1995    }
1996
1997    /// Start the call on `agent` as a background job `meta` describes.
1998    fn start_as(
1999        &self,
2000        agent: &Offered,
2001        arguments: Value,
2002        context: &ToolContext,
2003        turn: Turn,
2004        meta: JobMeta,
2005    ) -> Result<Value, ToolError> {
2006        let tool: Arc<dyn Tool> = if matches!(turn, Turn::Queued(_)) {
2007            Arc::new(QueuedTool(Arc::clone(&self.inner)))
2008        } else {
2009            Arc::clone(&self.inner) as Arc<dyn Tool>
2010        };
2011        self.jobs.start(
2012            tool,
2013            AGENT_TOOL,
2014            &agent.name,
2015            arguments,
2016            context,
2017            turn,
2018            None,
2019            meta,
2020        )
2021    }
2022
2023    /// Run the owner's call from the workbench as a background job of this
2024    /// session: no approval request, since it is the owner's own action,
2025    /// and no report turn, since the owner watches it. A call continuing a
2026    /// conversation that runs a turn steers it when `mode` allows and the
2027    /// agent can take the prompt, and otherwise queues behind it. `meta`'s
2028    /// worktree is recorded when the call makes none of its own, and its
2029    /// `resumed_from` as given; it is always the owner's.
2030    pub(crate) async fn owner_call(
2031        &self,
2032        mut arguments: Value,
2033        workspace: std::path::PathBuf,
2034        mode: scv_protocol::SendMode,
2035        meta: JobMeta,
2036    ) -> Result<OwnerCall, ToolError> {
2037        use scv_protocol::SendMode;
2038        let context = ToolContext::new(workspace, CancellationToken::new());
2039        self.inner.route(&arguments)?;
2040        self.inner.risk_for(&arguments, true)?;
2041        let worktree = if parse_args::<AgentArgs>(&arguments)?.worktree {
2042            self.jobs.has_room()?;
2043            self.inner
2044                .prepare_worktree(&mut arguments, &context)
2045                .await?
2046        } else {
2047            None
2048        };
2049        let (agent, routed) = self.inner.route(&arguments)?;
2050        let meta = JobMeta {
2051            origin: if meta.keep_origin {
2052                meta.origin
2053            } else {
2054                JobOrigin::Owner
2055            },
2056            worktree: worktree.or(meta.worktree),
2057            resumed_from: meta.resumed_from,
2058            keep_origin: meta.keep_origin,
2059            accepted_follow_up: meta.accepted_follow_up,
2060        };
2061        let Some(handle) = parse_args::<AgentArgs>(&routed)?
2062            .session
2063            .filter(|session| conversation::is_handle(session))
2064        else {
2065            return self
2066                .start_as(agent, arguments, &context, Turn::New, meta)
2067                .map(OwnerCall::Job);
2068        };
2069        let mut meta = meta;
2070        if let Some(cwd) = self.conversations.cwd(&handle) {
2071            if arguments.get("cwd").and_then(Value::as_str).is_none()
2072                && let Some(object) = arguments.as_object_mut()
2073            {
2074                object.insert("cwd".into(), cwd.display().to_string().into());
2075            }
2076            if meta.worktree.is_none() {
2077                meta.worktree = self.inner.worktree_at(&cwd);
2078            }
2079        }
2080        let place = self.jobs.lanes.join(&handle);
2081        let busy = agent.backend.busy(&routed)?;
2082        if !busy && !self.conversations.knows(&handle) {
2083            return Err(ToolError::invalid_arguments(format!(
2084                "no conversation {handle} in this session; it ended, or it was forgotten after it sat idle"
2085            )));
2086        }
2087        if place.ahead == 0 && !busy {
2088            return self
2089                .start_as(agent, arguments, &context, Turn::Next(place), meta)
2090                .map(OwnerCall::Job);
2091        }
2092        if mode != SendMode::Queue
2093            && !self.jobs.lanes.has_takeover(&handle)
2094            && agent
2095                .backend
2096                .steer(routed.clone(), context.clone())
2097                .await?
2098                .is_some()
2099        {
2100            return Ok(OwnerCall::Steered);
2101        }
2102        if mode == SendMode::Steer {
2103            return Err(ToolError::failed(format!(
2104                "conversation {handle} cannot take a prompt into its running turn (its agent \
2105                 does not steer, or no turn is running yet); send it queued instead"
2106            )));
2107        }
2108        queue_room(agent, &handle, &place)?;
2109        self.start_as(agent, arguments, &context, Turn::Queued(place), meta)
2110            .map(OwnerCall::Job)
2111    }
2112
2113    /// Whether `agent` is offered and takes `option` (`model`, `effort`,
2114    /// or `session`).
2115    pub(crate) fn takes(&self, agent: &str, option: &str) -> bool {
2116        self.inner.find(agent).is_some_and(|offered| match option {
2117            "model" => offered.accepts.model,
2118            "effort" => offered.accepts.effort,
2119            "session" => offered.accepts.session,
2120            _ => false,
2121        })
2122    }
2123
2124    /// Whether the session has conversation `handle`, so the owner can send
2125    /// it a prompt.
2126    pub(crate) fn knows(&self, handle: &str) -> bool {
2127        self.conversations.knows(handle)
2128    }
2129
2130    /// Take in conversation `vendor` that `agent` saved, so a call can
2131    /// continue it in `cwd`, when that agent can continue a conversation
2132    /// from its own session ID alone. Returns the handle, which is the one
2133    /// this session already holds for that agent, session, and directory.
2134    pub(crate) fn adopt(&self, agent: &str, cwd: &std::path::Path, vendor: &str) -> Option<String> {
2135        let offered = self.inner.find(agent)?;
2136        if !offered.accepts.session || !offered.backend.adoptable() {
2137            return None;
2138        }
2139        self.conversations.adopt(agent, cwd, vendor).ok()
2140    }
2141
2142    /// Take conversation `handle` over in its agent's own terminal for the
2143    /// owner, sized `size`, as SCV session `session`: refused, with why,
2144    /// when the agent's terminal may not be offered, or the conversation
2145    /// runs a turn, has turns waiting, or has no agent session to open.
2146    /// Calls that arrive meanwhile queue behind the takeover in its lane.
2147    pub(crate) async fn take_over(
2148        &self,
2149        handle: &str,
2150        session: &str,
2151        size: (u16, u16),
2152    ) -> Result<Arc<takeover::Takeover>, String> {
2153        let (agent, launch) = self.launch_for(handle)?;
2154        let place = self.jobs.lanes.join_takeover(handle);
2155        if place.ahead > 0 {
2156            return Err(waiting(handle));
2157        }
2158        self.open_terminal(&agent, &launch, handle, place, session, size)
2159            .await
2160    }
2161
2162    /// Hold conversation `handle`'s lane for the owner's takeover right
2163    /// behind its running turn, ahead of any prompt queued after it, and
2164    /// stop nothing yet: [`Self::take_over_reserved`] stops the background
2165    /// job running that turn. Refused, with why and with nothing held, as
2166    /// [`Self::take_over`] refuses but for the running turn, when that turn
2167    /// is no background job (a model's call, or a reviewed job's), and when
2168    /// no turn runs but a job's call is first in the lane: one about to
2169    /// start its turn (a queued prompt), or one whose turn just ended.
2170    pub(crate) fn reserve_takeover(&self, handle: &str) -> Result<Reserved, String> {
2171        let (agent, launch) = self.launch_for(handle)?;
2172        let obstacles = self
2173            .conversations
2174            .obstacles(handle)
2175            .ok_or_else(|| conversation::unknown_conversation(handle))?;
2176        if let Some(obstacle) = obstacles
2177            .into_iter()
2178            .find(|obstacle| *obstacle != Obstacle::Running)
2179        {
2180            return Err(obstacle.refusal(handle));
2181        }
2182        let (place, job) = self
2183            .jobs
2184            .lanes
2185            .reserve_takeover(handle)
2186            .map_err(|why| match why {
2187                Unreserved::TakenOver => Obstacle::TakenOver.refusal(handle),
2188                Unreserved::NotAJob => not_a_job(handle),
2189            })?;
2190        // Seen with the place held, so the turn is that of the call ahead
2191        // of it: no prompt queued behind can start one in between.
2192        let running = self.conversations.is_busy(handle);
2193        match &job {
2194            // A turn that holds no place in the lane is no job's to stop.
2195            None if running => return Err(not_a_job(handle)),
2196            Some(job) if !running => return Err(between_turns(handle, job)),
2197            _ => {}
2198        }
2199        Ok(Reserved {
2200            handle: handle.to_owned(),
2201            agent,
2202            launch,
2203            place,
2204            job,
2205        })
2206    }
2207
2208    /// Take the conversation `reserved` holds the lane of over for the
2209    /// owner: stop the background job running its turn, wait until that
2210    /// turn has ended and the takeover is first in the lane, and open the
2211    /// terminal. However it fails, the lane is let go and the conversation
2212    /// runs its queued prompts as before.
2213    pub(crate) async fn take_over_reserved(
2214        &self,
2215        reserved: Reserved,
2216        session: &str,
2217        size: (u16, u16),
2218    ) -> Result<Arc<takeover::Takeover>, String> {
2219        let Reserved {
2220            handle,
2221            agent,
2222            launch,
2223            place,
2224            job,
2225        } = reserved;
2226        let Some(job) = job else {
2227            return self
2228                .open_terminal(&agent, &launch, &handle, place, session, size)
2229                .await;
2230        };
2231        // One that finished meanwhile has let its turn go already.
2232        let _ = self.jobs.cancel_for_owner(&job);
2233        let never = CancellationToken::new();
2234        let ended = tokio::time::timeout(INTERRUPT_WAIT, async {
2235            place.wait_first(&never).await?;
2236            self.conversations.wait_idle(&handle, &never).await
2237        })
2238        .await;
2239        if ended.is_err() {
2240            return Err(format!(
2241                "Stopped {job}, but its turn had not ended after {} seconds, so {handle} was not \
2242                 taken over; attach once it is idle",
2243                INTERRUPT_WAIT.as_secs()
2244            ));
2245        }
2246        self.open_terminal(&agent, &launch, &handle, place, session, size)
2247            .await
2248            .map_err(|error| {
2249                format!(
2250                    "Stopped {job}, but {handle} could not be taken over: {error}. Its next \
2251                     prompt runs as before"
2252                )
2253            })
2254    }
2255
2256    /// Whether the owner could take conversation `handle` over in its
2257    /// agent's own terminal now, as [`Self::take_over`] and
2258    /// [`Self::reserve_takeover`] would decide; `None` when the session does
2259    /// not have it. The workbench adds what only it knows: how many
2260    /// takeovers run, and approvals waiting for the owner.
2261    pub(crate) fn attach_offer(&self, handle: &str) -> Option<AttachOffer> {
2262        let obstacles = self.conversations.obstacles(handle)?;
2263        let offer = |state, detail: Option<String>| AttachOffer {
2264            state,
2265            detail,
2266            interrupt: false,
2267            job: None,
2268        };
2269        if let Err(why) = self.launch_for(handle) {
2270            return Some(offer(AttachState::Ineligible, Some(why)));
2271        }
2272        if obstacles.contains(&Obstacle::TakenOver) || self.jobs.lanes.has_takeover(handle) {
2273            return Some(offer(AttachState::TakenOver, None));
2274        }
2275        if let Some(obstacle) = obstacles
2276            .iter()
2277            .find(|obstacle| **obstacle != Obstacle::Running)
2278        {
2279            return Some(offer(obstacle.state(), Some(obstacle.refusal(handle))));
2280        }
2281        let running = obstacles.contains(&Obstacle::Running);
2282        Some(match self.jobs.lanes.front(handle) {
2283            None if !running => offer(AttachState::Ready, None),
2284            None => offer(AttachState::Busy, Some(Obstacle::Running.refusal(handle))),
2285            Some(Front::Job(job)) if running => AttachOffer {
2286                interrupt: true,
2287                job: Some(job),
2288                ..offer(AttachState::Busy, Some(waiting(handle)))
2289            },
2290            Some(Front::Job(job)) => offer(AttachState::Busy, Some(between_turns(handle, &job))),
2291            Some(Front::Call | Front::Takeover) => {
2292                offer(AttachState::Busy, Some(not_a_job(handle)))
2293            }
2294        })
2295    }
2296
2297    /// The agent of conversation `handle` and how its terminal opens, or
2298    /// why it is never offered.
2299    fn launch_for(&self, handle: &str) -> Result<(String, takeover::TakeoverLaunch), String> {
2300        let agent = self
2301            .conversations
2302            .agent_of(handle)
2303            .ok_or_else(|| conversation::unknown_conversation(handle))?;
2304        let offered = self
2305            .inner
2306            .find(&agent)
2307            .ok_or_else(|| format!("{agent} is not offered in this session"))?;
2308        let launch = offered.backend.takeover()?;
2309        Ok((agent, launch))
2310    }
2311
2312    /// Hold conversation `handle`, whose lane `place` holds, stop what it
2313    /// kept between turns, and open `launch`'s terminal on it.
2314    async fn open_terminal(
2315        &self,
2316        agent: &str,
2317        launch: &takeover::TakeoverLaunch,
2318        handle: &str,
2319        place: Place,
2320        session: &str,
2321        size: (u16, u16),
2322    ) -> Result<Arc<takeover::Takeover>, String> {
2323        let mut hold = self.conversations.take_over(handle)?;
2324        if let Some(attachment) = hold.attachment.take() {
2325            match self.inner.find(agent) {
2326                Some(offered) => offered.backend.release(attachment).await,
2327                None => drop(attachment),
2328            }
2329        }
2330        takeover::start(launch, hold, place, session, size).await
2331    }
2332}
2333
2334/// The owner's takeover of a conversation whose lane is held right behind
2335/// its running turn, which is not stopped yet.
2336pub(crate) struct Reserved {
2337    handle: String,
2338    agent: String,
2339    launch: takeover::TakeoverLaunch,
2340    place: Place,
2341    /// The background job running the conversation's turn, which taking
2342    /// it over stops; `None` when it runs none.
2343    pub(crate) job: Option<String>,
2344}
2345
2346impl std::fmt::Debug for Reserved {
2347    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2348        formatter
2349            .debug_struct("Reserved")
2350            .field("handle", &self.handle)
2351            .field("job", &self.job)
2352            .finish_non_exhaustive()
2353    }
2354}
2355
2356/// How long the owner's interrupting takeover waits for the turn it
2357/// stopped to end before it gives the conversation's lane back.
2358const INTERRUPT_WAIT: Duration = Duration::from_secs(20);
2359
2360/// The refusal to take conversation `handle` over while calls are in its
2361/// lane.
2362fn waiting(handle: &str) -> String {
2363    format!(
2364        "{handle} has a turn running or waiting; attach once it is idle, or stop its jobs first"
2365    )
2366}
2367
2368/// The refusal to stop conversation `handle`'s running turn for a takeover
2369/// when that turn is no background job.
2370fn not_a_job(handle: &str) -> String {
2371    format!(
2372        "{handle} is running a turn that is no background job SCV can stop for you (a model's \
2373         call in its session, or a reviewed job's round); attach once it is idle"
2374    )
2375}
2376
2377/// The refusal to stop `job`, first in conversation `handle`'s lane, for a
2378/// takeover while no turn runs: its turn is about to start or just ended,
2379/// and a prompt that waits is never stopped.
2380fn between_turns(handle: &str, job: &str) -> String {
2381    format!(
2382        "{handle} is between turns: {job} is about to start the next one, or has just ended \
2383         its own, and only a running turn is stopped for a takeover; attach again in a moment"
2384    )
2385}
2386
2387#[derive(Deserialize)]
2388#[serde(deny_unknown_fields)]
2389struct WaitArgs {
2390    job: String,
2391    timeout_seconds: Option<u64>,
2392}
2393
2394#[derive(Deserialize)]
2395#[serde(deny_unknown_fields)]
2396struct StatusArgs {
2397    job: Option<String>,
2398}
2399
2400/// `agent_wait`: block until a background job finishes, or the timeout.
2401pub(crate) struct WaitTool {
2402    pub(crate) jobs: Arc<BackgroundJobs>,
2403    pub(crate) timeouts: Timeouts,
2404}
2405
2406#[async_trait]
2407impl Tool for WaitTool {
2408    fn spec(&self) -> ToolSpec {
2409        let mut timeout = timeout_schema(self.timeouts);
2410        timeout["description"] = format!(
2411            "Seconds to wait before returning the job still running. Defaults to {}; at most {}.",
2412            self.timeouts.default.min(self.timeouts.max).as_secs(),
2413            self.timeouts.max.as_secs()
2414        )
2415        .into();
2416        ToolSpec {
2417            name: "agent_wait".into(),
2418            description: "Wait for a background agent job (the `job` handle an agent call with \
2419                background: true returned) to finish, and return its result. Returns early with \
2420                status running when the timeout passes. Waiting holds your turn open, so the \
2421                user cannot reach you meanwhile; usually let SCV report the result instead."
2422                .into(),
2423            parameters: json!({
2424                "type":"object",
2425                "properties":{"job":{"type":"string"},"timeout_seconds":timeout},
2426                "required":["job"],
2427                "additionalProperties":false
2428            }),
2429        }
2430    }
2431
2432    fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
2433        let args: WaitArgs = parse_args(arguments)?;
2434        self.timeouts.resolve(args.timeout_seconds)?;
2435        Ok(ToolRisk::ReadOnly)
2436    }
2437
2438    fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
2439        let args: WaitArgs = parse_args(arguments)?;
2440        Ok(format!(
2441            "Wait for background job {}",
2442            bounded(&args.job, 64)
2443        ))
2444    }
2445
2446    async fn execute(
2447        &self,
2448        arguments: Value,
2449        context: ToolContext,
2450    ) -> Result<ToolOutput, ToolError> {
2451        let args: WaitArgs = parse_args(&arguments)?;
2452        let limit = self.timeouts.resolve(args.timeout_seconds)?;
2453        let value = self
2454            .jobs
2455            .wait(&args.job, limit, &context.cancellation, &context.call_id)
2456            .await?;
2457        Ok(ToolOutput::success(value.to_string()))
2458    }
2459}
2460
2461/// `agent_status`: describe one background job or all of them.
2462pub(crate) struct StatusTool {
2463    pub(crate) jobs: Arc<BackgroundJobs>,
2464}
2465
2466#[async_trait]
2467impl Tool for StatusTool {
2468    fn spec(&self) -> ToolSpec {
2469        ToolSpec {
2470            name: "agent_status".into(),
2471            description: "Show this session's background agent jobs: running ones with their \
2472                latest progress, finished ones with their result. Pass job for one job."
2473                .into(),
2474            parameters: json!({
2475                "type":"object",
2476                "properties":{"job":{"type":"string"}},
2477                "additionalProperties":false
2478            }),
2479        }
2480    }
2481
2482    fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
2483        let _: StatusArgs = parse_args(arguments)?;
2484        Ok(ToolRisk::ReadOnly)
2485    }
2486
2487    fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
2488        let args: StatusArgs = parse_args(arguments)?;
2489        Ok(args.job.map_or_else(
2490            || "List background jobs".into(),
2491            |job| format!("Show background job {}", bounded(&job, 64)),
2492        ))
2493    }
2494
2495    async fn execute(
2496        &self,
2497        arguments: Value,
2498        context: ToolContext,
2499    ) -> Result<ToolOutput, ToolError> {
2500        let args: StatusArgs = parse_args(&arguments)?;
2501        let value = self.jobs.describe(args.job.as_deref(), &context.call_id)?;
2502        Ok(ToolOutput::success(value.to_string()))
2503    }
2504}
2505
2506#[derive(Deserialize)]
2507#[serde(deny_unknown_fields)]
2508struct CancelArgs {
2509    job: String,
2510}
2511
2512/// `agent_cancel`: stop one running background job.
2513pub(crate) struct CancelTool {
2514    pub(crate) jobs: Arc<BackgroundJobs>,
2515}
2516
2517#[async_trait]
2518impl Tool for CancelTool {
2519    fn spec(&self) -> ToolSpec {
2520        ToolSpec {
2521            name: "agent_cancel".into(),
2522            description: "Stop a running background agent job (the `job` handle an agent call \
2523                with background: true returned), for example when the user no longer wants \
2524                it. The agent and every process it started are stopped; work it already wrote \
2525                stays. Returns the job with status cancelled, and no report turn follows."
2526                .into(),
2527            parameters: json!({
2528                "type":"object",
2529                "properties":{"job":{"type":"string"}},
2530                "required":["job"],
2531                "additionalProperties":false
2532            }),
2533        }
2534    }
2535
2536    fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
2537        let _: CancelArgs = parse_args(arguments)?;
2538        Ok(ToolRisk::Process)
2539    }
2540
2541    fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
2542        let args: CancelArgs = parse_args(arguments)?;
2543        Ok(format!("Stop background job {}", bounded(&args.job, 64)))
2544    }
2545
2546    async fn execute(
2547        &self,
2548        arguments: Value,
2549        context: ToolContext,
2550    ) -> Result<ToolOutput, ToolError> {
2551        let args: CancelArgs = parse_args(&arguments)?;
2552        let value = self.jobs.cancel(&args.job, &context.call_id).await?;
2553        Ok(ToolOutput::success(value.to_string()))
2554    }
2555}
2556
2557#[cfg(test)]
2558mod tests;