1pub(in crate::delegate) mod lane;
26mod owed;
27
28use std::{
29 collections::HashSet,
30 path::PathBuf,
31 sync::{Arc, Mutex, Weak},
32 time::{Duration, Instant},
33};
34
35use async_trait::async_trait;
36use scv_core::{
37 ApprovalGate, ProgressSink, Tool, ToolApprovals, ToolContext, ToolError, ToolOutput, ToolRisk,
38 ToolSpec,
39};
40use scv_protocol::{JobChange, JobOutcome, JobReport, JobStatus, describe_reports};
41use serde::Deserialize;
42use serde_json::{Map, Value, json};
43use tokio::sync::{mpsc, watch};
44use tokio_util::sync::CancellationToken;
45
46use self::lane::{Lanes, Place};
47use crate::{
48 BusyBehavior,
49 args::{Timeouts, bounded, parse_args, timeout_schema},
50 delegate::{
51 agent::{AGENT_TOOL, AgentTool, Offered},
52 conversation::{self, ConversationStore},
53 output::AgentReply,
54 request::AgentArgs,
55 review::{self, Record, ReviewedTool},
56 },
57 sync::lock,
58};
59
60const MAX_FINISHED: usize = 16;
62const REPORT_REPLY_CHARS: usize = 6000;
64const REPORT_MAX_JOBS: usize = 4;
66const REPORT_RETRY_DELAYS: [Duration; 2] = [Duration::from_secs(30), Duration::from_secs(120)];
70const CANCEL_SETTLE: Duration = Duration::from_secs(10);
72const MAX_PENDING_CHANGES: usize = 64;
75const TASK_CHARS: usize = 80;
77
78pub struct BackgroundJobs {
81 limit: usize,
82 state: Mutex<JobsState>,
83 cancellation: CancellationToken,
84 finished: Option<mpsc::UnboundedSender<()>>,
86 approvals: Option<Arc<dyn ApprovalGate>>,
89 lanes: Arc<Lanes>,
91}
92
93impl std::fmt::Debug for BackgroundJobs {
94 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
95 formatter
96 .debug_struct("BackgroundJobs")
97 .field("limit", &self.limit)
98 .finish_non_exhaustive()
99 }
100}
101
102#[derive(Default)]
103struct JobsState {
104 next: u64,
105 jobs: Vec<Job>,
106 changes: Vec<(String, JobChange)>,
108 owed: owed::Owed,
110}
111
112impl JobsState {
113 fn refresh(&mut self, call_id: &str, outcome: JobOutcome) {
116 if let Some((_, change)) = self
117 .changes
118 .iter_mut()
119 .rev()
120 .find(|(call, change)| call == call_id && change.job == outcome.job)
121 {
122 change.outcome = Some(outcome);
123 }
124 }
125
126 fn record(&mut self, call_id: &str, change: JobChange) {
129 if call_id.is_empty() {
130 return;
131 }
132 if self.changes.len() >= MAX_PENDING_CHANGES {
133 let (call, dropped) = self.changes.remove(0);
134 self.owed.evicted(&call, &dropped.job);
135 }
136 self.changes.push((call_id.to_owned(), change));
137 }
138}
139
140struct Job {
141 id: String,
142 tool: String,
144 agent: String,
146 task: String,
148 started: Instant,
149 progress: ProgressSink,
150 last_progress: Option<String>,
151 cancel: CancellationToken,
153 cancelled: bool,
155 queued: bool,
158 outcome: Option<Outcome>,
159 reported: bool,
164 reporting: bool,
166 report_attempts: u32,
168 report_due: Option<tokio::time::Instant>,
170 done: watch::Receiver<bool>,
171 review: Option<Arc<Record>>,
173 cancelling: bool,
176}
177
178struct Waiting<'a> {
182 jobs: &'a BackgroundJobs,
183 job: String,
184 call: String,
185 owed: bool,
187 settled: bool,
188}
189
190impl Waiting<'_> {
191 fn settle(mut self) {
196 self.settled = true;
197 let mut state = self.jobs.state();
198 let Some(entry) = state.jobs.iter_mut().find(|entry| entry.id == self.job) else {
199 return;
200 };
201 entry.cancelling = false;
202 if !self.owed {
203 return;
204 }
205 let pending = entry.outcome.is_none().then(|| entry.outcome()).flatten();
206 let pending = pending.map(|mut outcome| {
207 outcome.review.journal_pending = true;
208 outcome
209 });
210 if let Some(outcome) = state.owed.waited(&self.job, &self.call, pending) {
211 state.refresh(&self.call, outcome);
212 }
213 }
214}
215
216impl Drop for Waiting<'_> {
217 fn drop(&mut self) {
218 if self.settled {
219 return;
220 }
221 let mut state = self.jobs.state();
222 if let Some(entry) = state.jobs.iter_mut().find(|entry| entry.id == self.job) {
223 entry.cancelling = false;
224 }
225 if !self.owed {
226 return;
227 }
228 let (call, job) = (&self.call, &self.job);
230 state
231 .changes
232 .retain(|(recorded, change)| !(recorded == call && &change.job == job));
233 let due = state.owed.abandoned(job, call);
234 drop(state);
235 if due {
236 self.jobs.wake();
237 }
238 }
239}
240
241struct Outcome {
242 output: ToolOutput,
243 elapsed: Duration,
244}
245
246#[derive(Debug, PartialEq, Eq)]
248pub enum ReportFailure {
249 Retry(Duration),
251 GiveUp {
254 attempts: u32,
255 reports: Vec<JobReport>,
256 },
257 Settled,
259}
260
261impl Drop for BackgroundJobs {
262 fn drop(&mut self) {
263 self.cancellation.cancel();
264 }
265}
266
267impl BackgroundJobs {
268 pub fn new(limit: usize, finished: Option<mpsc::UnboundedSender<()>>) -> Self {
270 Self {
271 limit,
272 state: Mutex::default(),
273 cancellation: CancellationToken::new(),
274 finished,
275 approvals: None,
276 lanes: Arc::default(),
277 }
278 }
279
280 #[must_use]
283 pub fn with_approvals(mut self, gate: Arc<dyn ApprovalGate>) -> Self {
284 self.approvals = Some(gate);
285 self
286 }
287
288 pub(crate) fn limit(&self) -> usize {
289 self.limit
290 }
291
292 fn state(&self) -> std::sync::MutexGuard<'_, JobsState> {
293 lock(&self.state)
294 }
295
296 #[allow(
306 clippy::too_many_arguments,
307 reason = "a job's call, its context, its turn, and its review, each its own"
308 )]
309 pub(in crate::delegate) fn start(
310 self: &Arc<Self>,
311 tool: Arc<dyn Tool>,
312 name: &str,
313 agent: &str,
314 arguments: Value,
315 context: &ToolContext,
316 turn: Turn,
317 review: Option<&Arc<ReviewedTool>>,
318 ) -> Result<Value, ToolError> {
319 let task = task_line(
320 arguments
321 .get("prompt")
322 .and_then(Value::as_str)
323 .unwrap_or_default(),
324 );
325 let queued = matches!(turn, Turn::Queued(_));
326 let reserved = match review {
327 Some(review) => {
328 let id = {
329 let mut state = self.state();
330 self.room(&state, queued)?;
331 state.next += 1;
332 format!("job-{}", state.next)
333 };
334 review.begin(&id)?;
335 Some(id)
336 }
337 None => None,
338 };
339 let (id, progress, done_tx, cancellation) = {
340 let mut state = self.state();
341 self.room(&state, queued)?;
343 let id = reserved.unwrap_or_else(|| {
344 state.next += 1;
345 format!("job-{}", state.next)
346 });
347 let progress = ProgressSink::buffered();
348 let (done_tx, done) = watch::channel(false);
349 let cancel = self.cancellation.child_token();
350 state.record(
351 &context.call_id,
352 JobChange {
353 job: id.clone(),
354 tool: name.to_owned(),
355 agent: agent.to_owned(),
356 status: JobStatus::Running,
357 task: task.clone(),
358 journal: review.map(|review| review.journal_id()),
359 outcome: None,
360 },
361 );
362 state.jobs.push(Job {
363 id: id.clone(),
364 tool: name.to_owned(),
365 agent: agent.to_owned(),
366 task,
367 started: Instant::now(),
368 progress: progress.clone(),
369 last_progress: None,
370 cancel: cancel.clone(),
371 cancelled: false,
372 queued,
373 outcome: None,
374 reported: false,
375 reporting: false,
376 report_attempts: 0,
377 report_due: None,
378 done,
379 review: review.map(|review| Arc::clone(review.record())),
380 cancelling: false,
381 });
382 (id, progress, done_tx, cancel)
383 };
384 let jobs = Arc::downgrade(self);
385 let job = id.clone();
386 let approvals = self.approvals.clone();
387 let workspace = context.workspace.clone();
388 let reviewed = review.map(Arc::clone);
389 tokio::spawn(async move {
390 let started = Instant::now();
391 let mut context = ToolContext::new(workspace, cancellation);
392 context.background_job = true;
393 if let Some(gate) = &approvals {
396 context.approvals = ToolApprovals::new(Arc::clone(gate), job.clone());
397 }
398 context.progress = progress;
399 let place = turn.place();
400 let output = match &place {
401 Some(place) if queued => {
402 context.progress.report(&format!(
403 "queued: waits for conversation {}'s running turn",
404 place.handle()
405 ));
406 match place.wait_first(&context.cancellation).await {
407 Ok(()) => {
408 context.progress.take();
409 context.progress.report(&format!(
410 "started its turn in conversation {}",
411 place.handle()
412 ));
413 tool.execute(arguments, context).await
414 }
415 Err(error) => match &reviewed {
418 Some(review) => Ok(review.unstarted(&error)),
419 None => Err(error),
420 },
421 }
422 }
423 _ => tool.execute(arguments, context).await,
424 }
425 .unwrap_or_else(ToolOutput::from);
426 drop(place);
429 finish(&jobs, &job, output, started.elapsed());
430 let _ = done_tx.send(true);
431 });
432 let mut started = json!({
433 "job": id,
434 "agent": agent,
435 "status": "running",
436 "background": true,
437 "note": "The agent is working in the background. SCV reports the result in a new \
438 turn when it finishes. agent_status shows its progress and agent_cancel \
439 stops it."
440 });
441 if queued {
442 started["queued"] = true.into();
443 started["note"] = "The conversation is busy, so this prompt is queued: it runs in \
444 the background after the turns ahead of it, and SCV reports the result in a new \
445 turn when it finishes. agent_status shows where it stands and agent_cancel \
446 withdraws it."
447 .into();
448 }
449 Ok(started)
450 }
451
452 fn room(&self, state: &JobsState, queued: bool) -> Result<(), ToolError> {
455 let running = state
456 .jobs
457 .iter()
458 .filter(|job| job.outcome.is_none() && !job.queued)
459 .count();
460 if !queued && running >= self.limit {
461 return Err(ToolError::limit(format!(
462 "{running} background jobs are already running, the limit \
463 (agent.max_background). Start this one after a job finishes, or \
464 stop one with agent_cancel if the user no longer needs it."
465 )));
466 }
467 Ok(())
468 }
469
470 async fn cancel(&self, job: &str, call_id: &str) -> Result<Value, ToolError> {
481 let (mut done, owed) = {
482 let mut state = self.state();
483 let index = state
484 .jobs
485 .iter()
486 .position(|candidate| candidate.id == job)
487 .ok_or_else(|| unknown_job(job))?;
488 let entry = &mut state.jobs[index];
489 if entry.outcome.is_some() {
490 let (mut value, change) = entry.describe();
491 value["note"] = "The job had already finished.".into();
492 if let Some(change) = change {
493 state.record(call_id, change);
494 }
495 return Ok(value);
496 }
497 entry.cancelled = true;
498 entry.cancel.cancel();
499 entry.cancelling = true;
500 let done = entry.done.clone();
501 let mut owed = false;
502 if !entry.reported {
503 entry.reported = true;
504 let change = entry.change(JobStatus::Cancelled);
505 owed = entry.review.is_some();
506 state.record(call_id, change);
507 if owed {
508 state.owed.cancelling(job, call_id);
509 }
510 }
511 (done, owed)
512 };
513 let waiting = Waiting {
514 jobs: self,
515 job: job.to_owned(),
516 call: call_id.to_owned(),
517 owed,
518 settled: false,
519 };
520 let _ = tokio::time::timeout(CANCEL_SETTLE, done.wait_for(|finished| *finished)).await;
521 waiting.settle();
522 self.describe(Some(job), call_id)
523 }
524
525 pub fn take_updates(&self) -> Vec<JobOutcome> {
529 self.state().owed.take_updates()
530 }
531
532 pub fn updated(&self, jobs: &[String], delivered: bool) {
535 self.state().owed.updated(jobs, delivered);
536 }
537
538 pub fn published(&self, call_id: &str, delivered: bool) {
544 let due = self.state().owed.sent(call_id, delivered);
545 if due {
546 self.wake();
547 }
548 }
549
550 pub fn turn_ended(&self) {
553 let due = self.state().owed.turn_ended();
554 if due {
555 self.wake();
556 }
557 }
558
559 #[cfg(any(test, feature = "test-support"))]
566 pub fn owe_for_tests(&self, call_id: &str, change: JobChange, last: Option<JobOutcome>) {
567 let mut state = self.state();
568 state.owed.owe(call_id, &change, last);
569 state.record(call_id, change);
570 }
571
572 #[cfg(any(test, feature = "test-support"))]
574 pub fn stop_for_tests(&self, outcome: &JobOutcome) {
575 let due = self.state().owed.stopped(&outcome.job, outcome);
576 if due {
577 self.wake();
578 }
579 }
580
581 #[cfg(any(test, feature = "test-support"))]
585 pub fn start_for_tests(self: &Arc<Self>, agent: &str) -> tokio::sync::oneshot::Sender<()> {
586 struct Waits(Mutex<Option<tokio::sync::oneshot::Receiver<()>>>);
587 #[async_trait]
588 impl Tool for Waits {
589 fn spec(&self) -> ToolSpec {
590 ToolSpec {
591 name: AGENT_TOOL.into(),
592 description: String::new(),
593 parameters: json!({"type":"object"}),
594 }
595 }
596 fn risk(&self, _arguments: &Value) -> Result<ToolRisk, ToolError> {
597 Ok(ToolRisk::Delegate)
598 }
599 fn approval_summary(&self, _arguments: &Value) -> Result<String, ToolError> {
600 Ok(String::new())
601 }
602 async fn execute(
603 &self,
604 _arguments: Value,
605 _context: ToolContext,
606 ) -> Result<ToolOutput, ToolError> {
607 let release = lock(&self.0).take();
608 if let Some(release) = release {
609 let _ = release.await;
610 }
611 Ok(ToolOutput::success(
612 r#"{"status":"completed","reply":"done"}"#,
613 ))
614 }
615 }
616 let (release, waiting) = tokio::sync::oneshot::channel();
617 let context = ToolContext::new(std::env::temp_dir(), CancellationToken::new());
618 self.start(
619 Arc::new(Waits(Mutex::new(Some(waiting)))),
620 AGENT_TOOL,
621 agent,
622 json!({"prompt": "wait"}),
623 &context,
624 Turn::New,
625 None,
626 )
627 .expect("a test job starts");
628 release
629 }
630
631 fn wake(&self) {
633 if let Some(finished) = &self.finished {
634 let _ = finished.send(());
635 }
636 }
637
638 async fn wait(
641 &self,
642 job: &str,
643 limit: Duration,
644 cancellation: &CancellationToken,
645 call_id: &str,
646 ) -> Result<Value, ToolError> {
647 let mut done = self
648 .state()
649 .jobs
650 .iter()
651 .find(|candidate| candidate.id == job)
652 .map(|candidate| candidate.done.clone())
653 .ok_or_else(|| unknown_job(job))?;
654 tokio::select! {
655 () = cancellation.cancelled() => return Err(ToolError::cancelled("wait cancelled")),
656 _ = tokio::time::timeout(limit, done.wait_for(|finished| *finished)) => {}
657 }
658 self.describe(Some(job), call_id)
659 }
660
661 fn describe(&self, job: Option<&str>, call_id: &str) -> Result<Value, ToolError> {
664 let mut state = self.state();
665 let mut changes = Vec::new();
666 let value = if let Some(job) = job {
667 let entry = state
668 .jobs
669 .iter_mut()
670 .find(|candidate| candidate.id == job)
671 .ok_or_else(|| unknown_job(job))?;
672 let (value, change) = entry.describe();
673 changes.extend(change);
674 value
675 } else {
676 let jobs: Vec<Value> = state
677 .jobs
678 .iter_mut()
679 .map(|entry| {
680 let (value, change) = entry.describe();
681 changes.extend(change);
682 value
683 })
684 .collect();
685 json!({ "jobs": jobs })
686 };
687 for change in changes {
688 state.record(call_id, change);
689 }
690 Ok(value)
691 }
692
693 pub fn take_changes(&self, call_id: &str) -> Vec<JobChange> {
699 let mut state = self.state();
700 let mut taken = Vec::new();
701 state.changes.retain(|(call, change)| {
702 if call == call_id {
703 taken.push(change.clone());
704 false
705 } else {
706 true
707 }
708 });
709 for change in &mut taken {
710 state.owed.take(call_id, change);
711 }
712 taken
713 }
714
715 pub fn take_unreported(&self) -> Vec<JobReport> {
720 let now = tokio::time::Instant::now();
721 let mut state = self.state();
722 state
723 .jobs
724 .iter_mut()
725 .filter(|job| job.awaits_report() && job.report_due.is_none_or(|due| due <= now))
726 .take(REPORT_MAX_JOBS)
727 .filter_map(|job| {
728 job.reporting = true;
729 job.report()
730 })
731 .collect()
732 }
733
734 pub fn report_settled(&self, jobs: &[String]) {
737 let mut state = self.state();
738 for job in state.jobs.iter_mut().filter(|job| jobs.contains(&job.id)) {
739 job.reporting = false;
740 job.reported = true;
741 job.report_due = None;
742 }
743 }
744
745 pub fn report_failed(&self, jobs: &[String], retry: bool) -> ReportFailure {
752 let mut state = self.state();
753 let mut failed: Vec<&mut Job> = state
754 .jobs
755 .iter_mut()
756 .filter(|job| job.reporting && jobs.contains(&job.id))
757 .collect();
758 for job in &mut failed {
759 job.reporting = false;
760 job.report_attempts += 1;
761 }
762 failed.retain(|job| !job.reported);
764 let Some(attempts) = failed.iter().map(|job| job.report_attempts).max() else {
765 return ReportFailure::Settled;
766 };
767 let delay = usize::try_from(attempts - 1)
768 .ok()
769 .and_then(|earlier| REPORT_RETRY_DELAYS.get(earlier));
770 if retry && let Some(&delay) = delay {
771 let due = tokio::time::Instant::now() + delay;
772 for job in failed {
773 job.report_due = Some(due);
774 }
775 return ReportFailure::Retry(delay);
776 }
777 let reports = failed
778 .into_iter()
779 .filter_map(|job| {
780 job.reported = true;
781 job.report_due = None;
782 job.report()
783 })
784 .collect();
785 ReportFailure::GiveUp { attempts, reports }
786 }
787
788 pub fn retry_now(&self) -> bool {
792 let mut state = self.state();
793 let mut waiting = false;
794 for job in state.jobs.iter_mut().filter(|job| job.awaits_report()) {
795 waiting |= job.report_due.take().is_some();
796 }
797 waiting
798 }
799
800 pub fn next_retry(&self) -> Option<tokio::time::Instant> {
803 self.state()
804 .jobs
805 .iter()
806 .filter(|job| job.awaits_report())
807 .filter_map(|job| job.report_due)
808 .min()
809 }
810
811 pub fn pending(&self) -> usize {
816 let state = self.state();
817 let mut pending: HashSet<&str> = state
818 .jobs
819 .iter()
820 .filter(|job| job.outcome.is_none() || !job.reported)
821 .map(|job| job.id.as_str())
822 .collect();
823 pending.extend(state.owed.jobs());
824 pending.len()
825 }
826
827 pub fn running(&self) -> usize {
829 self.state()
830 .jobs
831 .iter()
832 .filter(|job| job.outcome.is_none())
833 .count()
834 }
835}
836
837pub(in crate::delegate) enum Turn {
839 New,
841 Next(Place),
843 Queued(Place),
845}
846
847impl Turn {
848 fn place(self) -> Option<Place> {
849 match self {
850 Self::New => None,
851 Self::Next(place) | Self::Queued(place) => Some(place),
852 }
853 }
854}
855
856fn finish(jobs: &Weak<BackgroundJobs>, id: &str, output: ToolOutput, elapsed: Duration) {
857 let Some(jobs) = jobs.upgrade() else {
859 return;
860 };
861 {
862 let mut state = jobs.state();
863 let mut last = None;
864 if let Some(job) = state.jobs.iter_mut().find(|job| job.id == id) {
865 job.last_progress = job.progress.take().or(job.last_progress.take());
866 job.outcome = Some(Outcome { output, elapsed });
867 job.reported |= job.cancelled;
869 last = job.outcome();
870 }
871 if let Some(last) = last {
873 state.owed.stopped(id, &last);
874 }
875 let finished = state
877 .jobs
878 .iter()
879 .filter(|job| job.outcome.is_some())
880 .count();
881 let mut excess = finished.saturating_sub(MAX_FINISHED);
882 state.jobs.retain(|job| {
883 if excess > 0 && job.outcome.is_some() && job.reported && !job.cancelling {
884 excess -= 1;
885 false
886 } else {
887 true
888 }
889 });
890 }
891 if let Some(finished) = &jobs.finished {
892 let _ = finished.send(());
893 }
894}
895
896impl Job {
897 fn awaits_report(&self) -> bool {
899 self.outcome.is_some() && !self.reported && !self.reporting
900 }
901
902 fn report(&self) -> Option<JobReport> {
904 let outcome = self.outcome.as_ref()?;
905 let reply = AgentReply::read(&result_value(&outcome.output)).unwrap_or_default();
906 Some(JobReport {
907 job: self.id.clone(),
908 agent: self.agent.clone(),
909 task: self.task.clone(),
910 status: job_status(&outcome.output, &reply),
911 session: reply.session,
912 reply: bounded(
913 reply
914 .reply
915 .as_deref()
916 .unwrap_or(outcome.output.content.as_str()),
917 REPORT_REPLY_CHARS,
918 ),
919 outcome: self.outcome(),
920 })
921 }
922
923 fn outcome(&self) -> Option<JobOutcome> {
926 self.review.as_ref().map(|record| {
927 let (review, landing) = record
928 .live
929 .snapshot(self.cancelled || self.cancel.is_cancelled());
930 JobOutcome {
931 job: self.id.clone(),
932 review,
933 landing,
934 }
935 })
936 }
937
938 fn change(&self, status: JobStatus) -> JobChange {
941 JobChange {
942 job: self.id.clone(),
943 tool: self.tool.clone(),
944 agent: self.agent.clone(),
945 status,
946 task: self.task.clone(),
947 journal: None,
948 outcome: if status == JobStatus::Running {
949 None
950 } else {
951 self.outcome()
952 },
953 }
954 }
955
956 fn describe(&mut self) -> (Value, Option<JobChange>) {
959 if let Some(line) = self.progress.take() {
960 self.last_progress = Some(line);
961 }
962 let mut change = None;
963 let mut value = Map::new();
964 value.insert("job".into(), self.id.clone().into());
965 value.insert("agent".into(), self.agent.clone().into());
966 match &self.outcome {
967 None => {
968 value.insert("status".into(), "running".into());
969 value.insert(
970 "elapsed_seconds".into(),
971 self.started.elapsed().as_secs().into(),
972 );
973 if let Some(progress) = &self.last_progress {
974 value.insert("progress".into(), progress.clone().into());
975 }
976 if let Some(record) = &self.review {
977 value.insert("review".into(), record.live.status());
978 }
979 }
980 Some(outcome) => {
981 let result = result_value(&outcome.output);
982 let status = if self.cancelled {
983 JobStatus::Cancelled
984 } else {
985 job_status(
986 &outcome.output,
987 &AgentReply::read(&result).unwrap_or_default(),
988 )
989 };
990 value.insert("status".into(), status.as_str().into());
991 value.insert("elapsed_seconds".into(), outcome.elapsed.as_secs().into());
992 value.insert("result".into(), result);
993 if !self.reported {
994 self.reported = true;
995 change = Some(self.change(status));
996 }
997 }
998 }
999 (Value::Object(value), change)
1000 }
1001}
1002
1003fn result_value(output: &ToolOutput) -> Value {
1005 match serde_json::from_str::<Value>(&output.content) {
1006 Ok(value @ Value::Object(_)) => value,
1007 _ => json!({ "reply": output.content, "is_error": output.is_error() }),
1008 }
1009}
1010
1011fn job_status(output: &ToolOutput, reply: &AgentReply) -> JobStatus {
1014 reply.status.unwrap_or(if output.is_error() {
1015 JobStatus::Failed
1016 } else {
1017 JobStatus::Completed
1018 })
1019}
1020
1021fn task_line(prompt: &str) -> String {
1024 let line = prompt
1025 .lines()
1026 .map(str::trim)
1027 .find(|line| !line.is_empty())
1028 .unwrap_or_default();
1029 let mut task: String = line
1030 .chars()
1031 .filter(|character| !character.is_control())
1032 .take(TASK_CHARS)
1033 .collect();
1034 if line.chars().count() > TASK_CHARS {
1035 task.push('…');
1036 }
1037 task
1038}
1039
1040fn unknown_job(job: &str) -> ToolError {
1041 ToolError::invalid_arguments(format!(
1042 "unknown background job {:?}; agent_status lists this session's jobs",
1043 bounded(job, 64)
1044 ))
1045}
1046
1047const REVIEWED_NOTE: &str = " SCV shows the user its own Review and Landing lines for \
1050 reviewed jobs. If you mention the outcome, repeat them as given; never call unapproved \
1051 work approved or done, and never call landed work approved. If a reviewer declined, tell \
1052 the user what it said. If a landing is NOT confirmed, or the reviewer could NOT confirm \
1053 it, say so plainly, and don't fix, revert, or land anything on your own.";
1054
1055fn reviewed_note(reports: &[JobReport]) -> &'static str {
1056 if reports.iter().any(|report| report.outcome.is_some()) {
1057 REVIEWED_NOTE
1058 } else {
1059 ""
1060 }
1061}
1062
1063pub fn report_prompt(reports: &[JobReport]) -> String {
1065 format!(
1066 "[SCV background report] Delegated work you started in the background has \
1067 finished. The user did not send this message: tell them briefly what \
1068 happened and the key result.{}\n\n{}",
1069 reviewed_note(reports),
1070 describe_reports(reports)
1071 )
1072}
1073
1074pub fn delivered_note(reports: &[JobReport], error: &str) -> String {
1077 format!(
1078 "[SCV background report, already delivered] Delegated work you started in \
1079 the background has finished. Your report of it failed ({error}), so SCV \
1080 sent the user the results below directly. The user did not send this \
1081 message; repeat the results only if they ask.{}\n\n{}",
1082 reviewed_note(reports),
1083 describe_reports(reports)
1084 )
1085}
1086
1087pub(crate) struct BackgroundCapable {
1090 pub(crate) inner: Arc<AgentTool>,
1091 pub(crate) jobs: Arc<BackgroundJobs>,
1092 pub(crate) conversations: Arc<ConversationStore>,
1094 pub(crate) reviews: Option<PathBuf>,
1097 pub(crate) session: Option<String>,
1099}
1100
1101struct QueuedTool(Arc<AgentTool>);
1104
1105#[async_trait]
1106impl Tool for QueuedTool {
1107 fn spec(&self) -> ToolSpec {
1108 self.0.spec()
1109 }
1110 fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
1111 self.0.risk(arguments)
1112 }
1113 fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
1114 self.0.approval_summary(arguments)
1115 }
1116 async fn execute(
1117 &self,
1118 arguments: Value,
1119 context: ToolContext,
1120 ) -> Result<ToolOutput, ToolError> {
1121 self.0.execute_when_idle(arguments, context).await
1122 }
1123}
1124
1125struct Split {
1127 arguments: Value,
1128 background: Option<bool>,
1130 review: Option<Value>,
1132}
1133
1134impl Split {
1135 fn new(arguments: &Value) -> Self {
1136 let mut arguments = arguments.clone();
1137 let object = arguments.as_object_mut();
1138 let (background, review) = match object {
1139 Some(object) => (
1140 object
1141 .remove("background")
1142 .map(|value| value.as_bool() == Some(true)),
1143 object.remove("review").filter(|review| !review.is_null()),
1144 ),
1145 None => (None, None),
1146 };
1147 Self {
1148 arguments,
1149 background,
1150 review,
1151 }
1152 }
1153
1154 fn background(&self) -> bool {
1155 self.background == Some(true)
1156 }
1157}
1158
1159#[async_trait]
1160impl Tool for BackgroundCapable {
1161 fn spec(&self) -> ToolSpec {
1162 let mut spec = self.inner.spec();
1163 let reviewed = self.offers_review();
1164 if let Some(properties) = spec
1165 .parameters
1166 .get_mut("properties")
1167 .and_then(Value::as_object_mut)
1168 {
1169 properties.insert(
1170 "background".into(),
1171 json!({
1172 "type":"boolean",
1173 "description":"Run in the background: the call returns a job handle at once, \
1174 the user can keep talking to you while the agent works, and SCV reports \
1175 the result in a new turn when it finishes. Use it for any substantial \
1176 task; run in the foreground only for quick work whose result you need \
1177 within this turn."
1178 }),
1179 );
1180 if reviewed {
1181 properties.insert("review".into(), review::schema(&self.inner));
1182 }
1183 }
1184 spec.description.push_str(&format!(
1185 " Set background to true for anything beyond a quick task: the call returns a job \
1186 handle at once (at most {} running per session), SCV reports the result when the \
1187 job finishes, agent_status shows progress, and agent_cancel stops it. A background \
1188 job's own approval requests get only the answer this session would give without \
1189 asking a person.",
1190 self.jobs.limit()
1191 ));
1192 if reviewed {
1193 spec.description.push_str(
1194 " A call with review runs as a reviewed background job: an independent \
1195 reviewer checks the work in up to review.rounds rounds.",
1196 );
1197 }
1198 spec
1199 }
1200
1201 fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
1202 let split = Split::new(arguments);
1203 if let Some(review) = &split.review {
1204 self.plan(&split, review)?;
1205 }
1206 self.inner.risk_for(
1207 &split.arguments,
1208 split.background() || split.review.is_some(),
1209 )
1210 }
1211
1212 fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
1213 let split = Split::new(arguments);
1214 let background = split.background() || split.review.is_some();
1215 let mut summary = self
1216 .inner
1217 .approval_summary_for(&split.arguments, background)?;
1218 if let Some(review) = &split.review {
1219 let plan = self.plan(&split, review)?;
1220 summary.push_str(
1221 " Runs in the background: the call returns at once and the result is \
1222 reported when the review ends.",
1223 );
1224 summary.push_str(&review::approval_summary(&plan, &self.inner)?);
1225 } else if split.background() {
1226 summary.push_str(
1227 " Runs in the background: the call returns at once and the result is \
1228 reported when the agent finishes.",
1229 );
1230 }
1231 Ok(summary)
1232 }
1233
1234 async fn execute(
1235 &self,
1236 arguments: Value,
1237 context: ToolContext,
1238 ) -> Result<ToolOutput, ToolError> {
1239 let split = Split::new(&arguments);
1240 if let Some(review) = &split.review {
1241 return self.start_reviewed(&split, review, &context);
1242 }
1243 let background = split.background();
1244 let arguments = split.arguments;
1245 let (agent, routed) = self.inner.route(&arguments)?;
1246 self.inner.risk_for(&arguments, background)?;
1250 let Some(handle) = parse_args::<AgentArgs>(&routed)?
1251 .session
1252 .filter(|session| conversation::is_handle(session))
1253 else {
1254 return if background {
1255 self.start(agent, arguments, &context, Turn::New)
1256 } else {
1257 self.inner.execute(arguments, context).await
1258 };
1259 };
1260 let place = self.jobs.lanes.join(&handle);
1262 if place.ahead == 0 && !agent.backend.busy(&routed)? {
1263 return if background {
1264 self.start(agent, arguments, &context, Turn::Next(place))
1265 } else {
1266 let output = self.inner.execute(arguments, context).await;
1268 drop(place);
1269 output
1270 };
1271 }
1272 let behavior = match agent.on_busy(&routed)? {
1273 BusyBehavior::Steer => {
1274 if let Some(output) = agent.backend.steer(routed, context.clone()).await? {
1275 return Ok(output);
1276 }
1277 agent.busy.fallback()
1278 }
1279 behavior => behavior,
1280 };
1281 match behavior {
1282 BusyBehavior::Wait if !background => {
1284 place.wait_first(&context.cancellation).await?;
1285 let output = self.inner.execute_when_idle(arguments, context).await;
1286 drop(place);
1287 output
1288 }
1289 BusyBehavior::Fail => Err(busy(&handle, &place)),
1290 _ => {
1291 queue_room(agent, &handle, &place)?;
1292 self.start(agent, arguments, &context, Turn::Queued(place))
1293 }
1294 }
1295 }
1296}
1297
1298fn busy(handle: &str, place: &Place) -> ToolError {
1301 let waiting = place.ahead.saturating_sub(1);
1303 ToolError::failed(if waiting == 0 {
1304 format!("session busy: conversation {handle} is still running a turn")
1305 } else {
1306 format!(
1307 "session busy: conversation {handle} is still running a turn, and {waiting} more \
1308 prompts wait for it"
1309 )
1310 })
1311}
1312
1313fn queue_room(agent: &Offered, handle: &str, place: &Place) -> Result<(), ToolError> {
1315 let waiting = place.ahead.saturating_sub(1);
1316 if waiting >= agent.busy.max_queued_turns {
1317 return Err(ToolError::limit(format!(
1318 "session busy: conversation {handle} is running a turn and {waiting} prompts \
1319 already wait for it, of the {} agent.max_queued_turns allows. Send this one \
1320 once they have run, or stop one with agent_cancel.",
1321 agent.busy.max_queued_turns
1322 )));
1323 }
1324 Ok(())
1325}
1326
1327impl BackgroundCapable {
1328 #[cfg(test)]
1331 pub(crate) fn plain(inner: Arc<AgentTool>, jobs: Arc<BackgroundJobs>) -> Self {
1332 Self {
1333 inner,
1334 jobs,
1335 conversations: Arc::new(ConversationStore::new(
1336 conversation::ConversationLimits {
1337 max: 8,
1338 idle: Duration::from_secs(86400),
1339 },
1340 None,
1341 )),
1342 reviews: None,
1343 session: None,
1344 }
1345 }
1346
1347 fn offers_review(&self) -> bool {
1351 self.reviews.is_some() && !self.inner.continuing().is_empty()
1352 }
1353
1354 fn plan(&self, split: &Split, review: &Value) -> Result<review::Plan, ToolError> {
1356 if !self.offers_review() {
1357 return Err(ToolError::invalid_arguments(
1358 "review is not available in this session",
1359 ));
1360 }
1361 if split.background == Some(false) {
1362 return Err(ToolError::invalid_arguments(
1363 "a reviewed call runs as a background job; omit background or set it to true",
1364 ));
1365 }
1366 let (builder, routed) = self.inner.route(&split.arguments)?;
1367 builder.backend.risk(&routed)?;
1368 review::plan(
1369 &self.inner,
1370 builder,
1371 &routed,
1372 review,
1373 self.conversations.limits().max,
1374 )
1375 }
1376
1377 fn start_reviewed(
1382 &self,
1383 split: &Split,
1384 review: &Value,
1385 context: &ToolContext,
1386 ) -> Result<ToolOutput, ToolError> {
1387 let plan = self.plan(split, review)?;
1388 let arguments = split.arguments.clone();
1389 self.inner.risk_for(&arguments, true)?;
1390 let (agent, routed) = self.inner.route(&arguments)?;
1391 let turn = match parse_args::<AgentArgs>(&routed)?
1392 .session
1393 .filter(|session| conversation::is_handle(session))
1394 {
1395 None => Turn::New,
1396 Some(handle) => {
1397 let place = self.jobs.lanes.join(&handle);
1398 if place.ahead == 0 && !agent.backend.busy(&routed)? {
1399 Turn::Next(place)
1400 } else if agent.on_busy(&routed)? == BusyBehavior::Fail {
1401 return Err(busy(&handle, &place));
1402 } else {
1403 queue_room(agent, &handle, &place)?;
1404 Turn::Queued(place)
1405 }
1406 }
1407 };
1408 let dir = self.reviews.as_deref().ok_or_else(|| {
1409 ToolError::invalid_arguments("review is not available in this session")
1410 })?;
1411 let reviewers: Vec<String> = plan
1412 .reviewer_names()
1413 .into_iter()
1414 .map(str::to_owned)
1415 .collect();
1416 let (rounds, land) = (plan.rounds, plan.land);
1417 let tool = Arc::new(ReviewedTool::new(
1418 Arc::clone(&self.inner),
1419 Arc::clone(&self.jobs.lanes),
1420 Arc::clone(&self.conversations),
1421 plan,
1422 arguments.clone(),
1423 self.session.clone(),
1424 dir,
1425 )?);
1426 let queued = matches!(turn, Turn::Queued(_));
1427 let mut started = self.jobs.start(
1428 Arc::clone(&tool) as Arc<dyn Tool>,
1429 AGENT_TOOL,
1430 &agent.name,
1431 arguments,
1432 context,
1433 turn,
1434 Some(&tool),
1435 )?;
1436 let first = reviewers.first().cloned().unwrap_or_default();
1437 started["review"] = json!({
1438 "reviewers": reviewers,
1439 "rounds": rounds,
1440 "land": review::land_name(land),
1441 "journal": tool.journal_id(),
1442 });
1443 started["note"] = format!(
1444 "{}The agent works on it, then a fresh {first} reviews it (or the next reviewer \
1445 available), for up to {rounds} rounds. SCV reports the result, with its own Review \
1446 and Landing lines, in a new turn when the review ends. agent_status shows the \
1447 round and phase; agent_cancel stops the whole review.",
1448 if queued {
1449 "The conversation is busy, so this reviewed call is queued behind the turns \
1450 ahead of it. "
1451 } else {
1452 ""
1453 }
1454 )
1455 .into();
1456 Ok(ToolOutput::success(started.to_string()))
1457 }
1458
1459 fn start(
1461 &self,
1462 agent: &Offered,
1463 arguments: Value,
1464 context: &ToolContext,
1465 turn: Turn,
1466 ) -> Result<ToolOutput, ToolError> {
1467 let tool: Arc<dyn Tool> = if matches!(turn, Turn::Queued(_)) {
1468 Arc::new(QueuedTool(Arc::clone(&self.inner)))
1469 } else {
1470 Arc::clone(&self.inner) as Arc<dyn Tool>
1471 };
1472 let started = self.jobs.start(
1473 tool,
1474 AGENT_TOOL,
1475 &agent.name,
1476 arguments,
1477 context,
1478 turn,
1479 None,
1480 )?;
1481 Ok(ToolOutput::success(started.to_string()))
1482 }
1483}
1484
1485#[derive(Deserialize)]
1486#[serde(deny_unknown_fields)]
1487struct WaitArgs {
1488 job: String,
1489 timeout_seconds: Option<u64>,
1490}
1491
1492#[derive(Deserialize)]
1493#[serde(deny_unknown_fields)]
1494struct StatusArgs {
1495 job: Option<String>,
1496}
1497
1498pub(crate) struct WaitTool {
1500 pub(crate) jobs: Arc<BackgroundJobs>,
1501 pub(crate) timeouts: Timeouts,
1502}
1503
1504#[async_trait]
1505impl Tool for WaitTool {
1506 fn spec(&self) -> ToolSpec {
1507 let mut timeout = timeout_schema(self.timeouts);
1508 timeout["description"] = format!(
1509 "Seconds to wait before returning the job still running. Defaults to {}; at most {}.",
1510 self.timeouts.default.min(self.timeouts.max).as_secs(),
1511 self.timeouts.max.as_secs()
1512 )
1513 .into();
1514 ToolSpec {
1515 name: "agent_wait".into(),
1516 description: "Wait for a background agent job (the `job` handle an agent call with \
1517 background: true returned) to finish, and return its result. Returns early with \
1518 status running when the timeout passes. Waiting holds your turn open, so the \
1519 user cannot reach you meanwhile; usually let SCV report the result instead."
1520 .into(),
1521 parameters: json!({
1522 "type":"object",
1523 "properties":{"job":{"type":"string"},"timeout_seconds":timeout},
1524 "required":["job"],
1525 "additionalProperties":false
1526 }),
1527 }
1528 }
1529
1530 fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
1531 let args: WaitArgs = parse_args(arguments)?;
1532 self.timeouts.resolve(args.timeout_seconds)?;
1533 Ok(ToolRisk::ReadOnly)
1534 }
1535
1536 fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
1537 let args: WaitArgs = parse_args(arguments)?;
1538 Ok(format!(
1539 "Wait for background job {}",
1540 bounded(&args.job, 64)
1541 ))
1542 }
1543
1544 async fn execute(
1545 &self,
1546 arguments: Value,
1547 context: ToolContext,
1548 ) -> Result<ToolOutput, ToolError> {
1549 let args: WaitArgs = parse_args(&arguments)?;
1550 let limit = self.timeouts.resolve(args.timeout_seconds)?;
1551 let value = self
1552 .jobs
1553 .wait(&args.job, limit, &context.cancellation, &context.call_id)
1554 .await?;
1555 Ok(ToolOutput::success(value.to_string()))
1556 }
1557}
1558
1559pub(crate) struct StatusTool {
1561 pub(crate) jobs: Arc<BackgroundJobs>,
1562}
1563
1564#[async_trait]
1565impl Tool for StatusTool {
1566 fn spec(&self) -> ToolSpec {
1567 ToolSpec {
1568 name: "agent_status".into(),
1569 description: "Show this session's background agent jobs: running ones with their \
1570 latest progress, finished ones with their result. Pass job for one job."
1571 .into(),
1572 parameters: json!({
1573 "type":"object",
1574 "properties":{"job":{"type":"string"}},
1575 "additionalProperties":false
1576 }),
1577 }
1578 }
1579
1580 fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
1581 let _: StatusArgs = parse_args(arguments)?;
1582 Ok(ToolRisk::ReadOnly)
1583 }
1584
1585 fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
1586 let args: StatusArgs = parse_args(arguments)?;
1587 Ok(args.job.map_or_else(
1588 || "List background jobs".into(),
1589 |job| format!("Show background job {}", bounded(&job, 64)),
1590 ))
1591 }
1592
1593 async fn execute(
1594 &self,
1595 arguments: Value,
1596 context: ToolContext,
1597 ) -> Result<ToolOutput, ToolError> {
1598 let args: StatusArgs = parse_args(&arguments)?;
1599 let value = self.jobs.describe(args.job.as_deref(), &context.call_id)?;
1600 Ok(ToolOutput::success(value.to_string()))
1601 }
1602}
1603
1604#[derive(Deserialize)]
1605#[serde(deny_unknown_fields)]
1606struct CancelArgs {
1607 job: String,
1608}
1609
1610pub(crate) struct CancelTool {
1612 pub(crate) jobs: Arc<BackgroundJobs>,
1613}
1614
1615#[async_trait]
1616impl Tool for CancelTool {
1617 fn spec(&self) -> ToolSpec {
1618 ToolSpec {
1619 name: "agent_cancel".into(),
1620 description: "Stop a running background agent job (the `job` handle an agent call \
1621 with background: true returned), for example when the user no longer wants \
1622 it. The agent and every process it started are stopped; work it already wrote \
1623 stays. Returns the job with status cancelled, and no report turn follows."
1624 .into(),
1625 parameters: json!({
1626 "type":"object",
1627 "properties":{"job":{"type":"string"}},
1628 "required":["job"],
1629 "additionalProperties":false
1630 }),
1631 }
1632 }
1633
1634 fn risk(&self, arguments: &Value) -> Result<ToolRisk, ToolError> {
1635 let _: CancelArgs = parse_args(arguments)?;
1636 Ok(ToolRisk::Process)
1637 }
1638
1639 fn approval_summary(&self, arguments: &Value) -> Result<String, ToolError> {
1640 let args: CancelArgs = parse_args(arguments)?;
1641 Ok(format!("Stop background job {}", bounded(&args.job, 64)))
1642 }
1643
1644 async fn execute(
1645 &self,
1646 arguments: Value,
1647 context: ToolContext,
1648 ) -> Result<ToolOutput, ToolError> {
1649 let args: CancelArgs = parse_args(&arguments)?;
1650 let value = self.jobs.cancel(&args.job, &context.call_id).await?;
1651 Ok(ToolOutput::success(value.to_string()))
1652 }
1653}
1654
1655#[cfg(test)]
1656mod tests;