1use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
2#[cfg(not(windows))]
3use std::ffi::OsString;
4use std::fs;
5use std::io::Write;
6use std::path::{Path, PathBuf};
7#[cfg(windows)]
8use std::process::Command;
9use std::process::{Child, Stdio};
10use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
11#[cfg(unix)]
12use std::sync::OnceLock;
13use std::sync::{Arc, Mutex, RwLock};
14use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
15
16use rusqlite::Connection;
17use serde::Serialize;
18
19use crate::bash_permissions::PermissionAsk;
20use crate::compress::caps::DropClass;
21use crate::compress::CompressionResult;
22use crate::context::SharedProgressSender;
23use crate::db::compression_events::CompressionAggregateCache;
24use crate::harness::Harness;
25use crate::protocol::{BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, PushFrame};
26use crate::sandbox_spawn::SpawnPlan;
27
28#[cfg(windows)]
29use std::os::windows::process::CommandExt;
30
31use super::buffer::{combine_streams, BgBuffer, DiskTruncation, StreamKind, TokenCountInput};
32use super::output::{
33 cap_completion_output, cap_completion_output_with_marker, cap_final_output,
34 cap_final_output_with_marker, completion_preview_threshold, json_output_pointer, quote_path,
35 retained_json_output_pointer, COMPRESS_INPUT_CAP_BYTES, COMPRESS_INPUT_HEAD_BYTES,
36 COMPRESS_INPUT_TAIL_BYTES, FINAL_OUTPUT_CAP_BYTES, RAW_PASSTHROUGH_CAP_BYTES,
37 RAW_PASSTHROUGH_HEAD_BYTES, RAW_PASSTHROUGH_TAIL_BYTES, RUNNING_OUTPUT_PREVIEW_BYTES,
38 STRUCTURED_OUTPUT_CAP_BYTES,
39};
40use super::persistence::{
41 allocate_task_layout, delete_resolved_task, delete_task_bundle, discover_task_ids,
42 open_task_artifact, quarantine_invalid_entry, quarantine_task_layout, read_exit_marker,
43 read_task_at, resolve_task_layout, session_tasks_dir, uninitialized_layout_is_recent,
44 unix_millis, update_task_at, validate_task_id, write_kill_marker_if_absent, write_task_at,
45 BgMode, ExitMarker, PersistedTask, TaskArtifact, TaskIoHandles, TaskPaths,
46};
47use super::process::is_process_alive;
48#[cfg(unix)]
49use super::process::terminate_pgid;
50#[cfg(windows)]
51use super::process::terminate_pid;
52use super::pty_process::spawn_pty_for_command;
53use super::pty_runtime::PtyRuntime;
54use super::watches::{PatternMatch, WatchPattern, WatchRegistry};
55use super::{BgTaskInfo, BgTaskStatus};
56const DEFAULT_BG_TIMEOUT: Duration = Duration::from_secs(30 * 60);
59const STALE_RUNNING_AFTER: Duration = Duration::from_secs(24 * 60 * 60);
60const PERSISTED_GC_GRACE: Duration = Duration::from_secs(24 * 60 * 60);
61const QUARANTINE_GC_GRACE: Duration = Duration::from_secs(30 * 24 * 60 * 60);
62
63const TOKENIZE_CAP_BYTES_PER_STREAM: usize = 128 * 1024;
64
65#[derive(Debug, Clone, Serialize)]
66pub struct BgCompletion {
67 pub task_id: String,
68 #[serde(skip_serializing)]
71 pub session_id: String,
72 pub status: BgTaskStatus,
73 pub exit_code: Option<i32>,
74 pub command: String,
75 #[serde(default, skip_serializing_if = "String::is_empty")]
81 pub output_preview: String,
82 #[serde(default, skip_serializing_if = "is_false")]
87 pub output_truncated: bool,
88 #[serde(default, skip_serializing_if = "Option::is_none")]
91 pub original_tokens: Option<u32>,
92 #[serde(default, skip_serializing_if = "Option::is_none")]
95 pub compressed_tokens: Option<u32>,
96 #[serde(default, skip_serializing_if = "is_false")]
98 pub tokens_skipped: bool,
99}
100
101fn is_false(v: &bool) -> bool {
102 !*v
103}
104
105#[derive(Debug, Clone, Serialize)]
106pub struct BgTaskSnapshot {
107 #[serde(flatten)]
108 pub info: BgTaskInfo,
109 pub exit_code: Option<i32>,
110 pub child_pid: Option<u32>,
111 pub workdir: String,
112 pub output_preview: String,
113 pub output_truncated: bool,
114 pub output_path: Option<String>,
115 pub stderr_path: Option<String>,
116 #[serde(skip_serializing_if = "Option::is_none")]
117 pub pty_rows: Option<u16>,
118 #[serde(skip_serializing_if = "Option::is_none")]
119 pub pty_cols: Option<u16>,
120 #[serde(skip_serializing_if = "Option::is_none")]
121 pub pty_screen: Option<String>,
122 #[serde(default, skip_serializing_if = "Vec::is_empty")]
123 pub scanner_report: Vec<PermissionAsk>,
124 #[serde(default, skip_serializing_if = "is_false")]
125 pub sandbox_native: bool,
126 #[serde(default, skip_serializing_if = "is_false")]
127 pub sandbox_unavailable: bool,
128}
129
130#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
131pub struct BgTaskHealthCounts {
132 pub running: usize,
133 pub pending_completions: usize,
134}
135
136#[derive(Debug, Clone, Copy, PartialEq, Eq)]
137enum TerminalOutputKind {
138 Compressed,
139 Raw,
140 Structured,
141}
142
143#[derive(Debug, Clone, PartialEq, Eq)]
144struct TerminalOutputCache {
145 output_preview: String,
146 output_truncated: bool,
147 kind: TerminalOutputKind,
148 output_path: Option<String>,
149 stderr_path: Option<String>,
150 artifact_access: ArtifactRecoveryAccess,
151 recovery: Option<RecoveryContext>,
152}
153
154#[derive(Debug, Clone, PartialEq, Eq)]
155struct ArtifactRecoveryAccess {
156 task_id: String,
157 readable: bool,
158}
159
160#[derive(Debug, Clone, PartialEq, Eq)]
161struct RecoveryContext {
162 dropped_by_class: BTreeMap<DropClass, usize>,
163 had_inner_drop: bool,
164 offset_hint_eligible: bool,
165 offset_start_line: Option<usize>,
166 byte_truncated: bool,
167 disk_truncated_prefix_bytes: u64,
168 output_path: Option<String>,
169 stderr_path: Option<String>,
170 include_stderr_path: bool,
171 artifact_access: ArtifactRecoveryAccess,
172}
173
174fn optional_string_bytes(value: Option<&String>) -> u64 {
175 value
176 .map(|value| crate::memory::usize_to_u64(value.len()))
177 .unwrap_or(0)
178}
179
180fn terminal_output_cache_estimated_bytes(cache: &TerminalOutputCache) -> u64 {
181 let recovery_bytes = cache
182 .recovery
183 .as_ref()
184 .map(|recovery| {
185 crate::memory::usize_to_u64(recovery.dropped_by_class.len())
186 .saturating_mul(
187 (std::mem::size_of::<DropClass>() + std::mem::size_of::<usize>()) as u64,
188 )
189 .saturating_add(optional_string_bytes(recovery.output_path.as_ref()))
190 .saturating_add(optional_string_bytes(recovery.stderr_path.as_ref()))
191 .saturating_add(crate::memory::usize_to_u64(
192 recovery.artifact_access.task_id.len(),
193 ))
194 })
195 .unwrap_or(0);
196 (std::mem::size_of::<TerminalOutputCache>() as u64)
197 .saturating_add(crate::memory::usize_to_u64(cache.output_preview.len()))
198 .saturating_add(optional_string_bytes(cache.output_path.as_ref()))
199 .saturating_add(optional_string_bytes(cache.stderr_path.as_ref()))
200 .saturating_add(crate::memory::usize_to_u64(
201 cache.artifact_access.task_id.len(),
202 ))
203 .saturating_add(recovery_bytes)
204}
205
206fn completion_estimated_bytes(completion: &BgCompletion) -> u64 {
207 (std::mem::size_of::<BgCompletion>() as u64)
208 .saturating_add(crate::memory::usize_to_u64(completion.task_id.len()))
209 .saturating_add(crate::memory::usize_to_u64(completion.session_id.len()))
210 .saturating_add(crate::memory::usize_to_u64(completion.command.len()))
211 .saturating_add(crate::memory::usize_to_u64(completion.output_preview.len()))
212}
213
214impl RecoveryContext {
215 fn has_visible_drop(&self) -> bool {
216 self.byte_truncated
217 || self.disk_truncated_prefix_bytes > 0
218 || self.had_inner_drop
219 || !self.dropped_by_class.is_empty()
220 }
221}
222
223#[derive(Clone)]
224pub struct BgTaskRegistry {
225 pub(crate) inner: Arc<RegistryInner>,
226}
227
228pub(crate) struct RegistryInner {
229 pub(crate) tasks: Mutex<HashMap<String, Arc<BgTask>>>,
230 pub(crate) completions: Mutex<VecDeque<BgCompletion>>,
231 pub(crate) progress_sender: SharedProgressSender,
232 watchdog_started: AtomicBool,
233 pub(crate) shutdown: AtomicBool,
234 pub(crate) long_running_reminder_enabled: AtomicBool,
235 pub(crate) long_running_reminder_interval_ms: AtomicU64,
236 persisted_gc_started: AtomicBool,
237 #[cfg(test)]
238 persisted_gc_runs: AtomicU64,
239 pub(crate) compressor:
245 Mutex<Option<Box<dyn Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync>>>,
246 pub(crate) db_pool: RwLock<Option<Arc<Mutex<Connection>>>>,
247 pub(crate) db_harness: RwLock<Option<String>>,
248 pub(crate) compression_aggregates: Arc<CompressionAggregateCache>,
249 pub(crate) wake_tx: crossbeam_channel::Sender<()>,
250 pub(crate) wake_rx: crossbeam_channel::Receiver<()>,
251 pub(crate) watch_registry: Mutex<WatchRegistry>,
252 wait_detach_sessions: Mutex<HashSet<String>>,
253 active_wait_sessions: Mutex<HashMap<String, usize>>,
254}
255
256pub(crate) struct BgTask {
257 pub(crate) task_id: String,
258 pub(crate) session_id: String,
259 pub(crate) paths: TaskPaths,
260 artifact_root: PathBuf,
261 pub(crate) started: Instant,
262 pub(crate) last_reminder_at: Mutex<Option<Instant>>,
263 pub(crate) terminal_at: Mutex<Option<Instant>>,
264 pub(crate) state: Mutex<BgTaskState>,
265}
266
267pub(crate) enum TaskRuntime {
268 Piped(Option<Child>),
269 Pty(Option<PtyRuntime>),
270}
271
272pub(crate) struct BgTaskState {
273 pub(crate) metadata: PersistedTask,
274 pub(crate) runtime: TaskRuntime,
275 pub(crate) io_handles: Option<TaskIoHandles>,
278 pub(crate) detached: bool,
279 pub(crate) child_exit_observed: bool,
290 pub(crate) buffer: BgBuffer,
291 terminal_output_cache: Option<TerminalOutputCache>,
292 pub(crate) pending_terminal_override: Option<BgTaskStatus>,
294}
295
296fn completion_matches_session(completion: &BgCompletion, session_id: Option<&str>) -> bool {
297 session_id
298 .map(|session_id| completion.session_id == session_id)
299 .unwrap_or(true)
300}
301
302impl BgTaskRegistry {
303 pub fn new(progress_sender: SharedProgressSender) -> Self {
304 let (wake_tx, wake_rx) = crossbeam_channel::bounded(1);
305 Self {
306 inner: Arc::new(RegistryInner {
307 tasks: Mutex::new(HashMap::new()),
308 completions: Mutex::new(VecDeque::new()),
309 progress_sender,
310 watchdog_started: AtomicBool::new(false),
311 shutdown: AtomicBool::new(false),
312 long_running_reminder_enabled: AtomicBool::new(true),
313 long_running_reminder_interval_ms: AtomicU64::new(600_000),
314 persisted_gc_started: AtomicBool::new(false),
315 #[cfg(test)]
316 persisted_gc_runs: AtomicU64::new(0),
317 compressor: Mutex::new(None),
318 db_pool: RwLock::new(None),
319 db_harness: RwLock::new(None),
320 compression_aggregates: Arc::new(CompressionAggregateCache::default()),
321 wake_tx,
322 wake_rx,
323 watch_registry: Mutex::new(WatchRegistry::default()),
324 wait_detach_sessions: Mutex::new(HashSet::new()),
325 active_wait_sessions: Mutex::new(HashMap::new()),
326 }),
327 }
328 }
329
330 pub fn is_session_owned_artifact_path(&self, session_id: &str, path: &Path) -> bool {
338 let Ok(requested) = fs::canonicalize(path) else {
339 return false;
340 };
341 let Ok(tasks) = self.inner.tasks.lock() else {
342 return false;
343 };
344
345 tasks.values().any(|task| {
346 task.session_id == session_id
347 && [
348 &task.paths.stdout,
349 &task.paths.stderr,
350 &task.paths.exit,
351 &task.paths.pty,
352 ]
353 .into_iter()
354 .filter_map(|known| known.file_name())
355 .any(|name| task.artifact_root.join(name) == requested)
356 })
357 }
358
359 pub fn read_artifact_path(
360 &self,
361 session_id: &str,
362 path: &Path,
363 ) -> Option<Result<Vec<u8>, String>> {
364 let requested = fs::canonicalize(path).ok()?;
365 let tasks = self.inner.tasks.lock().ok()?;
366 let (task, artifact) = tasks.values().find_map(|task| {
367 if task.session_id != session_id {
368 return None;
369 }
370 TaskArtifact::ALL.into_iter().find_map(|artifact| {
371 let expected = task
372 .paths
373 .artifact_path(artifact)
374 .file_name()
375 .map(|name| task.artifact_root.join(name));
376 (expected.as_deref() == Some(requested.as_path()))
377 .then(|| (Arc::clone(task), artifact))
378 })
379 })?;
380 drop(tasks);
381 Some(self.read_artifact(&task.task_id, session_id, artifact))
382 }
383
384 pub fn read_artifact_range(
385 &self,
386 task_id: &str,
387 session_id: &str,
388 artifact: TaskArtifact,
389 offset: u64,
390 ) -> Result<(Vec<u8>, u64), String> {
391 validate_task_id(task_id).map_err(|error| error.to_string())?;
392 let task = self
393 .task_for_session(task_id, session_id)
394 .ok_or_else(|| "task_not_found".to_string())?;
395 let mut file = open_task_artifact(&task.paths, artifact)
396 .map_err(|error| format!("artifact_refused: {error}"))?;
397 let len = file
398 .len()
399 .map_err(|error| format!("artifact_refused: {error}"))?;
400 let start = offset.min(len);
401 let bytes = file
402 .read_range(start, len.saturating_sub(start))
403 .map_err(|error| format!("artifact_refused: {error}"))?;
404 Ok((bytes, len))
405 }
406
407 pub fn read_artifact(
408 &self,
409 task_id: &str,
410 session_id: &str,
411 artifact: TaskArtifact,
412 ) -> Result<Vec<u8>, String> {
413 validate_task_id(task_id).map_err(|error| error.to_string())?;
414 let task = self
415 .task_for_session(task_id, session_id)
416 .ok_or_else(|| "task_not_found".to_string())?;
417 let mut file = open_task_artifact(&task.paths, artifact)
418 .map_err(|error| format!("artifact_refused: {error}"))?;
419 file.read_all()
420 .map_err(|error| format!("artifact_refused: {error}"))
421 }
422
423 pub fn set_harness(&self, harness: Harness) {
424 if let Ok(mut slot) = self.inner.db_harness.write() {
425 *slot = Some(harness.storage_segment());
426 }
427 }
428
429 pub fn set_db_pool(&self, conn: Arc<Mutex<Connection>>) {
430 if let Ok(mut slot) = self.inner.db_pool.write() {
431 *slot = Some(conn);
432 }
433 self.inner.compression_aggregates.clear();
434 }
435
436 pub fn clear_db_pool(&self) {
437 if let Ok(mut slot) = self.inner.db_pool.write() {
438 *slot = None;
439 }
440 self.inner.compression_aggregates.clear();
441 }
442
443 pub(crate) fn compression_aggregate_cache(&self) -> Arc<CompressionAggregateCache> {
444 Arc::clone(&self.inner.compression_aggregates)
445 }
446
447 pub fn begin_wait_mode_session(&self, session_id: &str) {
448 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
449 *active.entry(session_id.to_string()).or_insert(0) += 1;
450 }
451 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
452 detach.remove(session_id);
453 }
454 }
455
456 pub fn end_wait_mode_session(&self, session_id: &str) {
457 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
458 match active.get_mut(session_id) {
459 Some(count) if *count > 1 => *count -= 1,
460 Some(_) => {
461 active.remove(session_id);
462 }
463 None => {}
464 }
465 }
466 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
467 detach.remove(session_id);
468 }
469 }
470
471 pub fn signal_wait_mode_detach(&self, session_id: &str) -> bool {
472 let is_waiting = self
473 .inner
474 .active_wait_sessions
475 .lock()
476 .map(|active| active.get(session_id).copied().unwrap_or(0) > 0)
477 .unwrap_or(false);
478 if !is_waiting {
479 return false;
480 }
481 self.inner
482 .wait_detach_sessions
483 .lock()
484 .map(|mut detach| detach.insert(session_id.to_string()))
485 .unwrap_or(false)
486 }
487
488 pub fn active_wait_session_count(&self) -> usize {
491 self.inner
492 .active_wait_sessions
493 .lock()
494 .map(|active| active.len())
495 .unwrap_or(0)
496 }
497
498 pub fn take_wait_mode_detach(&self, session_id: &str) -> bool {
499 self.inner
500 .wait_detach_sessions
501 .lock()
502 .map(|mut detach| detach.remove(session_id))
503 .unwrap_or(false)
504 }
505
506 pub fn set_compressor<F>(&self, compressor: F)
511 where
512 F: Fn(&str, String) -> CompressionResult + Send + Sync + 'static,
513 {
514 self.set_compressor_with_exit_code(move |command, output, _exit_code| {
515 compressor(command, output)
516 });
517 }
518
519 pub fn set_compressor_with_exit_code<F>(&self, compressor: F)
520 where
521 F: Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync + 'static,
522 {
523 if let Ok(mut slot) = self.inner.compressor.lock() {
524 *slot = Some(Box::new(compressor));
525 }
526 }
527
528 pub(crate) fn compress_output(
531 &self,
532 command: &str,
533 output: String,
534 exit_code: Option<i32>,
535 ) -> CompressionResult {
536 let Ok(slot) = self.inner.compressor.lock() else {
537 return CompressionResult::new(output);
538 };
539 match slot.as_ref() {
540 Some(compressor) => compressor(command, output, exit_code),
541 None => CompressionResult::new(output),
542 }
543 }
544
545 fn ensure_terminal_output_cache(&self, task: &Arc<BgTask>) -> Option<TerminalOutputCache> {
546 let (metadata, buffer) = {
547 let state = task.state.lock().ok()?;
548 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
549 return None;
550 }
551 if let Some(cache) = state.terminal_output_cache.clone() {
552 return Some(cache);
553 }
554 (state.metadata.clone(), state.buffer.clone())
555 };
556
557 let mut cap_buffer = buffer.clone();
558 let disk_truncation = cap_buffer.enforce_terminal_cap();
559 let cache = self.render_terminal_output(&metadata, &cap_buffer, disk_truncation);
560 let mut state = task.state.lock().ok()?;
561 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
562 return None;
563 }
564 if let Some(existing) = state.terminal_output_cache.clone() {
565 return Some(existing);
566 }
567 state.terminal_output_cache = Some(cache.clone());
568 Some(cache)
569 }
570
571 fn render_terminal_output(
572 &self,
573 metadata: &PersistedTask,
574 buffer: &BgBuffer,
575 disk_truncation: DiskTruncation,
576 ) -> TerminalOutputCache {
577 let output_readable = buffer
578 .output_path()
579 .is_some_and(|path| self.is_session_owned_artifact_path(&metadata.session_id, &path));
580 let stderr_readable = buffer
581 .stderr_path()
582 .map(|path| self.is_session_owned_artifact_path(&metadata.session_id, path))
583 .unwrap_or(true);
584 let artifact_access = ArtifactRecoveryAccess {
585 task_id: metadata.task_id.clone(),
586 readable: output_readable && stderr_readable,
587 };
588
589 if metadata.mode == BgMode::Pty {
590 return TerminalOutputCache {
591 output_preview: String::new(),
592 output_truncated: false,
593 kind: TerminalOutputKind::Raw,
594 output_path: buffer.output_path().map(|path| path.display().to_string()),
595 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
596 artifact_access,
597 recovery: None,
598 };
599 }
600
601 if let Some(structured) = render_structured_output(
602 &metadata.command,
603 buffer,
604 disk_truncation,
605 artifact_access.clone(),
606 ) {
607 return structured;
608 }
609
610 if !metadata.compressed {
611 return render_raw_passthrough(buffer, disk_truncation, artifact_access);
612 }
613
614 let raw = buffer.read_combined_head_tail(
615 COMPRESS_INPUT_CAP_BYTES,
616 COMPRESS_INPUT_HEAD_BYTES,
617 COMPRESS_INPUT_TAIL_BYTES,
618 );
619 let compressed = self.compress_output(&metadata.command, raw.text, metadata.exit_code);
620 render_compressed_with_recovery(
621 buffer,
622 compressed,
623 raw.truncated,
624 disk_truncation,
625 artifact_access,
626 )
627 }
628
629 fn snapshot_with_terminal_cache(
630 &self,
631 task: &Arc<BgTask>,
632 preview_bytes: usize,
633 ) -> BgTaskSnapshot {
634 let mut snapshot = task.snapshot(preview_bytes);
635 self.maybe_compress_snapshot(task, &mut snapshot);
636 snapshot
637 }
638
639 fn post_terminal_transition(&self, task: &Arc<BgTask>, emit_frame: bool) -> Result<(), String> {
640 let (metadata, buffer) = {
641 let state = task
642 .state
643 .lock()
644 .map_err(|_| "background task lock poisoned".to_string())?;
645 if !state.metadata.status.is_terminal() {
646 return Ok(());
647 }
648 (state.metadata.clone(), state.buffer.clone())
649 };
650
651 let cache = self.ensure_terminal_output_cache(task);
652 self.enqueue_completion_from_parts(
653 &metadata,
654 Some(&buffer),
655 None,
656 emit_frame,
657 cache.as_ref(),
658 );
659 Ok(())
660 }
661
662 fn persist_task(&self, paths: &TaskPaths, metadata: &PersistedTask) -> std::io::Result<()> {
663 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
664 write_task_at(&task, metadata)?;
665 self.dual_write_task(paths, metadata);
666 Ok(())
667 }
668
669 fn update_task_metadata<F>(
670 &self,
671 paths: &TaskPaths,
672 update: F,
673 ) -> std::io::Result<PersistedTask>
674 where
675 F: FnOnce(&mut PersistedTask),
676 {
677 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
678 let metadata = update_task_at(&task, update)?;
679 self.dual_write_task(paths, &metadata);
680 Ok(metadata)
681 }
682
683 fn dual_write_task(&self, paths: &TaskPaths, metadata: &PersistedTask) {
684 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
685 let Some(pool) = pool else {
686 return;
687 };
688 let harness = self
689 .inner
690 .db_harness
691 .read()
692 .ok()
693 .and_then(|slot| slot.clone());
694 let Some(harness) = harness else {
695 crate::slog_warn!(
696 "dual-write bash_task to DB skipped for {}: harness not configured",
697 metadata.task_id
698 );
699 return;
700 };
701 let row = match metadata.to_bash_task_row(&harness, paths) {
702 Ok(row) => row,
703 Err(error) => {
704 crate::slog_warn!(
705 "dual-write bash_task to DB failed for {}: {}",
706 metadata.task_id,
707 error
708 );
709 return;
710 }
711 };
712 let conn = match pool.lock() {
713 Ok(conn) => conn,
714 Err(_) => {
715 crate::slog_warn!(
716 "dual-write bash_task to DB failed for {}: db mutex poisoned",
717 metadata.task_id
718 );
719 return;
720 }
721 };
722 if let Err(error) = crate::db::bash_tasks::upsert_bash_task(&conn, &row) {
723 crate::slog_warn!(
724 "dual-write bash_task to DB failed for {}: {}",
725 metadata.task_id,
726 error
727 );
728 }
729 }
730
731 fn delete_gc_task_from_db(&self, metadata: &PersistedTask) {
732 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
733 let Some(pool) = pool else {
734 return;
735 };
736 let harness = self
737 .inner
738 .db_harness
739 .read()
740 .ok()
741 .and_then(|slot| slot.clone());
742 let Some(harness) = harness else {
743 crate::slog_warn!(
744 "GC bash_task DB delete skipped for {}: harness not configured",
745 metadata.task_id
746 );
747 return;
748 };
749 let conn = match pool.lock() {
750 Ok(conn) => conn,
751 Err(_) => {
752 crate::slog_warn!(
753 "GC bash_task DB delete failed for {}: db mutex poisoned",
754 metadata.task_id
755 );
756 return;
757 }
758 };
759 if let Err(error) = crate::db::bash_tasks::delete_delivered_terminal_bash_task(
760 &conn,
761 &harness,
762 &metadata.session_id,
763 &metadata.task_id,
764 ) {
765 crate::slog_warn!(
766 "GC bash_task DB delete failed for {}: {}",
767 metadata.task_id,
768 error
769 );
770 }
771 }
772
773 pub fn record_scanner_report(
774 &self,
775 task_id: &str,
776 session_id: &str,
777 scanner_report: Vec<PermissionAsk>,
778 ) -> Result<(), String> {
779 if scanner_report.is_empty() {
780 return Ok(());
781 }
782 let task = self.task_for_session(task_id, session_id).ok_or_else(|| {
783 "background task not found while recording scanner report".to_string()
784 })?;
785 let metadata = {
786 let mut state = task
787 .state
788 .lock()
789 .map_err(|_| "background task lock poisoned".to_string())?;
790 state.metadata.scanner_report = scanner_report;
791 state.metadata.clone()
792 };
793 self.persist_task(&task.paths, &metadata)
794 .map_err(|error| format!("failed to persist scanner report: {error}"))
795 }
796
797 pub fn configure_long_running_reminders(&self, enabled: bool, interval_ms: u64) {
798 self.inner
799 .long_running_reminder_enabled
800 .store(enabled, Ordering::SeqCst);
801 self.inner
802 .long_running_reminder_interval_ms
803 .store(interval_ms, Ordering::SeqCst);
804 }
805
806 #[cfg(unix)]
807 #[allow(clippy::too_many_arguments)]
808 pub fn spawn(
809 &self,
810 spawn_plan: SpawnPlan,
811 command: &str,
812 session_id: String,
813 workdir: PathBuf,
814 env: HashMap<String, String>,
815 timeout: Option<Duration>,
816 storage_dir: PathBuf,
817 max_running: usize,
818 notify_on_completion: bool,
819 compressed: bool,
820 project_root: Option<PathBuf>,
821 ) -> Result<String, String> {
822 self.start_watchdog();
823
824 let running = self.running_count();
825 if running >= max_running {
826 #[cfg(unix)]
827 if let Some(prepared) = spawn_plan.prepared_task() {
828 let _ = delete_resolved_task(&prepared.resolved_task());
829 }
830 return Err(format!(
831 "background bash task limit exceeded: {running} running (max {max_running})"
832 ));
833 }
834
835 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
836 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
837 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
838 (spawn_plan.clone(), prepared.resolved_task())
839 } else {
840 let task = allocate_task_layout(&storage_dir, &session_id)
841 .map_err(|error| format!("failed to create background task layout: {error}"))?;
842 let shell = resolve_posix_shell();
843 let root = project_root.as_deref().unwrap_or(&workdir);
844 let environment =
845 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
846 let prepared = match crate::sandbox_spawn::prepare_task_payload(
847 &task,
848 command.as_bytes(),
849 root,
850 &workdir,
851 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
852 &shell,
853 &environment,
854 ) {
855 Ok(prepared) => prepared,
856 Err(error) => {
857 let _ = delete_resolved_task(&task);
858 return Err(error);
859 }
860 };
861 let task = prepared.resolved_task();
862 (spawn_plan.with_prepared_task(prepared), task)
863 };
864 let task_id = task_layout.paths.task_id.clone();
865 let paths = task_layout.paths.clone();
866
867 if self.task(&task_id).is_some() {
868 let _ = delete_resolved_task(&task_layout);
869 return Err("background task id collided with a live task".to_string());
870 }
871
872 let mut metadata = PersistedTask::starting(
873 task_id.clone(),
874 session_id.clone(),
875 command.to_string(),
876 workdir.clone(),
877 project_root,
878 timeout_ms,
879 notify_on_completion,
880 compressed,
881 );
882 attach_sandbox_metadata(&mut metadata, &spawn_plan);
883 if let Err(error) = write_task_at(&task_layout, &metadata) {
884 let _ = delete_resolved_task(&task_layout);
885 return Err(format!(
886 "failed to persist background task metadata: {error}"
887 ));
888 }
889 self.dual_write_task(&paths, &metadata);
890
891 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pipes)
892 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
893 let child = match spawn_detached_child(
894 &spawn_plan,
895 command,
896 &paths,
897 &workdir,
898 &env,
899 &mut io_handles,
900 ) {
901 Ok(child) => child,
902 Err(error) => {
903 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
904 let _ = delete_task_bundle(&paths);
905 return Err(error);
906 }
907 };
908
909 let child_pid = child.id();
910 metadata.mark_running(child_pid, child_pid as i32);
911 self.persist_task(&paths, &metadata)
912 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
913
914 let task = Arc::new(BgTask {
915 task_id: task_id.clone(),
916 session_id,
917 paths: paths.clone(),
918 artifact_root: canonical_artifact_root(&paths),
919 started: Instant::now(),
920 last_reminder_at: Mutex::new(None),
921 terminal_at: Mutex::new(None),
922 state: Mutex::new(BgTaskState {
923 metadata,
924 runtime: TaskRuntime::Piped(Some(child)),
925 io_handles: Some(io_handles),
926 detached: false,
927 child_exit_observed: false,
928 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
929 terminal_output_cache: None,
930 pending_terminal_override: None,
931 }),
932 });
933
934 self.inner
935 .tasks
936 .lock()
937 .map_err(|_| "background task registry lock poisoned".to_string())?
938 .insert(task_id.clone(), task);
939
940 Ok(task_id)
941 }
942
943 #[allow(clippy::too_many_arguments)]
944 pub fn spawn_pty(
945 &self,
946 spawn_plan: SpawnPlan,
947 command: &str,
948 session_id: String,
949 workdir: PathBuf,
950 env: HashMap<String, String>,
951 timeout: Option<Duration>,
952 storage_dir: PathBuf,
953 max_running: usize,
954 notify_on_completion: bool,
955 compressed: bool,
956 project_root: Option<PathBuf>,
957 rows: u16,
958 cols: u16,
959 ) -> Result<String, String> {
960 self.start_watchdog();
961
962 let running = self.running_count();
963 if running >= max_running {
964 #[cfg(unix)]
965 if let Some(prepared) = spawn_plan.prepared_task() {
966 let _ = delete_resolved_task(&prepared.resolved_task());
967 }
968 return Err(format!(
969 "background bash task limit exceeded: {running} running (max {max_running})"
970 ));
971 }
972
973 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
974 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
975 #[cfg(unix)]
976 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
977 (spawn_plan.clone(), prepared.resolved_task())
978 } else {
979 let task = allocate_task_layout(&storage_dir, &session_id)
980 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
981 let shell = super::resolved_shell_path(true);
982 let root = project_root.as_deref().unwrap_or(&workdir);
983 let environment =
984 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
985 let prepared = match crate::sandbox_spawn::prepare_task_payload(
986 &task,
987 command.as_bytes(),
988 root,
989 &workdir,
990 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
991 &shell,
992 &environment,
993 ) {
994 Ok(prepared) => prepared,
995 Err(error) => {
996 let _ = delete_resolved_task(&task);
997 return Err(error);
998 }
999 };
1000 let task = prepared.resolved_task();
1001 (spawn_plan.with_prepared_task(prepared), task)
1002 };
1003 #[cfg(windows)]
1004 let task_layout = allocate_task_layout(&storage_dir, &session_id)
1005 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1006 let task_id = task_layout.paths.task_id.clone();
1007 let paths = task_layout.paths.clone();
1008
1009 let mut metadata = PersistedTask::starting(
1010 task_id.clone(),
1011 session_id.clone(),
1012 command.to_string(),
1013 workdir.clone(),
1014 project_root,
1015 timeout_ms,
1016 notify_on_completion,
1017 compressed,
1018 );
1019 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1020 metadata.mode = BgMode::Pty;
1021 metadata.pty_rows = Some(rows);
1022 metadata.pty_cols = Some(cols);
1023 if let Err(error) = write_task_at(&task_layout, &metadata) {
1024 let _ = delete_resolved_task(&task_layout);
1025 return Err(format!(
1026 "failed to persist background task metadata: {error}"
1027 ));
1028 }
1029 self.dual_write_task(&paths, &metadata);
1030 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pty)
1031 .map_err(|error| format!("failed to pre-open PTY output handles: {error}"))?;
1032
1033 let runtime = match spawn_pty_for_command(
1034 &spawn_plan,
1035 &task_id,
1036 &session_id,
1037 command,
1038 &paths,
1039 &workdir,
1040 &env,
1041 rows,
1042 cols,
1043 self.inner.wake_tx.clone(),
1044 &mut io_handles,
1045 ) {
1046 Ok(runtime) => runtime,
1047 Err(error) => {
1048 crate::slog_warn!(
1049 "failed to spawn PTY background bash task {task_id}; deleting partial bundle: {error}"
1050 );
1051 let _ = delete_task_bundle(&paths);
1052 return Err(error);
1053 }
1054 };
1055
1056 if let Some(child_pid) = runtime.child_pid {
1057 metadata.mark_running(child_pid, child_pid as i32);
1058 } else {
1059 metadata.status = BgTaskStatus::Running;
1060 metadata.pgid = None;
1061 }
1062 self.persist_task(&paths, &metadata)
1063 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1064
1065 let task = Arc::new(BgTask {
1066 task_id: task_id.clone(),
1067 session_id,
1068 paths: paths.clone(),
1069 artifact_root: canonical_artifact_root(&paths),
1070 started: Instant::now(),
1071 last_reminder_at: Mutex::new(None),
1072 terminal_at: Mutex::new(None),
1073 state: Mutex::new(BgTaskState {
1074 metadata,
1075 runtime: TaskRuntime::Pty(Some(runtime)),
1076 io_handles: Some(io_handles),
1077 detached: false,
1078 child_exit_observed: false,
1079 buffer: BgBuffer::registered(&paths, BgMode::Pty),
1080 terminal_output_cache: None,
1081 pending_terminal_override: None,
1082 }),
1083 });
1084
1085 self.inner
1086 .tasks
1087 .lock()
1088 .map_err(|_| "background task registry lock poisoned".to_string())?
1089 .insert(task_id.clone(), task);
1090
1091 Ok(task_id)
1092 }
1093
1094 #[cfg(windows)]
1095 #[allow(clippy::too_many_arguments)]
1096 pub fn spawn(
1097 &self,
1098 spawn_plan: SpawnPlan,
1099 command: &str,
1100 session_id: String,
1101 workdir: PathBuf,
1102 env: HashMap<String, String>,
1103 timeout: Option<Duration>,
1104 storage_dir: PathBuf,
1105 max_running: usize,
1106 notify_on_completion: bool,
1107 compressed: bool,
1108 project_root: Option<PathBuf>,
1109 ) -> Result<String, String> {
1110 self.start_watchdog();
1111
1112 let running = self.running_count();
1113 if running >= max_running {
1114 #[cfg(unix)]
1115 if let Some(prepared) = spawn_plan.prepared_task() {
1116 let _ = delete_resolved_task(&prepared.resolved_task());
1117 }
1118 return Err(format!(
1119 "background bash task limit exceeded: {running} running (max {max_running})"
1120 ));
1121 }
1122
1123 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1124 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1125 let task_layout = allocate_task_layout(&storage_dir, &session_id)
1126 .map_err(|error| format!("failed to create background task layout: {error}"))?;
1127 let task_id = task_layout.paths.task_id.clone();
1128 let paths = task_layout.paths.clone();
1129
1130 let mut metadata = PersistedTask::starting(
1131 task_id.clone(),
1132 session_id.clone(),
1133 command.to_string(),
1134 workdir.clone(),
1135 project_root,
1136 timeout_ms,
1137 notify_on_completion,
1138 compressed,
1139 );
1140 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1141 if let Err(error) = write_task_at(&task_layout, &metadata) {
1142 let _ = delete_resolved_task(&task_layout);
1143 return Err(format!(
1144 "failed to persist background task metadata: {error}"
1145 ));
1146 }
1147 self.dual_write_task(&paths, &metadata);
1148 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pipes)
1149 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
1150
1151 let child = match spawn_detached_child(
1152 &spawn_plan,
1153 command,
1154 &paths,
1155 &workdir,
1156 &env,
1157 &mut io_handles,
1158 ) {
1159 Ok(child) => child,
1160 Err(error) => {
1161 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
1162 let _ = delete_task_bundle(&paths);
1163 return Err(error);
1164 }
1165 };
1166
1167 let child_pid = child.id();
1168 metadata.status = BgTaskStatus::Running;
1169 metadata.child_pid = Some(child_pid);
1170 metadata.pgid = None;
1171 self.persist_task(&paths, &metadata)
1172 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1173
1174 let task = Arc::new(BgTask {
1175 task_id: task_id.clone(),
1176 session_id,
1177 paths: paths.clone(),
1178 artifact_root: canonical_artifact_root(&paths),
1179 started: Instant::now(),
1180 last_reminder_at: Mutex::new(None),
1181 terminal_at: Mutex::new(None),
1182 state: Mutex::new(BgTaskState {
1183 metadata,
1184 runtime: TaskRuntime::Piped(Some(child)),
1185 io_handles: Some(io_handles),
1186 detached: false,
1187 child_exit_observed: false,
1188 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
1189 terminal_output_cache: None,
1190 pending_terminal_override: None,
1191 }),
1192 });
1193
1194 self.inner
1195 .tasks
1196 .lock()
1197 .map_err(|_| "background task registry lock poisoned".to_string())?
1198 .insert(task_id.clone(), task);
1199
1200 Ok(task_id)
1201 }
1202
1203 pub fn write_pty(
1204 &self,
1205 task_id: &str,
1206 session_id: &str,
1207 input: &[u8],
1208 ) -> Result<usize, String> {
1209 let task = self
1210 .task_for_session(task_id, session_id)
1211 .ok_or_else(|| "task_not_found".to_string())?;
1212
1213 let writer = {
1214 let state = task
1215 .state
1216 .lock()
1217 .map_err(|_| "background task lock poisoned".to_string())?;
1218 if state.metadata.mode != BgMode::Pty {
1219 return Err("task_not_pty".to_string());
1220 }
1221 if state.metadata.status.is_terminal() {
1222 return Err("task_exited".to_string());
1223 }
1224 match &state.runtime {
1225 TaskRuntime::Pty(Some(runtime)) => Arc::clone(&runtime.writer),
1226 TaskRuntime::Pty(None) => return Err("task_exited".to_string()),
1227 TaskRuntime::Piped(_) => return Err("task_not_pty".to_string()),
1228 }
1229 };
1230
1231 let mut writer = writer
1232 .lock()
1233 .map_err(|_| "PTY writer lock poisoned".to_string())?;
1234 writer
1235 .write_all(input)
1236 .map_err(|error| format!("failed to write to PTY: {error}"))?;
1237 writer
1238 .flush()
1239 .map_err(|error| format!("failed to flush PTY writer: {error}"))?;
1240 Ok(input.len())
1241 }
1242
1243 pub fn replay_session(&self, storage_dir: &Path, session_id: &str) -> Result<(), String> {
1244 self.replay_session_inner(storage_dir, session_id, None)
1245 }
1246
1247 pub fn replay_session_for_project(
1248 &self,
1249 storage_dir: &Path,
1250 session_id: &str,
1251 project_root: &Path,
1252 ) -> Result<(), String> {
1253 self.replay_session_inner(storage_dir, session_id, Some(project_root))
1254 }
1255
1256 fn replay_session_inner(
1257 &self,
1258 storage_dir: &Path,
1259 session_id: &str,
1260 project_root: Option<&Path>,
1261 ) -> Result<(), String> {
1262 self.start_watchdog();
1263 if !self.inner.persisted_gc_started.swap(true, Ordering::SeqCst) {
1264 if let Err(error) = self.maybe_gc_persisted(storage_dir) {
1265 crate::slog_warn!("failed to GC persisted background bash tasks: {error}");
1266 }
1267 }
1268
1269 let canonical_project = project_root.map(canonicalized_path);
1270 let tasks = match self.replay_session_from_db(session_id) {
1282 Some(Ok(tasks)) if !tasks.is_empty() => tasks,
1283 Some(Ok(_)) => {
1284 let disk_tasks = self.replay_session_from_disk(storage_dir, session_id)?;
1285 if !disk_tasks.is_empty() {
1286 crate::slog_info!(
1287 "bash task replay: 0 in DB for session {}, {} from disk fallback",
1288 session_id,
1289 disk_tasks.len()
1290 );
1291 }
1292 disk_tasks
1293 }
1294 Some(Err(error)) => {
1295 crate::slog_warn!(
1296 "bash task replay DB lookup failed for session {}; falling back to disk: {}",
1297 session_id,
1298 error
1299 );
1300 self.replay_session_from_disk(storage_dir, session_id)?
1301 }
1302 None => {
1303 self.replay_session_from_disk(storage_dir, session_id)?
1305 }
1306 };
1307
1308 for mut metadata in tasks {
1309 if metadata.session_id != session_id {
1310 continue;
1311 }
1312 if let Some(canonical_project) = canonical_project.as_deref() {
1313 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
1314 if metadata_project.as_deref() != Some(canonical_project) {
1315 continue;
1316 }
1317 }
1318
1319 if validate_task_id(&metadata.task_id).is_err() {
1320 crate::slog_warn!(
1321 "ignoring persisted background task with invalid id {:?}",
1322 metadata.task_id
1323 );
1324 continue;
1325 }
1326 let session_dir = session_tasks_dir(storage_dir, session_id);
1327 let resolved = match resolve_task_layout(&session_dir, &metadata.task_id) {
1328 Ok(task) => task,
1329 Err(error) => {
1330 crate::slog_warn!(
1331 "quarantining unresolved background task {}: {error}",
1332 metadata.task_id
1333 );
1334 let _ = quarantine_task_layout(
1335 storage_dir,
1336 &session_dir,
1337 &metadata.task_id,
1338 "invalid",
1339 );
1340 continue;
1341 }
1342 };
1343 match read_task_at(&resolved) {
1344 Ok(disk) if disk.task_id == metadata.task_id && disk.session_id == session_id => {}
1345 Ok(_) | Err(_) => {
1346 let _ = quarantine_task_layout(
1347 storage_dir,
1348 &session_dir,
1349 &metadata.task_id,
1350 "mismatch",
1351 );
1352 continue;
1353 }
1354 }
1355 let paths = resolved.paths;
1356 match metadata.status {
1357 BgTaskStatus::Starting => {
1358 let completion_was_delivered = metadata.completion_delivered;
1359 metadata.mark_terminal(
1360 BgTaskStatus::Failed,
1361 None,
1362 Some("spawn aborted".to_string()),
1363 );
1364 metadata.completion_delivered |= completion_was_delivered;
1365 let _ = self.persist_task(&paths, &metadata);
1366 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1367 self.insert_rehydrated_task(metadata, paths, true)?;
1368 }
1369 BgTaskStatus::Running | BgTaskStatus::Killing => {
1370 if metadata.mode == BgMode::Pty {
1371 if let Ok(Some(marker)) = read_exit_marker(&paths) {
1372 let completion_was_delivered = metadata.completion_delivered;
1373 metadata = terminal_metadata_from_marker(metadata, marker, None);
1374 metadata.completion_delivered |= completion_was_delivered;
1375 let _ = self.persist_task(&paths, &metadata);
1376 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1377 self.insert_rehydrated_task(metadata, paths, true)?;
1378 } else if metadata.status.is_terminal() {
1379 self.insert_rehydrated_task(metadata, paths, true)?;
1380 } else {
1381 let completion_was_delivered = metadata.completion_delivered;
1382 metadata.mark_terminal(
1383 BgTaskStatus::Killed,
1384 None,
1385 Some("pty_lost_on_bridge_restart".to_string()),
1386 );
1387 metadata.completion_delivered |= completion_was_delivered;
1388 let _ = self.persist_task(&paths, &metadata);
1389 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1390 self.insert_rehydrated_task(metadata, paths, true)?;
1391 }
1392 } else if self.running_metadata_is_stale(&metadata) {
1393 let completion_was_delivered = metadata.completion_delivered;
1394 metadata.mark_terminal(
1395 BgTaskStatus::Killed,
1396 None,
1397 Some("orphaned (>24h)".to_string()),
1398 );
1399 metadata.completion_delivered |= completion_was_delivered;
1400 let _ = write_kill_marker_if_absent(&paths);
1401 let _ = self.persist_task(&paths, &metadata);
1402 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1403 self.insert_rehydrated_task(metadata, paths, true)?;
1404 } else if let Ok(Some(marker)) = read_exit_marker(&paths) {
1405 let reason = (metadata.status == BgTaskStatus::Killing).then(|| {
1406 "recovered from inconsistent killing state on replay".to_string()
1407 });
1408 if reason.is_some() {
1409 crate::slog_warn!("background task {} had killing state with exit marker; preferring marker",
1410 metadata.task_id);
1411 }
1412 let completion_was_delivered = metadata.completion_delivered;
1413 metadata = terminal_metadata_from_marker(metadata, marker, reason);
1414 metadata.completion_delivered |= completion_was_delivered;
1415 let _ = self.persist_task(&paths, &metadata);
1416 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1417 self.insert_rehydrated_task(metadata, paths, true)?;
1418 } else if metadata.status == BgTaskStatus::Killing {
1419 let _ = write_kill_marker_if_absent(&paths);
1420 let completion_was_delivered = metadata.completion_delivered;
1421 metadata.mark_terminal(
1422 BgTaskStatus::Killed,
1423 None,
1424 Some("recovered from inconsistent killing state on replay".to_string()),
1425 );
1426 metadata.completion_delivered |= completion_was_delivered;
1427 let _ = self.persist_task(&paths, &metadata);
1428 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1429 self.insert_rehydrated_task(metadata, paths, true)?;
1430 } else if metadata.child_pid.is_some_and(|pid| !is_process_alive(pid)) {
1431 let completion_was_delivered = metadata.completion_delivered;
1432 metadata.mark_terminal(
1433 BgTaskStatus::Failed,
1434 None,
1435 Some("process exited without exit marker".to_string()),
1436 );
1437 metadata.completion_delivered |= completion_was_delivered;
1438 let _ = self.persist_task(&paths, &metadata);
1439 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1440 self.insert_rehydrated_task(metadata, paths, true)?;
1441 } else {
1442 self.insert_rehydrated_task(metadata, paths, true)?;
1443 }
1444 }
1445 _ if metadata.status.is_terminal() => {
1446 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1452 self.insert_rehydrated_task(metadata, paths, true)?;
1453 }
1454 _ => {}
1455 }
1456 }
1457
1458 Ok(())
1459 }
1460
1461 fn replay_session_from_db(
1462 &self,
1463 session_id: &str,
1464 ) -> Option<Result<Vec<PersistedTask>, String>> {
1465 let pool = self
1466 .inner
1467 .db_pool
1468 .read()
1469 .ok()
1470 .and_then(|slot| slot.clone())?;
1471 let harness = self
1472 .inner
1473 .db_harness
1474 .read()
1475 .ok()
1476 .and_then(|slot| slot.clone())?;
1477 let conn = match pool.lock() {
1478 Ok(conn) => conn,
1479 Err(_) => return Some(Err("db mutex poisoned".to_string())),
1480 };
1481 Some(
1482 crate::db::bash_tasks::list_bash_tasks_for_session(&conn, &harness, session_id)
1483 .map(|rows| rows.into_iter().map(PersistedTask::from).collect())
1484 .map_err(|error| error.to_string()),
1485 )
1486 }
1487
1488 fn replay_session_from_disk(
1489 &self,
1490 storage_dir: &Path,
1491 session_id: &str,
1492 ) -> Result<Vec<PersistedTask>, String> {
1493 let dir = session_tasks_dir(storage_dir, session_id);
1494 if !dir.exists() {
1495 return Ok(Vec::new());
1496 }
1497
1498 let (task_ids, invalid_entries) = discover_task_ids(&dir)
1499 .map_err(|error| format!("failed to discover background task layouts: {error}"))?;
1500 for entry in invalid_entries {
1501 if let Err(error) = quarantine_invalid_entry(storage_dir, &dir, &entry) {
1502 crate::slog_warn!(
1503 "failed to quarantine invalid background task entry {:?}: {error}",
1504 entry
1505 );
1506 }
1507 }
1508
1509 let mut tasks = Vec::new();
1510 for task_id in task_ids {
1511 let task = match resolve_task_layout(&dir, &task_id) {
1512 Ok(task) => task,
1513 Err(error)
1514 if error.kind() == std::io::ErrorKind::NotFound
1515 && uninitialized_layout_is_recent(
1516 &dir,
1517 &task_id,
1518 Duration::from_secs(5 * 60),
1519 )
1520 .unwrap_or(false) =>
1521 {
1522 continue;
1523 }
1524 Err(error) => {
1525 crate::slog_warn!(
1526 "quarantining unresolved background task {task_id} during replay: {error}"
1527 );
1528 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
1529 continue;
1530 }
1531 };
1532 match read_task_at(&task) {
1533 Ok(metadata) if metadata.session_id == session_id => tasks.push(metadata),
1534 Ok(_) => {
1535 crate::slog_warn!(
1536 "quarantining background task {task_id} with mismatched session metadata"
1537 );
1538 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "mismatch");
1539 }
1540 Err(error) => {
1541 crate::slog_warn!(
1542 "quarantining invalid background task metadata {task_id} during replay: {error}"
1543 );
1544 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
1545 }
1546 }
1547 }
1548 Ok(tasks)
1549 }
1550
1551 pub fn register_watch(
1552 &self,
1553 task_id: String,
1554 pattern: WatchPattern,
1555 once: bool,
1556 ) -> Result<String, &'static str> {
1557 let task = self.task(&task_id).ok_or("task_not_found")?;
1558 validate_task_id(&task_id).map_err(|_| "invalid_task_id")?;
1559 let (mode, terminal_at_registration) = task
1560 .state
1561 .lock()
1562 .map(|state| {
1563 (
1564 state.metadata.mode.clone(),
1565 state.metadata.status.is_terminal(),
1566 )
1567 })
1568 .map_err(|_| "background_task_lock_poisoned")?;
1569 let mut stdout = (mode == BgMode::Pipes)
1570 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
1571 .transpose()
1572 .map_err(|_| "artifact_refused")?;
1573 let mut stderr = (mode == BgMode::Pipes)
1574 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
1575 .transpose()
1576 .map_err(|_| "artifact_refused")?;
1577 let mut pty = (mode == BgMode::Pty)
1578 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
1579 .transpose()
1580 .map_err(|_| "artifact_refused")?;
1581
1582 let mut terminal_matches = Vec::new();
1583 let scanned_terminal = terminal_at_registration;
1584 let watch_id = {
1585 let mut registry = self
1586 .inner
1587 .watch_registry
1588 .lock()
1589 .map_err(|_| "watch_registry_poisoned")?;
1590 let watch_id = registry.register(task_id.clone(), pattern, once)?;
1591 match &mode {
1592 BgMode::Pipes => {
1593 let stdout_key = format!("{task_id}:stdout");
1594 let stderr_key = format!("{task_id}:stderr");
1595 if terminal_at_registration {
1596 registry.set_file_cursor(&stdout_key, 0);
1597 registry.set_file_cursor(&stderr_key, 0);
1598 terminal_matches.extend(registry.scan_file_new_bytes(
1599 &stdout_key,
1600 &task_id,
1601 stdout.as_mut().expect("pipe stdout opened"),
1602 ));
1603 terminal_matches.extend(registry.scan_file_new_bytes(
1604 &stderr_key,
1605 &task_id,
1606 stderr.as_mut().expect("pipe stderr opened"),
1607 ));
1608 } else {
1609 registry.prime_file_cursor(
1610 &stdout_key,
1611 stdout.as_ref().expect("pipe stdout opened"),
1612 );
1613 registry.prime_file_cursor(
1614 &stderr_key,
1615 stderr.as_ref().expect("pipe stderr opened"),
1616 );
1617 }
1618 }
1619 BgMode::Pty => {
1620 let pty_key = format!("{task_id}:pty");
1621 if terminal_at_registration {
1622 registry.set_file_cursor(&pty_key, 0);
1623 terminal_matches.extend(registry.scan_file_new_bytes(
1624 &pty_key,
1625 &task_id,
1626 pty.as_mut().expect("PTY artifact opened"),
1627 ));
1628 } else {
1629 registry.prime_file_cursor(
1630 &pty_key,
1631 pty.as_ref().expect("PTY artifact opened"),
1632 );
1633 }
1634 }
1635 }
1636 watch_id
1637 };
1638
1639 if task.is_terminal() {
1640 if !scanned_terminal {
1641 terminal_matches = {
1642 let mut registry = self
1643 .inner
1644 .watch_registry
1645 .lock()
1646 .map_err(|_| "watch_registry_poisoned")?;
1647 match &mode {
1648 BgMode::Pipes => {
1649 let stdout_key = format!("{task_id}:stdout");
1650 let stderr_key = format!("{task_id}:stderr");
1651 registry.set_file_cursor(&stdout_key, 0);
1652 registry.set_file_cursor(&stderr_key, 0);
1653 let mut matches = registry.scan_file_new_bytes(
1654 &stdout_key,
1655 &task_id,
1656 stdout.as_mut().expect("pipe stdout opened"),
1657 );
1658 matches.extend(registry.scan_file_new_bytes(
1659 &stderr_key,
1660 &task_id,
1661 stderr.as_mut().expect("pipe stderr opened"),
1662 ));
1663 matches
1664 }
1665 BgMode::Pty => {
1666 let pty_key = format!("{task_id}:pty");
1667 registry.set_file_cursor(&pty_key, 0);
1668 registry.scan_file_new_bytes(
1669 &pty_key,
1670 &task_id,
1671 pty.as_mut().expect("PTY artifact opened"),
1672 )
1673 }
1674 }
1675 };
1676 }
1677
1678 let (watch_controlled, watch_matched) = self.task_watch_state(&task_id);
1679 if terminal_matches.is_empty() && (!watch_controlled || watch_matched) {
1680 if watch_matched {
1681 let _ = task.set_completion_delivered(true, self);
1682 self.clear_task_watch_state(&task_id);
1683 }
1684 return Ok(watch_id);
1685 }
1686
1687 let completion = self
1688 .remove_pending_completion(&task_id)
1689 .or_else(|| self.completion_snapshot_for_task(&task));
1690 if terminal_matches.is_empty() {
1691 if let Some(completion) = completion.as_ref() {
1692 self.emit_bash_watch_exit(completion);
1693 }
1694 } else {
1695 for pattern_match in terminal_matches {
1696 self.emit_bash_pattern_match(&task.session_id, pattern_match);
1697 }
1698 }
1699 let _ = task.set_completion_delivered(true, self);
1700 self.clear_task_watch_state(&task_id);
1701 }
1702
1703 Ok(watch_id)
1704 }
1705
1706 pub fn unregister_watch(&self, task_id: &str, watch_id: &str) {
1707 if let Ok(mut registry) = self.inner.watch_registry.lock() {
1708 registry.unregister(task_id, watch_id);
1709 }
1710 }
1711
1712 pub fn active_watch_count(&self, task_id: &str) -> usize {
1713 self.inner
1714 .watch_registry
1715 .lock()
1716 .map(|registry| registry.active_count(task_id))
1717 .unwrap_or(0)
1718 }
1719
1720 fn task_watch_state(&self, task_id: &str) -> (bool, bool) {
1721 self.inner
1722 .watch_registry
1723 .lock()
1724 .map(|registry| {
1725 (
1726 registry.has_controlled_task(task_id),
1727 registry.has_matched_task(task_id),
1728 )
1729 })
1730 .unwrap_or((false, false))
1731 }
1732
1733 fn task_has_watch_control(&self, task_id: &str) -> bool {
1734 self.inner
1735 .watch_registry
1736 .lock()
1737 .map(|registry| registry.has_controlled_task(task_id))
1738 .unwrap_or(false)
1739 }
1740
1741 fn clear_task_watch_state(&self, task_id: &str) {
1742 if let Ok(mut registry) = self.inner.watch_registry.lock() {
1743 registry.clear_task(task_id);
1744 }
1745 }
1746
1747 pub(crate) fn scan_task_watch_output(&self, task: &Arc<BgTask>) {
1748 let mode = match task.state.lock() {
1749 Ok(state) => state.metadata.mode.clone(),
1750 Err(_) => return,
1751 };
1752 let mut stdout = (mode == BgMode::Pipes)
1753 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
1754 .transpose()
1755 .ok()
1756 .flatten();
1757 let mut stderr = (mode == BgMode::Pipes)
1758 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
1759 .transpose()
1760 .ok()
1761 .flatten();
1762 let mut pty = (mode == BgMode::Pty)
1763 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
1764 .transpose()
1765 .ok()
1766 .flatten();
1767 let mut matches = Vec::new();
1768 if let Ok(mut registry) = self.inner.watch_registry.lock() {
1769 match mode {
1770 BgMode::Pipes => {
1771 let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) else {
1772 return;
1773 };
1774 let stdout_key = format!("{}:stdout", task.task_id);
1775 let stderr_key = format!("{}:stderr", task.task_id);
1776 matches.extend(registry.scan_file_new_bytes(
1777 &stdout_key,
1778 &task.task_id,
1779 stdout,
1780 ));
1781 matches.extend(registry.scan_file_new_bytes(
1782 &stderr_key,
1783 &task.task_id,
1784 stderr,
1785 ));
1786 }
1787 BgMode::Pty => {
1788 let Some(pty) = pty.as_mut() else {
1789 return;
1790 };
1791 let pty_key = format!("{}:pty", task.task_id);
1792 matches.extend(registry.scan_file_new_bytes(&pty_key, &task.task_id, pty));
1793 }
1794 }
1795 }
1796 for pattern_match in matches {
1797 self.emit_bash_pattern_match(&task.session_id, pattern_match);
1798 }
1799 }
1800
1801 pub fn status(
1802 &self,
1803 task_id: &str,
1804 session_id: &str,
1805 project_root: Option<&Path>,
1806 storage_dir: Option<&Path>,
1807 preview_bytes: usize,
1808 ) -> Option<BgTaskSnapshot> {
1809 validate_task_id(task_id).ok()?;
1810 let mut task = self.task_for_session(task_id, session_id);
1811 if task.is_none() {
1812 if let Some(storage_dir) = storage_dir {
1813 let _ = if let Some(project_root) = project_root {
1814 self.replay_session_for_project(storage_dir, session_id, project_root)
1815 } else {
1816 self.replay_session(storage_dir, session_id)
1817 };
1818 task = self.task_for_session(task_id, session_id);
1819 }
1820 }
1821 let Some(task) = task else {
1822 return self.status_relaxed(
1823 task_id,
1824 session_id,
1825 project_root?,
1826 storage_dir?,
1827 preview_bytes,
1828 );
1829 };
1830 let _ = self.poll_task(&task);
1831 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
1832 }
1833
1834 fn status_relaxed_task(
1835 &self,
1836 task_id: &str,
1837 project_root: &Path,
1838 storage_dir: &Path,
1839 ) -> Option<Arc<BgTask>> {
1840 validate_task_id(task_id).ok()?;
1841 let canonical_project = canonicalized_path(project_root);
1842 match self.lookup_relaxed_task_from_db(task_id, project_root) {
1843 Some(Ok(Some(metadata))) => {
1844 if let Some(task) = self.task(task_id) {
1845 let matches_project = task
1846 .state
1847 .lock()
1848 .map(|state| {
1849 state
1850 .metadata
1851 .project_root
1852 .as_deref()
1853 .map(canonicalized_path)
1854 .as_deref()
1855 == Some(canonical_project.as_path())
1856 })
1857 .unwrap_or(false);
1858 return matches_project.then_some(task);
1859 }
1860 let resolved = resolve_task_layout(
1861 &session_tasks_dir(storage_dir, &metadata.session_id),
1862 &metadata.task_id,
1863 )
1864 .ok()?;
1865 let disk = read_task_at(&resolved).ok()?;
1866 if disk.task_id != metadata.task_id || disk.session_id != metadata.session_id {
1867 return None;
1868 }
1869 if self
1870 .insert_rehydrated_task(metadata, resolved.paths, true)
1871 .is_err()
1872 {
1873 return None;
1874 }
1875 return self.task(task_id);
1876 }
1877 Some(Ok(None)) => {
1878 crate::slog_info!(
1879 "bash task relaxed DB miss for {}; falling back to disk",
1880 task_id
1881 );
1882 }
1883 Some(Err(error)) => {
1884 crate::slog_warn!(
1885 "bash task relaxed DB lookup failed for {}; falling back to disk: {}",
1886 task_id,
1887 error
1888 );
1889 }
1890 None => {
1891 crate::slog_info!(
1892 "bash task relaxed DB unavailable for {}; falling back to disk",
1893 task_id
1894 );
1895 }
1896 }
1897 let root = storage_dir.join("bash-tasks");
1898 let entries = fs::read_dir(&root).ok()?;
1899 for entry in entries.flatten() {
1900 let dir = entry.path();
1901 if !dir.is_dir() {
1902 continue;
1903 }
1904 let resolved = match resolve_task_layout(&dir, task_id) {
1905 Ok(task) => task,
1906 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
1907 Err(error) => {
1908 crate::slog_warn!(
1909 "quarantining unresolved background task {task_id} during relaxed lookup: {error}"
1910 );
1911 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
1912 continue;
1913 }
1914 };
1915 let metadata = match read_task_at(&resolved) {
1916 Ok(metadata) => metadata,
1917 Err(error) => {
1918 crate::slog_warn!(
1919 "quarantining invalid background task metadata {task_id} during relaxed lookup: {error}"
1920 );
1921 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
1922 continue;
1923 }
1924 };
1925 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
1926 if metadata_project.as_deref() != Some(canonical_project.as_path()) {
1927 continue;
1928 }
1929 if let Some(task) = self.task(task_id) {
1930 let matches_project = task
1931 .state
1932 .lock()
1933 .map(|state| {
1934 state
1935 .metadata
1936 .project_root
1937 .as_deref()
1938 .map(canonicalized_path)
1939 .as_deref()
1940 == Some(canonical_project.as_path())
1941 })
1942 .unwrap_or(false);
1943 return matches_project.then_some(task);
1944 }
1945 if self
1946 .insert_rehydrated_task(metadata, resolved.paths, true)
1947 .is_err()
1948 {
1949 return None;
1950 }
1951 return self.task(task_id);
1952 }
1953 None
1954 }
1955
1956 fn lookup_relaxed_task_from_db(
1957 &self,
1958 task_id: &str,
1959 project_root: &Path,
1960 ) -> Option<Result<Option<PersistedTask>, String>> {
1961 let pool = self
1962 .inner
1963 .db_pool
1964 .read()
1965 .ok()
1966 .and_then(|slot| slot.clone())?;
1967 let harness = self
1968 .inner
1969 .db_harness
1970 .read()
1971 .ok()
1972 .and_then(|slot| slot.clone())?;
1973 let conn = match pool.lock() {
1974 Ok(conn) => conn,
1975 Err(_) => return Some(Err("db mutex poisoned".to_string())),
1976 };
1977 let project_key = crate::path_identity::project_scope_key(project_root);
1978 Some(
1979 crate::db::bash_tasks::find_bash_task_for_project(
1980 &conn,
1981 &harness,
1982 &project_key,
1983 task_id,
1984 )
1985 .map(|row| row.map(PersistedTask::from))
1986 .map_err(|error| error.to_string()),
1987 )
1988 }
1989
1990 pub(super) fn status_relaxed(
1991 &self,
1992 task_id: &str,
1993 _session_id: &str,
1994 project_root: &Path,
1995 storage_dir: &Path,
1996 preview_bytes: usize,
1997 ) -> Option<BgTaskSnapshot> {
1998 let task = self.status_relaxed_task(task_id, project_root, storage_dir)?;
1999 let _ = self.poll_task(&task);
2000 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
2001 }
2002
2003 pub fn kill_relaxed(
2004 &self,
2005 task_id: &str,
2006 project_root: &Path,
2007 storage_dir: &Path,
2008 ) -> Result<BgTaskSnapshot, String> {
2009 let task = self
2010 .status_relaxed_task(task_id, project_root, storage_dir)
2011 .ok_or_else(|| format!("background task not found: {task_id}"))?;
2012 self.kill_with_status(task_id, &task.session_id, BgTaskStatus::Killed)
2013 }
2014
2015 pub fn maybe_gc_persisted(&self, storage_dir: &Path) -> Result<usize, String> {
2016 #[cfg(test)]
2017 self.inner.persisted_gc_runs.fetch_add(1, Ordering::SeqCst);
2018
2019 let mut deleted = 0usize;
2020
2021 let root = storage_dir.join("bash-tasks");
2022 if root.exists() {
2023 let session_dirs = fs::read_dir(&root).map_err(|e| {
2024 format!(
2025 "failed to read background task root {}: {e}",
2026 root.display()
2027 )
2028 })?;
2029 for session_entry in session_dirs.flatten() {
2030 let session_dir = session_entry.path();
2031 if !session_dir.is_dir() {
2032 continue;
2033 }
2034 let (task_ids, invalid_entries) = match discover_task_ids(&session_dir) {
2035 Ok(discovery) => discovery,
2036 Err(error) => {
2037 crate::slog_warn!(
2038 "failed to discover background task session {}: {error}",
2039 session_dir.display()
2040 );
2041 continue;
2042 }
2043 };
2044 for entry in invalid_entries {
2045 let _ = quarantine_invalid_entry(storage_dir, &session_dir, &entry);
2046 }
2047 for task_id in task_ids {
2048 let resolved = match resolve_task_layout(&session_dir, &task_id) {
2049 Ok(task) => task,
2050 Err(error)
2051 if error.kind() == std::io::ErrorKind::NotFound
2052 && uninitialized_layout_is_recent(
2053 &session_dir,
2054 &task_id,
2055 Duration::from_secs(5 * 60),
2056 )
2057 .unwrap_or(false) =>
2058 {
2059 continue;
2060 }
2061 Err(error) => {
2062 crate::slog_warn!(
2063 "quarantining unresolved background task {task_id}: {error}"
2064 );
2065 quarantine_task_layout(storage_dir, &session_dir, &task_id, "invalid")
2066 .map_err(|error| error.to_string())?;
2067 continue;
2068 }
2069 };
2070 if modified_within(&resolved.paths.json, PERSISTED_GC_GRACE) {
2071 continue;
2072 }
2073 let metadata = match read_task_at(&resolved) {
2074 Ok(metadata) => metadata,
2075 Err(error) => {
2076 crate::slog_warn!(
2077 "quarantining corrupt background task metadata {task_id}: {error}"
2078 );
2079 quarantine_task_layout(storage_dir, &session_dir, &task_id, "corrupt")
2080 .map_err(|error| error.to_string())?;
2081 continue;
2082 }
2083 };
2084 if !(metadata.status.is_terminal() && metadata.completion_delivered) {
2085 continue;
2086 }
2087 match delete_task_bundle(&resolved.paths) {
2088 Ok(()) => {
2089 self.delete_gc_task_from_db(&metadata);
2090 deleted += 1;
2091 log::debug!(
2092 "deleted persisted background task bundle {}",
2093 metadata.task_id
2094 );
2095 }
2096 Err(error) => {
2097 crate::slog_warn!(
2098 "failed to delete background task bundle {}: {error}",
2099 metadata.task_id
2100 );
2101 }
2102 }
2103 }
2104 }
2105 }
2106 gc_quarantine(storage_dir);
2107 Ok(deleted)
2108 }
2109
2110 pub fn list(&self, preview_bytes: usize) -> Vec<BgTaskSnapshot> {
2111 let tasks = self
2112 .inner
2113 .tasks
2114 .lock()
2115 .map(|tasks| tasks.values().cloned().collect::<Vec<_>>())
2116 .unwrap_or_default();
2117 tasks
2118 .into_iter()
2119 .map(|task| {
2120 let _ = self.poll_task(&task);
2121 self.snapshot_with_terminal_cache(&task, preview_bytes)
2122 })
2123 .collect()
2124 }
2125
2126 fn maybe_compress_snapshot(&self, task: &Arc<BgTask>, snapshot: &mut BgTaskSnapshot) {
2132 if !snapshot.info.status.is_terminal() || snapshot.info.mode == BgMode::Pty {
2133 return;
2134 }
2135 if let Some(cache) = self.ensure_terminal_output_cache(task) {
2136 snapshot.output_preview = cache.output_preview;
2137 snapshot.output_truncated = cache.output_truncated;
2138 }
2139 }
2140
2141 pub fn kill(&self, task_id: &str, session_id: &str) -> Result<BgTaskSnapshot, String> {
2142 self.kill_with_status(task_id, session_id, BgTaskStatus::Killed)
2143 }
2144
2145 pub fn promote(&self, task_id: &str, session_id: &str) -> Result<bool, String> {
2146 let task = self
2147 .task_for_session(task_id, session_id)
2148 .ok_or_else(|| format!("background task not found: {task_id}"))?;
2149 let terminal_after_promote = {
2150 let mut state = task
2151 .state
2152 .lock()
2153 .map_err(|_| "background task lock poisoned".to_string())?;
2154 let updated = self
2155 .update_task_metadata(&task.paths, |metadata| {
2156 metadata.notify_on_completion = true;
2157 metadata.completion_delivered = false;
2158 })
2159 .map_err(|e| format!("failed to promote background task: {e}"))?;
2160 state.metadata = updated;
2161 state.metadata.status.is_terminal()
2162 };
2163 if terminal_after_promote {
2164 self.post_terminal_transition(&task, true)?;
2165 }
2166 Ok(true)
2167 }
2168
2169 pub(crate) fn kill_for_timeout(&self, task_id: &str, session_id: &str) -> Result<(), String> {
2170 self.kill_with_status(task_id, session_id, BgTaskStatus::TimedOut)
2171 .map(|_| ())
2172 }
2173
2174 pub fn cleanup_finished(&self, older_than: Duration) {
2175 let cutoff = Instant::now().checked_sub(older_than);
2176 let removable_paths: Vec<(String, TaskPaths)> =
2177 if let Ok(mut tasks) = self.inner.tasks.lock() {
2178 let removable = tasks
2179 .iter()
2180 .filter_map(|(task_id, task)| {
2181 let delivered_terminal = task
2182 .state
2183 .lock()
2184 .map(|state| {
2185 state.metadata.status.is_terminal()
2186 && state.metadata.completion_delivered
2187 })
2188 .unwrap_or(false);
2189 if !delivered_terminal {
2190 return None;
2191 }
2192
2193 let terminal_at = task.terminal_at.lock().ok().and_then(|at| *at);
2194 let expired = match (terminal_at, cutoff) {
2195 (Some(terminal_at), Some(cutoff)) => terminal_at <= cutoff,
2196 (Some(_), None) => true,
2197 (None, _) => false,
2198 };
2199 expired.then(|| task_id.clone())
2200 })
2201 .collect::<Vec<_>>();
2202
2203 removable
2204 .into_iter()
2205 .filter_map(|task_id| {
2206 tasks
2207 .remove(&task_id)
2208 .map(|task| (task_id, task.paths.clone()))
2209 })
2210 .collect()
2211 } else {
2212 Vec::new()
2213 };
2214
2215 for (task_id, paths) in removable_paths {
2216 match delete_task_bundle(&paths) {
2217 Ok(()) => log::debug!("deleted persisted background task bundle {task_id}"),
2218 Err(error) => crate::slog_warn!(
2219 "failed to delete persisted background task bundle {task_id}: {error}"
2220 ),
2221 }
2222 }
2223 }
2224
2225 pub fn drain_completions(&self) -> Vec<BgCompletion> {
2226 self.drain_completions_for_session(None)
2227 }
2228
2229 pub fn drain_completions_for_session(&self, session_id: Option<&str>) -> Vec<BgCompletion> {
2230 let completions = match self.inner.completions.lock() {
2231 Ok(completions) => completions,
2232 Err(_) => return Vec::new(),
2233 };
2234
2235 completions
2236 .iter()
2237 .filter(|completion| completion_matches_session(completion, session_id))
2238 .cloned()
2239 .collect()
2240 }
2241
2242 pub fn has_completions_for_session(&self, session_id: Option<&str>) -> bool {
2243 match self.inner.completions.lock() {
2244 Ok(completions) => completions
2245 .iter()
2246 .any(|completion| completion_matches_session(completion, session_id)),
2247 Err(_) => true,
2251 }
2252 }
2253
2254 pub fn ack_completions_for_session(
2255 &self,
2256 session_id: Option<&str>,
2257 task_ids: &[String],
2258 ) -> Vec<String> {
2259 if task_ids.is_empty() {
2260 return Vec::new();
2261 }
2262 let requested_task_ids = task_ids.iter().map(String::as_str).collect::<HashSet<_>>();
2263 let mut completion_sessions = HashMap::new();
2264 if let Ok(mut completions) = self.inner.completions.lock() {
2265 completions.retain(|completion| {
2266 let session_matches = session_id
2267 .map(|session_id| completion.session_id == session_id)
2268 .unwrap_or(true);
2269 if session_matches && requested_task_ids.contains(completion.task_id.as_str()) {
2270 completion_sessions
2271 .insert(completion.task_id.clone(), completion.session_id.clone());
2272 false
2273 } else {
2274 true
2275 }
2276 });
2277 }
2278
2279 let mut delivered = Vec::new();
2280 for task_id in task_ids {
2281 let task = if let Some(session_id) = session_id {
2282 self.task_for_session(task_id, session_id)
2283 } else if let Some(completion_session_id) = completion_sessions.get(task_id) {
2284 self.task_for_session(task_id, completion_session_id)
2285 } else {
2286 self.task(task_id)
2287 };
2288 if let Some(task) = task {
2289 if task.set_completion_delivered(true, self).is_ok() {
2290 delivered.push(task_id.clone());
2291 }
2292 }
2293 }
2294
2295 delivered
2296 }
2297
2298 pub fn pending_completions_for_session(&self, session_id: &str) -> Vec<BgCompletion> {
2299 self.inner
2300 .completions
2301 .lock()
2302 .map(|completions| {
2303 completions
2304 .iter()
2305 .filter(|completion| completion.session_id == session_id)
2306 .cloned()
2307 .collect()
2308 })
2309 .unwrap_or_default()
2310 }
2311
2312 fn remove_pending_completion(&self, task_id: &str) -> Option<BgCompletion> {
2313 let mut completions = self.inner.completions.lock().ok()?;
2314 let idx = completions
2315 .iter()
2316 .position(|completion| completion.task_id == task_id)?;
2317 completions.remove(idx)
2318 }
2319
2320 fn completion_snapshot_for_task(&self, task: &Arc<BgTask>) -> Option<BgCompletion> {
2321 let snapshot = self.snapshot_with_terminal_cache(task, RUNNING_OUTPUT_PREVIEW_BYTES);
2322 if !snapshot.info.status.is_terminal() {
2323 return None;
2324 }
2325 let (output_preview, output_truncated) = if snapshot.info.mode == BgMode::Pty {
2326 (String::new(), false)
2327 } else {
2328 self.ensure_terminal_output_cache(task)
2329 .map(|cache| completion_preview_for_cache(&cache, snapshot.exit_code))
2330 .unwrap_or_else(|| (String::new(), false))
2331 };
2332 Some(BgCompletion {
2333 task_id: snapshot.info.task_id,
2334 session_id: task.session_id.clone(),
2335 status: snapshot.info.status,
2336 exit_code: snapshot.exit_code,
2337 command: snapshot.info.command,
2338 output_preview,
2339 output_truncated,
2340 original_tokens: None,
2341 compressed_tokens: None,
2342 tokens_skipped: false,
2343 })
2344 }
2345
2346 pub fn detach(&self) {
2347 self.inner.shutdown.store(true, Ordering::SeqCst);
2348 if let Ok(mut tasks) = self.inner.tasks.lock() {
2349 for task in tasks.values() {
2350 if let Ok(mut state) = task.state.lock() {
2351 match &mut state.runtime {
2352 TaskRuntime::Piped(child) => *child = None,
2353 TaskRuntime::Pty(runtime) => *runtime = None,
2354 }
2355 state.detached = true;
2356 }
2357 }
2358 tasks.clear();
2359 }
2360 }
2361
2362 pub fn shutdown(&self) {
2363 let tasks = self
2364 .inner
2365 .tasks
2366 .lock()
2367 .map(|tasks| {
2368 tasks
2369 .values()
2370 .map(|task| (task.task_id.clone(), task.session_id.clone()))
2371 .collect::<Vec<_>>()
2372 })
2373 .unwrap_or_default();
2374 for (task_id, session_id) in tasks {
2375 let _ = self.kill(&task_id, &session_id);
2376 }
2377 }
2378
2379 pub(crate) fn poll_task(&self, task: &Arc<BgTask>) -> Result<(), String> {
2380 if let Ok(state) = task.state.lock() {
2381 if let TaskRuntime::Pty(Some(pty)) = &state.runtime {
2382 if !pty.exit_observed.load(Ordering::SeqCst) {
2390 return Ok(());
2391 }
2392 }
2393 }
2394 let marker = match read_exit_marker(&task.paths) {
2395 Ok(Some(marker)) => marker,
2396 Ok(None) => return Ok(()),
2397 Err(error) => return Err(format!("failed to read exit marker: {error}")),
2398 };
2399 self.finalize_from_marker(task, marker, None)
2400 }
2401
2402 pub(crate) fn reap_child(&self, task: &Arc<BgTask>) {
2403 let mut needs_completion = false;
2404 {
2405 let Ok(mut state) = task.state.lock() else {
2406 return;
2407 };
2408 match &mut state.runtime {
2409 TaskRuntime::Piped(child_slot) => {
2410 if let Some(child) = child_slot.as_mut() {
2411 if let Ok(Some(status)) = child.try_wait() {
2412 *child_slot = None;
2413 state.detached = true;
2414 state.child_exit_observed = true;
2415 if let Some(handles) = state.io_handles.as_mut() {
2416 if handles.artifact_len(TaskArtifact::Exit).unwrap_or(1) == 0 {
2417 let marker = status
2418 .code()
2419 .map(|code| code.to_string())
2420 .unwrap_or_else(|| "1".to_string());
2421 let _ = handles.write(TaskArtifact::Exit, marker.as_bytes());
2422 }
2423 }
2424 }
2425 } else if state.detached {
2426 let child_known_dead = state.child_exit_observed
2427 || state
2428 .metadata
2429 .child_pid
2430 .is_some_and(|pid| !is_process_alive(pid));
2431 if child_known_dead {
2432 needs_completion =
2433 self.fail_without_exit_marker_if_needed(task, &mut state);
2434 }
2435 }
2436 }
2437 TaskRuntime::Pty(Some(pty)) => {
2438 if pty.exit_observed.load(Ordering::SeqCst) {
2439 drop(state);
2440 let _ = self.poll_task(task);
2441 return;
2442 }
2443 }
2444 TaskRuntime::Pty(None) => {}
2445 }
2446 }
2447 if needs_completion {
2448 let _ = self.post_terminal_transition(task, true);
2449 }
2450 }
2451
2452 fn fail_without_exit_marker_if_needed(
2453 &self,
2454 task: &Arc<BgTask>,
2455 state: &mut BgTaskState,
2456 ) -> bool {
2457 if state.metadata.status.is_terminal() {
2458 return false;
2459 }
2460 if matches!(read_exit_marker(&task.paths), Ok(Some(_))) {
2461 return false;
2462 }
2463 let watch_controlled = self.task_has_watch_control(&task.task_id);
2464 let updated = self.update_task_metadata(&task.paths, |metadata| {
2465 metadata.mark_terminal(
2466 BgTaskStatus::Failed,
2467 None,
2468 Some("process exited without exit marker".to_string()),
2469 );
2470 if watch_controlled {
2471 metadata.completion_delivered = true;
2472 }
2473 });
2474 if let Ok(metadata) = updated {
2475 state.pending_terminal_override = None;
2476 state.metadata = metadata;
2477 task.mark_terminal_now();
2478 return true;
2479 }
2480 false
2481 }
2482
2483 pub(crate) fn running_tasks(&self) -> Vec<Arc<BgTask>> {
2484 self.inner
2485 .tasks
2486 .lock()
2487 .map(|tasks| {
2488 tasks
2489 .values()
2490 .filter(|task| task.is_running())
2491 .cloned()
2492 .collect()
2493 })
2494 .unwrap_or_default()
2495 }
2496
2497 fn insert_rehydrated_task(
2498 &self,
2499 metadata: PersistedTask,
2500 paths: TaskPaths,
2501 detached: bool,
2502 ) -> Result<(), String> {
2503 let task_id = metadata.task_id.clone();
2504 let session_id = metadata.session_id.clone();
2505 let started = started_instant_from_unix_millis(metadata.started_at);
2506 let suppress_replayed_running_reminder = metadata.status == BgTaskStatus::Running;
2507 let mode = metadata.mode.clone();
2508 let task = Arc::new(BgTask {
2509 task_id: task_id.clone(),
2510 session_id,
2511 paths: paths.clone(),
2512 artifact_root: canonical_artifact_root(&paths),
2513 started,
2514 last_reminder_at: Mutex::new(suppress_replayed_running_reminder.then(Instant::now)),
2515 terminal_at: Mutex::new(metadata.status.is_terminal().then(Instant::now)),
2516 state: Mutex::new(BgTaskState {
2517 metadata,
2518 runtime: if mode == BgMode::Pty {
2519 TaskRuntime::Pty(None)
2520 } else {
2521 TaskRuntime::Piped(None)
2522 },
2523 io_handles: None,
2524 detached,
2525 child_exit_observed: false,
2532 buffer: BgBuffer::registered(&paths, mode.clone()),
2533 terminal_output_cache: None,
2534 pending_terminal_override: None,
2535 }),
2536 });
2537 self.inner
2538 .tasks
2539 .lock()
2540 .map_err(|_| "background task registry lock poisoned".to_string())?
2541 .insert(task_id, task);
2542 Ok(())
2543 }
2544
2545 fn kill_with_status(
2546 &self,
2547 task_id: &str,
2548 session_id: &str,
2549 terminal_status: BgTaskStatus,
2550 ) -> Result<BgTaskSnapshot, String> {
2551 let task = self
2552 .task_for_session(task_id, session_id)
2553 .ok_or_else(|| format!("background task not found: {task_id}"))?;
2554 let mut terminalized = false;
2555
2556 {
2557 let mut state = task
2558 .state
2559 .lock()
2560 .map_err(|_| "background task lock poisoned".to_string())?;
2561 if state.metadata.status.is_terminal() {
2562 state.pending_terminal_override = None;
2563 } else if let Ok(Some(marker)) = read_exit_marker(&task.paths) {
2564 state.metadata =
2565 terminal_metadata_from_marker(state.metadata.clone(), marker, None);
2566 if self.task_has_watch_control(&task.task_id) {
2567 state.metadata.completion_delivered = true;
2568 }
2569 state.pending_terminal_override = None;
2570 task.mark_terminal_now();
2571 match &mut state.runtime {
2572 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
2579 TaskRuntime::Pty(runtime) => *runtime = None,
2580 }
2581 state.detached = true;
2582 self.persist_task(&task.paths, &state.metadata)
2583 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
2584 terminalized = true;
2585 } else {
2586 let was_already_killing = state.metadata.status == BgTaskStatus::Killing;
2587 if !was_already_killing {
2588 state.metadata.status = BgTaskStatus::Killing;
2589 self.persist_task(&task.paths, &state.metadata)
2590 .map_err(|e| format!("failed to persist killing state: {e}"))?;
2591 }
2592
2593 #[cfg(unix)]
2594 let pgid = state.metadata.pgid;
2595 #[cfg(windows)]
2596 let child_pid = state.metadata.child_pid;
2597 if !was_already_killing
2598 && state.metadata.mode == BgMode::Pty
2599 && terminal_status == BgTaskStatus::TimedOut
2600 {
2601 state.pending_terminal_override = Some(BgTaskStatus::TimedOut);
2602 }
2603
2604 #[cfg(windows)]
2605 let mut pty_forced_terminal_status: Option<BgTaskStatus> = None;
2606
2607 match &mut state.runtime {
2608 TaskRuntime::Piped(child_slot) => {
2609 #[cfg(unix)]
2610 if let Some(pgid) = pgid {
2611 terminate_pgid(pgid, child_slot.as_mut());
2612 }
2613 #[cfg(windows)]
2614 if let Some(child) = child_slot.as_mut() {
2615 super::process::terminate_process(child);
2616 } else if let Some(pid) = child_pid {
2617 terminate_pid(pid);
2618 }
2619 if let Some(child) = child_slot.as_mut() {
2620 let _ = child.wait();
2621 }
2622 *child_slot = None;
2623 state.detached = true;
2624
2625 if let Some(handles) = state.io_handles.as_mut() {
2626 handles.write(TaskArtifact::Exit, b"killed").map_err(|e| {
2627 format!("failed to write retained kill marker: {e}")
2628 })?;
2629 } else {
2630 write_kill_marker_if_absent(&task.paths)
2631 .map_err(|e| format!("failed to write kill marker: {e}"))?;
2632 }
2633
2634 let exit_code = terminal_exit_code_for_status(&terminal_status);
2635 state
2636 .metadata
2637 .mark_terminal(terminal_status, exit_code, None);
2638 if self.task_has_watch_control(&task.task_id) {
2639 state.metadata.completion_delivered = true;
2640 }
2641 state.pending_terminal_override = None;
2642 task.mark_terminal_now();
2643 self.persist_task(&task.paths, &state.metadata)
2644 .map_err(|e| format!("failed to persist killed state: {e}"))?;
2645 terminalized = true;
2646 }
2647 TaskRuntime::Pty(Some(pty)) => {
2648 pty.was_killed.store(true, Ordering::SeqCst);
2649 if let Err(error) = pty.killer.kill() {
2650 crate::slog_warn!(
2651 "[pty-kill] {task_id} ChildKiller::kill failed: {error}"
2652 );
2653 }
2654 if let Some(pid) = pty.child_pid {
2655 #[cfg(unix)]
2656 terminate_pgid(pid as i32, None);
2657 #[cfg(windows)]
2658 terminate_pid(pid);
2659 }
2660 drop(pty.master.take());
2661
2662 #[cfg(windows)]
2663 {
2664 let default_status = if terminal_status == BgTaskStatus::TimedOut {
2665 BgTaskStatus::TimedOut
2666 } else {
2667 BgTaskStatus::Killed
2668 };
2669 pty_forced_terminal_status = Some(
2670 state
2671 .pending_terminal_override
2672 .take()
2673 .unwrap_or(default_status),
2674 );
2675 }
2676 }
2677 TaskRuntime::Pty(None) => {}
2678 }
2679
2680 #[cfg(windows)]
2681 if let Some(target_status) = pty_forced_terminal_status {
2682 if !task.paths.exit.exists() {
2683 write_kill_marker_if_absent(&task.paths)
2684 .map_err(|e| format!("failed to write kill marker: {e}"))?;
2685 }
2686
2687 let exit_code = terminal_exit_code_for_status(&target_status);
2688 state.metadata.mark_terminal(target_status, exit_code, None);
2689 if self.task_has_watch_control(&task.task_id) {
2690 state.metadata.completion_delivered = true;
2691 }
2692 state.pending_terminal_override = None;
2693 task.mark_terminal_now();
2694 if let TaskRuntime::Pty(runtime) = &mut state.runtime {
2695 *runtime = None;
2696 }
2697 state.detached = true;
2698 self.persist_task(&task.paths, &state.metadata)
2699 .map_err(|e| format!("failed to persist killed PTY state: {e}"))?;
2700 terminalized = true;
2701 }
2702 }
2703 }
2704
2705 if terminalized {
2706 self.post_terminal_transition(&task, true)?;
2707 }
2708 Ok(self.snapshot_with_terminal_cache(&task, RUNNING_OUTPUT_PREVIEW_BYTES))
2709 }
2710
2711 fn finalize_from_marker(
2712 &self,
2713 task: &Arc<BgTask>,
2714 marker: ExitMarker,
2715 reason: Option<String>,
2716 ) -> Result<(), String> {
2717 let watch_controlled = self.task_has_watch_control(&task.task_id);
2718 let mut pty_reader_done = None;
2719 {
2720 let mut state = task
2721 .state
2722 .lock()
2723 .map_err(|_| "background task lock poisoned".to_string())?;
2724 if state.metadata.status.is_terminal() {
2725 state.pending_terminal_override = None;
2726 return Ok(());
2727 }
2728
2729 let pending_override = state.pending_terminal_override.take();
2730 let is_pty = state.metadata.mode == BgMode::Pty;
2731 let updated = self
2732 .update_task_metadata(&task.paths, |metadata| {
2733 let mut new_metadata = if is_pty && marker == ExitMarker::Killed {
2734 let mut metadata = metadata.clone();
2735 let target_status = pending_override.unwrap_or(BgTaskStatus::Killed);
2736 let exit_code = terminal_exit_code_for_status(&target_status);
2737 metadata.mark_terminal(target_status, exit_code, reason);
2738 metadata
2739 } else {
2740 terminal_metadata_from_marker(metadata.clone(), marker, reason)
2741 };
2742 if watch_controlled {
2743 new_metadata.completion_delivered = true;
2744 }
2745 *metadata = new_metadata;
2746 })
2747 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
2748 state.metadata = updated;
2749 task.mark_terminal_now();
2750 match &mut state.runtime {
2751 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
2756 TaskRuntime::Pty(runtime) => {
2757 pty_reader_done = runtime
2758 .as_ref()
2759 .map(|runtime| Arc::clone(&runtime.reader_done));
2760 *runtime = None;
2761 }
2762 }
2763 state.detached = true;
2764 }
2765
2766 if let Some(reader_done) = pty_reader_done {
2767 let deadline = Instant::now() + Duration::from_millis(200);
2768 while !reader_done.load(Ordering::SeqCst) && Instant::now() < deadline {
2769 std::thread::sleep(Duration::from_millis(10));
2770 }
2771 }
2772
2773 self.scan_task_watch_output(task);
2776
2777 self.post_terminal_transition(task, true)
2778 }
2779
2780 fn enqueue_completion_if_needed(
2781 &self,
2782 metadata: &PersistedTask,
2783 paths: Option<&TaskPaths>,
2784 emit_frame: bool,
2785 ) {
2786 if metadata.status.is_terminal() && !metadata.completion_delivered {
2787 let cache =
2788 paths.and_then(|paths| self.render_terminal_output_from_paths(metadata, paths));
2789 self.enqueue_completion_from_parts(metadata, None, paths, emit_frame, cache.as_ref());
2790 }
2791 }
2792
2793 fn render_terminal_output_from_paths(
2794 &self,
2795 metadata: &PersistedTask,
2796 paths: &TaskPaths,
2797 ) -> Option<TerminalOutputCache> {
2798 if metadata.mode == BgMode::Pty {
2799 return None;
2800 }
2801 let mut buffer = BgBuffer::registered(paths, BgMode::Pipes);
2802 let disk_truncation = buffer.enforce_terminal_cap();
2803 Some(self.render_terminal_output(metadata, &buffer, disk_truncation))
2804 }
2805
2806 fn enqueue_completion_from_parts(
2807 &self,
2808 metadata: &PersistedTask,
2809 buffer: Option<&BgBuffer>,
2810 paths: Option<&TaskPaths>,
2811 emit_frame: bool,
2812 terminal_render: Option<&TerminalOutputCache>,
2813 ) {
2814 if !metadata.status.is_terminal() {
2825 return;
2826 }
2827
2828 let owned_buffer = if buffer.is_none() && metadata.mode != BgMode::Pty {
2829 paths.map(|paths| BgBuffer::registered(paths, BgMode::Pipes))
2830 } else {
2831 None
2832 };
2833 let render_buffer = buffer.or(owned_buffer.as_ref());
2834 let owned_render = if terminal_render.is_none() {
2835 render_buffer.map(|buffer| {
2836 let mut capped_buffer = buffer.clone();
2837 let disk_truncation = capped_buffer.enforce_terminal_cap();
2838 self.render_terminal_output(metadata, &capped_buffer, disk_truncation)
2839 })
2840 } else {
2841 None
2842 };
2843 let render = terminal_render.or(owned_render.as_ref());
2844
2845 let (output_preview, output_truncated) = render
2849 .map(|cache| completion_preview_for_cache(cache, metadata.exit_code))
2850 .unwrap_or_else(|| (String::new(), false));
2851
2852 let token_counts = self.completion_token_counts(
2853 metadata,
2854 buffer,
2855 paths,
2856 render.map(|render| render.output_preview.as_str()),
2857 );
2858 let completion = BgCompletion {
2859 task_id: metadata.task_id.clone(),
2860 session_id: metadata.session_id.clone(),
2861 status: metadata.status.clone(),
2862 exit_code: metadata.exit_code,
2863 command: metadata.command.clone(),
2864 output_preview,
2865 output_truncated,
2866 original_tokens: token_counts.original_tokens,
2867 compressed_tokens: token_counts.compressed_tokens,
2868 tokens_skipped: token_counts.tokens_skipped,
2869 };
2870
2871 self.record_compression_event_if_applicable(metadata, &token_counts);
2882
2883 let (watch_controlled, watch_matched) = self.task_watch_state(&metadata.task_id);
2884 if watch_controlled {
2885 if emit_frame && !watch_matched {
2886 self.emit_bash_watch_exit(&completion);
2887 }
2888 self.clear_task_watch_state(&metadata.task_id);
2889 return;
2890 }
2891
2892 if metadata.completion_delivered {
2902 return;
2903 }
2904
2905 let pushed = if let Ok(mut completions) = self.inner.completions.lock() {
2908 if completions
2909 .iter()
2910 .any(|existing| existing.task_id == metadata.task_id)
2911 {
2912 false
2913 } else {
2914 completions.push_back(completion.clone());
2915 true
2916 }
2917 } else {
2918 false
2919 };
2920
2921 if pushed && emit_frame {
2922 self.emit_bash_completed(completion);
2923 }
2924 }
2925
2926 fn record_compression_event_if_applicable(
2927 &self,
2928 metadata: &PersistedTask,
2929 token_counts: &CompletionTokenCounts,
2930 ) {
2931 if metadata.mode == BgMode::Pty {
2932 return;
2933 }
2934
2935 let (original_tokens, compressed_tokens, original_bytes, compressed_bytes) = match (
2936 token_counts.original_tokens,
2937 token_counts.compressed_tokens,
2938 token_counts.original_bytes,
2939 token_counts.compressed_bytes,
2940 ) {
2941 (
2942 Some(original_tokens),
2943 Some(compressed_tokens),
2944 Some(original_bytes),
2945 Some(compressed_bytes),
2946 ) => (
2947 original_tokens,
2948 compressed_tokens,
2949 original_bytes,
2950 compressed_bytes,
2951 ),
2952 _ => {
2953 crate::slog_warn!(
2954 "compression event skipped for {}: token counts unavailable (likely spill file missing or unreadable)",
2955 metadata.task_id
2956 );
2957 return;
2958 }
2959 };
2960
2961 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
2962 let Some(pool) = pool else {
2963 crate::slog_warn!(
2964 "compression event skipped for {}: db_pool not initialized — was configure run?",
2965 metadata.task_id
2966 );
2967 return;
2968 };
2969 let harness = self
2970 .inner
2971 .db_harness
2972 .read()
2973 .ok()
2974 .and_then(|slot| slot.clone());
2975 let Some(harness) = harness else {
2976 crate::slog_warn!(
2977 "compression event insert skipped for {}: harness not configured",
2978 metadata.task_id
2979 );
2980 return;
2981 };
2982
2983 let project_root = metadata
2984 .project_root
2985 .as_deref()
2986 .unwrap_or(&metadata.workdir);
2987 let project_key = crate::path_identity::project_scope_key(project_root);
2988 let row = crate::db::compression_events::CompressionEventRow {
2989 harness: &harness,
2990 session_id: Some(&metadata.session_id),
2991 project_key: &project_key,
2992 tool: "bash",
2993 task_id: Some(&metadata.task_id),
2994 command: Some(&metadata.command),
2995 compressor: if metadata.compressed {
2996 "registry"
2997 } else {
2998 "none"
2999 },
3000 original_bytes,
3001 compressed_bytes,
3002 original_tokens,
3003 compressed_tokens,
3004 created_at: unix_millis() as i64,
3005 };
3006
3007 let conn = match pool.lock() {
3008 Ok(conn) => conn,
3009 Err(_) => {
3010 crate::slog_warn!(
3011 "compression event insert failed for {}: db mutex poisoned",
3012 metadata.task_id
3013 );
3014 return;
3015 }
3016 };
3017 match crate::db::compression_events::insert_compression_event(&conn, &row) {
3018 Ok(Some(row_id)) => {
3019 self.inner
3023 .compression_aggregates
3024 .record_successful_insert(&conn, &row, row_id);
3025 crate::slog_debug!(
3029 "compression event recorded for {} (project={}, session={}, {} → {} tokens)",
3030 metadata.task_id,
3031 project_key,
3032 metadata.session_id,
3033 original_tokens,
3034 compressed_tokens
3035 );
3036 }
3037 Ok(None) => {
3038 crate::slog_debug!(
3039 "duplicate compression event ignored for {} (project={}, session={})",
3040 metadata.task_id,
3041 project_key,
3042 metadata.session_id
3043 );
3044 }
3045 Err(error) => {
3046 crate::slog_warn!(
3047 "compression event insert failed for {}: {}",
3048 metadata.task_id,
3049 error
3050 );
3051 }
3052 }
3053 }
3054
3055 fn emit_bash_pattern_match(&self, session_id: &str, pattern_match: PatternMatch) {
3056 let Ok(progress_sender) = self
3057 .inner
3058 .progress_sender
3059 .lock()
3060 .map(|sender| sender.clone())
3061 else {
3062 return;
3063 };
3064 if let Some(sender) = progress_sender.as_ref() {
3065 sender(PushFrame::BashPatternMatch(BashPatternMatchFrame::new(
3066 pattern_match.task_id,
3067 session_id.to_string(),
3068 pattern_match.watch_id,
3069 pattern_match.match_text,
3070 pattern_match.match_offset,
3071 pattern_match.context,
3072 pattern_match.once,
3073 )));
3074 }
3075 }
3076
3077 fn emit_bash_watch_exit(&self, completion: &BgCompletion) {
3078 let Ok(progress_sender) = self
3079 .inner
3080 .progress_sender
3081 .lock()
3082 .map(|sender| sender.clone())
3083 else {
3084 return;
3085 };
3086 let Some(sender) = progress_sender.as_ref() else {
3087 return;
3088 };
3089 let status = completion_status_text(&completion.status, completion.exit_code);
3090 let preview = completion.output_preview.trim_end();
3091 let context = if preview.is_empty() {
3092 format!("task {} exited ({status})", completion.task_id)
3093 } else {
3094 format!(
3095 "task {} exited ({status})
3096{preview}",
3097 completion.task_id
3098 )
3099 };
3100 sender(PushFrame::BashPatternMatch(
3101 BashPatternMatchFrame::task_exit(
3102 completion.task_id.clone(),
3103 completion.session_id.clone(),
3104 format!("exited ({status})"),
3105 context,
3106 ),
3107 ));
3108 }
3109
3110 fn emit_bash_completed(&self, completion: BgCompletion) {
3111 let Ok(progress_sender) = self
3112 .inner
3113 .progress_sender
3114 .lock()
3115 .map(|sender| sender.clone())
3116 else {
3117 return;
3118 };
3119 let Some(sender) = progress_sender.as_ref() else {
3120 return;
3121 };
3122 sender(PushFrame::BashCompleted(BashCompletedFrame::new(
3130 completion.task_id,
3131 completion.session_id,
3132 completion.status,
3133 completion.exit_code,
3134 completion.command,
3135 completion.output_preview,
3136 completion.output_truncated,
3137 completion.original_tokens,
3138 completion.compressed_tokens,
3139 completion.tokens_skipped,
3140 )));
3141 }
3142
3143 fn completion_token_counts(
3144 &self,
3145 metadata: &PersistedTask,
3146 buffer: Option<&BgBuffer>,
3147 paths: Option<&TaskPaths>,
3148 rendered_output: Option<&str>,
3149 ) -> CompletionTokenCounts {
3150 if metadata.mode == BgMode::Pty {
3151 return CompletionTokenCounts::skipped();
3152 }
3153
3154 let raw = match buffer {
3155 Some(buffer) => buffer.read_for_token_count(TOKENIZE_CAP_BYTES_PER_STREAM),
3156 None => paths
3157 .map(|paths| {
3158 read_for_token_count_from_disk(metadata, paths, TOKENIZE_CAP_BYTES_PER_STREAM)
3159 })
3160 .unwrap_or(TokenCountInput::Skipped),
3161 };
3162
3163 let TokenCountInput::Text(raw_output) = raw else {
3164 return CompletionTokenCounts::skipped();
3165 };
3166
3167 let original_tokens = token_count_u32(&raw_output);
3168 let original_bytes = raw_output.len() as i64;
3169 let compressed_output = rendered_output.unwrap_or(&raw_output);
3170 let compressed_tokens = token_count_u32(compressed_output);
3171 let compressed_bytes = compressed_output.len() as i64;
3172 CompletionTokenCounts {
3173 original_tokens: Some(original_tokens),
3174 compressed_tokens: Some(compressed_tokens),
3175 original_bytes: Some(original_bytes),
3176 compressed_bytes: Some(compressed_bytes),
3177 tokens_skipped: false,
3178 }
3179 }
3180
3181 pub(crate) fn maybe_emit_long_running_reminder(&self, task: &Arc<BgTask>) {
3182 if !self
3183 .inner
3184 .long_running_reminder_enabled
3185 .load(Ordering::SeqCst)
3186 {
3187 return;
3188 }
3189 let interval_ms = self
3190 .inner
3191 .long_running_reminder_interval_ms
3192 .load(Ordering::SeqCst);
3193 if interval_ms == 0 {
3194 return;
3195 }
3196 let interval = Duration::from_millis(interval_ms);
3197 let now = Instant::now();
3198 let Ok(mut last_reminder_at) = task.last_reminder_at.lock() else {
3199 return;
3200 };
3201 let since = last_reminder_at.unwrap_or(task.started);
3202 if now.duration_since(since) < interval {
3203 return;
3204 }
3205 let command = task
3206 .state
3207 .lock()
3208 .map(|state| state.metadata.command.clone())
3209 .unwrap_or_default();
3210 *last_reminder_at = Some(now);
3211 self.emit_bash_long_running(BashLongRunningFrame::new(
3212 task.task_id.clone(),
3213 task.session_id.clone(),
3214 command,
3215 task.started.elapsed().as_millis() as u64,
3216 ));
3217 }
3218
3219 fn emit_bash_long_running(&self, frame: BashLongRunningFrame) {
3220 let Ok(progress_sender) = self
3221 .inner
3222 .progress_sender
3223 .lock()
3224 .map(|sender| sender.clone())
3225 else {
3226 return;
3227 };
3228 if let Some(sender) = progress_sender.as_ref() {
3229 sender(PushFrame::BashLongRunning(frame));
3230 }
3231 }
3232
3233 fn task(&self, task_id: &str) -> Option<Arc<BgTask>> {
3234 validate_task_id(task_id).ok()?;
3235 self.inner
3236 .tasks
3237 .lock()
3238 .ok()
3239 .and_then(|tasks| tasks.get(task_id).cloned())
3240 }
3241
3242 fn task_for_session(&self, task_id: &str, session_id: &str) -> Option<Arc<BgTask>> {
3243 self.task(task_id)
3244 .filter(|task| task.session_id == session_id)
3245 }
3246
3247 pub fn try_health_counts(&self) -> Option<BgTaskHealthCounts> {
3248 let running = self
3249 .inner
3250 .tasks
3251 .try_lock()
3252 .ok()
3253 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())?;
3254 let pending_completions = self.inner.completions.try_lock().ok().map(|q| q.len())?;
3255 Some(BgTaskHealthCounts {
3256 running,
3257 pending_completions,
3258 })
3259 }
3260
3261 pub fn estimated_memory(&self) -> crate::memory::MemoryEstimate {
3265 let tasks = match self.inner.tasks.try_lock() {
3266 Ok(tasks) => tasks.values().cloned().collect::<Vec<_>>(),
3267 Err(_) => return crate::memory::MemoryEstimate::busy(),
3268 };
3269 let mut bytes = 0u64;
3270 let mut terminal_output_caches = 0usize;
3271 let mut sessions = HashSet::new();
3272 for task in &tasks {
3273 sessions.insert(task.session_id.clone());
3274 let state = match task.state.try_lock() {
3275 Ok(state) => state,
3276 Err(_) => return crate::memory::MemoryEstimate::busy(),
3277 };
3278 if let Some(cache) = state.terminal_output_cache.as_ref() {
3279 terminal_output_caches = terminal_output_caches.saturating_add(1);
3280 bytes = bytes.saturating_add(terminal_output_cache_estimated_bytes(cache));
3281 }
3282 }
3283 let completion_count = match self.inner.completions.try_lock() {
3284 Ok(completions) => {
3285 for completion in completions.iter() {
3286 sessions.insert(completion.session_id.clone());
3287 bytes = bytes.saturating_add(completion_estimated_bytes(completion));
3288 }
3289 completions.len()
3290 }
3291 Err(_) => return crate::memory::MemoryEstimate::busy(),
3292 };
3293
3294 crate::memory::MemoryEstimate::estimated(bytes)
3295 .count("tasks", tasks.len())
3296 .count("sessions", sessions.len())
3297 .count("terminal_output_caches", terminal_output_caches)
3298 .count("completion_caches", completion_count)
3299 .count_u64("output_ring_bytes", 0)
3300 }
3301
3302 fn running_count(&self) -> usize {
3303 self.inner
3304 .tasks
3305 .lock()
3306 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())
3307 .unwrap_or(0)
3308 }
3309
3310 fn start_watchdog(&self) {
3311 if !self.inner.watchdog_started.swap(true, Ordering::SeqCst) {
3312 super::watchdog::start(self.clone());
3313 }
3314 }
3315
3316 fn running_metadata_is_stale(&self, metadata: &PersistedTask) -> bool {
3317 unix_millis().saturating_sub(metadata.started_at) > STALE_RUNNING_AFTER.as_millis() as u64
3318 }
3319
3320 #[cfg(test)]
3321 pub fn task_json_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
3322 self.task_for_session(task_id, session_id)
3323 .map(|task| task.paths.json.clone())
3324 }
3325
3326 #[cfg(test)]
3327 pub fn task_exit_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
3328 self.task_for_session(task_id, session_id)
3329 .map(|task| task.paths.exit.clone())
3330 }
3331}
3332
3333fn canonical_artifact_root(paths: &TaskPaths) -> PathBuf {
3334 fs::canonicalize(&paths.io_dir).unwrap_or_else(|_| paths.io_dir.clone())
3335}
3336
3337fn render_compressed_with_recovery(
3338 buffer: &BgBuffer,
3339 mut compressed: CompressionResult,
3340 input_truncated: bool,
3341 disk_truncation: DiskTruncation,
3342 artifact_access: ArtifactRecoveryAccess,
3343) -> TerminalOutputCache {
3344 let had_trailing_newline = compressed.text.ends_with('\n');
3352 let mut text = strip_plain_truncation_marker_lines(&compressed.text)
3353 .trim_end()
3354 .to_string();
3355 if had_trailing_newline && !text.is_empty() {
3356 text.push('\n');
3357 }
3358 compressed.text = text;
3359
3360 let output_path = buffer.output_path().map(|path| path.display().to_string());
3361 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
3362 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
3363 let mut recovery = RecoveryContext {
3364 dropped_by_class: compressed.dropped_by_class,
3365 had_inner_drop: compressed.had_inner_drop,
3366 offset_hint_eligible: compressed.offset_hint_eligible,
3367 offset_start_line: compressed.offset_start_line,
3368 byte_truncated: input_truncated,
3369 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
3370 output_path: output_path.clone(),
3371 stderr_path: stderr_path.clone(),
3372 include_stderr_path,
3373 artifact_access: artifact_access.clone(),
3374 };
3375
3376 let (output_preview, output_truncated) =
3377 render_body_with_recovery_marker(&compressed.text, &mut recovery);
3378 TerminalOutputCache {
3379 output_preview,
3380 output_truncated,
3381 kind: TerminalOutputKind::Compressed,
3382 output_path,
3383 stderr_path,
3384 artifact_access,
3385 recovery: Some(recovery),
3386 }
3387}
3388
3389fn render_body_with_recovery_marker(body: &str, recovery: &mut RecoveryContext) -> (String, bool) {
3390 render_body_with_recovery_marker_at_cap(
3391 body,
3392 recovery,
3393 FINAL_OUTPUT_CAP_BYTES,
3394 cap_final_output,
3395 cap_final_output_with_marker,
3396 )
3397}
3398
3399fn render_raw_body_with_recovery_marker(
3400 body: &str,
3401 recovery: &mut RecoveryContext,
3402) -> (String, bool) {
3403 render_body_with_recovery_marker_at_cap(
3404 body,
3405 recovery,
3406 RAW_PASSTHROUGH_CAP_BYTES,
3407 |input| {
3408 super::output::cap_head_tail(
3409 input,
3410 RAW_PASSTHROUGH_CAP_BYTES,
3411 RAW_PASSTHROUGH_HEAD_BYTES,
3412 RAW_PASSTHROUGH_TAIL_BYTES,
3413 )
3414 },
3415 |input, marker| {
3416 super::output::cap_head_tail_with_marker(
3417 input,
3418 RAW_PASSTHROUGH_CAP_BYTES,
3419 RAW_PASSTHROUGH_HEAD_BYTES,
3420 RAW_PASSTHROUGH_TAIL_BYTES,
3421 marker,
3422 )
3423 },
3424 )
3425}
3426
3427fn render_body_with_recovery_marker_at_cap<F, G>(
3428 body: &str,
3429 recovery: &mut RecoveryContext,
3430 cap_bytes: usize,
3431 cap_plain: F,
3432 cap_with_marker: G,
3433) -> (String, bool)
3434where
3435 F: Fn(&str) -> super::output::CappedText,
3436 G: Fn(&str, &str) -> super::output::CappedText,
3437{
3438 let needs_marker = recovery.has_visible_drop();
3439 if body.len() > cap_bytes {
3440 recovery.byte_truncated = true;
3441 if let Some(marker) = recovery_marker(recovery) {
3442 let capped = cap_with_marker(body, &marker);
3443 return (capped.text, true);
3444 }
3445 let capped = cap_plain(body);
3446 return (capped.text, capped.truncated || needs_marker);
3447 }
3448
3449 if !needs_marker {
3450 return (body.to_string(), false);
3451 }
3452
3453 let Some(marker) = recovery_marker(recovery) else {
3454 return (body.to_string(), true);
3455 };
3456 let with_marker = append_recovery_marker(body, &marker);
3457 if with_marker.len() <= cap_bytes {
3458 return (with_marker, true);
3459 }
3460
3461 recovery.byte_truncated = true;
3462 let marker = recovery_marker(recovery).unwrap_or(marker);
3463 let capped = cap_with_marker(body, &marker);
3464 (capped.text, true)
3465}
3466
3467fn append_recovery_marker(body: &str, marker: &str) -> String {
3468 if body.is_empty() {
3469 return marker.to_string();
3470 }
3471 let mut output = body.trim_end().to_string();
3472 output.push('\n');
3473 output.push_str(marker);
3474 output
3475}
3476
3477fn recovery_marker(recovery: &RecoveryContext) -> Option<String> {
3478 let mut parts = Vec::new();
3479 for (class, count) in &recovery.dropped_by_class {
3480 let label = if *count == 1 {
3481 class.singular()
3482 } else {
3483 class.plural()
3484 };
3485 parts.push(format!("+{count} more {label}"));
3486 }
3487 if recovery.byte_truncated {
3488 parts.push("truncated output".to_string());
3489 }
3490 let disk_truncated_prefix_bytes = recovery.disk_truncated_prefix_bytes;
3491 if disk_truncated_prefix_bytes > 0 {
3492 parts.push(format!(
3493 "truncated {disk_truncated_prefix_bytes} bytes from saved output prefix"
3494 ));
3495 } else if recovery.had_inner_drop && parts.is_empty() {
3496 parts.push("omitted output".to_string());
3497 }
3498
3499 if parts.is_empty() {
3500 return None;
3501 }
3502
3503 let hint = recovery_hint(recovery);
3504 Some(format!("[{}; {hint}]", parts.join(", ")))
3505}
3506
3507fn bash_status_recovery_hint(access: &ArtifactRecoveryAccess) -> String {
3508 let task_id = serde_json::to_string(&access.task_id)
3509 .unwrap_or_else(|_| format!("\"{}\"", access.task_id));
3510 format!("use bash_status({{taskId: {task_id}}})")
3511}
3512
3513fn recovery_hint(recovery: &RecoveryContext) -> String {
3514 if !recovery.artifact_access.readable {
3515 return bash_status_recovery_hint(&recovery.artifact_access);
3516 }
3517
3518 if recovery.offset_hint_eligible
3522 && !recovery.byte_truncated
3523 && recovery.dropped_by_class.is_empty()
3524 && !recovery.include_stderr_path
3525 {
3526 if let (Some(path), Some(line)) =
3527 (recovery.output_path.as_deref(), recovery.offset_start_line)
3528 {
3529 return format!("see remaining: tail -n +{line} {}", quote_path(path));
3530 }
3531 }
3532
3533 let mut paths = Vec::new();
3534 if let Some(path) = recovery.output_path.as_deref() {
3535 paths.push(path);
3536 }
3537 if recovery.include_stderr_path {
3538 if let Some(path) = recovery.stderr_path.as_deref() {
3539 if !paths.contains(&path) {
3540 paths.push(path);
3541 }
3542 }
3543 }
3544
3545 if paths.is_empty() {
3546 return "full output unavailable".to_string();
3547 }
3548
3549 let reads = paths
3550 .into_iter()
3551 .map(|path| format!("read {}", quote_path(path)))
3552 .collect::<Vec<_>>()
3553 .join(" and ");
3554 if recovery.disk_truncated_prefix_bytes > 0 {
3555 format!("retained output: {reads}")
3556 } else {
3557 format!("full output: {reads}")
3558 }
3559}
3560
3561fn strip_plain_truncation_marker_lines(input: &str) -> String {
3562 input
3563 .lines()
3564 .filter(|line| !is_plain_truncation_marker(line.trim()))
3565 .collect::<Vec<_>>()
3566 .join("\n")
3567}
3568
3569fn strip_recovery_marker_lines(input: &str) -> String {
3570 input
3571 .lines()
3572 .filter(|line| !is_recovery_marker(line.trim()))
3573 .collect::<Vec<_>>()
3574 .join("\n")
3575}
3576
3577fn is_plain_truncation_marker(line: &str) -> bool {
3578 let Some(rest) = line.strip_prefix("...<truncated ") else {
3579 return false;
3580 };
3581 let Some(bytes) = rest.strip_suffix(" bytes>...") else {
3582 return false;
3583 };
3584 !bytes.is_empty() && bytes.chars().all(|ch| ch.is_ascii_digit())
3585}
3586
3587fn is_recovery_marker(line: &str) -> bool {
3588 line.starts_with('[')
3589 && line.ends_with(']')
3590 && (line.contains("full output: read ")
3591 || line.contains("retained output: read ")
3592 || line.contains("see remaining: tail -n +")
3593 || line.contains("use bash_status({taskId:")
3594 || line.contains("full output unavailable"))
3595}
3596
3597fn structured_output_pointer(
3598 total_bytes: u64,
3599 output_path: &str,
3600 truncated_prefix_bytes: u64,
3601 artifact_access: &ArtifactRecoveryAccess,
3602) -> String {
3603 if artifact_access.readable {
3604 return if truncated_prefix_bytes > 0 {
3605 retained_json_output_pointer(total_bytes, output_path, truncated_prefix_bytes)
3606 } else {
3607 json_output_pointer(total_bytes, output_path)
3608 };
3609 }
3610
3611 let kb = total_bytes.div_ceil(1024);
3612 let hint = bash_status_recovery_hint(artifact_access);
3613 if truncated_prefix_bytes > 0 {
3614 format!(
3615 "[JSON output {kb} KB; truncated {truncated_prefix_bytes} bytes from saved output prefix; retained output: {hint}]"
3616 )
3617 } else {
3618 format!("[JSON output {kb} KB; full output: {hint}]")
3619 }
3620}
3621
3622fn render_structured_output(
3623 command: &str,
3624 buffer: &BgBuffer,
3625 disk_truncation: DiskTruncation,
3626 artifact_access: ArtifactRecoveryAccess,
3627) -> Option<TerminalOutputCache> {
3628 if !is_gh_structured_command(command) {
3629 return None;
3630 }
3631
3632 let output_path = buffer
3633 .output_path()
3634 .map(|path| path.display().to_string())?;
3635 let stdout_bytes = buffer.stream_len(StreamKind::Stdout);
3636 if stdout_bytes == 0 {
3637 return None;
3638 }
3639
3640 if stdout_bytes > STRUCTURED_OUTPUT_CAP_BYTES as u64 {
3641 if !stream_starts_like_json(buffer, StreamKind::Stdout) {
3642 return None;
3643 }
3644 let output_preview = structured_output_pointer(
3645 stdout_bytes,
3646 &output_path,
3647 disk_truncation.total_prefix_bytes(),
3648 &artifact_access,
3649 );
3650 return Some(TerminalOutputCache {
3651 output_preview,
3652 output_truncated: true,
3653 kind: TerminalOutputKind::Structured,
3654 output_path: Some(output_path),
3655 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
3656 artifact_access,
3657 recovery: None,
3658 });
3659 }
3660
3661 let stdout = buffer.read_stream_bounded(StreamKind::Stdout, STRUCTURED_OUTPUT_CAP_BYTES);
3662 if stdout.truncated || !is_structured_body(&stdout.text) {
3663 return None;
3664 }
3665
3666 Some(TerminalOutputCache {
3667 output_preview: stdout.text,
3668 output_truncated: false,
3669 kind: TerminalOutputKind::Structured,
3670 output_path: Some(output_path),
3671 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
3672 artifact_access,
3673 recovery: None,
3674 })
3675}
3676
3677fn render_raw_passthrough(
3678 buffer: &BgBuffer,
3679 disk_truncation: DiskTruncation,
3680 artifact_access: ArtifactRecoveryAccess,
3681) -> TerminalOutputCache {
3682 let raw = buffer.read_combined_head_tail(
3683 RAW_PASSTHROUGH_CAP_BYTES,
3684 RAW_PASSTHROUGH_HEAD_BYTES,
3685 RAW_PASSTHROUGH_TAIL_BYTES,
3686 );
3687 let output_path = buffer.output_path().map(|path| path.display().to_string());
3688 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
3689 if !raw.truncated && disk_truncation.total_prefix_bytes() == 0 {
3690 return TerminalOutputCache {
3691 output_preview: raw.text,
3692 output_truncated: false,
3693 kind: TerminalOutputKind::Raw,
3694 output_path,
3695 stderr_path,
3696 artifact_access,
3697 recovery: None,
3698 };
3699 }
3700
3701 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
3702 let mut recovery = RecoveryContext {
3703 dropped_by_class: BTreeMap::new(),
3704 had_inner_drop: false,
3705 offset_hint_eligible: false,
3706 offset_start_line: None,
3707 byte_truncated: raw.truncated,
3708 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
3709 output_path: output_path.clone(),
3710 stderr_path: stderr_path.clone(),
3711 include_stderr_path,
3712 artifact_access: artifact_access.clone(),
3713 };
3714 let (output_preview, output_truncated) =
3715 render_raw_body_with_recovery_marker(&raw.text, &mut recovery);
3716 TerminalOutputCache {
3717 output_preview,
3718 output_truncated,
3719 kind: TerminalOutputKind::Raw,
3720 output_path,
3721 stderr_path,
3722 artifact_access,
3723 recovery: Some(recovery),
3724 }
3725}
3726
3727fn completion_preview_for_cache(
3728 cache: &TerminalOutputCache,
3729 exit_code: Option<i32>,
3730) -> (String, bool) {
3731 let exit_ok = exit_code == Some(0);
3734 let threshold = completion_preview_threshold(exit_ok);
3735 if cache.kind == TerminalOutputKind::Structured && cache.output_preview.len() > threshold {
3736 if let Some(path) = cache.output_path.as_deref() {
3737 return (
3738 structured_output_pointer(
3739 cache.output_preview.len() as u64,
3740 path,
3741 0,
3742 &cache.artifact_access,
3743 ),
3744 true,
3745 );
3746 }
3747 return (cache.output_preview.clone(), cache.output_truncated);
3748 }
3749
3750 if let Some(recovery) = cache.recovery.as_ref() {
3751 if cache.output_preview.len() <= threshold {
3752 return (cache.output_preview.clone(), cache.output_truncated);
3753 }
3754 let body = strip_recovery_marker_lines(&cache.output_preview);
3755 let mut completion_recovery = recovery.clone();
3756 completion_recovery.byte_truncated = true;
3757 if let Some(marker) = recovery_marker(&completion_recovery) {
3758 let capped = cap_completion_output_with_marker(&body, &marker, exit_ok);
3759 return (capped.text, true);
3760 }
3761 }
3762
3763 let capped = cap_completion_output(&cache.output_preview, exit_ok);
3764 (capped.text, cache.output_truncated || capped.truncated)
3765}
3766
3767fn is_gh_structured_command(command: &str) -> bool {
3768 let Some(normalized) = crate::compress::plain_command_for_structured_output(command) else {
3769 return false;
3770 };
3771 let tokens = shell_words_for_flags(&normalized);
3772 let Some(head) = tokens.first() else {
3773 return false;
3774 };
3775 let head_name = Path::new(head)
3776 .file_name()
3777 .and_then(|name| name.to_str())
3778 .unwrap_or(head);
3779 if !(head_name == "gh" || head_name.eq_ignore_ascii_case("gh.exe")) {
3780 return false;
3781 }
3782 tokens.iter().any(|token| {
3783 matches!(token.as_str(), "--json" | "--jq" | "--template")
3784 || token.starts_with("--json=")
3785 || token.starts_with("--jq=")
3786 || token.starts_with("--template=")
3787 })
3788}
3789
3790fn shell_words_for_flags(command: &str) -> Vec<String> {
3791 let mut words = Vec::new();
3792 let mut current = String::new();
3793 let mut in_single = false;
3794 let mut in_double = false;
3795 let mut escaped = false;
3796
3797 for ch in command.chars() {
3798 if escaped {
3799 current.push(ch);
3800 escaped = false;
3801 continue;
3802 }
3803 if ch == '\\' && !in_single {
3804 escaped = true;
3805 continue;
3806 }
3807 if ch == '\'' && !in_double {
3808 in_single = !in_single;
3809 continue;
3810 }
3811 if ch == '"' && !in_single {
3812 in_double = !in_double;
3813 continue;
3814 }
3815 if ch.is_whitespace() && !in_single && !in_double {
3816 if !current.is_empty() {
3817 words.push(std::mem::take(&mut current));
3818 }
3819 continue;
3820 }
3821 if matches!(ch, ';' | '&' | '|') && !in_single && !in_double {
3822 if !current.is_empty() {
3823 words.push(std::mem::take(&mut current));
3824 }
3825 continue;
3826 }
3827 current.push(ch);
3828 }
3829 if !current.is_empty() {
3830 words.push(current);
3831 }
3832 words
3833}
3834
3835fn is_structured_body(body: &str) -> bool {
3836 let trimmed = body.trim();
3837 if trimmed.is_empty() {
3838 return false;
3839 }
3840 if serde_json::from_str::<serde_json::Value>(trimmed).is_ok() {
3841 return true;
3842 }
3843
3844 let mut saw_line = false;
3845 for line in trimmed
3846 .lines()
3847 .map(str::trim)
3848 .filter(|line| !line.is_empty())
3849 {
3850 saw_line = true;
3851 if serde_json::from_str::<serde_json::Value>(line).is_err() {
3852 return false;
3853 }
3854 }
3855 saw_line
3856}
3857
3858fn stream_starts_like_json(buffer: &BgBuffer, stream: StreamKind) -> bool {
3859 buffer
3860 .read_stream_bounded(stream, 512)
3861 .text
3862 .chars()
3863 .find(|ch| !ch.is_whitespace())
3864 .is_some_and(|ch| matches!(ch, '{' | '[' | '"' | '-' | '0'..='9' | 't' | 'f' | 'n'))
3865}
3866
3867struct CompletionTokenCounts {
3868 original_tokens: Option<u32>,
3869 compressed_tokens: Option<u32>,
3870 original_bytes: Option<i64>,
3871 compressed_bytes: Option<i64>,
3872 tokens_skipped: bool,
3873}
3874
3875impl CompletionTokenCounts {
3876 fn skipped() -> Self {
3877 Self {
3878 original_tokens: None,
3879 compressed_tokens: None,
3880 original_bytes: None,
3881 compressed_bytes: None,
3882 tokens_skipped: true,
3883 }
3884 }
3885}
3886
3887fn completion_status_text(status: &BgTaskStatus, exit_code: Option<i32>) -> String {
3888 match status {
3889 BgTaskStatus::TimedOut => "timed out".to_string(),
3890 BgTaskStatus::Killed => "killed".to_string(),
3891 _ => exit_code
3892 .map(|code| format!("exit {code}"))
3893 .unwrap_or_else(|| format!("{status:?}").to_lowercase()),
3894 }
3895}
3896
3897fn token_count_u32(text: &str) -> u32 {
3898 aft_tokenizer::count_tokens(text)
3899 .try_into()
3900 .unwrap_or(u32::MAX)
3901}
3902
3903impl Default for BgTaskRegistry {
3904 fn default() -> Self {
3905 Self::new(Arc::new(Mutex::new(None)))
3906 }
3907}
3908
3909fn modified_within(path: &Path, grace: Duration) -> bool {
3910 fs::metadata(path)
3911 .and_then(|metadata| metadata.modified())
3912 .ok()
3913 .and_then(|modified| SystemTime::now().duration_since(modified).ok())
3914 .map(|age| age < grace)
3915 .unwrap_or(false)
3916}
3917
3918fn canonicalized_path(path: &Path) -> PathBuf {
3919 fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf())
3920}
3921
3922fn started_instant_from_unix_millis(started_at: u64) -> Instant {
3923 let now_ms = SystemTime::now()
3924 .duration_since(UNIX_EPOCH)
3925 .ok()
3926 .map(|duration| duration.as_millis() as u64)
3927 .unwrap_or(started_at);
3928 let elapsed_ms = now_ms.saturating_sub(started_at);
3929 Instant::now()
3930 .checked_sub(Duration::from_millis(elapsed_ms))
3931 .unwrap_or_else(Instant::now)
3932}
3933
3934fn gc_quarantine(storage_dir: &Path) {
3935 let quarantine_root = storage_dir.join("bash-tasks-quarantine");
3936 let Ok(session_dirs) = fs::read_dir(&quarantine_root) else {
3937 return;
3938 };
3939 for session_entry in session_dirs.flatten() {
3940 let session_quarantine_dir = session_entry.path();
3941 if !session_quarantine_dir.is_dir() {
3942 continue;
3943 }
3944 let entries = match fs::read_dir(&session_quarantine_dir) {
3945 Ok(entries) => entries,
3946 Err(error) => {
3947 crate::slog_warn!(
3948 "failed to read background task quarantine dir {}: {error}",
3949 session_quarantine_dir.display()
3950 );
3951 continue;
3952 }
3953 };
3954 for entry in entries.flatten() {
3955 let path = entry.path();
3956 if modified_within(&path, QUARANTINE_GC_GRACE) {
3957 continue;
3958 }
3959 let result = if path.is_dir() {
3960 fs::remove_dir_all(&path)
3961 } else {
3962 fs::remove_file(&path)
3963 };
3964 match result {
3965 Ok(()) => log::debug!(
3966 "deleted old background task quarantine entry {}",
3967 path.display()
3968 ),
3969 Err(error) => crate::slog_warn!(
3970 "failed to delete old background task quarantine entry {}: {error}",
3971 path.display()
3972 ),
3973 }
3974 }
3975 let _ = fs::remove_dir(&session_quarantine_dir);
3976 }
3977 let _ = fs::remove_dir(&quarantine_root);
3978}
3979
3980fn read_for_token_count_from_disk(
3981 metadata: &PersistedTask,
3982 paths: &TaskPaths,
3983 max_bytes_per_stream: usize,
3984) -> TokenCountInput {
3985 if metadata.mode == BgMode::Pty {
3986 return TokenCountInput::Skipped;
3987 }
3988 let stdout = read_file_tail_capped(paths, TaskArtifact::Stdout, max_bytes_per_stream);
3995 let stderr = read_file_tail_capped(paths, TaskArtifact::Stderr, max_bytes_per_stream);
3996 match (stdout, stderr) {
3997 (Ok(stdout), Ok(stderr)) => TokenCountInput::Text(combine_streams(
3998 String::from_utf8_lossy(&stdout).as_ref(),
3999 String::from_utf8_lossy(&stderr).as_ref(),
4000 )),
4001 (Ok(stdout), Err(_)) => TokenCountInput::Text(combine_streams(
4002 String::from_utf8_lossy(&stdout).as_ref(),
4003 "",
4004 )),
4005 (Err(_), Ok(stderr)) => TokenCountInput::Text(combine_streams(
4006 "",
4007 String::from_utf8_lossy(&stderr).as_ref(),
4008 )),
4009 (Err(_), Err(_)) => TokenCountInput::Skipped,
4010 }
4011}
4012
4013fn read_file_tail_capped(
4014 paths: &TaskPaths,
4015 artifact: TaskArtifact,
4016 max_bytes: usize,
4017) -> std::io::Result<Vec<u8>> {
4018 let mut file = open_task_artifact(paths, artifact)?;
4019 file.tail(max_bytes).map(|(bytes, _)| bytes)
4020}
4021
4022impl BgTask {
4023 fn snapshot(&self, preview_bytes: usize) -> BgTaskSnapshot {
4024 let state = self
4025 .state
4026 .lock()
4027 .unwrap_or_else(|poison| poison.into_inner());
4028 self.snapshot_locked(&state, preview_bytes)
4029 }
4030
4031 fn snapshot_locked(&self, state: &BgTaskState, preview_bytes: usize) -> BgTaskSnapshot {
4032 let metadata = &state.metadata;
4033 let duration_ms = metadata.duration_ms.or_else(|| {
4034 metadata
4035 .status
4036 .is_terminal()
4037 .then(|| self.started.elapsed().as_millis() as u64)
4038 });
4039 let (output_preview, output_truncated) = if metadata.mode == BgMode::Pty {
4040 (String::new(), false)
4041 } else if metadata.status.is_terminal() {
4042 state
4043 .terminal_output_cache
4044 .as_ref()
4045 .map(|cache| (cache.output_preview.clone(), cache.output_truncated))
4046 .unwrap_or_else(|| (String::new(), false))
4047 } else if preview_bytes == 0 {
4048 (String::new(), false)
4049 } else {
4050 state.buffer.read_tail(preview_bytes)
4051 };
4052 BgTaskSnapshot {
4053 info: BgTaskInfo {
4054 task_id: self.task_id.clone(),
4055 status: metadata.status.clone(),
4056 command: metadata.command.clone(),
4057 mode: metadata.mode.clone(),
4058 started_at: metadata.started_at,
4059 duration_ms,
4060 },
4061 exit_code: metadata.exit_code,
4062 child_pid: metadata.child_pid,
4063 workdir: metadata.workdir.display().to_string(),
4064 output_preview,
4065 output_truncated,
4066 output_path: state
4067 .buffer
4068 .output_path()
4069 .map(|path| path.display().to_string()),
4070 stderr_path: state
4071 .buffer
4072 .stderr_path()
4073 .map(|path| path.display().to_string()),
4074 pty_rows: (metadata.mode == BgMode::Pty).then_some(metadata.pty_rows.unwrap_or(24)),
4075 pty_cols: (metadata.mode == BgMode::Pty).then_some(metadata.pty_cols.unwrap_or(80)),
4076 pty_screen: None,
4077 scanner_report: metadata.scanner_report.clone(),
4078 sandbox_native: metadata.sandbox_native,
4079 sandbox_unavailable: metadata.sandbox_native
4080 && open_task_artifact(&self.paths, TaskArtifact::SandboxUnavailable)
4081 .and_then(|mut file| file.read_all())
4082 .is_ok_and(|bytes| bytes == b"sandbox_unavailable"),
4083 }
4084 }
4085
4086 pub(crate) fn is_running(&self) -> bool {
4087 self.state
4088 .lock()
4089 .map(|state| {
4090 state.metadata.status == BgTaskStatus::Running
4091 || (state.metadata.mode == BgMode::Pty
4092 && state.metadata.status == BgTaskStatus::Killing)
4093 })
4094 .unwrap_or(false)
4095 }
4096
4097 fn is_terminal(&self) -> bool {
4098 self.state
4099 .lock()
4100 .map(|state| state.metadata.status.is_terminal())
4101 .unwrap_or(false)
4102 }
4103
4104 fn mark_terminal_now(&self) {
4105 if let Ok(mut terminal_at) = self.terminal_at.lock() {
4106 if terminal_at.is_none() {
4107 *terminal_at = Some(Instant::now());
4108 }
4109 }
4110 }
4111
4112 fn set_completion_delivered(
4113 &self,
4114 delivered: bool,
4115 registry: &BgTaskRegistry,
4116 ) -> Result<(), String> {
4117 let mut state = self
4118 .state
4119 .lock()
4120 .map_err(|_| "background task lock poisoned".to_string())?;
4121 let updated = registry
4122 .update_task_metadata(&self.paths, |metadata| {
4123 metadata.completion_delivered = delivered;
4124 })
4125 .map_err(|e| format!("failed to update completion delivery: {e}"))?;
4126 state.metadata = updated;
4127 Ok(())
4128 }
4129}
4130
4131#[cfg(unix)]
4152fn reap_piped_child(child_slot: &mut Option<Child>) {
4153 if let Some(mut child) = child_slot.take() {
4154 if matches!(child.try_wait(), Ok(None)) {
4155 let _ = child.wait();
4156 }
4157 }
4158}
4159
4160#[cfg(windows)]
4165fn reap_piped_child(child_slot: &mut Option<Child>) {
4166 *child_slot = None;
4167}
4168
4169fn terminal_metadata_from_marker(
4170 mut metadata: PersistedTask,
4171 marker: ExitMarker,
4172 reason: Option<String>,
4173) -> PersistedTask {
4174 match marker {
4175 ExitMarker::Code(code) => {
4176 let status = if code == 0 {
4177 BgTaskStatus::Completed
4178 } else {
4179 BgTaskStatus::Failed
4180 };
4181 metadata.mark_terminal(status, Some(code), reason);
4182 }
4183 ExitMarker::Killed => metadata.mark_terminal(
4184 BgTaskStatus::Killed,
4185 terminal_exit_code_for_status(&BgTaskStatus::Killed),
4186 reason,
4187 ),
4188 }
4189 metadata
4190}
4191
4192fn terminal_exit_code_for_status(status: &BgTaskStatus) -> Option<i32> {
4193 match status {
4194 BgTaskStatus::TimedOut => Some(124),
4195 BgTaskStatus::Killed => Some(137),
4196 _ => None,
4197 }
4198}
4199
4200fn attach_sandbox_metadata(metadata: &mut PersistedTask, spawn_plan: &SpawnPlan) {
4201 metadata.sandbox_native = spawn_plan.is_native_launcher();
4202 metadata.sandbox_temp_dir = spawn_plan.temp_dir().map(Path::to_path_buf);
4203}
4204
4205#[cfg(unix)]
4206pub(crate) fn resolve_posix_shell() -> PathBuf {
4207 static POSIX_SHELL: OnceLock<PathBuf> = OnceLock::new();
4208 POSIX_SHELL
4209 .get_or_init(|| {
4210 std::env::var_os("BASH")
4211 .filter(|value| !value.is_empty())
4212 .map(PathBuf::from)
4213 .filter(|path| path.exists())
4214 .or_else(|| which::which("bash").ok())
4215 .or_else(|| which::which("zsh").ok())
4216 .unwrap_or_else(|| PathBuf::from("/bin/sh"))
4217 })
4218 .clone()
4219}
4220
4221#[cfg(windows)]
4222fn detached_shell_command_for(
4223 shell: crate::windows_shell::WindowsShell,
4224 command: &str,
4225 exit_path: &Path,
4226 paths: &TaskPaths,
4227 creation_flags: u32,
4228) -> Result<Command, String> {
4229 use crate::windows_shell::WindowsShell;
4230 let wrapper_body = shell.wrapper_script_bytes(command, exit_path);
4243 let wrapper_ext = match shell {
4244 WindowsShell::Pwsh | WindowsShell::Powershell => "ps1",
4245 WindowsShell::Cmd => "bat",
4246 WindowsShell::Posix(_) => "sh",
4250 };
4251 let wrapper_path = paths.dir.join(format!(
4252 "{}.{}",
4253 paths
4254 .json
4255 .file_stem()
4256 .and_then(|s| s.to_str())
4257 .unwrap_or("wrapper"),
4258 wrapper_ext
4259 ));
4260 fs::write(&wrapper_path, wrapper_body)
4261 .map_err(|e| format!("failed to write background bash wrapper script: {e}"))?;
4262
4263 let mut cmd = Command::new(shell.binary().as_ref());
4264 match shell {
4265 WindowsShell::Pwsh | WindowsShell::Powershell => {
4266 cmd.args([
4269 "-NoLogo",
4270 "-NoProfile",
4271 "-NonInteractive",
4272 "-ExecutionPolicy",
4273 "Bypass",
4274 "-File",
4275 ]);
4276 cmd.arg(&wrapper_path);
4277 }
4278 WindowsShell::Cmd => {
4279 cmd.args(["/D", "/C"]);
4286 cmd.arg(&wrapper_path);
4287 }
4288 WindowsShell::Posix(_) => {
4289 cmd.arg(&wrapper_path);
4294 }
4295 }
4296
4297 cmd.creation_flags(creation_flags);
4301 Ok(cmd)
4302}
4303
4304fn spawn_detached_child(
4320 spawn_plan: &SpawnPlan,
4321 command: &str,
4322 paths: &TaskPaths,
4323 workdir: &Path,
4324 env: &HashMap<String, String>,
4325 io_handles: &mut TaskIoHandles,
4326) -> Result<std::process::Child, String> {
4327 #[cfg(not(windows))]
4328 let _ = command;
4329 #[cfg(not(windows))]
4330 {
4331 use std::os::fd::AsRawFd;
4332
4333 let stdout = io_handles
4334 .clone_file(TaskArtifact::Stdout)
4335 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
4336 let stderr = io_handles
4337 .clone_file(TaskArtifact::Stderr)
4338 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
4339 let prepared = spawn_plan
4340 .prepared_task()
4341 .ok_or_else(|| "background task payload was not prepared".to_string())?;
4342 let payload = prepared.invocation()?;
4343 let exit = io_handles
4344 .inheritable_file(TaskArtifact::Exit)
4345 .map_err(|e| format!("failed to inherit exit marker handle: {e}"))?;
4346 let failure = io_handles
4347 .inheritable_file(TaskArtifact::SandboxUnavailable)
4348 .map_err(|e| format!("failed to inherit sandbox failure marker handle: {e}"))?;
4349 let shell = spawn_plan
4350 .host_shell_path()
4351 .map(Path::to_path_buf)
4352 .unwrap_or_else(resolve_posix_shell);
4353 let args = vec![
4354 OsString::from("-c"),
4355 payload.wrapper_text.clone(),
4356 OsString::from("aft-payload-wrapper"),
4357 shell.as_os_str().to_os_string(),
4358 payload.command_text.clone(),
4359 OsString::from(crate::sandbox_spawn::CHILD_EXIT_FD.to_string()),
4360 ];
4361 let (mut child_command, profile_handle) = crate::sandbox_spawn::detached_command_for_plan(
4362 spawn_plan,
4363 std::ffi::OsStr::new("/bin/sh"),
4364 &args,
4365 &paths.json,
4366 crate::sandbox_spawn::CHILD_EXIT_FD,
4367 crate::sandbox_spawn::CHILD_FAILURE_FD,
4368 )?;
4369 crate::sandbox_spawn::apply_marker_fd_allowlist(
4370 &mut child_command,
4371 exit.as_raw_fd(),
4372 failure.as_raw_fd(),
4373 )?;
4374 child_command
4375 .current_dir(workdir)
4376 .envs(env)
4377 .stdin(Stdio::null())
4378 .stdout(Stdio::from(stdout))
4379 .stderr(Stdio::from(stderr));
4380 crate::sandbox_spawn::apply_sandbox_environment(spawn_plan, &mut child_command, env);
4381 let child = child_command
4382 .spawn()
4383 .map_err(|e| format!("failed to spawn background bash command: {e}"));
4384 drop((payload, exit, failure, profile_handle));
4385 child
4386 }
4387 #[cfg(windows)]
4388 {
4389 use crate::windows_shell::shell_candidates;
4390 match spawn_plan {
4391 SpawnPlan::Unsandboxed | SpawnPlan::Host { .. } => {}
4392 SpawnPlan::Refused { code, .. } => return Err((*code).to_string()),
4393 SpawnPlan::Launcher { .. } => return Err("sandbox_unavailable".to_string()),
4394 }
4395 let candidates: Vec<crate::windows_shell::WindowsShell> = shell_candidates();
4406 const FLAG_CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200;
4419 const FLAG_CREATE_BREAKAWAY_FROM_JOB: u32 = 0x0100_0000;
4420 const FLAG_CREATE_NO_WINDOW: u32 = 0x0800_0000;
4421 let with_breakaway =
4422 FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP | FLAG_CREATE_BREAKAWAY_FROM_JOB;
4423 let without_breakaway = FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP;
4424 let mut last_error: Option<String> = None;
4425 for (idx, shell) in candidates.iter().enumerate() {
4426 for &flags in &[with_breakaway, without_breakaway] {
4430 let stdout = io_handles
4433 .clone_file(TaskArtifact::Stdout)
4434 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
4435 let stderr = io_handles
4436 .clone_file(TaskArtifact::Stderr)
4437 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
4438 let mut cmd =
4439 detached_shell_command_for(shell.clone(), command, &paths.exit, paths, flags)?;
4440 cmd.current_dir(workdir)
4441 .envs(env)
4442 .stdin(Stdio::null())
4443 .stdout(Stdio::from(stdout))
4444 .stderr(Stdio::from(stderr));
4445 match cmd.spawn() {
4446 Ok(child) => {
4447 if idx > 0 {
4448 crate::slog_warn!("background bash spawn fell back to {} after {} earlier candidate(s) failed; \
4449 the cached PATH probe disagreed with runtime spawn — likely PATH \
4450 inheritance, antivirus / AppLocker / Defender ASR, or sandbox policy.",
4451 shell.binary(),
4452 idx);
4453 }
4454 if flags == without_breakaway {
4455 crate::slog_warn!(
4456 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected \
4457 (likely a restrictive Job Object — CI sandbox or MDM policy). \
4458 Spawned without breakaway; the bg task will be torn down if the \
4459 AFT process group is killed."
4460 );
4461 }
4462 return Ok(child);
4463 }
4464 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
4465 crate::slog_warn!("background bash spawn: {} returned NotFound at runtime — trying next candidate",
4466 shell.binary());
4467 last_error = Some(format!("{}: {e}", shell.binary()));
4468 break;
4471 }
4472 Err(e) if flags == with_breakaway && e.raw_os_error() == Some(5) => {
4473 crate::slog_warn!(
4475 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected with \
4476 Access Denied — retrying {} without breakaway",
4477 shell.binary()
4478 );
4479 last_error = Some(format!("{}: {e}", shell.binary()));
4480 continue;
4481 }
4482 Err(e) => {
4483 return Err(format!(
4484 "failed to spawn background bash command via {}: {e}",
4485 shell.binary()
4486 ));
4487 }
4488 }
4489 }
4490 }
4491 Err(format!(
4492 "failed to spawn background bash command: no Windows shell could be spawned. \
4493 Last error: {}. PATH-probed candidates: {:?}",
4494 last_error.unwrap_or_else(|| "no candidates were attempted".to_string()),
4495 candidates.iter().map(|s| s.binary()).collect::<Vec<_>>()
4496 ))
4497 }
4498}
4499
4500#[cfg(test)]
4501fn random_slug() -> String {
4502 let mut bytes = [0u8; 8];
4510 getrandom::fill(&mut bytes).unwrap_or_else(|_| {
4512 let t = SystemTime::now()
4514 .duration_since(UNIX_EPOCH)
4515 .map(|d| d.as_nanos() as u64)
4516 .unwrap_or(0);
4517 let p = u64::from(std::process::id());
4518 bytes.copy_from_slice(&(t ^ p.rotate_left(32)).to_le_bytes());
4519 });
4520 let hex: String = bytes.iter().map(|b| format!("{b:02x}")).collect();
4522 format!("bash-{hex}")
4523}
4524
4525#[cfg(test)]
4526mod tests {
4527 use std::collections::HashMap;
4528 use std::fs;
4529 #[cfg(unix)]
4530 use std::os::unix::fs::PermissionsExt;
4531 use std::sync::atomic::{AtomicBool, AtomicUsize};
4532 use std::sync::{Arc, Mutex};
4533 use std::time::{Duration, Instant};
4534
4535 use super::*;
4536 use crate::bash_background::persistence::{read_task, task_paths, write_task};
4537
4538 #[cfg(unix)]
4539 const QUICK_SUCCESS_COMMAND: &str = "true";
4540 #[cfg(windows)]
4541 const QUICK_SUCCESS_COMMAND: &str = "cmd /c exit 0";
4542
4543 #[cfg(unix)]
4544 const LONG_RUNNING_COMMAND: &str = "sleep 5";
4545 #[cfg(windows)]
4546 const LONG_RUNNING_COMMAND: &str = "cmd /c timeout /t 5 /nobreak > nul";
4547
4548 #[test]
4549 fn bash_memory_estimate_is_zero_when_empty_and_nonzero_for_completion_cache() {
4550 let registry = BgTaskRegistry::default();
4551 assert_eq!(registry.estimated_memory().estimated_bytes, Some(0));
4552 registry
4553 .inner
4554 .completions
4555 .lock()
4556 .unwrap()
4557 .push_back(BgCompletion {
4558 task_id: "bash-memory".to_string(),
4559 session_id: "session-memory".to_string(),
4560 status: BgTaskStatus::Completed,
4561 exit_code: Some(0),
4562 command: "printf memory".to_string(),
4563 output_preview: "resident completion output".to_string(),
4564 output_truncated: false,
4565 original_tokens: None,
4566 compressed_tokens: None,
4567 tokens_skipped: false,
4568 });
4569 let estimate = registry.estimated_memory();
4570 assert!(estimate.estimated_bytes.unwrap() > 0);
4571 assert_eq!(estimate.counts["completion_caches"], 1);
4572 assert_eq!(estimate.counts["sessions"], 1);
4573 }
4574
4575 #[test]
4576 fn gh_structured_detection_rejects_piped_commands() {
4577 assert!(is_gh_structured_command(
4578 "gh issue list --json number,title"
4579 ));
4580 assert!(is_gh_structured_command(
4581 "cd repo && gh issue list --json number,title"
4582 ));
4583
4584 assert!(!is_gh_structured_command(
4585 "gh issue list --json number,title | jq '.[]'"
4586 ));
4587 assert!(!is_gh_structured_command(
4588 "gh issue list --json number,title |"
4589 ));
4590 }
4591
4592 fn insert_terminal_piped_task(
4593 registry: &BgTaskRegistry,
4594 dir: &tempfile::TempDir,
4595 command: &str,
4596 stdout: &str,
4597 stderr: &str,
4598 compressed: bool,
4599 ) -> (String, Arc<BgTask>) {
4600 let task_id = random_slug();
4601 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
4602 fs::create_dir_all(&paths.dir).unwrap();
4603 fs::write(&paths.stdout, stdout).unwrap();
4604 fs::write(&paths.stderr, stderr).unwrap();
4605 let mut metadata = PersistedTask::starting(
4606 task_id.clone(),
4607 "session".to_string(),
4608 command.to_string(),
4609 dir.path().to_path_buf(),
4610 Some(dir.path().to_path_buf()),
4611 Some(30_000),
4612 true,
4613 compressed,
4614 );
4615 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
4616 write_task(&paths.json, &metadata).unwrap();
4617 registry
4618 .insert_rehydrated_task(metadata, paths, true)
4619 .expect("insert terminal task");
4620 let task = registry.task_for_session(&task_id, "session").unwrap();
4621 (task_id, task)
4622 }
4623
4624 #[test]
4625 fn bash_zero_preview_running_status_skips_output_read_while_explicit_preview_reads() {
4626 let registry = BgTaskRegistry::default();
4627 let dir = tempfile::tempdir().unwrap();
4628 let task_id = random_slug();
4629 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
4630 fs::create_dir_all(&paths.dir).unwrap();
4631 fs::write(&paths.stdout, "live output\n").unwrap();
4632 fs::write(&paths.stderr, "").unwrap();
4633 let stdout_path = paths.stdout.clone();
4634 let mut metadata = PersistedTask::starting(
4635 task_id.clone(),
4636 "session".to_string(),
4637 "sleep 60".to_string(),
4638 dir.path().to_path_buf(),
4639 Some(dir.path().to_path_buf()),
4640 Some(30_000),
4641 true,
4642 false,
4643 );
4644 metadata.status = BgTaskStatus::Running;
4645 write_task(&paths.json, &metadata).unwrap();
4646 registry
4647 .insert_rehydrated_task(metadata, paths, false)
4648 .expect("insert running task");
4649
4650 crate::bash_background::buffer::reset_tail_read_count(&stdout_path);
4651 for _ in 0..5 {
4652 let snapshot = registry
4653 .status(&task_id, "session", Some(dir.path()), Some(dir.path()), 0)
4654 .expect("running snapshot");
4655 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
4656 assert!(snapshot.output_preview.is_empty());
4657 }
4658 assert_eq!(
4659 crate::bash_background::buffer::tail_read_count(&stdout_path),
4660 0
4661 );
4662
4663 let snapshot = registry
4664 .status(
4665 &task_id,
4666 "session",
4667 Some(dir.path()),
4668 Some(dir.path()),
4669 RUNNING_OUTPUT_PREVIEW_BYTES,
4670 )
4671 .expect("explicit running snapshot");
4672 assert_eq!(snapshot.output_preview, "live output\n");
4673 assert_eq!(
4674 crate::bash_background::buffer::tail_read_count(&stdout_path),
4675 1
4676 );
4677 }
4678
4679 #[test]
4680 fn artifact_read_capability_requires_exact_canonical_path_and_session() {
4681 let registry = BgTaskRegistry::default();
4682 let dir = tempfile::tempdir().unwrap();
4683 let (_task_id, task) = insert_terminal_piped_task(
4684 ®istry,
4685 &dir,
4686 "printf output",
4687 "stdout\n",
4688 "stderr\n",
4689 true,
4690 );
4691 fs::write(&task.paths.exit, "0\n").unwrap();
4692
4693 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stdout));
4694 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stderr));
4695 assert!(registry.is_session_owned_artifact_path("session", &task.paths.exit));
4696 assert!(!registry.is_session_owned_artifact_path("different-session", &task.paths.stdout));
4697 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.json));
4698
4699 let unregistered = task.paths.dir.join("unregistered-output");
4700 fs::write(&unregistered, "not a task artifact\n").unwrap();
4701 assert!(!registry.is_session_owned_artifact_path("session", &unregistered));
4702 }
4703
4704 #[cfg(unix)]
4705 #[test]
4706 fn artifact_directory_symlink_does_not_create_a_prefix_exception() {
4707 let registry = BgTaskRegistry::default();
4708 let dir = tempfile::tempdir().unwrap();
4709 let project = dir.path().join("project");
4710 fs::create_dir_all(&project).unwrap();
4711 let (_task_id, task) =
4712 insert_terminal_piped_task(®istry, &dir, "printf output", "stdout\n", "", true);
4713 let link = project.join("task-artifacts");
4714 std::os::unix::fs::symlink(&task.paths.dir, &link).unwrap();
4715 let unregistered = task.paths.dir.join("unregistered-output");
4716 fs::write(&unregistered, "not registered\n").unwrap();
4717
4718 assert!(!registry.is_session_owned_artifact_path("session", &link));
4719 assert!(
4720 !registry.is_session_owned_artifact_path("session", &link.join("unregistered-output"))
4721 );
4722 assert!(registry.is_session_owned_artifact_path(
4723 "session",
4724 &link.join(task.paths.stdout.file_name().unwrap())
4725 ));
4726
4727 let outside = dir.path().join("outside-secret");
4728 fs::write(&outside, "must stay private\n").unwrap();
4729 fs::remove_file(&task.paths.stdout).unwrap();
4730 std::os::unix::fs::symlink(&outside, &task.paths.stdout).unwrap();
4731 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.stdout));
4732 }
4733
4734 #[test]
4735 fn recovery_footer_uses_bash_status_when_artifact_is_not_registered() {
4736 let registry = BgTaskRegistry::default();
4737 let dir = tempfile::tempdir().unwrap();
4738 let task_id = "bash-1111111111111111";
4739 let paths = task_paths(dir.path(), "session", task_id).unwrap();
4740 fs::create_dir_all(&paths.dir).unwrap();
4741 fs::write(
4742 &paths.stdout,
4743 format!("{}tail\n", "output-line\n".repeat(2_000)),
4744 )
4745 .unwrap();
4746 fs::write(&paths.stderr, "").unwrap();
4747 let mut metadata = PersistedTask::starting(
4748 task_id.to_string(),
4749 "session".to_string(),
4750 "printf output".to_string(),
4751 dir.path().to_path_buf(),
4752 Some(dir.path().to_path_buf()),
4753 Some(30_000),
4754 true,
4755 true,
4756 );
4757 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
4758 write_task(&paths.json, &metadata).unwrap();
4759
4760 let cache = registry
4761 .render_terminal_output_from_paths(&metadata, &paths)
4762 .expect("terminal render");
4763
4764 assert!(cache
4765 .output_preview
4766 .contains("use bash_status({taskId: \"bash-1111111111111111\"})"));
4767 assert!(!cache.output_preview.contains("full output: read "));
4768 }
4769
4770 fn insert_terminal_pty_task(
4771 registry: &BgTaskRegistry,
4772 dir: &tempfile::TempDir,
4773 pty_output: &str,
4774 ) -> (String, Arc<BgTask>) {
4775 let task_id = random_slug();
4776 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
4777 fs::create_dir_all(&paths.dir).unwrap();
4778 fs::write(&paths.pty, pty_output).unwrap();
4779 let mut metadata = PersistedTask::starting(
4780 task_id.clone(),
4781 "session".to_string(),
4782 "python".to_string(),
4783 dir.path().to_path_buf(),
4784 Some(dir.path().to_path_buf()),
4785 Some(30_000),
4786 true,
4787 true,
4788 );
4789 metadata.mode = BgMode::Pty;
4790 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
4791 write_task(&paths.json, &metadata).unwrap();
4792 registry
4793 .insert_rehydrated_task(metadata, paths, true)
4794 .expect("insert terminal pty task");
4795 let task = registry.task_for_session(&task_id, "session").unwrap();
4796 (task_id, task)
4797 }
4798
4799 #[cfg(unix)]
4800 fn wait_for_terminal_snapshot(
4801 registry: &BgTaskRegistry,
4802 task_id: &str,
4803 session_id: &str,
4804 project: &Path,
4805 storage: &Path,
4806 ) -> BgTaskSnapshot {
4807 let started = Instant::now();
4808 loop {
4809 let snapshot = registry
4810 .status(task_id, session_id, Some(project), Some(storage), 4096)
4811 .expect("spawned task should be visible to status");
4812 if snapshot.info.status.is_terminal() {
4813 return snapshot;
4814 }
4815 assert!(
4816 started.elapsed() < Duration::from_secs(10),
4817 "timed out waiting for task {task_id} to finish; last status={:?}",
4818 snapshot.info.status
4819 );
4820 std::thread::sleep(Duration::from_millis(50));
4821 }
4822 }
4823
4824 fn write_running_project_task(storage: &Path, project: &Path, session: &str, task_id: &str) {
4825 let paths = task_paths(storage, session, task_id).unwrap();
4826 let mut metadata = PersistedTask::starting(
4827 task_id.to_string(),
4828 session.to_string(),
4829 "sleep 60".to_string(),
4830 project.to_path_buf(),
4831 Some(project.to_path_buf()),
4832 Some(30_000),
4833 true,
4834 true,
4835 );
4836 metadata.status = BgTaskStatus::Running;
4837 write_task(&paths.json, &metadata).unwrap();
4838 fs::write(&paths.stdout, "still running\n").unwrap();
4839 fs::write(&paths.stderr, "").unwrap();
4840 }
4841
4842 #[test]
4843 fn status_replay_filters_same_session_by_project_root() {
4844 let project_a = tempfile::tempdir().unwrap();
4845 let project_b = tempfile::tempdir().unwrap();
4846 let storage = tempfile::tempdir().unwrap();
4847 let session = "shared-session";
4848 let task_id = "bash-2222222222222222";
4849 write_running_project_task(storage.path(), project_a.path(), session, task_id);
4850
4851 let actor_b = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
4852 assert!(actor_b
4853 .status(
4854 task_id,
4855 session,
4856 Some(project_b.path()),
4857 Some(storage.path()),
4858 1024,
4859 )
4860 .is_none());
4861 assert!(actor_b.task_for_session(task_id, session).is_none());
4862
4863 let actor_a = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
4864 let snapshot = actor_a
4865 .status(
4866 task_id,
4867 session,
4868 Some(project_a.path()),
4869 Some(storage.path()),
4870 1024,
4871 )
4872 .expect("owning project should replay its task");
4873 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
4874 }
4875
4876 #[cfg(unix)]
4877 #[test]
4878 fn multiline_pipeline_stdout_persists_all_lines_after_terminal_status() {
4879 let cases = [
4880 (
4881 "long-first",
4882 "sleep 0.5; printf 'one\\n' | cat\nprintf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
4883 vec!["one", "1", "three"],
4884 ),
4885 (
4886 "short-first",
4887 "printf 'one\\n' | cat\nsleep 0.2; printf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
4888 vec!["one", "1", "three"],
4889 ),
4890 (
4891 "failing-middle",
4892 "sleep 0.2; printf 'one\\n' | cat\nfalse; printf 'after-false\\n' | cat\nprintf 'three\\n' | cat",
4893 vec!["one", "after-false", "three"],
4894 ),
4895 ];
4896
4897 for (name, command, expected_lines) in cases {
4898 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
4899 let dir = tempfile::tempdir().unwrap();
4900 let session_id = format!("session-{name}");
4901 let task_id = registry
4902 .spawn(
4903 SpawnPlan::Unsandboxed,
4904 command,
4905 session_id.clone(),
4906 dir.path().to_path_buf(),
4907 HashMap::new(),
4908 Some(Duration::from_secs(30)),
4909 dir.path().to_path_buf(),
4910 10,
4911 true,
4912 true,
4913 Some(dir.path().to_path_buf()),
4914 )
4915 .unwrap();
4916
4917 let snapshot = wait_for_terminal_snapshot(
4918 ®istry,
4919 &task_id,
4920 &session_id,
4921 dir.path(),
4922 dir.path(),
4923 );
4924 assert_eq!(
4925 snapshot.info.status,
4926 BgTaskStatus::Completed,
4927 "{name}: task should complete; snapshot={snapshot:?}"
4928 );
4929 assert_eq!(
4930 snapshot.exit_code,
4931 Some(0),
4932 "{name}: script should use the final command's exit code"
4933 );
4934
4935 let stdout = String::from_utf8(
4936 registry
4937 .read_artifact(&task_id, &session_id, TaskArtifact::Stdout)
4938 .expect("read validated stdout artifact"),
4939 )
4940 .expect("stdout is UTF-8");
4941 let lines: Vec<&str> = stdout.lines().collect();
4942 assert_eq!(
4943 lines, expected_lines,
4944 "{name}: raw stdout artifact must include every newline-separated command's output"
4945 );
4946 }
4947 }
4948
4949 #[test]
4950 fn recognizes_all_recovery_marker_forms() {
4951 assert!(is_recovery_marker(
4952 "[truncated output; full output: read \"/tmp/out\"]"
4953 ));
4954 assert!(is_recovery_marker(
4955 "[omitted output; see remaining: tail -n +42 \"/tmp/out\"]"
4956 ));
4957 assert!(is_recovery_marker(
4958 "[truncated output; full output unavailable]"
4959 ));
4960 assert!(is_recovery_marker(
4961 r#"[truncated 123 bytes from saved output prefix; retained output: read "/tmp/out"]"#
4962 ));
4963 }
4964
4965 #[test]
4966 fn recovery_marker_reports_disk_prefix_truncation_as_retained_output() {
4967 let recovery = RecoveryContext {
4968 dropped_by_class: BTreeMap::new(),
4969 had_inner_drop: false,
4970 offset_hint_eligible: false,
4971 offset_start_line: None,
4972 byte_truncated: false,
4973 disk_truncated_prefix_bytes: 4096,
4974 output_path: Some("/tmp/stdout".to_string()),
4975 stderr_path: None,
4976 include_stderr_path: false,
4977 artifact_access: ArtifactRecoveryAccess {
4978 task_id: "bash-test".to_string(),
4979 readable: true,
4980 },
4981 };
4982
4983 let marker = recovery_marker(&recovery).expect("disk truncation must emit marker");
4984
4985 assert!(marker.contains("truncated 4096 bytes from saved output prefix"));
4986 assert!(marker.contains(r#"retained output: read "/tmp/stdout""#));
4987 assert!(!marker.contains("full output: read"));
4988 }
4989
4990 #[test]
4991 fn killed_exit_marker_sets_nonzero_sentinel_exit_code() {
4992 let metadata = PersistedTask::starting(
4993 "task".to_string(),
4994 "session".to_string(),
4995 "cargo test".to_string(),
4996 PathBuf::from("/tmp"),
4997 None,
4998 None,
4999 true,
5000 true,
5001 );
5002
5003 let terminal = terminal_metadata_from_marker(metadata, ExitMarker::Killed, None);
5004
5005 assert_eq!(terminal.status, BgTaskStatus::Killed);
5006 assert_eq!(terminal.exit_code, Some(137));
5007 }
5008
5009 #[test]
5010 fn terminal_status_polls_use_cached_render_once_and_off_lock() {
5011 let registry = BgTaskRegistry::default();
5012 let dir = tempfile::tempdir().unwrap();
5013 let (_task_id, task) = insert_terminal_piped_task(
5014 ®istry,
5015 &dir,
5016 "custom-tool --verbose",
5017 &"stdout line\n".repeat(200_000),
5018 "",
5019 true,
5020 );
5021 let calls = Arc::new(AtomicUsize::new(0));
5022 let saw_unlocked_state = Arc::new(AtomicBool::new(false));
5023 let task_holder = Arc::new(Mutex::new(Some(Arc::clone(&task))));
5024 let calls_for_closure = Arc::clone(&calls);
5025 let unlocked_for_closure = Arc::clone(&saw_unlocked_state);
5026 let task_for_closure = Arc::clone(&task_holder);
5027 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
5028 calls_for_closure.fetch_add(1, Ordering::SeqCst);
5029 if let Some(task) = task_for_closure.lock().unwrap().as_ref() {
5030 if task.state.try_lock().is_ok() {
5031 unlocked_for_closure.store(true, Ordering::SeqCst);
5032 }
5033 }
5034 CompressionResult::new(format!("compressed {} bytes", output.len()))
5035 });
5036
5037 let first = registry
5038 .status(
5039 &task.task_id,
5040 "session",
5041 None,
5042 Some(dir.path()),
5043 RUNNING_OUTPUT_PREVIEW_BYTES,
5044 )
5045 .unwrap();
5046 let second = registry
5047 .status(
5048 &task.task_id,
5049 "session",
5050 None,
5051 Some(dir.path()),
5052 RUNNING_OUTPUT_PREVIEW_BYTES,
5053 )
5054 .unwrap();
5055 let listed = registry.list(RUNNING_OUTPUT_PREVIEW_BYTES);
5056
5057 assert_eq!(
5058 calls.load(Ordering::SeqCst),
5059 1,
5060 "terminal render must be cached"
5061 );
5062 assert!(
5063 saw_unlocked_state.load(Ordering::SeqCst),
5064 "compressor must run after releasing the task state lock"
5065 );
5066 assert!(first.output_preview.starts_with("compressed "));
5067 assert_eq!(second.output_preview, first.output_preview);
5068 assert_eq!(listed[0].output_preview, first.output_preview);
5069 }
5070
5071 #[test]
5072 fn completion_preview_success_keeps_tail_only() {
5073 let registry = BgTaskRegistry::default();
5078 let dir = tempfile::tempdir().unwrap();
5079 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
5080 let (_task_id, task) =
5081 insert_terminal_piped_task(®istry, &dir, "cat big.log", &output, "", false);
5082
5083 registry.post_terminal_transition(&task, true).unwrap();
5084 let completions = registry.drain_completions_for_session(Some("session"));
5085 assert_eq!(completions.len(), 1);
5086 let preview = &completions[0].output_preview;
5087 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
5088 assert!(!preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
5089 assert!(completions[0].output_truncated);
5090 }
5091
5092 #[test]
5093 fn completion_preview_failure_keeps_head_and_tail() {
5094 let registry = BgTaskRegistry::default();
5097 let dir = tempfile::tempdir().unwrap();
5098 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
5099 let task_id = random_slug();
5100 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5101 fs::create_dir_all(&paths.dir).unwrap();
5102 fs::write(&paths.stdout, &output).unwrap();
5103 fs::write(&paths.stderr, "").unwrap();
5104 let mut metadata = PersistedTask::starting(
5105 task_id.clone(),
5106 "session".to_string(),
5107 "cat big.log".to_string(),
5108 dir.path().to_path_buf(),
5109 Some(dir.path().to_path_buf()),
5110 Some(30_000),
5111 true,
5112 false,
5113 );
5114 metadata.mark_terminal(BgTaskStatus::Failed, Some(1), None);
5115 write_task(&paths.json, &metadata).unwrap();
5116 registry
5117 .insert_rehydrated_task(metadata, paths, true)
5118 .expect("insert terminal task");
5119 let task = registry.task_for_session(&task_id, "session").unwrap();
5120
5121 registry.post_terminal_transition(&task, true).unwrap();
5122 let completions = registry.drain_completions_for_session(Some("session"));
5123 assert_eq!(completions.len(), 1);
5124 let preview = &completions[0].output_preview;
5125 assert!(preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
5126 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
5127 }
5128
5129 #[test]
5130 fn has_completions_for_session_matches_pending_delivery() {
5131 let registry = BgTaskRegistry::default();
5132 assert!(!registry.has_completions_for_session(Some("session")));
5133 assert!(!registry.has_completions_for_session(None));
5134
5135 let dir = tempfile::tempdir().unwrap();
5136 let (_task_id, task) =
5137 insert_terminal_piped_task(®istry, &dir, QUICK_SUCCESS_COMMAND, "done\n", "", false);
5138 registry.post_terminal_transition(&task, true).unwrap();
5139
5140 assert!(registry.has_completions_for_session(Some("session")));
5141 assert!(registry.has_completions_for_session(None));
5142 assert!(!registry.has_completions_for_session(Some("other-session")));
5143
5144 let completions = registry.drain_completions_for_session(Some("session"));
5145 assert_eq!(completions.len(), 1);
5146 assert_eq!(completions[0].task_id, task.task_id);
5147 }
5148
5149 #[test]
5150 fn structured_gh_json_survives_intact_and_ignores_stderr() {
5151 let registry = BgTaskRegistry::default();
5152 let dir = tempfile::tempdir().unwrap();
5153 let calls = Arc::new(AtomicUsize::new(0));
5154 let calls_for_closure = Arc::clone(&calls);
5155 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
5156 calls_for_closure.fetch_add(1, Ordering::SeqCst);
5157 CompressionResult::new(output)
5158 });
5159 let (task_id, _task) = insert_terminal_piped_task(
5160 ®istry,
5161 &dir,
5162 "gh pr view 123 --json body",
5163 "{\"body\":\"hello\"}",
5164 "warning: stderr must not join json",
5165 true,
5166 );
5167
5168 let snapshot = registry
5169 .status(
5170 &task_id,
5171 "session",
5172 None,
5173 Some(dir.path()),
5174 RUNNING_OUTPUT_PREVIEW_BYTES,
5175 )
5176 .unwrap();
5177
5178 assert_eq!(snapshot.output_preview, "{\"body\":\"hello\"}");
5179 assert!(!snapshot.output_preview.contains("warning"));
5180 assert!(!snapshot.output_truncated);
5181 assert_eq!(
5182 calls.load(Ordering::SeqCst),
5183 0,
5184 "structured JSON bypasses compression"
5185 );
5186 }
5187
5188 #[test]
5189 fn registry_emits_single_recovery_marker_for_class_drops() {
5190 let registry = BgTaskRegistry::default();
5191 let dir = tempfile::tempdir().unwrap();
5192 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
5193 let mut dropped = BTreeMap::new();
5194 dropped.insert(DropClass::Error, 18);
5195 dropped.insert(DropClass::Warning, 6);
5196 CompressionResult::with_class_drops("kept diagnostic", dropped)
5197 });
5198 let (task_id, task) =
5199 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
5200
5201 let snapshot = registry
5202 .status(
5203 &task_id,
5204 "session",
5205 None,
5206 Some(dir.path()),
5207 RUNNING_OUTPUT_PREVIEW_BYTES,
5208 )
5209 .unwrap();
5210
5211 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
5212 assert!(snapshot.output_preview.contains("+18 more errors"));
5213 assert!(snapshot.output_preview.contains("+6 more warnings"));
5214 assert!(snapshot
5215 .output_preview
5216 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
5217 assert!(!snapshot.output_preview.contains("tail -n +"));
5218 assert!(snapshot.output_truncated);
5219 }
5220
5221 #[test]
5222 fn registry_marker_reports_semantic_and_byte_drops_once() {
5223 let registry = BgTaskRegistry::default();
5224 let dir = tempfile::tempdir().unwrap();
5225 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
5226 let mut dropped = BTreeMap::new();
5227 dropped.insert(DropClass::Error, 1);
5228 CompressionResult::with_class_drops(
5229 format!("HEAD-SIGNAL\n{}TAIL-SIGNAL", "middle\n".repeat(8_000)),
5230 dropped,
5231 )
5232 });
5233 let (task_id, _task) =
5234 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
5235
5236 let snapshot = registry
5237 .status(
5238 &task_id,
5239 "session",
5240 None,
5241 Some(dir.path()),
5242 RUNNING_OUTPUT_PREVIEW_BYTES,
5243 )
5244 .unwrap();
5245
5246 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
5247 assert!(snapshot.output_preview.contains("+1 more error"));
5248 assert!(snapshot.output_preview.contains("truncated output"));
5249 assert!(snapshot.output_preview.contains("HEAD-SIGNAL"));
5250 assert!(snapshot.output_preview.contains("TAIL-SIGNAL"));
5251 assert!(!snapshot.output_preview.contains("...<truncated"));
5252 assert!(snapshot.output_truncated);
5253 }
5254
5255 #[test]
5256 fn cargo_stderr_class_drops_name_both_capture_paths() {
5257 let registry = BgTaskRegistry::default();
5258 let dir = tempfile::tempdir().unwrap();
5259 let filter_registry = crate::compress::toml_filter::FilterRegistry::default();
5260 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
5261 crate::compress::compress_with_registry_exit_code(
5262 command,
5263 &output,
5264 exit_code,
5265 &filter_registry,
5266 )
5267 });
5268 let stderr = (0..22)
5269 .map(|index| {
5270 format!(
5271 "error: cargo failure {index}\n --> src/lib.rs:{}:1\n |\n{} | boom\n",
5272 index + 1,
5273 index + 1
5274 )
5275 })
5276 .collect::<Vec<_>>()
5277 .join("\n");
5278 let (task_id, task) = insert_terminal_piped_task(
5279 ®istry,
5280 &dir,
5281 "cargo check",
5282 "Finished dev [unoptimized] target(s) in 0.01s\n",
5283 &stderr,
5284 true,
5285 );
5286
5287 let snapshot = registry
5288 .status(
5289 &task_id,
5290 "session",
5291 None,
5292 Some(dir.path()),
5293 RUNNING_OUTPUT_PREVIEW_BYTES,
5294 )
5295 .unwrap();
5296
5297 assert!(snapshot.output_preview.contains("+2 more errors"));
5298 assert!(snapshot
5299 .output_preview
5300 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
5301 assert!(snapshot
5302 .output_preview
5303 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
5304 assert!(!snapshot.output_preview.contains("tail -n +"));
5305 }
5306
5307 #[test]
5308 fn over_ceiling_structured_json_uses_pointer_not_partial_json() {
5309 let registry = BgTaskRegistry::default();
5310 let dir = tempfile::tempdir().unwrap();
5311 let body = format!("{{\"body\":\"{}\"}}", "x".repeat(60 * 1024));
5312 let (task_id, task) = insert_terminal_piped_task(
5313 ®istry,
5314 &dir,
5315 "cd /repo && gh pr view 123 --json body",
5316 &body,
5317 "",
5318 true,
5319 );
5320
5321 let snapshot = registry
5322 .status(
5323 &task_id,
5324 "session",
5325 None,
5326 Some(dir.path()),
5327 RUNNING_OUTPUT_PREVIEW_BYTES,
5328 )
5329 .unwrap();
5330
5331 assert!(snapshot.output_preview.starts_with("[JSON output "));
5332 assert!(snapshot
5333 .output_preview
5334 .contains(&task.paths.stdout.display().to_string()));
5335 assert!(!snapshot.output_preview.contains(&"x".repeat(1024)));
5336 assert!(snapshot.output_truncated);
5337 }
5338
5339 #[test]
5340 fn toml_strip_tail_cap_uses_full_output_hint_not_offset_hint() {
5341 let registry = BgTaskRegistry::default();
5342 let dir = tempfile::tempdir().unwrap();
5343 let filter_registry = crate::compress::toml_filter::build_registry(
5344 crate::compress::builtin_filters::ALL,
5345 None,
5346 None,
5347 );
5348 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
5349 crate::compress::compress_with_registry_exit_code(
5350 command,
5351 &output,
5352 exit_code,
5353 &filter_registry,
5354 )
5355 });
5356 let stdout = format!(
5357 "make[1]: Entering directory `/tmp`\n{}",
5358 (0..100)
5359 .map(|index| format!("compile line {index}"))
5360 .collect::<Vec<_>>()
5361 .join("\n")
5362 );
5363 let (task_id, task) =
5364 insert_terminal_piped_task(®istry, &dir, "make all", &stdout, "", true);
5365
5366 let snapshot = registry
5367 .status(
5368 &task_id,
5369 "session",
5370 None,
5371 Some(dir.path()),
5372 RUNNING_OUTPUT_PREVIEW_BYTES,
5373 )
5374 .unwrap();
5375
5376 assert!(snapshot.output_preview.contains("compile line 99"));
5377 assert!(snapshot.output_preview.contains(&format!(
5378 "full output: read \"{}\"",
5379 task.paths.stdout.display()
5380 )));
5381 assert!(!snapshot
5382 .output_preview
5383 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
5384 assert!(!snapshot.output_preview.contains("tail -n +"));
5385 }
5386
5387 #[test]
5388 fn compressed_false_raw_passthrough_uses_wider_head_tail_cap() {
5389 let registry = BgTaskRegistry::default();
5390 let dir = tempfile::tempdir().unwrap();
5391 let output = format!("RAW-HEAD\n{}RAW-TAIL\n", "raw-middle\n".repeat(8_000));
5392 let (task_id, task) =
5393 insert_terminal_piped_task(®istry, &dir, "cat raw.log", &output, "RAW-ERR\n", false);
5394
5395 let snapshot = registry
5396 .status(
5397 &task_id,
5398 "session",
5399 None,
5400 Some(dir.path()),
5401 RUNNING_OUTPUT_PREVIEW_BYTES,
5402 )
5403 .unwrap();
5404
5405 assert!(snapshot.output_preview.contains("RAW-HEAD"));
5406 assert!(snapshot.output_preview.contains("RAW-TAIL"));
5407 assert!(snapshot.output_preview.contains("truncated output"));
5408 assert!(snapshot
5409 .output_preview
5410 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
5411 assert!(snapshot
5412 .output_preview
5413 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
5414 assert!(!snapshot.output_preview.contains("tail -n +"));
5415 assert!(snapshot.output_preview.len() > 16 * 1024);
5416 assert!(snapshot.output_truncated);
5417 }
5418
5419 #[test]
5420 fn pty_terminal_snapshot_bypasses_line_compression() {
5421 let registry = BgTaskRegistry::default();
5422 let dir = tempfile::tempdir().unwrap();
5423 let calls = Arc::new(AtomicUsize::new(0));
5424 let calls_for_closure = Arc::clone(&calls);
5425 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
5426 calls_for_closure.fetch_add(1, Ordering::SeqCst);
5427 CompressionResult::new(output)
5428 });
5429 let (task_id, _task) = insert_terminal_pty_task(®istry, &dir, "raw\u{1b}[31m pty bytes");
5430
5431 let snapshot = registry
5432 .status(
5433 &task_id,
5434 "session",
5435 None,
5436 Some(dir.path()),
5437 RUNNING_OUTPUT_PREVIEW_BYTES,
5438 )
5439 .unwrap();
5440
5441 assert_eq!(snapshot.info.mode, BgMode::Pty);
5442 assert_eq!(snapshot.output_preview, "");
5443 assert_eq!(calls.load(Ordering::SeqCst), 0);
5444 }
5445
5446 #[test]
5447 fn pty_dimensions_are_persisted_and_returned_in_snapshot() {
5448 let registry = BgTaskRegistry::default();
5449 let dir = tempfile::tempdir().unwrap();
5450 let task_id = registry
5451 .spawn_pty(
5452 SpawnPlan::Unsandboxed,
5453 QUICK_SUCCESS_COMMAND,
5454 "session".to_string(),
5455 dir.path().to_path_buf(),
5456 HashMap::new(),
5457 Some(Duration::from_secs(30)),
5458 dir.path().to_path_buf(),
5459 10,
5460 true,
5461 false,
5462 Some(dir.path().to_path_buf()),
5463 50,
5464 120,
5465 )
5466 .unwrap();
5467
5468 let resolved =
5469 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
5470 let metadata = read_task_at(&resolved).unwrap();
5471 assert_eq!(
5472 metadata.schema_version,
5473 crate::bash_background::persistence::SCHEMA_VERSION
5474 );
5475 assert_eq!(metadata.mode, BgMode::Pty);
5476 assert_eq!(metadata.pty_rows, Some(50));
5477 assert_eq!(metadata.pty_cols, Some(120));
5478
5479 let snapshot = registry
5480 .status(&task_id, "session", None, Some(dir.path()), 1024)
5481 .unwrap();
5482 assert_eq!(snapshot.pty_rows, Some(50));
5483 assert_eq!(snapshot.pty_cols, Some(120));
5484 }
5485
5486 fn spawn_dead_child() -> std::process::Child {
5491 #[cfg(unix)]
5492 let mut cmd = std::process::Command::new("true");
5493 #[cfg(windows)]
5494 let mut cmd = {
5495 let mut c = std::process::Command::new("cmd");
5496 c.args(["/c", "exit", "0"]);
5497 c
5498 };
5499 cmd.stdin(std::process::Stdio::null());
5500 cmd.stdout(std::process::Stdio::null());
5501 cmd.stderr(std::process::Stdio::null());
5502 let mut child = cmd.spawn().expect("spawn replacement child for reap test");
5503 let started = Instant::now();
5512 loop {
5513 match child.try_wait() {
5514 Ok(Some(_)) => break,
5515 Ok(None) => {
5516 if started.elapsed() > Duration::from_secs(5) {
5517 panic!("dead-child stand-in did not exit within 5s");
5518 }
5519 std::thread::sleep(Duration::from_millis(10));
5520 }
5521 Err(error) => panic!("dead-child try_wait failed: {error}"),
5522 }
5523 }
5524 child
5525 }
5526
5527 #[test]
5528 fn ack_marks_delivered_even_when_completion_was_already_consumed_locally() {
5529 let registry = BgTaskRegistry::default();
5530 let dir = tempfile::tempdir().unwrap();
5531 let task_id = registry
5532 .spawn(
5533 SpawnPlan::Unsandboxed,
5534 LONG_RUNNING_COMMAND,
5535 "session".to_string(),
5536 dir.path().to_path_buf(),
5537 HashMap::new(),
5538 Some(Duration::from_secs(30)),
5539 dir.path().to_path_buf(),
5540 10,
5541 true,
5542 false,
5543 Some(dir.path().to_path_buf()),
5544 )
5545 .unwrap();
5546 registry
5547 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
5548 .unwrap();
5549 assert_eq!(
5550 registry
5551 .drain_completions_for_session(Some("session"))
5552 .len(),
5553 1
5554 );
5555
5556 registry.inner.completions.lock().unwrap().clear();
5559
5560 assert_eq!(
5561 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
5562 vec![task_id.clone()]
5563 );
5564 assert!(registry
5565 .drain_completions_for_session(Some("session"))
5566 .is_empty());
5567
5568 let resolved =
5569 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
5570 let metadata = read_task_at(&resolved).unwrap();
5571 assert!(metadata.completion_delivered);
5572
5573 let replayed = BgTaskRegistry::default();
5574 replayed
5575 .replay_session_inner(dir.path(), "session", None)
5576 .unwrap();
5577 assert!(replayed
5578 .drain_completions_for_session(Some("session"))
5579 .is_empty());
5580 }
5581
5582 #[test]
5583 fn register_watch_rejects_unknown_task() {
5584 let registry = BgTaskRegistry::default();
5585
5586 let result = registry.register_watch(
5587 "missing-task".to_string(),
5588 WatchPattern::Substring("READY".into()),
5589 true,
5590 );
5591
5592 assert_eq!(result, Err("task_not_found"));
5593 }
5594
5595 #[test]
5596 fn register_watch_on_terminal_task_scans_existing_output() {
5597 let frames = Arc::new(Mutex::new(Vec::new()));
5598 let captured = Arc::clone(&frames);
5599 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
5600 captured.lock().unwrap().push(frame);
5601 })
5602 as Box<dyn Fn(PushFrame) + Send + Sync>);
5603 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
5604 let dir = tempfile::tempdir().unwrap();
5605 let task_id = registry
5606 .spawn(
5607 SpawnPlan::Unsandboxed,
5608 LONG_RUNNING_COMMAND,
5609 "session".to_string(),
5610 dir.path().to_path_buf(),
5611 HashMap::new(),
5612 Some(Duration::from_secs(30)),
5613 dir.path().to_path_buf(),
5614 10,
5615 true,
5616 false,
5617 Some(dir.path().to_path_buf()),
5618 )
5619 .unwrap();
5620 registry
5621 .inner
5622 .shutdown
5623 .store(true, std::sync::atomic::Ordering::SeqCst);
5624 let task = registry.task_for_session(&task_id, "session").unwrap();
5625 std::fs::write(&task.paths.stdout, "READY\n").unwrap();
5626 registry
5627 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
5628 .unwrap();
5629 frames.lock().unwrap().clear();
5630 registry.inner.completions.lock().unwrap().clear();
5631
5632 registry
5633 .register_watch(
5634 task_id.clone(),
5635 WatchPattern::Substring("READY".into()),
5636 true,
5637 )
5638 .unwrap();
5639
5640 let frames = frames.lock().unwrap();
5641 let frame = frames
5642 .iter()
5643 .find_map(|frame| match frame {
5644 PushFrame::BashPatternMatch(frame) => Some(frame),
5645 _ => None,
5646 })
5647 .expect("terminal watch registration should emit pattern frame");
5648 assert_eq!(frame.reason, "pattern_match");
5649 assert_eq!(frame.task_id, task_id);
5650 assert_eq!(frame.session_id, "session");
5651 assert_eq!(frame.match_text, "READY");
5652 assert_eq!(frame.match_offset, 0);
5653 assert_eq!(registry.active_watch_count(&frame.task_id), 0);
5654 let metadata = read_task(&task.paths.json).unwrap();
5655 assert!(metadata.completion_delivered);
5656 }
5657
5658 #[test]
5659 fn cleanup_finished_removes_terminal_tasks_older_than_threshold() {
5660 let registry = BgTaskRegistry::default();
5661 let dir = tempfile::tempdir().unwrap();
5662 let task_id = registry
5663 .spawn(
5664 SpawnPlan::Unsandboxed,
5665 QUICK_SUCCESS_COMMAND,
5666 "session".to_string(),
5667 dir.path().to_path_buf(),
5668 HashMap::new(),
5669 Some(Duration::from_secs(30)),
5670 dir.path().to_path_buf(),
5671 10,
5672 true,
5673 false,
5674 Some(dir.path().to_path_buf()),
5675 )
5676 .unwrap();
5677 registry
5678 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
5679 .unwrap();
5680 let completions = registry.drain_completions_for_session(Some("session"));
5681 assert_eq!(completions.len(), 1);
5682 assert_eq!(
5683 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
5684 vec![task_id.clone()]
5685 );
5686
5687 registry.cleanup_finished(Duration::ZERO);
5688
5689 assert!(registry.inner.tasks.lock().unwrap().is_empty());
5690 }
5691
5692 #[test]
5693 fn cleanup_finished_retains_undelivered_terminals() {
5694 let registry = BgTaskRegistry::default();
5695 let dir = tempfile::tempdir().unwrap();
5696 let task_id = registry
5697 .spawn(
5698 SpawnPlan::Unsandboxed,
5699 QUICK_SUCCESS_COMMAND,
5700 "session".to_string(),
5701 dir.path().to_path_buf(),
5702 HashMap::new(),
5703 Some(Duration::from_secs(30)),
5704 dir.path().to_path_buf(),
5705 10,
5706 true,
5707 false,
5708 Some(dir.path().to_path_buf()),
5709 )
5710 .unwrap();
5711 registry
5712 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
5713 .unwrap();
5714
5715 registry.cleanup_finished(Duration::ZERO);
5716
5717 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
5718 }
5719
5720 #[test]
5728 fn reap_child_marks_failed_when_child_exits_without_exit_marker() {
5729 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5730 let dir = tempfile::tempdir().unwrap();
5731 let task_id = registry
5732 .spawn(
5733 SpawnPlan::Unsandboxed,
5734 QUICK_SUCCESS_COMMAND,
5735 "session".to_string(),
5736 dir.path().to_path_buf(),
5737 HashMap::new(),
5738 Some(Duration::from_secs(30)),
5739 dir.path().to_path_buf(),
5740 10,
5741 true,
5742 false,
5743 Some(dir.path().to_path_buf()),
5744 )
5745 .unwrap();
5746
5747 let task = registry.task_for_session(&task_id, "session").unwrap();
5748
5749 let started = Instant::now();
5754 loop {
5755 let exited = {
5756 let mut state = task.state.lock().unwrap();
5757 match &mut state.runtime {
5758 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
5759 _ => true,
5760 }
5761 };
5762 if exited {
5763 break;
5764 }
5765 assert!(
5766 started.elapsed() < Duration::from_secs(5),
5767 "child should exit quickly"
5768 );
5769 std::thread::sleep(Duration::from_millis(20));
5770 }
5771
5772 registry
5780 .inner
5781 .shutdown
5782 .store(true, std::sync::atomic::Ordering::SeqCst);
5783 std::thread::sleep(Duration::from_millis(550));
5787
5788 let _ = std::fs::remove_file(&task.paths.exit);
5791
5792 {
5807 let mut state = task.state.lock().unwrap();
5808 state.metadata.status = BgTaskStatus::Running;
5809 state.metadata.status_reason = None;
5810 state.metadata.exit_code = None;
5811 state.metadata.finished_at = None;
5812 state.metadata.duration_ms = None;
5813 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
5816 .expect("persist reset Running metadata for reap_child test");
5817 if matches!(state.runtime, TaskRuntime::Piped(None)) {
5821 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
5822 }
5823 }
5824 *task.terminal_at.lock().unwrap() = None;
5827
5828 assert!(
5831 task.is_running(),
5832 "precondition: metadata.status == Running"
5833 );
5834 assert!(
5835 !task.paths.exit.exists(),
5836 "precondition: exit marker absent"
5837 );
5838
5839 registry.reap_child(&task);
5844
5845 {
5846 let state = task.state.lock().unwrap();
5847 assert_eq!(
5848 state.metadata.status,
5849 BgTaskStatus::Running,
5850 "first reap must leave status Running while waiting one pass for marker"
5851 );
5852 assert_eq!(
5853 state.metadata.status_reason, None,
5854 "first reap must not record a failure reason"
5855 );
5856 assert!(
5857 matches!(state.runtime, TaskRuntime::Piped(None)),
5858 "child handle must be released after first reap"
5859 );
5860 assert!(
5861 state.detached,
5862 "task must be marked detached after first reap"
5863 );
5864 }
5865
5866 registry.reap_child(&task);
5870
5871 let state = task.state.lock().unwrap();
5872 assert!(
5873 state.metadata.status.is_terminal(),
5874 "second reap must transition to terminal when PID dead and no marker. Got status={:?}",
5875 state.metadata.status
5876 );
5877 assert_eq!(
5878 state.metadata.status,
5879 BgTaskStatus::Failed,
5880 "must specifically be Failed (not Killed): status={:?}",
5881 state.metadata.status
5882 );
5883 assert_eq!(
5884 state.metadata.status_reason.as_deref(),
5885 Some("process exited without exit marker"),
5886 "reason must match replay path's wording: {:?}",
5887 state.metadata.status_reason
5888 );
5889 assert!(
5890 matches!(state.runtime, TaskRuntime::Piped(None)),
5891 "child handle must stay released after second reap"
5892 );
5893 assert!(
5894 state.detached,
5895 "task must remain detached after second reap"
5896 );
5897 }
5898
5899 #[test]
5904 fn reap_child_preserves_running_when_exit_marker_exists() {
5905 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5906 let dir = tempfile::tempdir().unwrap();
5907 let task_id = registry
5908 .spawn(
5909 SpawnPlan::Unsandboxed,
5910 QUICK_SUCCESS_COMMAND,
5911 "session".to_string(),
5912 dir.path().to_path_buf(),
5913 HashMap::new(),
5914 Some(Duration::from_secs(30)),
5915 dir.path().to_path_buf(),
5916 10,
5917 true,
5918 false,
5919 Some(dir.path().to_path_buf()),
5920 )
5921 .unwrap();
5922
5923 let task = registry.task_for_session(&task_id, "session").unwrap();
5924
5925 let started = Instant::now();
5928 loop {
5929 let exited = {
5930 let mut state = task.state.lock().unwrap();
5931 match &mut state.runtime {
5932 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
5933 _ => true,
5934 }
5935 };
5936 if exited && task.paths.exit.exists() {
5937 break;
5938 }
5939 assert!(
5940 started.elapsed() < Duration::from_secs(5),
5941 "child should exit and write marker quickly"
5942 );
5943 std::thread::sleep(Duration::from_millis(20));
5944 }
5945
5946 registry
5952 .inner
5953 .shutdown
5954 .store(true, std::sync::atomic::Ordering::SeqCst);
5955 std::thread::sleep(Duration::from_millis(550));
5956
5957 {
5963 let mut state = task.state.lock().unwrap();
5964 state.metadata.status = BgTaskStatus::Running;
5965 state.metadata.status_reason = None;
5966 if matches!(state.runtime, TaskRuntime::Piped(None)) {
5967 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
5968 }
5969 }
5970 *task.terminal_at.lock().unwrap() = None;
5971 if !task.paths.exit.exists() {
5974 std::fs::write(&task.paths.exit, "0").expect("write replacement exit marker");
5975 }
5976
5977 registry.reap_child(&task);
5981
5982 let state = task.state.lock().unwrap();
5983 assert!(
5984 matches!(state.runtime, TaskRuntime::Piped(None)),
5985 "child handle still released even when marker exists"
5986 );
5987 assert!(
5988 state.detached,
5989 "task still marked detached even when marker exists"
5990 );
5991 assert_eq!(
5996 state.metadata.status,
5997 BgTaskStatus::Running,
5998 "reap_child must defer to poll_task when marker exists"
5999 );
6000 }
6001
6002 #[cfg(unix)]
6006 fn pid_stat(pid: u32) -> Option<String> {
6007 let output = std::process::Command::new("ps")
6008 .args(["-o", "stat=", "-p", &pid.to_string()])
6009 .output()
6010 .ok()?;
6011 if !output.status.success() {
6012 return None;
6013 }
6014 let stat = String::from_utf8_lossy(&output.stdout).trim().to_string();
6015 if stat.is_empty() {
6016 None
6017 } else {
6018 Some(stat)
6019 }
6020 }
6021
6022 #[cfg(unix)]
6024 fn is_zombie(pid: u32) -> bool {
6025 pid_stat(pid).is_some_and(|stat| stat.starts_with('Z'))
6026 }
6027
6028 #[cfg(unix)]
6034 fn spawn_unreaped_zombie() -> std::process::Child {
6035 let child = std::process::Command::new("true")
6036 .stdin(std::process::Stdio::null())
6037 .stdout(std::process::Stdio::null())
6038 .stderr(std::process::Stdio::null())
6039 .spawn()
6040 .expect("spawn zombie stand-in");
6041 let pid = child.id();
6042 let started = Instant::now();
6043 while !is_zombie(pid) {
6044 assert!(
6045 started.elapsed() < Duration::from_secs(5),
6046 "stand-in child should become a zombie within 5s"
6047 );
6048 std::thread::sleep(Duration::from_millis(10));
6049 }
6050 child
6052 }
6053
6054 #[cfg(unix)]
6064 #[test]
6065 fn finalize_from_marker_reaps_child_no_zombie() {
6066 use std::sync::atomic::Ordering;
6067
6068 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6069 let dir = tempfile::tempdir().unwrap();
6070 let task_id = registry
6071 .spawn(
6072 SpawnPlan::Unsandboxed,
6073 QUICK_SUCCESS_COMMAND,
6074 "session".to_string(),
6075 dir.path().to_path_buf(),
6076 HashMap::new(),
6077 Some(Duration::from_secs(30)),
6078 dir.path().to_path_buf(),
6079 10,
6080 true,
6081 false,
6082 Some(dir.path().to_path_buf()),
6083 )
6084 .unwrap();
6085
6086 registry.inner.shutdown.store(true, Ordering::SeqCst);
6090 std::thread::sleep(Duration::from_millis(550));
6091
6092 let task = registry.task_for_session(&task_id, "session").unwrap();
6093
6094 let started = Instant::now();
6098 while !task.paths.exit.exists() {
6099 assert!(
6100 started.elapsed() < Duration::from_secs(5),
6101 "exit marker should land quickly for `true`"
6102 );
6103 std::thread::sleep(Duration::from_millis(20));
6104 }
6105
6106 let zombie_pid;
6112 {
6113 let mut state = task.state.lock().unwrap();
6114 state.metadata.status = BgTaskStatus::Running;
6115 state.metadata.status_reason = None;
6116 state.metadata.exit_code = None;
6117 state.metadata.finished_at = None;
6118 state.metadata.duration_ms = None;
6119 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
6120 .expect("persist reset Running metadata");
6121 let zombie = spawn_unreaped_zombie();
6122 zombie_pid = zombie.id();
6123 state.runtime = TaskRuntime::Piped(Some(zombie));
6124 }
6125 *task.terminal_at.lock().unwrap() = None;
6126
6127 assert!(
6129 is_zombie(zombie_pid),
6130 "precondition: stand-in child {zombie_pid} must be a zombie before finalize"
6131 );
6132
6133 registry.poll_task(&task).unwrap();
6136
6137 {
6138 let state = task.state.lock().unwrap();
6139 assert!(
6140 matches!(state.runtime, TaskRuntime::Piped(None)),
6141 "child handle must be released after marker finalize"
6142 );
6143 assert!(
6144 state.metadata.status.is_terminal(),
6145 "task must be terminal after marker finalize: {:?}",
6146 state.metadata.status
6147 );
6148 }
6149
6150 assert!(
6153 !is_zombie(zombie_pid),
6154 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
6155 after the exit-marker terminal transition"
6156 );
6157 }
6158
6159 #[cfg(unix)]
6163 #[test]
6164 fn kill_with_existing_marker_reaps_child_no_zombie() {
6165 use std::sync::atomic::Ordering;
6166
6167 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6168 let dir = tempfile::tempdir().unwrap();
6169 let task_id = registry
6170 .spawn(
6171 SpawnPlan::Unsandboxed,
6172 QUICK_SUCCESS_COMMAND,
6173 "session".to_string(),
6174 dir.path().to_path_buf(),
6175 HashMap::new(),
6176 Some(Duration::from_secs(30)),
6177 dir.path().to_path_buf(),
6178 10,
6179 true,
6180 false,
6181 Some(dir.path().to_path_buf()),
6182 )
6183 .unwrap();
6184
6185 registry.inner.shutdown.store(true, Ordering::SeqCst);
6186 std::thread::sleep(Duration::from_millis(550));
6187
6188 let task = registry.task_for_session(&task_id, "session").unwrap();
6189
6190 let started = Instant::now();
6191 while !task.paths.exit.exists() {
6192 assert!(
6193 started.elapsed() < Duration::from_secs(5),
6194 "exit marker should land quickly for `true`"
6195 );
6196 std::thread::sleep(Duration::from_millis(20));
6197 }
6198
6199 let zombie_pid;
6200 {
6201 let mut state = task.state.lock().unwrap();
6202 state.metadata.status = BgTaskStatus::Running;
6203 state.metadata.status_reason = None;
6204 state.metadata.exit_code = None;
6205 state.metadata.finished_at = None;
6206 state.metadata.duration_ms = None;
6207 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
6208 .expect("persist reset Running metadata");
6209 let zombie = spawn_unreaped_zombie();
6210 zombie_pid = zombie.id();
6211 state.runtime = TaskRuntime::Piped(Some(zombie));
6212 }
6213 *task.terminal_at.lock().unwrap() = None;
6214
6215 assert!(
6216 is_zombie(zombie_pid),
6217 "precondition: stand-in child {zombie_pid} must be a zombie before kill"
6218 );
6219
6220 registry
6222 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6223 .expect("kill should succeed");
6224
6225 {
6226 let state = task.state.lock().unwrap();
6227 assert!(
6228 matches!(state.runtime, TaskRuntime::Piped(None)),
6229 "child handle must be released after marker-aware kill"
6230 );
6231 assert!(state.metadata.status.is_terminal());
6232 }
6233
6234 assert!(
6235 !is_zombie(zombie_pid),
6236 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
6237 after a marker-aware kill"
6238 );
6239 }
6240
6241 #[test]
6242 fn cleanup_finished_keeps_running_tasks() {
6243 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6244 let dir = tempfile::tempdir().unwrap();
6245 let task_id = registry
6246 .spawn(
6247 SpawnPlan::Unsandboxed,
6248 LONG_RUNNING_COMMAND,
6249 "session".to_string(),
6250 dir.path().to_path_buf(),
6251 HashMap::new(),
6252 Some(Duration::from_secs(30)),
6253 dir.path().to_path_buf(),
6254 10,
6255 true,
6256 false,
6257 Some(dir.path().to_path_buf()),
6258 )
6259 .unwrap();
6260
6261 registry.cleanup_finished(Duration::ZERO);
6262
6263 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
6264 let _ = registry.kill(&task_id, "session");
6265 }
6266
6267 #[cfg(unix)]
6268 #[test]
6269 fn rehydrating_sandboxed_task_never_respawns_persisted_command() {
6270 let project = tempfile::tempdir().unwrap();
6271 let storage = tempfile::tempdir().unwrap();
6272 let sandbox_temp = storage.path().join("sandbox-temp");
6273 fs::create_dir(&sandbox_temp).unwrap();
6274 let launcher_script = project.path().join("sandbox-launch");
6275 let launcher = PathBuf::from("/bin/sh");
6276 fs::write(
6277 &launcher_script,
6278 "while [ \"$#\" -gt 0 ]; do\n if [ \"$1\" = -- ]; then\n shift\n exec \"$@\"\n fi\n shift\ndone\nexit 78\n",
6279 )
6280 .unwrap();
6281 let mut permissions = fs::metadata(&launcher_script).unwrap().permissions();
6282 permissions.set_mode(0o700);
6283 fs::set_permissions(&launcher_script, permissions).unwrap();
6284
6285 let profile = crate::sandbox_profile::SandboxProfile::build(
6286 vec![project.path().to_path_buf()],
6287 Vec::new(),
6288 Vec::new(),
6289 Vec::new(),
6290 Vec::new(),
6291 Vec::new(),
6292 Vec::new(),
6293 sandbox_temp,
6294 )
6295 .unwrap();
6296 let plan = SpawnPlan::launcher_for_test(profile, launcher);
6297 let spawn_marker = project.path().join("spawn-count");
6298 let stop_marker = project.path().join("stop-command");
6299 let quote =
6300 |path: &Path| format!("'{}'", path.display().to_string().replace('\'', "'\\''"));
6301 let command = format!(
6302 "printf 'spawn\\n' >> {}; while [ ! -e {} ]; do sleep 0.05; done",
6303 quote(&spawn_marker),
6304 quote(&stop_marker)
6305 );
6306
6307 let original = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6308 let task_id = original
6309 .spawn(
6310 plan,
6311 &command,
6312 "sandbox-rehydrate".to_string(),
6313 project.path().to_path_buf(),
6314 HashMap::new(),
6315 Some(Duration::from_secs(30)),
6316 storage.path().to_path_buf(),
6317 10,
6318 true,
6319 false,
6320 Some(project.path().to_path_buf()),
6321 )
6322 .unwrap();
6323 let started = Instant::now();
6324 while !spawn_marker.exists() {
6325 assert!(
6326 started.elapsed() < Duration::from_secs(20),
6327 "original sandboxed task did not start"
6328 );
6329 std::thread::sleep(Duration::from_millis(10));
6330 }
6331 original.detach();
6332
6333 let restarted = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6334 restarted
6335 .replay_session(storage.path(), "sandbox-rehydrate")
6336 .unwrap();
6337 let replayed = restarted
6338 .status(
6339 &task_id,
6340 "sandbox-rehydrate",
6341 Some(project.path()),
6342 Some(storage.path()),
6343 4096,
6344 )
6345 .expect("rehydrated sandbox task");
6346 assert_eq!(replayed.info.status, BgTaskStatus::Running);
6347 assert!(replayed.sandbox_native);
6348
6349 std::thread::sleep(Duration::from_millis(650));
6350 assert_eq!(
6351 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
6352 1,
6353 "registry replay must observe the persisted process without spawning its command"
6354 );
6355
6356 fs::write(&stop_marker, "stop").unwrap();
6357 let terminal = wait_for_terminal_snapshot(
6358 &restarted,
6359 &task_id,
6360 "sandbox-rehydrate",
6361 project.path(),
6362 storage.path(),
6363 );
6364 assert_eq!(terminal.info.status, BgTaskStatus::Completed);
6365 assert_eq!(
6366 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
6367 1
6368 );
6369 restarted.detach();
6370 }
6371
6372 #[cfg(windows)]
6373 fn wait_for_file(path: &Path) -> String {
6374 let started = Instant::now();
6381 loop {
6382 if let Ok(content) = fs::read_to_string(path) {
6383 if !content.trim().is_empty() {
6384 return content;
6385 }
6386 }
6387 assert!(
6388 started.elapsed() < Duration::from_secs(30),
6389 "timed out waiting for non-empty {}",
6390 path.display()
6391 );
6392 std::thread::sleep(Duration::from_millis(100));
6393 }
6394 }
6395
6396 #[cfg(windows)]
6397 fn spawn_windows_registry_command(
6398 command: &str,
6399 ) -> (BgTaskRegistry, tempfile::TempDir, String) {
6400 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6401 let dir = tempfile::tempdir().unwrap();
6402 let task_id = registry
6403 .spawn(
6404 SpawnPlan::Unsandboxed,
6405 command,
6406 "session".to_string(),
6407 dir.path().to_path_buf(),
6408 HashMap::new(),
6409 Some(Duration::from_secs(30)),
6410 dir.path().to_path_buf(),
6411 10,
6412 false,
6413 false,
6414 Some(dir.path().to_path_buf()),
6415 )
6416 .unwrap();
6417 (registry, dir, task_id)
6418 }
6419
6420 #[cfg(windows)]
6421 #[test]
6422 fn windows_spawn_writes_exit_marker_for_zero_exit() {
6423 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 0");
6424 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
6425
6426 let content = wait_for_file(&exit_path);
6427
6428 assert_eq!(content.trim(), "0");
6429 }
6430
6431 #[cfg(windows)]
6432 #[test]
6433 fn windows_spawn_writes_exit_marker_for_nonzero_exit() {
6434 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 42");
6435 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
6436
6437 let content = wait_for_file(&exit_path);
6438
6439 assert_eq!(content.trim(), "42");
6440 }
6441
6442 #[cfg(windows)]
6443 #[test]
6444 fn windows_spawn_captures_stdout_to_disk() {
6445 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c echo hello");
6446 let task = registry.task_for_session(&task_id, "session").unwrap();
6447 let stdout_path = task.paths.stdout.clone();
6448 let exit_path = task.paths.exit.clone();
6449
6450 let _ = wait_for_file(&exit_path);
6451 let stdout = fs::read_to_string(stdout_path).expect("read stdout");
6452
6453 assert!(stdout.contains("hello"), "stdout was {stdout:?}");
6454 }
6455
6456 #[cfg(windows)]
6457 #[test]
6458 fn windows_spawn_uses_pwsh_when_available() {
6459 let candidates = crate::windows_shell::shell_candidates_with(
6463 |binary| match binary {
6464 "pwsh.exe" => Some(std::path::PathBuf::from(r"C:\pwsh\pwsh.exe")),
6465 "powershell.exe" => Some(std::path::PathBuf::from(r"C:\ps\powershell.exe")),
6466 _ => None,
6467 },
6468 || None,
6469 );
6470 let shell = candidates.first().expect("at least one candidate").clone();
6471 assert_eq!(shell, crate::windows_shell::WindowsShell::Pwsh);
6472 assert_eq!(shell.binary().as_ref(), "pwsh.exe");
6473 }
6474
6475 #[cfg(windows)]
6478 #[test]
6479 fn windows_shell_cmd_wrapper_writes_marker_via_temp_rename() {
6480 let exit_path = Path::new(r"C:\Temp\bash-test.exit");
6481 let script =
6482 crate::windows_shell::WindowsShell::Cmd.wrapper_script("cmd /c exit 42", exit_path);
6483
6484 assert!(
6485 script.contains("set CODE=%ERRORLEVEL%"),
6486 "wrapper must capture the child exit code: {script}"
6487 );
6488 assert!(
6489 script.contains("exit /B %CODE%"),
6490 "wrapper must propagate the child exit code: {script}"
6491 );
6492 assert!(
6497 script.contains("bash-test.exit"),
6498 "wrapper must target the exit marker path: {script}"
6499 );
6500 assert!(
6501 script.contains("move /Y"),
6502 "wrapper must write the marker atomically via temp-file + rename: {script}"
6503 );
6504 }
6505
6506 #[cfg(windows)]
6512 #[test]
6513 fn windows_shell_cmd_bg_command_uses_minimal_cmd_flags() {
6514 use crate::windows_shell::WindowsShell;
6515 let cmd = WindowsShell::Cmd.bg_command("echo wrapped");
6516 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
6517 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
6518 assert_eq!(
6519 args_strs,
6520 vec!["/D", "/S", "/C", "echo wrapped"],
6521 "Cmd::bg_command must prepend /D /S /C"
6522 );
6523 }
6524
6525 #[cfg(windows)]
6528 #[test]
6529 fn windows_shell_pwsh_bg_command_uses_standard_args() {
6530 use crate::windows_shell::WindowsShell;
6531 let cmd = WindowsShell::Pwsh.bg_command("Get-Date");
6532 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
6533 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
6534 assert!(
6535 args_strs.contains(&"-Command"),
6536 "Pwsh::bg_command must use -Command: {args_strs:?}"
6537 );
6538 assert!(
6539 args_strs.contains(&"Get-Date"),
6540 "Pwsh::bg_command must include the user command body"
6541 );
6542 }
6543}