1use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
2use std::fs;
3use std::io::{Read, Write};
4use std::path::{Path, PathBuf};
5use std::process::{Child, Command, Stdio};
6use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
7#[cfg(unix)]
8use std::sync::OnceLock;
9use std::sync::{Arc, Mutex, RwLock};
10use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
11
12use rusqlite::Connection;
13use serde::Serialize;
14
15use crate::compress::caps::DropClass;
16use crate::compress::CompressionResult;
17use crate::context::SharedProgressSender;
18use crate::harness::Harness;
19use crate::protocol::{BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, PushFrame};
20
21#[cfg(unix)]
22use std::os::unix::process::CommandExt;
23#[cfg(windows)]
24use std::os::windows::process::CommandExt;
25
26use super::buffer::{combine_streams, BgBuffer, DiskTruncation, StreamKind, TokenCountInput};
27use super::output::{
28 cap_completion_output, cap_completion_output_with_marker, cap_final_output,
29 cap_final_output_with_marker, completion_preview_threshold, json_output_pointer, quote_path,
30 retained_json_output_pointer, COMPRESS_INPUT_CAP_BYTES, COMPRESS_INPUT_HEAD_BYTES,
31 COMPRESS_INPUT_TAIL_BYTES, FINAL_OUTPUT_CAP_BYTES, RAW_PASSTHROUGH_CAP_BYTES,
32 RAW_PASSTHROUGH_HEAD_BYTES, RAW_PASSTHROUGH_TAIL_BYTES, RUNNING_OUTPUT_PREVIEW_BYTES,
33 STRUCTURED_OUTPUT_CAP_BYTES,
34};
35use super::persistence::{
36 create_capture_file, delete_task_bundle, read_exit_marker, read_task, session_tasks_dir,
37 task_paths, unix_millis, update_task, write_kill_marker_if_absent, write_task, BgMode,
38 ExitMarker, PersistedTask, TaskPaths,
39};
40use super::process::is_process_alive;
41#[cfg(unix)]
42use super::process::terminate_pgid;
43#[cfg(windows)]
44use super::process::terminate_pid;
45use super::pty_process::spawn_pty_for_command;
46use super::pty_runtime::PtyRuntime;
47use super::watches::{PatternMatch, WatchPattern, WatchRegistry};
48use super::{BgTaskInfo, BgTaskStatus};
49const DEFAULT_BG_TIMEOUT: Duration = Duration::from_secs(30 * 60);
52const STALE_RUNNING_AFTER: Duration = Duration::from_secs(24 * 60 * 60);
53const PERSISTED_GC_GRACE: Duration = Duration::from_secs(24 * 60 * 60);
54const QUARANTINE_GC_GRACE: Duration = Duration::from_secs(30 * 24 * 60 * 60);
55
56const TOKENIZE_CAP_BYTES_PER_STREAM: usize = 128 * 1024;
57
58#[derive(Debug, Clone, Serialize)]
59pub struct BgCompletion {
60 pub task_id: String,
61 #[serde(skip_serializing)]
64 pub session_id: String,
65 pub status: BgTaskStatus,
66 pub exit_code: Option<i32>,
67 pub command: String,
68 #[serde(default, skip_serializing_if = "String::is_empty")]
74 pub output_preview: String,
75 #[serde(default, skip_serializing_if = "is_false")]
80 pub output_truncated: bool,
81 #[serde(default, skip_serializing_if = "Option::is_none")]
84 pub original_tokens: Option<u32>,
85 #[serde(default, skip_serializing_if = "Option::is_none")]
88 pub compressed_tokens: Option<u32>,
89 #[serde(default, skip_serializing_if = "is_false")]
91 pub tokens_skipped: bool,
92}
93
94fn is_false(v: &bool) -> bool {
95 !*v
96}
97
98#[derive(Debug, Clone, Serialize)]
99pub struct BgTaskSnapshot {
100 #[serde(flatten)]
101 pub info: BgTaskInfo,
102 pub exit_code: Option<i32>,
103 pub child_pid: Option<u32>,
104 pub workdir: String,
105 pub output_preview: String,
106 pub output_truncated: bool,
107 pub output_path: Option<String>,
108 pub stderr_path: Option<String>,
109 #[serde(skip_serializing_if = "Option::is_none")]
110 pub pty_rows: Option<u16>,
111 #[serde(skip_serializing_if = "Option::is_none")]
112 pub pty_cols: Option<u16>,
113 #[serde(skip_serializing_if = "Option::is_none")]
114 pub pty_screen: Option<String>,
115}
116
117#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
118pub struct BgTaskHealthCounts {
119 pub running: usize,
120 pub pending_completions: usize,
121}
122
123#[derive(Debug, Clone, Copy, PartialEq, Eq)]
124enum TerminalOutputKind {
125 Compressed,
126 Raw,
127 Structured,
128}
129
130#[derive(Debug, Clone, PartialEq, Eq)]
131struct TerminalOutputCache {
132 output_preview: String,
133 output_truncated: bool,
134 kind: TerminalOutputKind,
135 output_path: Option<String>,
136 stderr_path: Option<String>,
137 recovery: Option<RecoveryContext>,
138}
139
140#[derive(Debug, Clone, PartialEq, Eq)]
141struct RecoveryContext {
142 dropped_by_class: BTreeMap<DropClass, usize>,
143 had_inner_drop: bool,
144 offset_hint_eligible: bool,
145 offset_start_line: Option<usize>,
146 byte_truncated: bool,
147 disk_truncated_prefix_bytes: u64,
148 output_path: Option<String>,
149 stderr_path: Option<String>,
150 include_stderr_path: bool,
151}
152
153impl RecoveryContext {
154 fn has_visible_drop(&self) -> bool {
155 self.byte_truncated
156 || self.disk_truncated_prefix_bytes > 0
157 || self.had_inner_drop
158 || !self.dropped_by_class.is_empty()
159 }
160}
161
162#[derive(Clone)]
163pub struct BgTaskRegistry {
164 pub(crate) inner: Arc<RegistryInner>,
165}
166
167pub(crate) struct RegistryInner {
168 pub(crate) tasks: Mutex<HashMap<String, Arc<BgTask>>>,
169 pub(crate) completions: Mutex<VecDeque<BgCompletion>>,
170 pub(crate) progress_sender: SharedProgressSender,
171 watchdog_started: AtomicBool,
172 pub(crate) shutdown: AtomicBool,
173 pub(crate) long_running_reminder_enabled: AtomicBool,
174 pub(crate) long_running_reminder_interval_ms: AtomicU64,
175 persisted_gc_started: AtomicBool,
176 #[cfg(test)]
177 persisted_gc_runs: AtomicU64,
178 pub(crate) compressor:
184 Mutex<Option<Box<dyn Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync>>>,
185 pub(crate) db_pool: RwLock<Option<Arc<Mutex<Connection>>>>,
186 pub(crate) db_harness: RwLock<Option<String>>,
187 pub(crate) wake_tx: crossbeam_channel::Sender<()>,
188 pub(crate) wake_rx: crossbeam_channel::Receiver<()>,
189 pub(crate) watch_registry: Mutex<WatchRegistry>,
190}
191
192pub(crate) struct BgTask {
193 pub(crate) task_id: String,
194 pub(crate) session_id: String,
195 pub(crate) paths: TaskPaths,
196 pub(crate) started: Instant,
197 pub(crate) last_reminder_at: Mutex<Option<Instant>>,
198 pub(crate) terminal_at: Mutex<Option<Instant>>,
199 pub(crate) state: Mutex<BgTaskState>,
200}
201
202pub(crate) enum TaskRuntime {
203 Piped(Option<Child>),
204 Pty(Option<PtyRuntime>),
205}
206
207pub(crate) struct BgTaskState {
208 pub(crate) metadata: PersistedTask,
209 pub(crate) runtime: TaskRuntime,
210 pub(crate) detached: bool,
211 pub(crate) child_exit_observed: bool,
222 pub(crate) buffer: BgBuffer,
223 terminal_output_cache: Option<TerminalOutputCache>,
224 pub(crate) pending_terminal_override: Option<BgTaskStatus>,
226}
227
228fn completion_matches_session(completion: &BgCompletion, session_id: Option<&str>) -> bool {
229 session_id
230 .map(|session_id| completion.session_id == session_id)
231 .unwrap_or(true)
232}
233
234impl BgTaskRegistry {
235 pub fn new(progress_sender: SharedProgressSender) -> Self {
236 let (wake_tx, wake_rx) = crossbeam_channel::bounded(1);
237 Self {
238 inner: Arc::new(RegistryInner {
239 tasks: Mutex::new(HashMap::new()),
240 completions: Mutex::new(VecDeque::new()),
241 progress_sender,
242 watchdog_started: AtomicBool::new(false),
243 shutdown: AtomicBool::new(false),
244 long_running_reminder_enabled: AtomicBool::new(true),
245 long_running_reminder_interval_ms: AtomicU64::new(600_000),
246 persisted_gc_started: AtomicBool::new(false),
247 #[cfg(test)]
248 persisted_gc_runs: AtomicU64::new(0),
249 compressor: Mutex::new(None),
250 db_pool: RwLock::new(None),
251 db_harness: RwLock::new(None),
252 wake_tx,
253 wake_rx,
254 watch_registry: Mutex::new(WatchRegistry::default()),
255 }),
256 }
257 }
258
259 pub fn set_harness(&self, harness: Harness) {
260 if let Ok(mut slot) = self.inner.db_harness.write() {
261 *slot = Some(harness.storage_segment());
262 }
263 }
264
265 pub fn set_db_pool(&self, conn: Arc<Mutex<Connection>>) {
266 if let Ok(mut slot) = self.inner.db_pool.write() {
267 *slot = Some(conn);
268 }
269 }
270
271 pub fn clear_db_pool(&self) {
272 if let Ok(mut slot) = self.inner.db_pool.write() {
273 *slot = None;
274 }
275 }
276
277 pub fn set_compressor<F>(&self, compressor: F)
282 where
283 F: Fn(&str, String) -> CompressionResult + Send + Sync + 'static,
284 {
285 self.set_compressor_with_exit_code(move |command, output, _exit_code| {
286 compressor(command, output)
287 });
288 }
289
290 pub fn set_compressor_with_exit_code<F>(&self, compressor: F)
291 where
292 F: Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync + 'static,
293 {
294 if let Ok(mut slot) = self.inner.compressor.lock() {
295 *slot = Some(Box::new(compressor));
296 }
297 }
298
299 pub(crate) fn compress_output(
302 &self,
303 command: &str,
304 output: String,
305 exit_code: Option<i32>,
306 ) -> CompressionResult {
307 let Ok(slot) = self.inner.compressor.lock() else {
308 return CompressionResult::new(output);
309 };
310 match slot.as_ref() {
311 Some(compressor) => compressor(command, output, exit_code),
312 None => CompressionResult::new(output),
313 }
314 }
315
316 fn ensure_terminal_output_cache(&self, task: &Arc<BgTask>) -> Option<TerminalOutputCache> {
317 let (metadata, buffer) = {
318 let state = task.state.lock().ok()?;
319 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
320 return None;
321 }
322 if let Some(cache) = state.terminal_output_cache.clone() {
323 return Some(cache);
324 }
325 (state.metadata.clone(), state.buffer.clone())
326 };
327
328 let mut cap_buffer = buffer.clone();
329 let disk_truncation = cap_buffer.enforce_terminal_cap();
330 let cache = self.render_terminal_output(&metadata, &cap_buffer, disk_truncation);
331 let mut state = task.state.lock().ok()?;
332 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
333 return None;
334 }
335 if let Some(existing) = state.terminal_output_cache.clone() {
336 return Some(existing);
337 }
338 state.terminal_output_cache = Some(cache.clone());
339 Some(cache)
340 }
341
342 fn render_terminal_output(
343 &self,
344 metadata: &PersistedTask,
345 buffer: &BgBuffer,
346 disk_truncation: DiskTruncation,
347 ) -> TerminalOutputCache {
348 if metadata.mode == BgMode::Pty {
349 return TerminalOutputCache {
350 output_preview: String::new(),
351 output_truncated: false,
352 kind: TerminalOutputKind::Raw,
353 output_path: buffer.output_path().map(|path| path.display().to_string()),
354 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
355 recovery: None,
356 };
357 }
358
359 if let Some(structured) =
360 render_structured_output(&metadata.command, buffer, disk_truncation)
361 {
362 return structured;
363 }
364
365 if !metadata.compressed {
366 return render_raw_passthrough(buffer, disk_truncation);
367 }
368
369 let raw = buffer.read_combined_head_tail(
370 COMPRESS_INPUT_CAP_BYTES,
371 COMPRESS_INPUT_HEAD_BYTES,
372 COMPRESS_INPUT_TAIL_BYTES,
373 );
374 let compressed = self.compress_output(&metadata.command, raw.text, metadata.exit_code);
375 render_compressed_with_recovery(buffer, compressed, raw.truncated, disk_truncation)
376 }
377
378 fn snapshot_with_terminal_cache(
379 &self,
380 task: &Arc<BgTask>,
381 preview_bytes: usize,
382 ) -> BgTaskSnapshot {
383 let mut snapshot = task.snapshot(preview_bytes);
384 self.maybe_compress_snapshot(task, &mut snapshot);
385 snapshot
386 }
387
388 fn post_terminal_transition(&self, task: &Arc<BgTask>, emit_frame: bool) -> Result<(), String> {
389 let (metadata, buffer) = {
390 let state = task
391 .state
392 .lock()
393 .map_err(|_| "background task lock poisoned".to_string())?;
394 if !state.metadata.status.is_terminal() {
395 return Ok(());
396 }
397 (state.metadata.clone(), state.buffer.clone())
398 };
399
400 let cache = self.ensure_terminal_output_cache(task);
401 self.enqueue_completion_from_parts(
402 &metadata,
403 Some(&buffer),
404 None,
405 emit_frame,
406 cache.as_ref(),
407 );
408 Ok(())
409 }
410
411 fn persist_task(&self, paths: &TaskPaths, metadata: &PersistedTask) -> std::io::Result<()> {
412 write_task(&paths.json, metadata)?;
413 self.dual_write_task(paths, metadata);
414 Ok(())
415 }
416
417 fn update_task_metadata<F>(
418 &self,
419 paths: &TaskPaths,
420 update: F,
421 ) -> std::io::Result<PersistedTask>
422 where
423 F: FnOnce(&mut PersistedTask),
424 {
425 let metadata = update_task(&paths.json, update)?;
426 self.dual_write_task(paths, &metadata);
427 Ok(metadata)
428 }
429
430 fn dual_write_task(&self, paths: &TaskPaths, metadata: &PersistedTask) {
431 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
432 let Some(pool) = pool else {
433 return;
434 };
435 let harness = self
436 .inner
437 .db_harness
438 .read()
439 .ok()
440 .and_then(|slot| slot.clone());
441 let Some(harness) = harness else {
442 crate::slog_warn!(
443 "dual-write bash_task to DB skipped for {}: harness not configured",
444 metadata.task_id
445 );
446 return;
447 };
448 let row = match metadata.to_bash_task_row(&harness, paths) {
449 Ok(row) => row,
450 Err(error) => {
451 crate::slog_warn!(
452 "dual-write bash_task to DB failed for {}: {}",
453 metadata.task_id,
454 error
455 );
456 return;
457 }
458 };
459 let conn = match pool.lock() {
460 Ok(conn) => conn,
461 Err(_) => {
462 crate::slog_warn!(
463 "dual-write bash_task to DB failed for {}: db mutex poisoned",
464 metadata.task_id
465 );
466 return;
467 }
468 };
469 if let Err(error) = crate::db::bash_tasks::upsert_bash_task(&conn, &row) {
470 crate::slog_warn!(
471 "dual-write bash_task to DB failed for {}: {}",
472 metadata.task_id,
473 error
474 );
475 }
476 }
477
478 pub fn configure_long_running_reminders(&self, enabled: bool, interval_ms: u64) {
479 self.inner
480 .long_running_reminder_enabled
481 .store(enabled, Ordering::SeqCst);
482 self.inner
483 .long_running_reminder_interval_ms
484 .store(interval_ms, Ordering::SeqCst);
485 }
486
487 #[cfg(unix)]
488 #[allow(clippy::too_many_arguments)]
489 pub fn spawn(
490 &self,
491 command: &str,
492 session_id: String,
493 workdir: PathBuf,
494 env: HashMap<String, String>,
495 timeout: Option<Duration>,
496 storage_dir: PathBuf,
497 max_running: usize,
498 notify_on_completion: bool,
499 compressed: bool,
500 project_root: Option<PathBuf>,
501 ) -> Result<String, String> {
502 self.start_watchdog();
503
504 let running = self.running_count();
505 if running >= max_running {
506 return Err(format!(
507 "background bash task limit exceeded: {running} running (max {max_running})"
508 ));
509 }
510
511 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
512 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
513 let task_id = self.generate_unique_task_id()?;
514 let paths = task_paths(&storage_dir, &session_id, &task_id);
515 fs::create_dir_all(&paths.dir)
516 .map_err(|e| format!("failed to create background task dir: {e}"))?;
517
518 let mut metadata = PersistedTask::starting(
519 task_id.clone(),
520 session_id.clone(),
521 command.to_string(),
522 workdir.clone(),
523 project_root,
524 timeout_ms,
525 notify_on_completion,
526 compressed,
527 );
528 self.persist_task(&paths, &metadata)
529 .map_err(|e| format!("failed to persist background task metadata: {e}"))?;
530
531 create_capture_file(&paths.stdout)
535 .map_err(|e| format!("failed to create stdout capture file: {e}"))?;
536 create_capture_file(&paths.stderr)
537 .map_err(|e| format!("failed to create stderr capture file: {e}"))?;
538
539 let child = match spawn_detached_child(command, &paths, &workdir, &env) {
540 Ok(child) => child,
541 Err(error) => {
542 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
543 let _ = delete_task_bundle(&paths);
544 return Err(error);
545 }
546 };
547
548 let child_pid = child.id();
549 metadata.mark_running(child_pid, child_pid as i32);
550 self.persist_task(&paths, &metadata)
551 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
552
553 let task = Arc::new(BgTask {
554 task_id: task_id.clone(),
555 session_id,
556 paths: paths.clone(),
557 started: Instant::now(),
558 last_reminder_at: Mutex::new(None),
559 terminal_at: Mutex::new(None),
560 state: Mutex::new(BgTaskState {
561 metadata,
562 runtime: TaskRuntime::Piped(Some(child)),
563 detached: false,
564 child_exit_observed: false,
565 buffer: BgBuffer::new(paths.stdout.clone(), paths.stderr.clone()),
566 terminal_output_cache: None,
567 pending_terminal_override: None,
568 }),
569 });
570
571 self.inner
572 .tasks
573 .lock()
574 .map_err(|_| "background task registry lock poisoned".to_string())?
575 .insert(task_id.clone(), task);
576
577 Ok(task_id)
578 }
579
580 #[allow(clippy::too_many_arguments)]
581 pub fn spawn_pty(
582 &self,
583 command: &str,
584 session_id: String,
585 workdir: PathBuf,
586 env: HashMap<String, String>,
587 timeout: Option<Duration>,
588 storage_dir: PathBuf,
589 max_running: usize,
590 notify_on_completion: bool,
591 compressed: bool,
592 project_root: Option<PathBuf>,
593 rows: u16,
594 cols: u16,
595 ) -> Result<String, String> {
596 self.start_watchdog();
597
598 let running = self.running_count();
599 if running >= max_running {
600 return Err(format!(
601 "background bash task limit exceeded: {running} running (max {max_running})"
602 ));
603 }
604
605 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
606 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
607 let task_id = self.generate_unique_task_id()?;
608 let paths = task_paths(&storage_dir, &session_id, &task_id);
609 fs::create_dir_all(&paths.dir)
610 .map_err(|e| format!("failed to create background task dir: {e}"))?;
611
612 let mut metadata = PersistedTask::starting(
613 task_id.clone(),
614 session_id.clone(),
615 command.to_string(),
616 workdir.clone(),
617 project_root,
618 timeout_ms,
619 notify_on_completion,
620 compressed,
621 );
622 metadata.mode = BgMode::Pty;
623 metadata.pty_rows = Some(rows);
624 metadata.pty_cols = Some(cols);
625 self.persist_task(&paths, &metadata)
626 .map_err(|e| format!("failed to persist background task metadata: {e}"))?;
627 create_capture_file(&paths.pty)
628 .map_err(|e| format!("failed to create PTY capture file: {e}"))?;
629
630 let runtime = match spawn_pty_for_command(
631 &task_id,
632 &session_id,
633 command,
634 &paths,
635 &workdir,
636 &env,
637 rows,
638 cols,
639 self.inner.wake_tx.clone(),
640 ) {
641 Ok(runtime) => runtime,
642 Err(error) => {
643 crate::slog_warn!(
644 "failed to spawn PTY background bash task {task_id}; deleting partial bundle: {error}"
645 );
646 let _ = delete_task_bundle(&paths);
647 return Err(error);
648 }
649 };
650
651 if let Some(child_pid) = runtime.child_pid {
652 metadata.mark_running(child_pid, child_pid as i32);
653 } else {
654 metadata.status = BgTaskStatus::Running;
655 metadata.pgid = None;
656 }
657 self.persist_task(&paths, &metadata)
658 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
659
660 let task = Arc::new(BgTask {
661 task_id: task_id.clone(),
662 session_id,
663 paths: paths.clone(),
664 started: Instant::now(),
665 last_reminder_at: Mutex::new(None),
666 terminal_at: Mutex::new(None),
667 state: Mutex::new(BgTaskState {
668 metadata,
669 runtime: TaskRuntime::Pty(Some(runtime)),
670 detached: false,
671 child_exit_observed: false,
672 buffer: BgBuffer::pty(paths.pty.clone()),
673 terminal_output_cache: None,
674 pending_terminal_override: None,
675 }),
676 });
677
678 self.inner
679 .tasks
680 .lock()
681 .map_err(|_| "background task registry lock poisoned".to_string())?
682 .insert(task_id.clone(), task);
683
684 Ok(task_id)
685 }
686
687 #[cfg(windows)]
688 #[allow(clippy::too_many_arguments)]
689 pub fn spawn(
690 &self,
691 command: &str,
692 session_id: String,
693 workdir: PathBuf,
694 env: HashMap<String, String>,
695 timeout: Option<Duration>,
696 storage_dir: PathBuf,
697 max_running: usize,
698 notify_on_completion: bool,
699 compressed: bool,
700 project_root: Option<PathBuf>,
701 ) -> Result<String, String> {
702 self.start_watchdog();
703
704 let running = self.running_count();
705 if running >= max_running {
706 return Err(format!(
707 "background bash task limit exceeded: {running} running (max {max_running})"
708 ));
709 }
710
711 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
712 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
713 let task_id = self.generate_unique_task_id()?;
714 let paths = task_paths(&storage_dir, &session_id, &task_id);
715 fs::create_dir_all(&paths.dir)
716 .map_err(|e| format!("failed to create background task dir: {e}"))?;
717
718 let mut metadata = PersistedTask::starting(
719 task_id.clone(),
720 session_id.clone(),
721 command.to_string(),
722 workdir.clone(),
723 project_root,
724 timeout_ms,
725 notify_on_completion,
726 compressed,
727 );
728 self.persist_task(&paths, &metadata)
729 .map_err(|e| format!("failed to persist background task metadata: {e}"))?;
730
731 create_capture_file(&paths.stdout)
737 .map_err(|e| format!("failed to create stdout capture file: {e}"))?;
738 create_capture_file(&paths.stderr)
739 .map_err(|e| format!("failed to create stderr capture file: {e}"))?;
740
741 let child = match spawn_detached_child(command, &paths, &workdir, &env) {
742 Ok(child) => child,
743 Err(error) => {
744 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
745 let _ = delete_task_bundle(&paths);
746 return Err(error);
747 }
748 };
749
750 let child_pid = child.id();
751 metadata.status = BgTaskStatus::Running;
752 metadata.child_pid = Some(child_pid);
753 metadata.pgid = None;
754 self.persist_task(&paths, &metadata)
755 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
756
757 let task = Arc::new(BgTask {
758 task_id: task_id.clone(),
759 session_id,
760 paths: paths.clone(),
761 started: Instant::now(),
762 last_reminder_at: Mutex::new(None),
763 terminal_at: Mutex::new(None),
764 state: Mutex::new(BgTaskState {
765 metadata,
766 runtime: TaskRuntime::Piped(Some(child)),
767 detached: false,
768 child_exit_observed: false,
769 buffer: BgBuffer::new(paths.stdout.clone(), paths.stderr.clone()),
770 terminal_output_cache: None,
771 pending_terminal_override: None,
772 }),
773 });
774
775 self.inner
776 .tasks
777 .lock()
778 .map_err(|_| "background task registry lock poisoned".to_string())?
779 .insert(task_id.clone(), task);
780
781 Ok(task_id)
782 }
783
784 pub fn write_pty(
785 &self,
786 task_id: &str,
787 session_id: &str,
788 input: &[u8],
789 ) -> Result<usize, String> {
790 let task = self
791 .task_for_session(task_id, session_id)
792 .ok_or_else(|| "task_not_found".to_string())?;
793
794 let writer = {
795 let state = task
796 .state
797 .lock()
798 .map_err(|_| "background task lock poisoned".to_string())?;
799 if state.metadata.mode != BgMode::Pty {
800 return Err("task_not_pty".to_string());
801 }
802 if state.metadata.status.is_terminal() {
803 return Err("task_exited".to_string());
804 }
805 match &state.runtime {
806 TaskRuntime::Pty(Some(runtime)) => Arc::clone(&runtime.writer),
807 TaskRuntime::Pty(None) => return Err("task_exited".to_string()),
808 TaskRuntime::Piped(_) => return Err("task_not_pty".to_string()),
809 }
810 };
811
812 let mut writer = writer
813 .lock()
814 .map_err(|_| "PTY writer lock poisoned".to_string())?;
815 writer
816 .write_all(input)
817 .map_err(|error| format!("failed to write to PTY: {error}"))?;
818 writer
819 .flush()
820 .map_err(|error| format!("failed to flush PTY writer: {error}"))?;
821 Ok(input.len())
822 }
823
824 pub fn replay_session(&self, storage_dir: &Path, session_id: &str) -> Result<(), String> {
825 self.replay_session_inner(storage_dir, session_id, None)
826 }
827
828 pub fn replay_session_for_project(
829 &self,
830 storage_dir: &Path,
831 session_id: &str,
832 project_root: &Path,
833 ) -> Result<(), String> {
834 self.replay_session_inner(storage_dir, session_id, Some(project_root))
835 }
836
837 fn replay_session_inner(
838 &self,
839 storage_dir: &Path,
840 session_id: &str,
841 project_root: Option<&Path>,
842 ) -> Result<(), String> {
843 self.start_watchdog();
844 if !self.inner.persisted_gc_started.swap(true, Ordering::SeqCst) {
845 if let Err(error) = self.maybe_gc_persisted(storage_dir) {
846 crate::slog_warn!("failed to GC persisted background bash tasks: {error}");
847 }
848 }
849
850 let canonical_project = project_root.map(canonicalized_path);
851 let tasks = match self.replay_session_from_db(session_id) {
863 Some(Ok(tasks)) if !tasks.is_empty() => tasks,
864 Some(Ok(_)) => {
865 let disk_tasks = self.replay_session_from_disk(storage_dir, session_id)?;
866 if !disk_tasks.is_empty() {
867 crate::slog_info!(
868 "bash task replay: 0 in DB for session {}, {} from disk fallback",
869 session_id,
870 disk_tasks.len()
871 );
872 }
873 disk_tasks
874 }
875 Some(Err(error)) => {
876 crate::slog_warn!(
877 "bash task replay DB lookup failed for session {}; falling back to disk: {}",
878 session_id,
879 error
880 );
881 self.replay_session_from_disk(storage_dir, session_id)?
882 }
883 None => {
884 self.replay_session_from_disk(storage_dir, session_id)?
886 }
887 };
888
889 for mut metadata in tasks {
890 if metadata.session_id != session_id {
891 continue;
892 }
893 if let Some(canonical_project) = canonical_project.as_deref() {
894 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
895 if metadata_project.as_deref() != Some(canonical_project) {
896 continue;
897 }
898 }
899
900 let paths = task_paths(storage_dir, session_id, &metadata.task_id);
901 match metadata.status {
902 BgTaskStatus::Starting => {
903 let completion_was_delivered = metadata.completion_delivered;
904 metadata.mark_terminal(
905 BgTaskStatus::Failed,
906 None,
907 Some("spawn aborted".to_string()),
908 );
909 metadata.completion_delivered |= completion_was_delivered;
910 let _ = self.persist_task(&paths, &metadata);
911 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
912 self.insert_rehydrated_task(metadata, paths, true)?;
913 }
914 BgTaskStatus::Running | BgTaskStatus::Killing => {
915 if metadata.mode == BgMode::Pty {
916 if let Ok(Some(marker)) = read_exit_marker(&paths.exit) {
917 let completion_was_delivered = metadata.completion_delivered;
918 metadata = terminal_metadata_from_marker(metadata, marker, None);
919 metadata.completion_delivered |= completion_was_delivered;
920 let _ = self.persist_task(&paths, &metadata);
921 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
922 self.insert_rehydrated_task(metadata, paths, true)?;
923 } else if metadata.status.is_terminal() {
924 self.insert_rehydrated_task(metadata, paths, true)?;
925 } else {
926 let completion_was_delivered = metadata.completion_delivered;
927 metadata.mark_terminal(
928 BgTaskStatus::Killed,
929 None,
930 Some("pty_lost_on_bridge_restart".to_string()),
931 );
932 metadata.completion_delivered |= completion_was_delivered;
933 let _ = self.persist_task(&paths, &metadata);
934 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
935 self.insert_rehydrated_task(metadata, paths, true)?;
936 }
937 } else if self.running_metadata_is_stale(&metadata) {
938 let completion_was_delivered = metadata.completion_delivered;
939 metadata.mark_terminal(
940 BgTaskStatus::Killed,
941 None,
942 Some("orphaned (>24h)".to_string()),
943 );
944 metadata.completion_delivered |= completion_was_delivered;
945 if !paths.exit.exists() {
946 let _ = write_kill_marker_if_absent(&paths.exit);
947 }
948 let _ = self.persist_task(&paths, &metadata);
949 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
950 self.insert_rehydrated_task(metadata, paths, true)?;
951 } else if let Ok(Some(marker)) = read_exit_marker(&paths.exit) {
952 let reason = (metadata.status == BgTaskStatus::Killing).then(|| {
953 "recovered from inconsistent killing state on replay".to_string()
954 });
955 if reason.is_some() {
956 crate::slog_warn!("background task {} had killing state with exit marker; preferring marker",
957 metadata.task_id);
958 }
959 let completion_was_delivered = metadata.completion_delivered;
960 metadata = terminal_metadata_from_marker(metadata, marker, reason);
961 metadata.completion_delivered |= completion_was_delivered;
962 let _ = self.persist_task(&paths, &metadata);
963 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
964 self.insert_rehydrated_task(metadata, paths, true)?;
965 } else if metadata.status == BgTaskStatus::Killing {
966 if !paths.exit.exists() {
967 let _ = write_kill_marker_if_absent(&paths.exit);
968 }
969 let completion_was_delivered = metadata.completion_delivered;
970 metadata.mark_terminal(
971 BgTaskStatus::Killed,
972 None,
973 Some("recovered from inconsistent killing state on replay".to_string()),
974 );
975 metadata.completion_delivered |= completion_was_delivered;
976 let _ = self.persist_task(&paths, &metadata);
977 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
978 self.insert_rehydrated_task(metadata, paths, true)?;
979 } else if metadata.child_pid.is_some_and(|pid| !is_process_alive(pid)) {
980 let completion_was_delivered = metadata.completion_delivered;
981 metadata.mark_terminal(
982 BgTaskStatus::Failed,
983 None,
984 Some("process exited without exit marker".to_string()),
985 );
986 metadata.completion_delivered |= completion_was_delivered;
987 let _ = self.persist_task(&paths, &metadata);
988 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
989 self.insert_rehydrated_task(metadata, paths, true)?;
990 } else {
991 self.insert_rehydrated_task(metadata, paths, true)?;
992 }
993 }
994 _ if metadata.status.is_terminal() => {
995 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1001 self.insert_rehydrated_task(metadata, paths, true)?;
1002 }
1003 _ => {}
1004 }
1005 }
1006
1007 Ok(())
1008 }
1009
1010 fn replay_session_from_db(
1011 &self,
1012 session_id: &str,
1013 ) -> Option<Result<Vec<PersistedTask>, String>> {
1014 let pool = self
1015 .inner
1016 .db_pool
1017 .read()
1018 .ok()
1019 .and_then(|slot| slot.clone())?;
1020 let harness = self
1021 .inner
1022 .db_harness
1023 .read()
1024 .ok()
1025 .and_then(|slot| slot.clone())?;
1026 let conn = match pool.lock() {
1027 Ok(conn) => conn,
1028 Err(_) => return Some(Err("db mutex poisoned".to_string())),
1029 };
1030 Some(
1031 crate::db::bash_tasks::list_bash_tasks_for_session(&conn, &harness, session_id)
1032 .map(|rows| rows.into_iter().map(PersistedTask::from).collect())
1033 .map_err(|error| error.to_string()),
1034 )
1035 }
1036
1037 fn replay_session_from_disk(
1038 &self,
1039 storage_dir: &Path,
1040 session_id: &str,
1041 ) -> Result<Vec<PersistedTask>, String> {
1042 let dir = session_tasks_dir(storage_dir, session_id);
1043 if !dir.exists() {
1044 return Ok(Vec::new());
1045 }
1046
1047 let entries = fs::read_dir(&dir)
1048 .map_err(|e| format!("failed to read background task dir {}: {e}", dir.display()))?;
1049 let mut tasks = Vec::new();
1050 for entry in entries.flatten() {
1051 let path = entry.path();
1052 if path.extension().and_then(|extension| extension.to_str()) != Some("json") {
1053 continue;
1054 }
1055 match read_task(&path) {
1056 Ok(metadata) => tasks.push(metadata),
1057 Err(error) => {
1058 crate::slog_warn!(
1059 "quarantining invalid background task metadata {} during replay: {error}",
1060 path.display()
1061 );
1062 if let Err(quarantine_error) =
1063 quarantine_task_json(storage_dir, &dir, &path, QuarantineKind::Invalid)
1064 {
1065 crate::slog_warn!(
1066 "failed to quarantine invalid background task metadata {}: {quarantine_error}",
1067 path.display()
1068 );
1069 }
1070 }
1071 }
1072 }
1073 Ok(tasks)
1074 }
1075
1076 pub fn register_watch(
1077 &self,
1078 task_id: String,
1079 pattern: WatchPattern,
1080 once: bool,
1081 ) -> Result<String, &'static str> {
1082 let task = self.task(&task_id).ok_or("task_not_found")?;
1083 let (mode, terminal_at_registration, stdout, stderr, pty) = task
1084 .state
1085 .lock()
1086 .map(|state| {
1087 (
1088 state.metadata.mode.clone(),
1089 state.metadata.status.is_terminal(),
1090 task.paths.stdout.clone(),
1091 task.paths.stderr.clone(),
1092 task.paths.pty.clone(),
1093 )
1094 })
1095 .map_err(|_| "background_task_lock_poisoned")?;
1096
1097 let mut terminal_matches = Vec::new();
1098 let scanned_terminal = terminal_at_registration;
1099 let watch_id = {
1100 let mut registry = self
1101 .inner
1102 .watch_registry
1103 .lock()
1104 .map_err(|_| "watch_registry_poisoned")?;
1105 let watch_id = registry.register(task_id.clone(), pattern, once)?;
1106 match &mode {
1107 BgMode::Pipes => {
1108 let stdout_key = format!("{task_id}:stdout");
1109 let stderr_key = format!("{task_id}:stderr");
1110 if terminal_at_registration {
1111 registry.set_file_cursor(&stdout_key, 0);
1112 registry.set_file_cursor(&stderr_key, 0);
1113 terminal_matches.extend(registry.scan_file_new_bytes(
1114 &stdout_key,
1115 &task_id,
1116 &stdout,
1117 ));
1118 terminal_matches.extend(registry.scan_file_new_bytes(
1119 &stderr_key,
1120 &task_id,
1121 &stderr,
1122 ));
1123 } else {
1124 registry.prime_file_cursor(&stdout_key, &stdout);
1125 registry.prime_file_cursor(&stderr_key, &stderr);
1126 }
1127 }
1128 BgMode::Pty => {
1129 let pty_key = format!("{task_id}:pty");
1130 if terminal_at_registration {
1131 registry.set_file_cursor(&pty_key, 0);
1132 terminal_matches
1133 .extend(registry.scan_file_new_bytes(&pty_key, &task_id, &pty));
1134 } else {
1135 registry.prime_file_cursor(&pty_key, &pty);
1136 }
1137 }
1138 }
1139 watch_id
1140 };
1141
1142 if task.is_terminal() {
1143 if !scanned_terminal {
1144 terminal_matches = {
1145 let mut registry = self
1146 .inner
1147 .watch_registry
1148 .lock()
1149 .map_err(|_| "watch_registry_poisoned")?;
1150 match &mode {
1151 BgMode::Pipes => {
1152 let stdout_key = format!("{task_id}:stdout");
1153 let stderr_key = format!("{task_id}:stderr");
1154 registry.set_file_cursor(&stdout_key, 0);
1155 registry.set_file_cursor(&stderr_key, 0);
1156 let mut matches =
1157 registry.scan_file_new_bytes(&stdout_key, &task_id, &stdout);
1158 matches.extend(registry.scan_file_new_bytes(
1159 &stderr_key,
1160 &task_id,
1161 &stderr,
1162 ));
1163 matches
1164 }
1165 BgMode::Pty => {
1166 let pty_key = format!("{task_id}:pty");
1167 registry.set_file_cursor(&pty_key, 0);
1168 registry.scan_file_new_bytes(&pty_key, &task_id, &pty)
1169 }
1170 }
1171 };
1172 }
1173
1174 let (watch_controlled, watch_matched) = self.task_watch_state(&task_id);
1175 if terminal_matches.is_empty() && (!watch_controlled || watch_matched) {
1176 if watch_matched {
1177 let _ = task.set_completion_delivered(true, self);
1178 self.clear_task_watch_state(&task_id);
1179 }
1180 return Ok(watch_id);
1181 }
1182
1183 let completion = self
1184 .remove_pending_completion(&task_id)
1185 .or_else(|| self.completion_snapshot_for_task(&task));
1186 if terminal_matches.is_empty() {
1187 if let Some(completion) = completion.as_ref() {
1188 self.emit_bash_watch_exit(completion);
1189 }
1190 } else {
1191 for pattern_match in terminal_matches {
1192 self.emit_bash_pattern_match(&task.session_id, pattern_match);
1193 }
1194 }
1195 let _ = task.set_completion_delivered(true, self);
1196 self.clear_task_watch_state(&task_id);
1197 }
1198
1199 Ok(watch_id)
1200 }
1201
1202 pub fn unregister_watch(&self, task_id: &str, watch_id: &str) {
1203 if let Ok(mut registry) = self.inner.watch_registry.lock() {
1204 registry.unregister(task_id, watch_id);
1205 }
1206 }
1207
1208 pub fn active_watch_count(&self, task_id: &str) -> usize {
1209 self.inner
1210 .watch_registry
1211 .lock()
1212 .map(|registry| registry.active_count(task_id))
1213 .unwrap_or(0)
1214 }
1215
1216 fn task_watch_state(&self, task_id: &str) -> (bool, bool) {
1217 self.inner
1218 .watch_registry
1219 .lock()
1220 .map(|registry| {
1221 (
1222 registry.has_controlled_task(task_id),
1223 registry.has_matched_task(task_id),
1224 )
1225 })
1226 .unwrap_or((false, false))
1227 }
1228
1229 fn task_has_watch_control(&self, task_id: &str) -> bool {
1230 self.inner
1231 .watch_registry
1232 .lock()
1233 .map(|registry| registry.has_controlled_task(task_id))
1234 .unwrap_or(false)
1235 }
1236
1237 fn clear_task_watch_state(&self, task_id: &str) {
1238 if let Ok(mut registry) = self.inner.watch_registry.lock() {
1239 registry.clear_task(task_id);
1240 }
1241 }
1242
1243 pub(crate) fn scan_task_watch_output(&self, task: &Arc<BgTask>) {
1244 let (mode, stdout, stderr, pty) = match task.state.lock() {
1245 Ok(state) => (
1246 state.metadata.mode.clone(),
1247 task.paths.stdout.clone(),
1248 task.paths.stderr.clone(),
1249 task.paths.pty.clone(),
1250 ),
1251 Err(_) => return,
1252 };
1253 let mut matches = Vec::new();
1254 if let Ok(mut registry) = self.inner.watch_registry.lock() {
1255 match mode {
1256 BgMode::Pipes => {
1257 let stdout_key = format!("{}:stdout", task.task_id);
1258 let stderr_key = format!("{}:stderr", task.task_id);
1259 matches.extend(registry.scan_file_new_bytes(
1260 &stdout_key,
1261 &task.task_id,
1262 &stdout,
1263 ));
1264 matches.extend(registry.scan_file_new_bytes(
1265 &stderr_key,
1266 &task.task_id,
1267 &stderr,
1268 ));
1269 }
1270 BgMode::Pty => {
1271 let pty_key = format!("{}:pty", task.task_id);
1272 matches.extend(registry.scan_file_new_bytes(&pty_key, &task.task_id, &pty));
1273 }
1274 }
1275 }
1276 for pattern_match in matches {
1277 self.emit_bash_pattern_match(&task.session_id, pattern_match);
1278 }
1279 }
1280
1281 pub fn status(
1282 &self,
1283 task_id: &str,
1284 session_id: &str,
1285 project_root: Option<&Path>,
1286 storage_dir: Option<&Path>,
1287 preview_bytes: usize,
1288 ) -> Option<BgTaskSnapshot> {
1289 let mut task = self.task_for_session(task_id, session_id);
1290 if task.is_none() {
1291 if let Some(storage_dir) = storage_dir {
1292 let _ = if let Some(project_root) = project_root {
1293 self.replay_session_for_project(storage_dir, session_id, project_root)
1294 } else {
1295 self.replay_session(storage_dir, session_id)
1296 };
1297 task = self.task_for_session(task_id, session_id);
1298 }
1299 }
1300 let Some(task) = task else {
1301 return self.status_relaxed(
1302 task_id,
1303 session_id,
1304 project_root?,
1305 storage_dir?,
1306 preview_bytes,
1307 );
1308 };
1309 let _ = self.poll_task(&task);
1310 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
1311 }
1312
1313 fn status_relaxed_task(
1314 &self,
1315 task_id: &str,
1316 project_root: &Path,
1317 storage_dir: &Path,
1318 ) -> Option<Arc<BgTask>> {
1319 let canonical_project = canonicalized_path(project_root);
1320 match self.lookup_relaxed_task_from_db(task_id, project_root) {
1321 Some(Ok(Some(metadata))) => {
1322 if let Some(task) = self.task(task_id) {
1323 let matches_project = task
1324 .state
1325 .lock()
1326 .map(|state| {
1327 state
1328 .metadata
1329 .project_root
1330 .as_deref()
1331 .map(canonicalized_path)
1332 .as_deref()
1333 == Some(canonical_project.as_path())
1334 })
1335 .unwrap_or(false);
1336 return matches_project.then_some(task);
1337 }
1338 let paths = task_paths(storage_dir, &metadata.session_id, &metadata.task_id);
1339 if self.insert_rehydrated_task(metadata, paths, true).is_err() {
1340 return None;
1341 }
1342 return self.task(task_id);
1343 }
1344 Some(Ok(None)) => {
1345 crate::slog_info!(
1346 "bash task relaxed DB miss for {}; falling back to disk",
1347 task_id
1348 );
1349 }
1350 Some(Err(error)) => {
1351 crate::slog_warn!(
1352 "bash task relaxed DB lookup failed for {}; falling back to disk: {}",
1353 task_id,
1354 error
1355 );
1356 }
1357 None => {
1358 crate::slog_info!(
1359 "bash task relaxed DB unavailable for {}; falling back to disk",
1360 task_id
1361 );
1362 }
1363 }
1364 let root = storage_dir.join("bash-tasks");
1365 let entries = fs::read_dir(&root).ok()?;
1366 for entry in entries.flatten() {
1367 let dir = entry.path();
1368 if !dir.is_dir() {
1369 continue;
1370 }
1371 let path = dir.join(format!("{task_id}.json"));
1372 if !path.exists() {
1373 continue;
1374 }
1375 let metadata = match read_task(&path) {
1376 Ok(metadata) => metadata,
1377 Err(error) => {
1378 crate::slog_warn!(
1379 "quarantining invalid background task metadata {} during relaxed lookup: {error}",
1380 path.display()
1381 );
1382 if let Err(quarantine_error) =
1383 quarantine_task_json(storage_dir, &dir, &path, QuarantineKind::Invalid)
1384 {
1385 crate::slog_warn!(
1386 "failed to quarantine invalid background task metadata {}: {quarantine_error}",
1387 path.display()
1388 );
1389 }
1390 continue;
1391 }
1392 };
1393 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
1394 if metadata_project.as_deref() != Some(canonical_project.as_path()) {
1395 continue;
1396 }
1397 if let Some(task) = self.task(task_id) {
1398 let matches_project = task
1399 .state
1400 .lock()
1401 .map(|state| {
1402 state
1403 .metadata
1404 .project_root
1405 .as_deref()
1406 .map(canonicalized_path)
1407 .as_deref()
1408 == Some(canonical_project.as_path())
1409 })
1410 .unwrap_or(false);
1411 return matches_project.then_some(task);
1412 }
1413 let paths = task_paths(storage_dir, &metadata.session_id, &metadata.task_id);
1414 if self.insert_rehydrated_task(metadata, paths, true).is_err() {
1415 return None;
1416 }
1417 return self.task(task_id);
1418 }
1419 None
1420 }
1421
1422 fn lookup_relaxed_task_from_db(
1423 &self,
1424 task_id: &str,
1425 project_root: &Path,
1426 ) -> Option<Result<Option<PersistedTask>, String>> {
1427 let pool = self
1428 .inner
1429 .db_pool
1430 .read()
1431 .ok()
1432 .and_then(|slot| slot.clone())?;
1433 let harness = self
1434 .inner
1435 .db_harness
1436 .read()
1437 .ok()
1438 .and_then(|slot| slot.clone())?;
1439 let conn = match pool.lock() {
1440 Ok(conn) => conn,
1441 Err(_) => return Some(Err("db mutex poisoned".to_string())),
1442 };
1443 let project_key = crate::path_identity::project_scope_key(project_root);
1444 Some(
1445 crate::db::bash_tasks::find_bash_task_for_project(
1446 &conn,
1447 &harness,
1448 &project_key,
1449 task_id,
1450 )
1451 .map(|row| row.map(PersistedTask::from))
1452 .map_err(|error| error.to_string()),
1453 )
1454 }
1455
1456 pub(super) fn status_relaxed(
1457 &self,
1458 task_id: &str,
1459 _session_id: &str,
1460 project_root: &Path,
1461 storage_dir: &Path,
1462 preview_bytes: usize,
1463 ) -> Option<BgTaskSnapshot> {
1464 let task = self.status_relaxed_task(task_id, project_root, storage_dir)?;
1465 let _ = self.poll_task(&task);
1466 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
1467 }
1468
1469 pub fn kill_relaxed(
1470 &self,
1471 task_id: &str,
1472 project_root: &Path,
1473 storage_dir: &Path,
1474 ) -> Result<BgTaskSnapshot, String> {
1475 let task = self
1476 .status_relaxed_task(task_id, project_root, storage_dir)
1477 .ok_or_else(|| format!("background task not found: {task_id}"))?;
1478 self.kill_with_status(task_id, &task.session_id, BgTaskStatus::Killed)
1479 }
1480
1481 pub fn maybe_gc_persisted(&self, storage_dir: &Path) -> Result<usize, String> {
1482 #[cfg(test)]
1483 self.inner.persisted_gc_runs.fetch_add(1, Ordering::SeqCst);
1484
1485 let mut deleted = 0usize;
1486
1487 let root = storage_dir.join("bash-tasks");
1488 if root.exists() {
1489 let session_dirs = fs::read_dir(&root).map_err(|e| {
1490 format!(
1491 "failed to read background task root {}: {e}",
1492 root.display()
1493 )
1494 })?;
1495 for session_entry in session_dirs.flatten() {
1496 let session_dir = session_entry.path();
1497 if !session_dir.is_dir() {
1498 continue;
1499 }
1500 let task_entries = match fs::read_dir(&session_dir) {
1501 Ok(entries) => entries,
1502 Err(error) => {
1503 crate::slog_warn!(
1504 "failed to read background task session dir {}: {error}",
1505 session_dir.display()
1506 );
1507 continue;
1508 }
1509 };
1510 for task_entry in task_entries.flatten() {
1511 let json_path = task_entry.path();
1512 if json_path
1513 .extension()
1514 .and_then(|extension| extension.to_str())
1515 != Some("json")
1516 {
1517 continue;
1518 }
1519 if modified_within(&json_path, PERSISTED_GC_GRACE) {
1520 continue;
1521 }
1522 let metadata = match read_task(&json_path) {
1523 Ok(metadata) => metadata,
1524 Err(error) => {
1525 crate::slog_warn!(
1526 "quarantining corrupt background task metadata {}: {error}",
1527 json_path.display()
1528 );
1529 quarantine_task_json(
1530 storage_dir,
1531 &session_dir,
1532 &json_path,
1533 QuarantineKind::Corrupt,
1534 )?;
1535 continue;
1536 }
1537 };
1538 if !(metadata.status.is_terminal() && metadata.completion_delivered) {
1539 continue;
1540 }
1541 let paths = task_paths(storage_dir, &metadata.session_id, &metadata.task_id);
1542 match delete_task_bundle(&paths) {
1543 Ok(()) => {
1544 deleted += 1;
1545 log::debug!(
1546 "deleted persisted background task bundle {}",
1547 metadata.task_id
1548 );
1549 }
1550 Err(error) => {
1551 crate::slog_warn!(
1552 "failed to delete background task bundle {}: {error}",
1553 metadata.task_id
1554 );
1555 continue;
1556 }
1557 }
1558 }
1559 }
1560 }
1561 gc_quarantine(storage_dir);
1562 Ok(deleted)
1563 }
1564
1565 pub fn list(&self, preview_bytes: usize) -> Vec<BgTaskSnapshot> {
1566 let tasks = self
1567 .inner
1568 .tasks
1569 .lock()
1570 .map(|tasks| tasks.values().cloned().collect::<Vec<_>>())
1571 .unwrap_or_default();
1572 tasks
1573 .into_iter()
1574 .map(|task| {
1575 let _ = self.poll_task(&task);
1576 self.snapshot_with_terminal_cache(&task, preview_bytes)
1577 })
1578 .collect()
1579 }
1580
1581 fn maybe_compress_snapshot(&self, task: &Arc<BgTask>, snapshot: &mut BgTaskSnapshot) {
1587 if !snapshot.info.status.is_terminal() || snapshot.info.mode == BgMode::Pty {
1588 return;
1589 }
1590 if let Some(cache) = self.ensure_terminal_output_cache(task) {
1591 snapshot.output_preview = cache.output_preview;
1592 snapshot.output_truncated = cache.output_truncated;
1593 }
1594 }
1595
1596 pub fn kill(&self, task_id: &str, session_id: &str) -> Result<BgTaskSnapshot, String> {
1597 self.kill_with_status(task_id, session_id, BgTaskStatus::Killed)
1598 }
1599
1600 pub fn promote(&self, task_id: &str, session_id: &str) -> Result<bool, String> {
1601 let task = self
1602 .task_for_session(task_id, session_id)
1603 .ok_or_else(|| format!("background task not found: {task_id}"))?;
1604 let terminal_after_promote = {
1605 let mut state = task
1606 .state
1607 .lock()
1608 .map_err(|_| "background task lock poisoned".to_string())?;
1609 let updated = self
1610 .update_task_metadata(&task.paths, |metadata| {
1611 metadata.notify_on_completion = true;
1612 metadata.completion_delivered = false;
1613 })
1614 .map_err(|e| format!("failed to promote background task: {e}"))?;
1615 state.metadata = updated;
1616 state.metadata.status.is_terminal()
1617 };
1618 if terminal_after_promote {
1619 self.post_terminal_transition(&task, true)?;
1620 }
1621 Ok(true)
1622 }
1623
1624 pub(crate) fn kill_for_timeout(&self, task_id: &str, session_id: &str) -> Result<(), String> {
1625 self.kill_with_status(task_id, session_id, BgTaskStatus::TimedOut)
1626 .map(|_| ())
1627 }
1628
1629 pub fn cleanup_finished(&self, older_than: Duration) {
1630 let cutoff = Instant::now().checked_sub(older_than);
1631 let removable_paths: Vec<(String, TaskPaths)> =
1632 if let Ok(mut tasks) = self.inner.tasks.lock() {
1633 let removable = tasks
1634 .iter()
1635 .filter_map(|(task_id, task)| {
1636 let delivered_terminal = task
1637 .state
1638 .lock()
1639 .map(|state| {
1640 state.metadata.status.is_terminal()
1641 && state.metadata.completion_delivered
1642 })
1643 .unwrap_or(false);
1644 if !delivered_terminal {
1645 return None;
1646 }
1647
1648 let terminal_at = task.terminal_at.lock().ok().and_then(|at| *at);
1649 let expired = match (terminal_at, cutoff) {
1650 (Some(terminal_at), Some(cutoff)) => terminal_at <= cutoff,
1651 (Some(_), None) => true,
1652 (None, _) => false,
1653 };
1654 expired.then(|| task_id.clone())
1655 })
1656 .collect::<Vec<_>>();
1657
1658 removable
1659 .into_iter()
1660 .filter_map(|task_id| {
1661 tasks
1662 .remove(&task_id)
1663 .map(|task| (task_id, task.paths.clone()))
1664 })
1665 .collect()
1666 } else {
1667 Vec::new()
1668 };
1669
1670 for (task_id, paths) in removable_paths {
1671 match delete_task_bundle(&paths) {
1672 Ok(()) => log::debug!("deleted persisted background task bundle {task_id}"),
1673 Err(error) => crate::slog_warn!(
1674 "failed to delete persisted background task bundle {task_id}: {error}"
1675 ),
1676 }
1677 }
1678 }
1679
1680 pub fn drain_completions(&self) -> Vec<BgCompletion> {
1681 self.drain_completions_for_session(None)
1682 }
1683
1684 pub fn drain_completions_for_session(&self, session_id: Option<&str>) -> Vec<BgCompletion> {
1685 let completions = match self.inner.completions.lock() {
1686 Ok(completions) => completions,
1687 Err(_) => return Vec::new(),
1688 };
1689
1690 completions
1691 .iter()
1692 .filter(|completion| completion_matches_session(completion, session_id))
1693 .cloned()
1694 .collect()
1695 }
1696
1697 pub fn has_completions_for_session(&self, session_id: Option<&str>) -> bool {
1698 match self.inner.completions.lock() {
1699 Ok(completions) => completions
1700 .iter()
1701 .any(|completion| completion_matches_session(completion, session_id)),
1702 Err(_) => true,
1706 }
1707 }
1708
1709 pub fn ack_completions_for_session(
1710 &self,
1711 session_id: Option<&str>,
1712 task_ids: &[String],
1713 ) -> Vec<String> {
1714 if task_ids.is_empty() {
1715 return Vec::new();
1716 }
1717 let requested_task_ids = task_ids.iter().map(String::as_str).collect::<HashSet<_>>();
1718 let mut completion_sessions = HashMap::new();
1719 if let Ok(mut completions) = self.inner.completions.lock() {
1720 completions.retain(|completion| {
1721 let session_matches = session_id
1722 .map(|session_id| completion.session_id == session_id)
1723 .unwrap_or(true);
1724 if session_matches && requested_task_ids.contains(completion.task_id.as_str()) {
1725 completion_sessions
1726 .insert(completion.task_id.clone(), completion.session_id.clone());
1727 false
1728 } else {
1729 true
1730 }
1731 });
1732 }
1733
1734 let mut delivered = Vec::new();
1735 for task_id in task_ids {
1736 let task = if let Some(session_id) = session_id {
1737 self.task_for_session(task_id, session_id)
1738 } else if let Some(completion_session_id) = completion_sessions.get(task_id) {
1739 self.task_for_session(task_id, completion_session_id)
1740 } else {
1741 self.task(task_id)
1742 };
1743 if let Some(task) = task {
1744 if task.set_completion_delivered(true, self).is_ok() {
1745 delivered.push(task_id.clone());
1746 }
1747 }
1748 }
1749
1750 delivered
1751 }
1752
1753 pub fn pending_completions_for_session(&self, session_id: &str) -> Vec<BgCompletion> {
1754 self.inner
1755 .completions
1756 .lock()
1757 .map(|completions| {
1758 completions
1759 .iter()
1760 .filter(|completion| completion.session_id == session_id)
1761 .cloned()
1762 .collect()
1763 })
1764 .unwrap_or_default()
1765 }
1766
1767 fn remove_pending_completion(&self, task_id: &str) -> Option<BgCompletion> {
1768 let mut completions = self.inner.completions.lock().ok()?;
1769 let idx = completions
1770 .iter()
1771 .position(|completion| completion.task_id == task_id)?;
1772 completions.remove(idx)
1773 }
1774
1775 fn completion_snapshot_for_task(&self, task: &Arc<BgTask>) -> Option<BgCompletion> {
1776 let snapshot = self.snapshot_with_terminal_cache(task, RUNNING_OUTPUT_PREVIEW_BYTES);
1777 if !snapshot.info.status.is_terminal() {
1778 return None;
1779 }
1780 let (output_preview, output_truncated) = if snapshot.info.mode == BgMode::Pty {
1781 (String::new(), false)
1782 } else {
1783 self.ensure_terminal_output_cache(task)
1784 .map(|cache| completion_preview_for_cache(&cache, snapshot.exit_code))
1785 .unwrap_or_else(|| (String::new(), false))
1786 };
1787 Some(BgCompletion {
1788 task_id: snapshot.info.task_id,
1789 session_id: task.session_id.clone(),
1790 status: snapshot.info.status,
1791 exit_code: snapshot.exit_code,
1792 command: snapshot.info.command,
1793 output_preview,
1794 output_truncated,
1795 original_tokens: None,
1796 compressed_tokens: None,
1797 tokens_skipped: false,
1798 })
1799 }
1800
1801 pub fn detach(&self) {
1802 self.inner.shutdown.store(true, Ordering::SeqCst);
1803 if let Ok(mut tasks) = self.inner.tasks.lock() {
1804 for task in tasks.values() {
1805 if let Ok(mut state) = task.state.lock() {
1806 match &mut state.runtime {
1807 TaskRuntime::Piped(child) => *child = None,
1808 TaskRuntime::Pty(runtime) => *runtime = None,
1809 }
1810 state.detached = true;
1811 }
1812 }
1813 tasks.clear();
1814 }
1815 }
1816
1817 pub fn shutdown(&self) {
1818 let tasks = self
1819 .inner
1820 .tasks
1821 .lock()
1822 .map(|tasks| {
1823 tasks
1824 .values()
1825 .map(|task| (task.task_id.clone(), task.session_id.clone()))
1826 .collect::<Vec<_>>()
1827 })
1828 .unwrap_or_default();
1829 for (task_id, session_id) in tasks {
1830 let _ = self.kill(&task_id, &session_id);
1831 }
1832 }
1833
1834 pub(crate) fn poll_task(&self, task: &Arc<BgTask>) -> Result<(), String> {
1835 if let Ok(state) = task.state.lock() {
1836 if let TaskRuntime::Pty(Some(pty)) = &state.runtime {
1837 if !pty.exit_observed.load(Ordering::SeqCst) {
1845 return Ok(());
1846 }
1847 }
1848 }
1849 let marker = match read_exit_marker(&task.paths.exit) {
1850 Ok(Some(marker)) => marker,
1851 Ok(None) => return Ok(()),
1852 Err(error) => return Err(format!("failed to read exit marker: {error}")),
1853 };
1854 self.finalize_from_marker(task, marker, None)
1855 }
1856
1857 pub(crate) fn reap_child(&self, task: &Arc<BgTask>) {
1858 let mut needs_completion = false;
1859 {
1860 let Ok(mut state) = task.state.lock() else {
1861 return;
1862 };
1863 match &mut state.runtime {
1864 TaskRuntime::Piped(child_slot) => {
1865 if let Some(child) = child_slot.as_mut() {
1866 if matches!(child.try_wait(), Ok(Some(_))) {
1867 *child_slot = None;
1868 state.detached = true;
1869 state.child_exit_observed = true;
1870 }
1871 } else if state.detached {
1872 let child_known_dead = state.child_exit_observed
1873 || state
1874 .metadata
1875 .child_pid
1876 .is_some_and(|pid| !is_process_alive(pid));
1877 if child_known_dead {
1878 needs_completion =
1879 self.fail_without_exit_marker_if_needed(task, &mut state);
1880 }
1881 }
1882 }
1883 TaskRuntime::Pty(Some(pty)) => {
1884 if pty.exit_observed.load(Ordering::SeqCst) {
1885 drop(state);
1886 let _ = self.poll_task(task);
1887 return;
1888 }
1889 }
1890 TaskRuntime::Pty(None) => {}
1891 }
1892 }
1893 if needs_completion {
1894 let _ = self.post_terminal_transition(task, true);
1895 }
1896 }
1897
1898 fn fail_without_exit_marker_if_needed(
1899 &self,
1900 task: &Arc<BgTask>,
1901 state: &mut BgTaskState,
1902 ) -> bool {
1903 if state.metadata.status.is_terminal() {
1904 return false;
1905 }
1906 if matches!(read_exit_marker(&task.paths.exit), Ok(Some(_))) {
1907 return false;
1908 }
1909 let watch_controlled = self.task_has_watch_control(&task.task_id);
1910 let updated = self.update_task_metadata(&task.paths, |metadata| {
1911 metadata.mark_terminal(
1912 BgTaskStatus::Failed,
1913 None,
1914 Some("process exited without exit marker".to_string()),
1915 );
1916 if watch_controlled {
1917 metadata.completion_delivered = true;
1918 }
1919 });
1920 if let Ok(metadata) = updated {
1921 state.pending_terminal_override = None;
1922 state.metadata = metadata;
1923 task.mark_terminal_now();
1924 return true;
1925 }
1926 false
1927 }
1928
1929 pub(crate) fn running_tasks(&self) -> Vec<Arc<BgTask>> {
1930 self.inner
1931 .tasks
1932 .lock()
1933 .map(|tasks| {
1934 tasks
1935 .values()
1936 .filter(|task| task.is_running())
1937 .cloned()
1938 .collect()
1939 })
1940 .unwrap_or_default()
1941 }
1942
1943 fn insert_rehydrated_task(
1944 &self,
1945 metadata: PersistedTask,
1946 paths: TaskPaths,
1947 detached: bool,
1948 ) -> Result<(), String> {
1949 let task_id = metadata.task_id.clone();
1950 let session_id = metadata.session_id.clone();
1951 let started = started_instant_from_unix_millis(metadata.started_at);
1952 let suppress_replayed_running_reminder = metadata.status == BgTaskStatus::Running;
1953 let mode = metadata.mode.clone();
1954 let task = Arc::new(BgTask {
1955 task_id: task_id.clone(),
1956 session_id,
1957 paths: paths.clone(),
1958 started,
1959 last_reminder_at: Mutex::new(suppress_replayed_running_reminder.then(Instant::now)),
1960 terminal_at: Mutex::new(metadata.status.is_terminal().then(Instant::now)),
1961 state: Mutex::new(BgTaskState {
1962 metadata,
1963 runtime: if mode == BgMode::Pty {
1964 TaskRuntime::Pty(None)
1965 } else {
1966 TaskRuntime::Piped(None)
1967 },
1968 detached,
1969 child_exit_observed: false,
1976 buffer: if mode == BgMode::Pty {
1977 BgBuffer::pty(paths.pty.clone())
1978 } else {
1979 BgBuffer::new(paths.stdout.clone(), paths.stderr.clone())
1980 },
1981 terminal_output_cache: None,
1982 pending_terminal_override: None,
1983 }),
1984 });
1985 self.inner
1986 .tasks
1987 .lock()
1988 .map_err(|_| "background task registry lock poisoned".to_string())?
1989 .insert(task_id, task);
1990 Ok(())
1991 }
1992
1993 fn kill_with_status(
1994 &self,
1995 task_id: &str,
1996 session_id: &str,
1997 terminal_status: BgTaskStatus,
1998 ) -> Result<BgTaskSnapshot, String> {
1999 let task = self
2000 .task_for_session(task_id, session_id)
2001 .ok_or_else(|| format!("background task not found: {task_id}"))?;
2002 let mut terminalized = false;
2003
2004 {
2005 let mut state = task
2006 .state
2007 .lock()
2008 .map_err(|_| "background task lock poisoned".to_string())?;
2009 if state.metadata.status.is_terminal() {
2010 state.pending_terminal_override = None;
2011 } else if let Ok(Some(marker)) = read_exit_marker(&task.paths.exit) {
2012 state.metadata =
2013 terminal_metadata_from_marker(state.metadata.clone(), marker, None);
2014 if self.task_has_watch_control(&task.task_id) {
2015 state.metadata.completion_delivered = true;
2016 }
2017 state.pending_terminal_override = None;
2018 task.mark_terminal_now();
2019 match &mut state.runtime {
2020 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
2027 TaskRuntime::Pty(runtime) => *runtime = None,
2028 }
2029 state.detached = true;
2030 self.persist_task(&task.paths, &state.metadata)
2031 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
2032 terminalized = true;
2033 } else {
2034 let was_already_killing = state.metadata.status == BgTaskStatus::Killing;
2035 if !was_already_killing {
2036 state.metadata.status = BgTaskStatus::Killing;
2037 self.persist_task(&task.paths, &state.metadata)
2038 .map_err(|e| format!("failed to persist killing state: {e}"))?;
2039 }
2040
2041 #[cfg(unix)]
2042 let pgid = state.metadata.pgid;
2043 #[cfg(windows)]
2044 let child_pid = state.metadata.child_pid;
2045 if !was_already_killing
2046 && state.metadata.mode == BgMode::Pty
2047 && terminal_status == BgTaskStatus::TimedOut
2048 {
2049 state.pending_terminal_override = Some(BgTaskStatus::TimedOut);
2050 }
2051
2052 #[cfg(windows)]
2053 let mut pty_forced_terminal_status: Option<BgTaskStatus> = None;
2054
2055 match &mut state.runtime {
2056 TaskRuntime::Piped(child_slot) => {
2057 #[cfg(unix)]
2058 if let Some(pgid) = pgid {
2059 terminate_pgid(pgid, child_slot.as_mut());
2060 }
2061 #[cfg(windows)]
2062 if let Some(child) = child_slot.as_mut() {
2063 super::process::terminate_process(child);
2064 } else if let Some(pid) = child_pid {
2065 terminate_pid(pid);
2066 }
2067 if let Some(child) = child_slot.as_mut() {
2068 let _ = child.wait();
2069 }
2070 *child_slot = None;
2071 state.detached = true;
2072
2073 if !task.paths.exit.exists() {
2074 write_kill_marker_if_absent(&task.paths.exit)
2075 .map_err(|e| format!("failed to write kill marker: {e}"))?;
2076 }
2077
2078 let exit_code = terminal_exit_code_for_status(&terminal_status);
2079 state
2080 .metadata
2081 .mark_terminal(terminal_status, exit_code, None);
2082 if self.task_has_watch_control(&task.task_id) {
2083 state.metadata.completion_delivered = true;
2084 }
2085 state.pending_terminal_override = None;
2086 task.mark_terminal_now();
2087 self.persist_task(&task.paths, &state.metadata)
2088 .map_err(|e| format!("failed to persist killed state: {e}"))?;
2089 terminalized = true;
2090 }
2091 TaskRuntime::Pty(Some(pty)) => {
2092 pty.was_killed.store(true, Ordering::SeqCst);
2093 if let Err(error) = pty.killer.kill() {
2094 crate::slog_warn!(
2095 "[pty-kill] {task_id} ChildKiller::kill failed: {error}"
2096 );
2097 }
2098 if let Some(pid) = pty.child_pid {
2099 #[cfg(unix)]
2100 terminate_pgid(pid as i32, None);
2101 #[cfg(windows)]
2102 terminate_pid(pid);
2103 }
2104 drop(pty.master.take());
2105
2106 #[cfg(windows)]
2107 {
2108 let default_status = if terminal_status == BgTaskStatus::TimedOut {
2109 BgTaskStatus::TimedOut
2110 } else {
2111 BgTaskStatus::Killed
2112 };
2113 pty_forced_terminal_status = Some(
2114 state
2115 .pending_terminal_override
2116 .take()
2117 .unwrap_or(default_status),
2118 );
2119 }
2120 }
2121 TaskRuntime::Pty(None) => {}
2122 }
2123
2124 #[cfg(windows)]
2125 if let Some(target_status) = pty_forced_terminal_status {
2126 if !task.paths.exit.exists() {
2127 write_kill_marker_if_absent(&task.paths.exit)
2128 .map_err(|e| format!("failed to write kill marker: {e}"))?;
2129 }
2130
2131 let exit_code = terminal_exit_code_for_status(&target_status);
2132 state.metadata.mark_terminal(target_status, exit_code, None);
2133 if self.task_has_watch_control(&task.task_id) {
2134 state.metadata.completion_delivered = true;
2135 }
2136 state.pending_terminal_override = None;
2137 task.mark_terminal_now();
2138 if let TaskRuntime::Pty(runtime) = &mut state.runtime {
2139 *runtime = None;
2140 }
2141 state.detached = true;
2142 self.persist_task(&task.paths, &state.metadata)
2143 .map_err(|e| format!("failed to persist killed PTY state: {e}"))?;
2144 terminalized = true;
2145 }
2146 }
2147 }
2148
2149 if terminalized {
2150 self.post_terminal_transition(&task, true)?;
2151 }
2152 Ok(self.snapshot_with_terminal_cache(&task, RUNNING_OUTPUT_PREVIEW_BYTES))
2153 }
2154
2155 fn finalize_from_marker(
2156 &self,
2157 task: &Arc<BgTask>,
2158 marker: ExitMarker,
2159 reason: Option<String>,
2160 ) -> Result<(), String> {
2161 let watch_controlled = self.task_has_watch_control(&task.task_id);
2162 let mut pty_reader_done = None;
2163 {
2164 let mut state = task
2165 .state
2166 .lock()
2167 .map_err(|_| "background task lock poisoned".to_string())?;
2168 if state.metadata.status.is_terminal() {
2169 state.pending_terminal_override = None;
2170 return Ok(());
2171 }
2172
2173 let pending_override = state.pending_terminal_override.take();
2174 let is_pty = state.metadata.mode == BgMode::Pty;
2175 let updated = self
2176 .update_task_metadata(&task.paths, |metadata| {
2177 let mut new_metadata = if is_pty && marker == ExitMarker::Killed {
2178 let mut metadata = metadata.clone();
2179 let target_status = pending_override.unwrap_or(BgTaskStatus::Killed);
2180 let exit_code = terminal_exit_code_for_status(&target_status);
2181 metadata.mark_terminal(target_status, exit_code, reason);
2182 metadata
2183 } else {
2184 terminal_metadata_from_marker(metadata.clone(), marker, reason)
2185 };
2186 if watch_controlled {
2187 new_metadata.completion_delivered = true;
2188 }
2189 *metadata = new_metadata;
2190 })
2191 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
2192 state.metadata = updated;
2193 task.mark_terminal_now();
2194 match &mut state.runtime {
2195 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
2200 TaskRuntime::Pty(runtime) => {
2201 pty_reader_done = runtime
2202 .as_ref()
2203 .map(|runtime| Arc::clone(&runtime.reader_done));
2204 *runtime = None;
2205 }
2206 }
2207 state.detached = true;
2208 }
2209
2210 if let Some(reader_done) = pty_reader_done {
2211 let deadline = Instant::now() + Duration::from_millis(200);
2212 while !reader_done.load(Ordering::SeqCst) && Instant::now() < deadline {
2213 std::thread::sleep(Duration::from_millis(10));
2214 }
2215 }
2216
2217 self.scan_task_watch_output(task);
2220
2221 self.post_terminal_transition(task, true)
2222 }
2223
2224 fn enqueue_completion_if_needed(
2225 &self,
2226 metadata: &PersistedTask,
2227 paths: Option<&TaskPaths>,
2228 emit_frame: bool,
2229 ) {
2230 if metadata.status.is_terminal() && !metadata.completion_delivered {
2231 let cache =
2232 paths.and_then(|paths| self.render_terminal_output_from_paths(metadata, paths));
2233 self.enqueue_completion_from_parts(metadata, None, paths, emit_frame, cache.as_ref());
2234 }
2235 }
2236
2237 fn render_terminal_output_from_paths(
2238 &self,
2239 metadata: &PersistedTask,
2240 paths: &TaskPaths,
2241 ) -> Option<TerminalOutputCache> {
2242 if metadata.mode == BgMode::Pty {
2243 return None;
2244 }
2245 let mut buffer = BgBuffer::new(paths.stdout.clone(), paths.stderr.clone());
2246 let disk_truncation = buffer.enforce_terminal_cap();
2247 Some(self.render_terminal_output(metadata, &buffer, disk_truncation))
2248 }
2249
2250 fn enqueue_completion_from_parts(
2251 &self,
2252 metadata: &PersistedTask,
2253 buffer: Option<&BgBuffer>,
2254 paths: Option<&TaskPaths>,
2255 emit_frame: bool,
2256 terminal_render: Option<&TerminalOutputCache>,
2257 ) {
2258 if !metadata.status.is_terminal() {
2269 return;
2270 }
2271
2272 let owned_buffer = if buffer.is_none() && metadata.mode != BgMode::Pty {
2273 paths.map(|paths| BgBuffer::new(paths.stdout.clone(), paths.stderr.clone()))
2274 } else {
2275 None
2276 };
2277 let render_buffer = buffer.or(owned_buffer.as_ref());
2278 let owned_render = if terminal_render.is_none() {
2279 render_buffer.map(|buffer| {
2280 let mut capped_buffer = buffer.clone();
2281 let disk_truncation = capped_buffer.enforce_terminal_cap();
2282 self.render_terminal_output(metadata, &capped_buffer, disk_truncation)
2283 })
2284 } else {
2285 None
2286 };
2287 let render = terminal_render.or(owned_render.as_ref());
2288
2289 let (output_preview, output_truncated) = render
2293 .map(|cache| completion_preview_for_cache(cache, metadata.exit_code))
2294 .unwrap_or_else(|| (String::new(), false));
2295
2296 let token_counts = self.completion_token_counts(
2297 metadata,
2298 buffer,
2299 paths,
2300 render.map(|render| render.output_preview.as_str()),
2301 );
2302 let completion = BgCompletion {
2303 task_id: metadata.task_id.clone(),
2304 session_id: metadata.session_id.clone(),
2305 status: metadata.status.clone(),
2306 exit_code: metadata.exit_code,
2307 command: metadata.command.clone(),
2308 output_preview,
2309 output_truncated,
2310 original_tokens: token_counts.original_tokens,
2311 compressed_tokens: token_counts.compressed_tokens,
2312 tokens_skipped: token_counts.tokens_skipped,
2313 };
2314
2315 self.record_compression_event_if_applicable(metadata, &token_counts);
2326
2327 let (watch_controlled, watch_matched) = self.task_watch_state(&metadata.task_id);
2328 if watch_controlled {
2329 if emit_frame && !watch_matched {
2330 self.emit_bash_watch_exit(&completion);
2331 }
2332 self.clear_task_watch_state(&metadata.task_id);
2333 return;
2334 }
2335
2336 if metadata.completion_delivered {
2346 return;
2347 }
2348
2349 let pushed = if let Ok(mut completions) = self.inner.completions.lock() {
2352 if completions
2353 .iter()
2354 .any(|existing| existing.task_id == metadata.task_id)
2355 {
2356 false
2357 } else {
2358 completions.push_back(completion.clone());
2359 true
2360 }
2361 } else {
2362 false
2363 };
2364
2365 if pushed && emit_frame {
2366 self.emit_bash_completed(completion);
2367 }
2368 }
2369
2370 fn record_compression_event_if_applicable(
2371 &self,
2372 metadata: &PersistedTask,
2373 token_counts: &CompletionTokenCounts,
2374 ) {
2375 if metadata.mode == BgMode::Pty {
2376 return;
2377 }
2378
2379 let (original_tokens, compressed_tokens, original_bytes, compressed_bytes) = match (
2380 token_counts.original_tokens,
2381 token_counts.compressed_tokens,
2382 token_counts.original_bytes,
2383 token_counts.compressed_bytes,
2384 ) {
2385 (
2386 Some(original_tokens),
2387 Some(compressed_tokens),
2388 Some(original_bytes),
2389 Some(compressed_bytes),
2390 ) => (
2391 original_tokens,
2392 compressed_tokens,
2393 original_bytes,
2394 compressed_bytes,
2395 ),
2396 _ => {
2397 crate::slog_warn!(
2398 "compression event skipped for {}: token counts unavailable (likely spill file missing or unreadable)",
2399 metadata.task_id
2400 );
2401 return;
2402 }
2403 };
2404
2405 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
2406 let Some(pool) = pool else {
2407 crate::slog_warn!(
2408 "compression event skipped for {}: db_pool not initialized — was configure run?",
2409 metadata.task_id
2410 );
2411 return;
2412 };
2413 let harness = self
2414 .inner
2415 .db_harness
2416 .read()
2417 .ok()
2418 .and_then(|slot| slot.clone());
2419 let Some(harness) = harness else {
2420 crate::slog_warn!(
2421 "compression event insert skipped for {}: harness not configured",
2422 metadata.task_id
2423 );
2424 return;
2425 };
2426
2427 let project_root = metadata
2428 .project_root
2429 .as_deref()
2430 .unwrap_or(&metadata.workdir);
2431 let project_key = crate::path_identity::project_scope_key(project_root);
2432 let row = crate::db::compression_events::CompressionEventRow {
2433 harness: &harness,
2434 session_id: Some(&metadata.session_id),
2435 project_key: &project_key,
2436 tool: "bash",
2437 task_id: Some(&metadata.task_id),
2438 command: Some(&metadata.command),
2439 compressor: if metadata.compressed {
2440 "registry"
2441 } else {
2442 "none"
2443 },
2444 original_bytes,
2445 compressed_bytes,
2446 original_tokens,
2447 compressed_tokens,
2448 created_at: unix_millis() as i64,
2449 };
2450
2451 let conn = match pool.lock() {
2452 Ok(conn) => conn,
2453 Err(_) => {
2454 crate::slog_warn!(
2455 "compression event insert failed for {}: db mutex poisoned",
2456 metadata.task_id
2457 );
2458 return;
2459 }
2460 };
2461 match crate::db::compression_events::insert_compression_event(&conn, &row) {
2462 Ok(_) => {
2463 crate::slog_debug!(
2467 "compression event recorded for {} (project={}, session={}, {} → {} tokens)",
2468 metadata.task_id,
2469 project_key,
2470 metadata.session_id,
2471 original_tokens,
2472 compressed_tokens
2473 );
2474 }
2475 Err(error) => {
2476 crate::slog_warn!(
2477 "compression event insert failed for {}: {}",
2478 metadata.task_id,
2479 error
2480 );
2481 }
2482 }
2483 }
2484
2485 fn emit_bash_pattern_match(&self, session_id: &str, pattern_match: PatternMatch) {
2486 let Ok(progress_sender) = self
2487 .inner
2488 .progress_sender
2489 .lock()
2490 .map(|sender| sender.clone())
2491 else {
2492 return;
2493 };
2494 if let Some(sender) = progress_sender.as_ref() {
2495 sender(PushFrame::BashPatternMatch(BashPatternMatchFrame::new(
2496 pattern_match.task_id,
2497 session_id.to_string(),
2498 pattern_match.watch_id,
2499 pattern_match.match_text,
2500 pattern_match.match_offset,
2501 pattern_match.context,
2502 pattern_match.once,
2503 )));
2504 }
2505 }
2506
2507 fn emit_bash_watch_exit(&self, completion: &BgCompletion) {
2508 let Ok(progress_sender) = self
2509 .inner
2510 .progress_sender
2511 .lock()
2512 .map(|sender| sender.clone())
2513 else {
2514 return;
2515 };
2516 let Some(sender) = progress_sender.as_ref() else {
2517 return;
2518 };
2519 let status = completion_status_text(&completion.status, completion.exit_code);
2520 let preview = completion.output_preview.trim_end();
2521 let context = if preview.is_empty() {
2522 format!("task {} exited ({status})", completion.task_id)
2523 } else {
2524 format!(
2525 "task {} exited ({status})
2526{preview}",
2527 completion.task_id
2528 )
2529 };
2530 sender(PushFrame::BashPatternMatch(
2531 BashPatternMatchFrame::task_exit(
2532 completion.task_id.clone(),
2533 completion.session_id.clone(),
2534 format!("exited ({status})"),
2535 context,
2536 ),
2537 ));
2538 }
2539
2540 fn emit_bash_completed(&self, completion: BgCompletion) {
2541 let Ok(progress_sender) = self
2542 .inner
2543 .progress_sender
2544 .lock()
2545 .map(|sender| sender.clone())
2546 else {
2547 return;
2548 };
2549 let Some(sender) = progress_sender.as_ref() else {
2550 return;
2551 };
2552 sender(PushFrame::BashCompleted(BashCompletedFrame::new(
2560 completion.task_id,
2561 completion.session_id,
2562 completion.status,
2563 completion.exit_code,
2564 completion.command,
2565 completion.output_preview,
2566 completion.output_truncated,
2567 completion.original_tokens,
2568 completion.compressed_tokens,
2569 completion.tokens_skipped,
2570 )));
2571 }
2572
2573 fn completion_token_counts(
2574 &self,
2575 metadata: &PersistedTask,
2576 buffer: Option<&BgBuffer>,
2577 paths: Option<&TaskPaths>,
2578 rendered_output: Option<&str>,
2579 ) -> CompletionTokenCounts {
2580 if metadata.mode == BgMode::Pty {
2581 return CompletionTokenCounts::skipped();
2582 }
2583
2584 let raw = match buffer {
2585 Some(buffer) => buffer.read_for_token_count(TOKENIZE_CAP_BYTES_PER_STREAM),
2586 None => paths
2587 .map(|paths| {
2588 read_for_token_count_from_disk(metadata, paths, TOKENIZE_CAP_BYTES_PER_STREAM)
2589 })
2590 .unwrap_or(TokenCountInput::Skipped),
2591 };
2592
2593 let TokenCountInput::Text(raw_output) = raw else {
2594 return CompletionTokenCounts::skipped();
2595 };
2596
2597 let original_tokens = token_count_u32(&raw_output);
2598 let original_bytes = raw_output.len() as i64;
2599 let compressed_output = rendered_output.unwrap_or(&raw_output);
2600 let compressed_tokens = token_count_u32(compressed_output);
2601 let compressed_bytes = compressed_output.len() as i64;
2602 CompletionTokenCounts {
2603 original_tokens: Some(original_tokens),
2604 compressed_tokens: Some(compressed_tokens),
2605 original_bytes: Some(original_bytes),
2606 compressed_bytes: Some(compressed_bytes),
2607 tokens_skipped: false,
2608 }
2609 }
2610
2611 pub(crate) fn maybe_emit_long_running_reminder(&self, task: &Arc<BgTask>) {
2612 if !self
2613 .inner
2614 .long_running_reminder_enabled
2615 .load(Ordering::SeqCst)
2616 {
2617 return;
2618 }
2619 let interval_ms = self
2620 .inner
2621 .long_running_reminder_interval_ms
2622 .load(Ordering::SeqCst);
2623 if interval_ms == 0 {
2624 return;
2625 }
2626 let interval = Duration::from_millis(interval_ms);
2627 let now = Instant::now();
2628 let Ok(mut last_reminder_at) = task.last_reminder_at.lock() else {
2629 return;
2630 };
2631 let since = last_reminder_at.unwrap_or(task.started);
2632 if now.duration_since(since) < interval {
2633 return;
2634 }
2635 let command = task
2636 .state
2637 .lock()
2638 .map(|state| state.metadata.command.clone())
2639 .unwrap_or_default();
2640 *last_reminder_at = Some(now);
2641 self.emit_bash_long_running(BashLongRunningFrame::new(
2642 task.task_id.clone(),
2643 task.session_id.clone(),
2644 command,
2645 task.started.elapsed().as_millis() as u64,
2646 ));
2647 }
2648
2649 fn emit_bash_long_running(&self, frame: BashLongRunningFrame) {
2650 let Ok(progress_sender) = self
2651 .inner
2652 .progress_sender
2653 .lock()
2654 .map(|sender| sender.clone())
2655 else {
2656 return;
2657 };
2658 if let Some(sender) = progress_sender.as_ref() {
2659 sender(PushFrame::BashLongRunning(frame));
2660 }
2661 }
2662
2663 fn task(&self, task_id: &str) -> Option<Arc<BgTask>> {
2664 self.inner
2665 .tasks
2666 .lock()
2667 .ok()
2668 .and_then(|tasks| tasks.get(task_id).cloned())
2669 }
2670
2671 fn task_for_session(&self, task_id: &str, session_id: &str) -> Option<Arc<BgTask>> {
2672 self.task(task_id)
2673 .filter(|task| task.session_id == session_id)
2674 }
2675
2676 pub fn try_health_counts(&self) -> Option<BgTaskHealthCounts> {
2677 let running = self
2678 .inner
2679 .tasks
2680 .try_lock()
2681 .ok()
2682 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())?;
2683 let pending_completions = self.inner.completions.try_lock().ok().map(|q| q.len())?;
2684 Some(BgTaskHealthCounts {
2685 running,
2686 pending_completions,
2687 })
2688 }
2689
2690 fn running_count(&self) -> usize {
2691 self.inner
2692 .tasks
2693 .lock()
2694 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())
2695 .unwrap_or(0)
2696 }
2697
2698 fn start_watchdog(&self) {
2699 if !self.inner.watchdog_started.swap(true, Ordering::SeqCst) {
2700 super::watchdog::start(self.clone());
2701 }
2702 }
2703
2704 fn running_metadata_is_stale(&self, metadata: &PersistedTask) -> bool {
2705 unix_millis().saturating_sub(metadata.started_at) > STALE_RUNNING_AFTER.as_millis() as u64
2706 }
2707
2708 #[cfg(test)]
2709 pub fn task_json_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
2710 self.task_for_session(task_id, session_id)
2711 .map(|task| task.paths.json.clone())
2712 }
2713
2714 #[cfg(test)]
2715 pub fn task_exit_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
2716 self.task_for_session(task_id, session_id)
2717 .map(|task| task.paths.exit.clone())
2718 }
2719
2720 fn generate_unique_task_id(&self) -> Result<String, String> {
2722 for _ in 0..32 {
2723 let candidate = random_slug();
2724 let tasks = self
2725 .inner
2726 .tasks
2727 .lock()
2728 .map_err(|_| "background task registry lock poisoned".to_string())?;
2729 if tasks.contains_key(&candidate) {
2730 continue;
2731 }
2732 let completions = self
2733 .inner
2734 .completions
2735 .lock()
2736 .map_err(|_| "background completions lock poisoned".to_string())?;
2737 if completions
2738 .iter()
2739 .any(|completion| completion.task_id == candidate)
2740 {
2741 continue;
2742 }
2743 return Ok(candidate);
2744 }
2745 Err("failed to allocate unique background task id after 32 attempts".to_string())
2746 }
2747}
2748
2749fn render_compressed_with_recovery(
2750 buffer: &BgBuffer,
2751 mut compressed: CompressionResult,
2752 input_truncated: bool,
2753 disk_truncation: DiskTruncation,
2754) -> TerminalOutputCache {
2755 let had_trailing_newline = compressed.text.ends_with('\n');
2763 let mut text = strip_plain_truncation_marker_lines(&compressed.text)
2764 .trim_end()
2765 .to_string();
2766 if had_trailing_newline && !text.is_empty() {
2767 text.push('\n');
2768 }
2769 compressed.text = text;
2770
2771 let output_path = buffer.output_path().map(|path| path.display().to_string());
2772 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
2773 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
2774 let mut recovery = RecoveryContext {
2775 dropped_by_class: compressed.dropped_by_class,
2776 had_inner_drop: compressed.had_inner_drop,
2777 offset_hint_eligible: compressed.offset_hint_eligible,
2778 offset_start_line: compressed.offset_start_line,
2779 byte_truncated: input_truncated,
2780 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
2781 output_path: output_path.clone(),
2782 stderr_path: stderr_path.clone(),
2783 include_stderr_path,
2784 };
2785
2786 let (output_preview, output_truncated) =
2787 render_body_with_recovery_marker(&compressed.text, &mut recovery);
2788 TerminalOutputCache {
2789 output_preview,
2790 output_truncated,
2791 kind: TerminalOutputKind::Compressed,
2792 output_path,
2793 stderr_path,
2794 recovery: Some(recovery),
2795 }
2796}
2797
2798fn render_body_with_recovery_marker(body: &str, recovery: &mut RecoveryContext) -> (String, bool) {
2799 render_body_with_recovery_marker_at_cap(
2800 body,
2801 recovery,
2802 FINAL_OUTPUT_CAP_BYTES,
2803 cap_final_output,
2804 cap_final_output_with_marker,
2805 )
2806}
2807
2808fn render_raw_body_with_recovery_marker(
2809 body: &str,
2810 recovery: &mut RecoveryContext,
2811) -> (String, bool) {
2812 render_body_with_recovery_marker_at_cap(
2813 body,
2814 recovery,
2815 RAW_PASSTHROUGH_CAP_BYTES,
2816 |input| {
2817 super::output::cap_head_tail(
2818 input,
2819 RAW_PASSTHROUGH_CAP_BYTES,
2820 RAW_PASSTHROUGH_HEAD_BYTES,
2821 RAW_PASSTHROUGH_TAIL_BYTES,
2822 )
2823 },
2824 |input, marker| {
2825 super::output::cap_head_tail_with_marker(
2826 input,
2827 RAW_PASSTHROUGH_CAP_BYTES,
2828 RAW_PASSTHROUGH_HEAD_BYTES,
2829 RAW_PASSTHROUGH_TAIL_BYTES,
2830 marker,
2831 )
2832 },
2833 )
2834}
2835
2836fn render_body_with_recovery_marker_at_cap<F, G>(
2837 body: &str,
2838 recovery: &mut RecoveryContext,
2839 cap_bytes: usize,
2840 cap_plain: F,
2841 cap_with_marker: G,
2842) -> (String, bool)
2843where
2844 F: Fn(&str) -> super::output::CappedText,
2845 G: Fn(&str, &str) -> super::output::CappedText,
2846{
2847 let needs_marker = recovery.has_visible_drop();
2848 if body.len() > cap_bytes {
2849 recovery.byte_truncated = true;
2850 if let Some(marker) = recovery_marker(recovery) {
2851 let capped = cap_with_marker(body, &marker);
2852 return (capped.text, true);
2853 }
2854 let capped = cap_plain(body);
2855 return (capped.text, capped.truncated || needs_marker);
2856 }
2857
2858 if !needs_marker {
2859 return (body.to_string(), false);
2860 }
2861
2862 let Some(marker) = recovery_marker(recovery) else {
2863 return (body.to_string(), true);
2864 };
2865 let with_marker = append_recovery_marker(body, &marker);
2866 if with_marker.len() <= cap_bytes {
2867 return (with_marker, true);
2868 }
2869
2870 recovery.byte_truncated = true;
2871 let marker = recovery_marker(recovery).unwrap_or(marker);
2872 let capped = cap_with_marker(body, &marker);
2873 (capped.text, true)
2874}
2875
2876fn append_recovery_marker(body: &str, marker: &str) -> String {
2877 if body.is_empty() {
2878 return marker.to_string();
2879 }
2880 let mut output = body.trim_end().to_string();
2881 output.push('\n');
2882 output.push_str(marker);
2883 output
2884}
2885
2886fn recovery_marker(recovery: &RecoveryContext) -> Option<String> {
2887 let mut parts = Vec::new();
2888 for (class, count) in &recovery.dropped_by_class {
2889 let label = if *count == 1 {
2890 class.singular()
2891 } else {
2892 class.plural()
2893 };
2894 parts.push(format!("+{count} more {label}"));
2895 }
2896 if recovery.byte_truncated {
2897 parts.push("truncated output".to_string());
2898 }
2899 let disk_truncated_prefix_bytes = recovery.disk_truncated_prefix_bytes;
2900 if disk_truncated_prefix_bytes > 0 {
2901 parts.push(format!(
2902 "truncated {disk_truncated_prefix_bytes} bytes from saved output prefix"
2903 ));
2904 } else if recovery.had_inner_drop && parts.is_empty() {
2905 parts.push("omitted output".to_string());
2906 }
2907
2908 if parts.is_empty() {
2909 return None;
2910 }
2911
2912 let hint = recovery_hint(recovery);
2913 Some(format!("[{}; {hint}]", parts.join(", ")))
2914}
2915
2916fn recovery_hint(recovery: &RecoveryContext) -> String {
2917 if recovery.offset_hint_eligible
2921 && !recovery.byte_truncated
2922 && recovery.dropped_by_class.is_empty()
2923 && !recovery.include_stderr_path
2924 {
2925 if let (Some(path), Some(line)) =
2926 (recovery.output_path.as_deref(), recovery.offset_start_line)
2927 {
2928 return format!("see remaining: tail -n +{line} {}", quote_path(path));
2929 }
2930 }
2931
2932 let mut paths = Vec::new();
2933 if let Some(path) = recovery.output_path.as_deref() {
2934 paths.push(path);
2935 }
2936 if recovery.include_stderr_path {
2937 if let Some(path) = recovery.stderr_path.as_deref() {
2938 if !paths.contains(&path) {
2939 paths.push(path);
2940 }
2941 }
2942 }
2943
2944 if paths.is_empty() {
2945 return "full output unavailable".to_string();
2946 }
2947
2948 let reads = paths
2949 .into_iter()
2950 .map(|path| format!("read {}", quote_path(path)))
2951 .collect::<Vec<_>>()
2952 .join(" and ");
2953 if recovery.disk_truncated_prefix_bytes > 0 {
2954 format!("retained output: {reads}")
2955 } else {
2956 format!("full output: {reads}")
2957 }
2958}
2959
2960fn strip_plain_truncation_marker_lines(input: &str) -> String {
2961 input
2962 .lines()
2963 .filter(|line| !is_plain_truncation_marker(line.trim()))
2964 .collect::<Vec<_>>()
2965 .join("\n")
2966}
2967
2968fn strip_recovery_marker_lines(input: &str) -> String {
2969 input
2970 .lines()
2971 .filter(|line| !is_recovery_marker(line.trim()))
2972 .collect::<Vec<_>>()
2973 .join("\n")
2974}
2975
2976fn is_plain_truncation_marker(line: &str) -> bool {
2977 let Some(rest) = line.strip_prefix("...<truncated ") else {
2978 return false;
2979 };
2980 let Some(bytes) = rest.strip_suffix(" bytes>...") else {
2981 return false;
2982 };
2983 !bytes.is_empty() && bytes.chars().all(|ch| ch.is_ascii_digit())
2984}
2985
2986fn is_recovery_marker(line: &str) -> bool {
2987 line.starts_with('[')
2988 && line.ends_with(']')
2989 && (line.contains("full output: read ")
2990 || line.contains("retained output: read ")
2991 || line.contains("see remaining: tail -n +")
2992 || line.contains("full output unavailable"))
2993}
2994
2995fn render_structured_output(
2996 command: &str,
2997 buffer: &BgBuffer,
2998 disk_truncation: DiskTruncation,
2999) -> Option<TerminalOutputCache> {
3000 if !is_gh_structured_command(command) {
3001 return None;
3002 }
3003
3004 let output_path = buffer
3005 .output_path()
3006 .map(|path| path.display().to_string())?;
3007 let stdout_bytes = buffer.stream_len(StreamKind::Stdout);
3008 if stdout_bytes == 0 {
3009 return None;
3010 }
3011
3012 if stdout_bytes > STRUCTURED_OUTPUT_CAP_BYTES as u64 {
3013 if !stream_starts_like_json(buffer, StreamKind::Stdout) {
3014 return None;
3015 }
3016 let output_preview = if disk_truncation.total_prefix_bytes() > 0 {
3017 retained_json_output_pointer(
3018 stdout_bytes,
3019 &output_path,
3020 disk_truncation.total_prefix_bytes(),
3021 )
3022 } else {
3023 json_output_pointer(stdout_bytes, &output_path)
3024 };
3025 return Some(TerminalOutputCache {
3026 output_preview,
3027 output_truncated: true,
3028 kind: TerminalOutputKind::Structured,
3029 output_path: Some(output_path),
3030 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
3031 recovery: None,
3032 });
3033 }
3034
3035 let stdout = buffer.read_stream_bounded(StreamKind::Stdout, STRUCTURED_OUTPUT_CAP_BYTES);
3036 if stdout.truncated || !is_structured_body(&stdout.text) {
3037 return None;
3038 }
3039
3040 Some(TerminalOutputCache {
3041 output_preview: stdout.text,
3042 output_truncated: false,
3043 kind: TerminalOutputKind::Structured,
3044 output_path: Some(output_path),
3045 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
3046 recovery: None,
3047 })
3048}
3049
3050fn render_raw_passthrough(
3051 buffer: &BgBuffer,
3052 disk_truncation: DiskTruncation,
3053) -> TerminalOutputCache {
3054 let raw = buffer.read_combined_head_tail(
3055 RAW_PASSTHROUGH_CAP_BYTES,
3056 RAW_PASSTHROUGH_HEAD_BYTES,
3057 RAW_PASSTHROUGH_TAIL_BYTES,
3058 );
3059 let output_path = buffer.output_path().map(|path| path.display().to_string());
3060 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
3061 if !raw.truncated && disk_truncation.total_prefix_bytes() == 0 {
3062 return TerminalOutputCache {
3063 output_preview: raw.text,
3064 output_truncated: false,
3065 kind: TerminalOutputKind::Raw,
3066 output_path,
3067 stderr_path,
3068 recovery: None,
3069 };
3070 }
3071
3072 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
3073 let mut recovery = RecoveryContext {
3074 dropped_by_class: BTreeMap::new(),
3075 had_inner_drop: false,
3076 offset_hint_eligible: false,
3077 offset_start_line: None,
3078 byte_truncated: raw.truncated,
3079 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
3080 output_path: output_path.clone(),
3081 stderr_path: stderr_path.clone(),
3082 include_stderr_path,
3083 };
3084 let (output_preview, output_truncated) =
3085 render_raw_body_with_recovery_marker(&raw.text, &mut recovery);
3086 TerminalOutputCache {
3087 output_preview,
3088 output_truncated,
3089 kind: TerminalOutputKind::Raw,
3090 output_path,
3091 stderr_path,
3092 recovery: Some(recovery),
3093 }
3094}
3095
3096fn completion_preview_for_cache(
3097 cache: &TerminalOutputCache,
3098 exit_code: Option<i32>,
3099) -> (String, bool) {
3100 let exit_ok = exit_code == Some(0);
3103 let threshold = completion_preview_threshold(exit_ok);
3104 if cache.kind == TerminalOutputKind::Structured && cache.output_preview.len() > threshold {
3105 if let Some(path) = cache.output_path.as_deref() {
3106 return (
3107 json_output_pointer(cache.output_preview.len() as u64, path),
3108 true,
3109 );
3110 }
3111 return (cache.output_preview.clone(), cache.output_truncated);
3112 }
3113
3114 if let Some(recovery) = cache.recovery.as_ref() {
3115 if cache.output_preview.len() <= threshold {
3116 return (cache.output_preview.clone(), cache.output_truncated);
3117 }
3118 let body = strip_recovery_marker_lines(&cache.output_preview);
3119 let mut completion_recovery = recovery.clone();
3120 completion_recovery.byte_truncated = true;
3121 if let Some(marker) = recovery_marker(&completion_recovery) {
3122 let capped = cap_completion_output_with_marker(&body, &marker, exit_ok);
3123 return (capped.text, true);
3124 }
3125 }
3126
3127 let capped = cap_completion_output(&cache.output_preview, exit_ok);
3128 (capped.text, cache.output_truncated || capped.truncated)
3129}
3130
3131fn is_gh_structured_command(command: &str) -> bool {
3132 let Some(normalized) = crate::compress::plain_command_for_structured_output(command) else {
3133 return false;
3134 };
3135 let tokens = shell_words_for_flags(&normalized);
3136 let Some(head) = tokens.first() else {
3137 return false;
3138 };
3139 let head_name = Path::new(head)
3140 .file_name()
3141 .and_then(|name| name.to_str())
3142 .unwrap_or(head);
3143 if !(head_name == "gh" || head_name.eq_ignore_ascii_case("gh.exe")) {
3144 return false;
3145 }
3146 tokens.iter().any(|token| {
3147 matches!(token.as_str(), "--json" | "--jq" | "--template")
3148 || token.starts_with("--json=")
3149 || token.starts_with("--jq=")
3150 || token.starts_with("--template=")
3151 })
3152}
3153
3154fn shell_words_for_flags(command: &str) -> Vec<String> {
3155 let mut words = Vec::new();
3156 let mut current = String::new();
3157 let mut in_single = false;
3158 let mut in_double = false;
3159 let mut escaped = false;
3160
3161 for ch in command.chars() {
3162 if escaped {
3163 current.push(ch);
3164 escaped = false;
3165 continue;
3166 }
3167 if ch == '\\' && !in_single {
3168 escaped = true;
3169 continue;
3170 }
3171 if ch == '\'' && !in_double {
3172 in_single = !in_single;
3173 continue;
3174 }
3175 if ch == '"' && !in_single {
3176 in_double = !in_double;
3177 continue;
3178 }
3179 if ch.is_whitespace() && !in_single && !in_double {
3180 if !current.is_empty() {
3181 words.push(std::mem::take(&mut current));
3182 }
3183 continue;
3184 }
3185 if matches!(ch, ';' | '&' | '|') && !in_single && !in_double {
3186 if !current.is_empty() {
3187 words.push(std::mem::take(&mut current));
3188 }
3189 continue;
3190 }
3191 current.push(ch);
3192 }
3193 if !current.is_empty() {
3194 words.push(current);
3195 }
3196 words
3197}
3198
3199fn is_structured_body(body: &str) -> bool {
3200 let trimmed = body.trim();
3201 if trimmed.is_empty() {
3202 return false;
3203 }
3204 if serde_json::from_str::<serde_json::Value>(trimmed).is_ok() {
3205 return true;
3206 }
3207
3208 let mut saw_line = false;
3209 for line in trimmed
3210 .lines()
3211 .map(str::trim)
3212 .filter(|line| !line.is_empty())
3213 {
3214 saw_line = true;
3215 if serde_json::from_str::<serde_json::Value>(line).is_err() {
3216 return false;
3217 }
3218 }
3219 saw_line
3220}
3221
3222fn stream_starts_like_json(buffer: &BgBuffer, stream: StreamKind) -> bool {
3223 let path = match (buffer, stream) {
3224 (BgBuffer::Pipes { stdout_path, .. }, StreamKind::Stdout) => Some(stdout_path),
3225 (BgBuffer::Pipes { stderr_path, .. }, StreamKind::Stderr) => Some(stderr_path),
3226 (BgBuffer::Pty { combined_path }, _) => Some(combined_path),
3227 };
3228 let Some(path) = path else {
3229 return false;
3230 };
3231 let Ok(file) = std::fs::File::open(path) else {
3232 return false;
3233 };
3234 let mut limited = file.take(512);
3235 let mut bytes = Vec::new();
3236 if limited.read_to_end(&mut bytes).is_err() {
3237 return false;
3238 }
3239 String::from_utf8_lossy(&bytes)
3240 .chars()
3241 .find(|ch| !ch.is_whitespace())
3242 .is_some_and(|ch| matches!(ch, '{' | '[' | '"' | '-' | '0'..='9' | 't' | 'f' | 'n'))
3243}
3244
3245struct CompletionTokenCounts {
3246 original_tokens: Option<u32>,
3247 compressed_tokens: Option<u32>,
3248 original_bytes: Option<i64>,
3249 compressed_bytes: Option<i64>,
3250 tokens_skipped: bool,
3251}
3252
3253impl CompletionTokenCounts {
3254 fn skipped() -> Self {
3255 Self {
3256 original_tokens: None,
3257 compressed_tokens: None,
3258 original_bytes: None,
3259 compressed_bytes: None,
3260 tokens_skipped: true,
3261 }
3262 }
3263}
3264
3265fn completion_status_text(status: &BgTaskStatus, exit_code: Option<i32>) -> String {
3266 match status {
3267 BgTaskStatus::TimedOut => "timed out".to_string(),
3268 BgTaskStatus::Killed => "killed".to_string(),
3269 _ => exit_code
3270 .map(|code| format!("exit {code}"))
3271 .unwrap_or_else(|| format!("{status:?}").to_lowercase()),
3272 }
3273}
3274
3275fn token_count_u32(text: &str) -> u32 {
3276 aft_tokenizer::count_tokens(text)
3277 .try_into()
3278 .unwrap_or(u32::MAX)
3279}
3280
3281impl Default for BgTaskRegistry {
3282 fn default() -> Self {
3283 Self::new(Arc::new(Mutex::new(None)))
3284 }
3285}
3286
3287fn modified_within(path: &Path, grace: Duration) -> bool {
3288 fs::metadata(path)
3289 .and_then(|metadata| metadata.modified())
3290 .ok()
3291 .and_then(|modified| SystemTime::now().duration_since(modified).ok())
3292 .map(|age| age < grace)
3293 .unwrap_or(false)
3294}
3295
3296fn canonicalized_path(path: &Path) -> PathBuf {
3297 fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf())
3298}
3299
3300fn started_instant_from_unix_millis(started_at: u64) -> Instant {
3301 let now_ms = SystemTime::now()
3302 .duration_since(UNIX_EPOCH)
3303 .ok()
3304 .map(|duration| duration.as_millis() as u64)
3305 .unwrap_or(started_at);
3306 let elapsed_ms = now_ms.saturating_sub(started_at);
3307 Instant::now()
3308 .checked_sub(Duration::from_millis(elapsed_ms))
3309 .unwrap_or_else(Instant::now)
3310}
3311
3312fn gc_quarantine(storage_dir: &Path) {
3313 let quarantine_root = storage_dir.join("bash-tasks-quarantine");
3314 let Ok(session_dirs) = fs::read_dir(&quarantine_root) else {
3315 return;
3316 };
3317 for session_entry in session_dirs.flatten() {
3318 let session_quarantine_dir = session_entry.path();
3319 if !session_quarantine_dir.is_dir() {
3320 continue;
3321 }
3322 let entries = match fs::read_dir(&session_quarantine_dir) {
3323 Ok(entries) => entries,
3324 Err(error) => {
3325 crate::slog_warn!(
3326 "failed to read background task quarantine dir {}: {error}",
3327 session_quarantine_dir.display()
3328 );
3329 continue;
3330 }
3331 };
3332 for entry in entries.flatten() {
3333 let path = entry.path();
3334 if modified_within(&path, QUARANTINE_GC_GRACE) {
3335 continue;
3336 }
3337 let result = if path.is_dir() {
3338 fs::remove_dir_all(&path)
3339 } else {
3340 fs::remove_file(&path)
3341 };
3342 match result {
3343 Ok(()) => log::debug!(
3344 "deleted old background task quarantine entry {}",
3345 path.display()
3346 ),
3347 Err(error) => crate::slog_warn!(
3348 "failed to delete old background task quarantine entry {}: {error}",
3349 path.display()
3350 ),
3351 }
3352 }
3353 let _ = fs::remove_dir(&session_quarantine_dir);
3354 }
3355 let _ = fs::remove_dir(&quarantine_root);
3356}
3357
3358enum QuarantineKind {
3359 Corrupt,
3360 Invalid,
3361}
3362
3363fn quarantine_task_json(
3364 storage_dir: &Path,
3365 session_dir: &Path,
3366 json_path: &Path,
3367 kind: QuarantineKind,
3368) -> Result<(), String> {
3369 let session_hash = session_dir
3370 .file_name()
3371 .and_then(|name| name.to_str())
3372 .ok_or_else(|| {
3373 format!(
3374 "invalid background task session dir: {}",
3375 session_dir.display()
3376 )
3377 })?;
3378 let task_name = json_path
3379 .file_name()
3380 .and_then(|name| name.to_str())
3381 .ok_or_else(|| format!("invalid background task json path: {}", json_path.display()))?;
3382 let unix_ts = SystemTime::now()
3383 .duration_since(UNIX_EPOCH)
3384 .map(|duration| duration.as_secs())
3385 .unwrap_or(0);
3386 let quarantine_dir = storage_dir.join("bash-tasks-quarantine").join(session_hash);
3387 fs::create_dir_all(&quarantine_dir).map_err(|e| {
3388 format!(
3389 "failed to create background task quarantine dir {}: {e}",
3390 quarantine_dir.display()
3391 )
3392 })?;
3393 let target_name = quarantine_name(task_name, unix_ts, &kind);
3394 let target = quarantine_dir.join(target_name);
3395 fs::rename(json_path, &target).map_err(|e| {
3396 format!(
3397 "failed to quarantine background task metadata {} to {}: {e}",
3398 json_path.display(),
3399 target.display()
3400 )
3401 })?;
3402
3403 for sibling in task_sibling_paths(json_path) {
3404 if !sibling.exists() {
3405 continue;
3406 }
3407 let Some(sibling_name) = sibling.file_name().and_then(|name| name.to_str()) else {
3408 crate::slog_warn!(
3409 "skipping background task sibling with invalid name during quarantine: {}",
3410 sibling.display()
3411 );
3412 continue;
3413 };
3414 let sibling_target = quarantine_dir.join(quarantine_name(sibling_name, unix_ts, &kind));
3415 if let Err(error) = fs::rename(&sibling, &sibling_target) {
3416 crate::slog_warn!(
3417 "failed to quarantine background task sibling {} to {}: {error}",
3418 sibling.display(),
3419 sibling_target.display()
3420 );
3421 }
3422 }
3423
3424 let _ = fs::remove_dir(session_dir);
3425 Ok(())
3426}
3427
3428fn quarantine_name(file_name: &str, unix_ts: u64, kind: &QuarantineKind) -> String {
3429 match kind {
3430 QuarantineKind::Corrupt => format!("{file_name}.corrupt-{unix_ts}"),
3431 QuarantineKind::Invalid => {
3432 let path = Path::new(file_name);
3433 let stem = path.file_stem().and_then(|stem| stem.to_str());
3434 let extension = path.extension().and_then(|extension| extension.to_str());
3435 match (stem, extension) {
3436 (Some(stem), Some(extension)) => format!("{stem}.invalid.{unix_ts}.{extension}"),
3437 _ => format!("{file_name}.invalid.{unix_ts}"),
3438 }
3439 }
3440 }
3441}
3442
3443fn task_sibling_paths(json_path: &Path) -> Vec<PathBuf> {
3444 let Some(parent) = json_path.parent() else {
3445 return Vec::new();
3446 };
3447 let Some(stem) = json_path.file_stem().and_then(|stem| stem.to_str()) else {
3448 return Vec::new();
3449 };
3450 ["stdout", "stderr", "exit", "pty", "ps1", "bat", "sh"]
3451 .into_iter()
3452 .map(|extension| parent.join(format!("{stem}.{extension}")))
3453 .collect()
3454}
3455
3456fn read_for_token_count_from_disk(
3457 metadata: &PersistedTask,
3458 paths: &TaskPaths,
3459 max_bytes_per_stream: usize,
3460) -> TokenCountInput {
3461 if metadata.mode == BgMode::Pty {
3462 return TokenCountInput::Skipped;
3463 }
3464 let stdout = read_file_tail_capped(&paths.stdout, max_bytes_per_stream);
3471 let stderr = read_file_tail_capped(&paths.stderr, max_bytes_per_stream);
3472 match (stdout, stderr) {
3473 (Ok(stdout), Ok(stderr)) => TokenCountInput::Text(combine_streams(
3474 String::from_utf8_lossy(&stdout).as_ref(),
3475 String::from_utf8_lossy(&stderr).as_ref(),
3476 )),
3477 (Ok(stdout), Err(_)) => TokenCountInput::Text(combine_streams(
3478 String::from_utf8_lossy(&stdout).as_ref(),
3479 "",
3480 )),
3481 (Err(_), Ok(stderr)) => TokenCountInput::Text(combine_streams(
3482 "",
3483 String::from_utf8_lossy(&stderr).as_ref(),
3484 )),
3485 (Err(_), Err(_)) => TokenCountInput::Skipped,
3486 }
3487}
3488
3489fn read_file_tail_capped(path: &Path, max_bytes: usize) -> std::io::Result<Vec<u8>> {
3494 use std::io::{Read, Seek, SeekFrom};
3495 let mut file = std::fs::File::open(path)?;
3496 let len = file.metadata()?.len();
3497 let read_len = len.min(max_bytes as u64);
3498 if read_len > 0 && len > max_bytes as u64 {
3499 file.seek(SeekFrom::End(-(read_len as i64)))?;
3500 }
3501 let mut bytes = Vec::with_capacity(read_len as usize);
3502 file.read_to_end(&mut bytes)?;
3503 Ok(bytes)
3504}
3505
3506impl BgTask {
3507 fn snapshot(&self, preview_bytes: usize) -> BgTaskSnapshot {
3508 let state = self
3509 .state
3510 .lock()
3511 .unwrap_or_else(|poison| poison.into_inner());
3512 self.snapshot_locked(&state, preview_bytes)
3513 }
3514
3515 fn snapshot_locked(&self, state: &BgTaskState, preview_bytes: usize) -> BgTaskSnapshot {
3516 let metadata = &state.metadata;
3517 let duration_ms = metadata.duration_ms.or_else(|| {
3518 metadata
3519 .status
3520 .is_terminal()
3521 .then(|| self.started.elapsed().as_millis() as u64)
3522 });
3523 let (output_preview, output_truncated) = if metadata.mode == BgMode::Pty {
3524 (String::new(), false)
3525 } else if metadata.status.is_terminal() {
3526 state
3527 .terminal_output_cache
3528 .as_ref()
3529 .map(|cache| (cache.output_preview.clone(), cache.output_truncated))
3530 .unwrap_or_else(|| (String::new(), false))
3531 } else {
3532 state.buffer.read_tail(preview_bytes)
3533 };
3534 BgTaskSnapshot {
3535 info: BgTaskInfo {
3536 task_id: self.task_id.clone(),
3537 status: metadata.status.clone(),
3538 command: metadata.command.clone(),
3539 mode: metadata.mode.clone(),
3540 started_at: metadata.started_at,
3541 duration_ms,
3542 },
3543 exit_code: metadata.exit_code,
3544 child_pid: metadata.child_pid,
3545 workdir: metadata.workdir.display().to_string(),
3546 output_preview,
3547 output_truncated,
3548 output_path: state
3549 .buffer
3550 .output_path()
3551 .map(|path| path.display().to_string()),
3552 stderr_path: state
3553 .buffer
3554 .stderr_path()
3555 .map(|path| path.display().to_string()),
3556 pty_rows: (metadata.mode == BgMode::Pty).then_some(metadata.pty_rows.unwrap_or(24)),
3557 pty_cols: (metadata.mode == BgMode::Pty).then_some(metadata.pty_cols.unwrap_or(80)),
3558 pty_screen: None,
3559 }
3560 }
3561
3562 pub(crate) fn is_running(&self) -> bool {
3563 self.state
3564 .lock()
3565 .map(|state| {
3566 state.metadata.status == BgTaskStatus::Running
3567 || (state.metadata.mode == BgMode::Pty
3568 && state.metadata.status == BgTaskStatus::Killing)
3569 })
3570 .unwrap_or(false)
3571 }
3572
3573 fn is_terminal(&self) -> bool {
3574 self.state
3575 .lock()
3576 .map(|state| state.metadata.status.is_terminal())
3577 .unwrap_or(false)
3578 }
3579
3580 fn mark_terminal_now(&self) {
3581 if let Ok(mut terminal_at) = self.terminal_at.lock() {
3582 if terminal_at.is_none() {
3583 *terminal_at = Some(Instant::now());
3584 }
3585 }
3586 }
3587
3588 fn set_completion_delivered(
3589 &self,
3590 delivered: bool,
3591 registry: &BgTaskRegistry,
3592 ) -> Result<(), String> {
3593 let mut state = self
3594 .state
3595 .lock()
3596 .map_err(|_| "background task lock poisoned".to_string())?;
3597 let updated = registry
3598 .update_task_metadata(&self.paths, |metadata| {
3599 metadata.completion_delivered = delivered;
3600 })
3601 .map_err(|e| format!("failed to update completion delivery: {e}"))?;
3602 state.metadata = updated;
3603 Ok(())
3604 }
3605}
3606
3607#[cfg(unix)]
3628fn reap_piped_child(child_slot: &mut Option<Child>) {
3629 if let Some(mut child) = child_slot.take() {
3630 if matches!(child.try_wait(), Ok(None)) {
3631 let _ = child.wait();
3632 }
3633 }
3634}
3635
3636#[cfg(windows)]
3641fn reap_piped_child(child_slot: &mut Option<Child>) {
3642 *child_slot = None;
3643}
3644
3645fn terminal_metadata_from_marker(
3646 mut metadata: PersistedTask,
3647 marker: ExitMarker,
3648 reason: Option<String>,
3649) -> PersistedTask {
3650 match marker {
3651 ExitMarker::Code(code) => {
3652 let status = if code == 0 {
3653 BgTaskStatus::Completed
3654 } else {
3655 BgTaskStatus::Failed
3656 };
3657 metadata.mark_terminal(status, Some(code), reason);
3658 }
3659 ExitMarker::Killed => metadata.mark_terminal(
3660 BgTaskStatus::Killed,
3661 terminal_exit_code_for_status(&BgTaskStatus::Killed),
3662 reason,
3663 ),
3664 }
3665 metadata
3666}
3667
3668fn terminal_exit_code_for_status(status: &BgTaskStatus) -> Option<i32> {
3669 match status {
3670 BgTaskStatus::TimedOut => Some(124),
3671 BgTaskStatus::Killed => Some(137),
3672 _ => None,
3673 }
3674}
3675
3676#[cfg(unix)]
3677fn write_unix_command_script(command: &str, paths: &TaskPaths) -> Result<PathBuf, String> {
3678 let stem = paths
3679 .json
3680 .file_stem()
3681 .and_then(|stem| stem.to_str())
3682 .unwrap_or("wrapper");
3683 let script_path = paths.dir.join(format!("{stem}.sh"));
3684 fs::write(&script_path, command)
3685 .map_err(|e| format!("failed to write background bash command script: {e}"))?;
3686 Ok(script_path)
3687}
3688
3689#[cfg(unix)]
3690fn detached_shell_command(command_script: &Path, exit_path: &Path) -> Command {
3691 let shell = resolve_posix_shell();
3692 let mut cmd = Command::new(&shell);
3693 cmd.arg("-c")
3699 .arg(r#""$0" "$1"; code=$?; printf "%s" "$code" > "$2.tmp.$$"; mv -f "$2.tmp.$$" "$2""#)
3700 .arg(&shell)
3701 .arg(command_script)
3702 .arg(exit_path);
3703 unsafe {
3704 cmd.pre_exec(|| {
3705 if libc::setsid() == -1 {
3706 return Err(std::io::Error::last_os_error());
3707 }
3708 Ok(())
3709 });
3710 }
3711 cmd
3712}
3713
3714#[cfg(unix)]
3715fn resolve_posix_shell() -> PathBuf {
3716 static POSIX_SHELL: OnceLock<PathBuf> = OnceLock::new();
3717 POSIX_SHELL
3718 .get_or_init(|| {
3719 std::env::var_os("BASH")
3720 .filter(|value| !value.is_empty())
3721 .map(PathBuf::from)
3722 .filter(|path| path.exists())
3723 .or_else(|| which::which("bash").ok())
3724 .or_else(|| which::which("zsh").ok())
3725 .unwrap_or_else(|| PathBuf::from("/bin/sh"))
3726 })
3727 .clone()
3728}
3729
3730#[cfg(windows)]
3731fn detached_shell_command_for(
3732 shell: crate::windows_shell::WindowsShell,
3733 command: &str,
3734 exit_path: &Path,
3735 paths: &TaskPaths,
3736 creation_flags: u32,
3737) -> Result<Command, String> {
3738 use crate::windows_shell::WindowsShell;
3739 let wrapper_body = shell.wrapper_script_bytes(command, exit_path);
3752 let wrapper_ext = match shell {
3753 WindowsShell::Pwsh | WindowsShell::Powershell => "ps1",
3754 WindowsShell::Cmd => "bat",
3755 WindowsShell::Posix(_) => "sh",
3759 };
3760 let wrapper_path = paths.dir.join(format!(
3761 "{}.{}",
3762 paths
3763 .json
3764 .file_stem()
3765 .and_then(|s| s.to_str())
3766 .unwrap_or("wrapper"),
3767 wrapper_ext
3768 ));
3769 fs::write(&wrapper_path, wrapper_body)
3770 .map_err(|e| format!("failed to write background bash wrapper script: {e}"))?;
3771
3772 let mut cmd = Command::new(shell.binary().as_ref());
3773 match shell {
3774 WindowsShell::Pwsh | WindowsShell::Powershell => {
3775 cmd.args([
3778 "-NoLogo",
3779 "-NoProfile",
3780 "-NonInteractive",
3781 "-ExecutionPolicy",
3782 "Bypass",
3783 "-File",
3784 ]);
3785 cmd.arg(&wrapper_path);
3786 }
3787 WindowsShell::Cmd => {
3788 cmd.args(["/D", "/C"]);
3795 cmd.arg(&wrapper_path);
3796 }
3797 WindowsShell::Posix(_) => {
3798 cmd.arg(&wrapper_path);
3803 }
3804 }
3805
3806 cmd.creation_flags(creation_flags);
3810 Ok(cmd)
3811}
3812
3813fn spawn_detached_child(
3829 command: &str,
3830 paths: &TaskPaths,
3831 workdir: &Path,
3832 env: &HashMap<String, String>,
3833) -> Result<std::process::Child, String> {
3834 #[cfg(not(windows))]
3835 {
3836 let stdout = create_capture_file(&paths.stdout)
3837 .map_err(|e| format!("failed to open stdout capture file: {e}"))?;
3838 let stderr = create_capture_file(&paths.stderr)
3839 .map_err(|e| format!("failed to open stderr capture file: {e}"))?;
3840 let command_script = write_unix_command_script(command, paths)?;
3841 detached_shell_command(&command_script, &paths.exit)
3842 .current_dir(workdir)
3843 .envs(env)
3844 .stdin(Stdio::null())
3845 .stdout(Stdio::from(stdout))
3846 .stderr(Stdio::from(stderr))
3847 .spawn()
3848 .map_err(|e| format!("failed to spawn background bash command: {e}"))
3849 }
3850 #[cfg(windows)]
3851 {
3852 use crate::windows_shell::shell_candidates;
3853 let candidates: Vec<crate::windows_shell::WindowsShell> = shell_candidates();
3864 const FLAG_CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200;
3885 const FLAG_CREATE_BREAKAWAY_FROM_JOB: u32 = 0x0100_0000;
3886 const FLAG_CREATE_NO_WINDOW: u32 = 0x0800_0000;
3887 let with_breakaway =
3888 FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP | FLAG_CREATE_BREAKAWAY_FROM_JOB;
3889 let without_breakaway = FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP;
3890 let mut last_error: Option<String> = None;
3891 for (idx, shell) in candidates.iter().enumerate() {
3892 for &flags in &[with_breakaway, without_breakaway] {
3896 let stdout = create_capture_file(&paths.stdout)
3898 .map_err(|e| format!("failed to open stdout capture file: {e}"))?;
3899 let stderr = create_capture_file(&paths.stderr)
3900 .map_err(|e| format!("failed to open stderr capture file: {e}"))?;
3901 let mut cmd =
3902 detached_shell_command_for(shell.clone(), command, &paths.exit, paths, flags)?;
3903 cmd.current_dir(workdir)
3904 .envs(env)
3905 .stdin(Stdio::null())
3906 .stdout(Stdio::from(stdout))
3907 .stderr(Stdio::from(stderr));
3908 match cmd.spawn() {
3909 Ok(child) => {
3910 if idx > 0 {
3911 crate::slog_warn!("background bash spawn fell back to {} after {} earlier candidate(s) failed; \
3912 the cached PATH probe disagreed with runtime spawn — likely PATH \
3913 inheritance, antivirus / AppLocker / Defender ASR, or sandbox policy.",
3914 shell.binary(),
3915 idx);
3916 }
3917 if flags == without_breakaway {
3918 crate::slog_warn!(
3919 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected \
3920 (likely a restrictive Job Object — CI sandbox or MDM policy). \
3921 Spawned without breakaway; the bg task will be torn down if the \
3922 AFT process group is killed."
3923 );
3924 }
3925 return Ok(child);
3926 }
3927 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
3928 crate::slog_warn!("background bash spawn: {} returned NotFound at runtime — trying next candidate",
3929 shell.binary());
3930 last_error = Some(format!("{}: {e}", shell.binary()));
3931 break;
3934 }
3935 Err(e) if flags == with_breakaway && e.raw_os_error() == Some(5) => {
3936 crate::slog_warn!(
3938 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected with \
3939 Access Denied — retrying {} without breakaway",
3940 shell.binary()
3941 );
3942 last_error = Some(format!("{}: {e}", shell.binary()));
3943 continue;
3944 }
3945 Err(e) => {
3946 return Err(format!(
3947 "failed to spawn background bash command via {}: {e}",
3948 shell.binary()
3949 ));
3950 }
3951 }
3952 }
3953 }
3954 Err(format!(
3955 "failed to spawn background bash command: no Windows shell could be spawned. \
3956 Last error: {}. PATH-probed candidates: {:?}",
3957 last_error.unwrap_or_else(|| "no candidates were attempted".to_string()),
3958 candidates.iter().map(|s| s.binary()).collect::<Vec<_>>()
3959 ))
3960 }
3961}
3962
3963fn random_slug() -> String {
3964 let mut bytes = [0u8; 8];
3972 getrandom::fill(&mut bytes).unwrap_or_else(|_| {
3974 let t = SystemTime::now()
3976 .duration_since(UNIX_EPOCH)
3977 .map(|d| d.as_nanos() as u64)
3978 .unwrap_or(0);
3979 let p = u64::from(std::process::id());
3980 bytes.copy_from_slice(&(t ^ p.rotate_left(32)).to_le_bytes());
3981 });
3982 let hex: String = bytes.iter().map(|b| format!("{b:02x}")).collect();
3984 format!("bash-{hex}")
3985}
3986
3987#[cfg(test)]
3988mod tests {
3989 use std::collections::HashMap;
3990 use std::fs;
3991 use std::sync::atomic::{AtomicBool, AtomicUsize};
3992 use std::sync::{Arc, Mutex};
3993 use std::time::Duration;
3994 #[cfg(windows)]
3995 use std::time::Instant;
3996
3997 use super::*;
3998
3999 #[cfg(unix)]
4000 const QUICK_SUCCESS_COMMAND: &str = "true";
4001 #[cfg(windows)]
4002 const QUICK_SUCCESS_COMMAND: &str = "cmd /c exit 0";
4003
4004 #[cfg(unix)]
4005 const LONG_RUNNING_COMMAND: &str = "sleep 5";
4006 #[cfg(windows)]
4007 const LONG_RUNNING_COMMAND: &str = "cmd /c timeout /t 5 /nobreak > nul";
4008
4009 #[test]
4010 fn gh_structured_detection_rejects_piped_commands() {
4011 assert!(is_gh_structured_command(
4012 "gh issue list --json number,title"
4013 ));
4014 assert!(is_gh_structured_command(
4015 "cd repo && gh issue list --json number,title"
4016 ));
4017
4018 assert!(!is_gh_structured_command(
4019 "gh issue list --json number,title | jq '.[]'"
4020 ));
4021 assert!(!is_gh_structured_command(
4022 "gh issue list --json number,title |"
4023 ));
4024 }
4025
4026 fn insert_terminal_piped_task(
4027 registry: &BgTaskRegistry,
4028 dir: &tempfile::TempDir,
4029 command: &str,
4030 stdout: &str,
4031 stderr: &str,
4032 compressed: bool,
4033 ) -> (String, Arc<BgTask>) {
4034 let task_id = format!("bash-test-{}", random_slug());
4035 let paths = task_paths(dir.path(), "session", &task_id);
4036 fs::create_dir_all(&paths.dir).unwrap();
4037 fs::write(&paths.stdout, stdout).unwrap();
4038 fs::write(&paths.stderr, stderr).unwrap();
4039 let mut metadata = PersistedTask::starting(
4040 task_id.clone(),
4041 "session".to_string(),
4042 command.to_string(),
4043 dir.path().to_path_buf(),
4044 Some(dir.path().to_path_buf()),
4045 Some(30_000),
4046 true,
4047 compressed,
4048 );
4049 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
4050 write_task(&paths.json, &metadata).unwrap();
4051 registry
4052 .insert_rehydrated_task(metadata, paths, true)
4053 .expect("insert terminal task");
4054 let task = registry.task_for_session(&task_id, "session").unwrap();
4055 (task_id, task)
4056 }
4057
4058 fn insert_terminal_pty_task(
4059 registry: &BgTaskRegistry,
4060 dir: &tempfile::TempDir,
4061 pty_output: &str,
4062 ) -> (String, Arc<BgTask>) {
4063 let task_id = format!("bash-test-{}", random_slug());
4064 let paths = task_paths(dir.path(), "session", &task_id);
4065 fs::create_dir_all(&paths.dir).unwrap();
4066 fs::write(&paths.pty, pty_output).unwrap();
4067 let mut metadata = PersistedTask::starting(
4068 task_id.clone(),
4069 "session".to_string(),
4070 "python".to_string(),
4071 dir.path().to_path_buf(),
4072 Some(dir.path().to_path_buf()),
4073 Some(30_000),
4074 true,
4075 true,
4076 );
4077 metadata.mode = BgMode::Pty;
4078 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
4079 write_task(&paths.json, &metadata).unwrap();
4080 registry
4081 .insert_rehydrated_task(metadata, paths, true)
4082 .expect("insert terminal pty task");
4083 let task = registry.task_for_session(&task_id, "session").unwrap();
4084 (task_id, task)
4085 }
4086
4087 #[cfg(unix)]
4088 fn wait_for_terminal_snapshot(
4089 registry: &BgTaskRegistry,
4090 task_id: &str,
4091 session_id: &str,
4092 project: &Path,
4093 storage: &Path,
4094 ) -> BgTaskSnapshot {
4095 let started = Instant::now();
4096 loop {
4097 let snapshot = registry
4098 .status(task_id, session_id, Some(project), Some(storage), 4096)
4099 .expect("spawned task should be visible to status");
4100 if snapshot.info.status.is_terminal() {
4101 return snapshot;
4102 }
4103 assert!(
4104 started.elapsed() < Duration::from_secs(10),
4105 "timed out waiting for task {task_id} to finish; last status={:?}",
4106 snapshot.info.status
4107 );
4108 std::thread::sleep(Duration::from_millis(50));
4109 }
4110 }
4111
4112 fn write_running_project_task(storage: &Path, project: &Path, session: &str, task_id: &str) {
4113 let paths = task_paths(storage, session, task_id);
4114 let mut metadata = PersistedTask::starting(
4115 task_id.to_string(),
4116 session.to_string(),
4117 "sleep 60".to_string(),
4118 project.to_path_buf(),
4119 Some(project.to_path_buf()),
4120 Some(30_000),
4121 true,
4122 true,
4123 );
4124 metadata.status = BgTaskStatus::Running;
4125 write_task(&paths.json, &metadata).unwrap();
4126 fs::write(&paths.stdout, "still running\n").unwrap();
4127 fs::write(&paths.stderr, "").unwrap();
4128 }
4129
4130 #[test]
4131 fn status_replay_filters_same_session_by_project_root() {
4132 let project_a = tempfile::tempdir().unwrap();
4133 let project_b = tempfile::tempdir().unwrap();
4134 let storage = tempfile::tempdir().unwrap();
4135 let session = "shared-session";
4136 let task_id = "bash-project-a";
4137 write_running_project_task(storage.path(), project_a.path(), session, task_id);
4138
4139 let actor_b = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
4140 assert!(actor_b
4141 .status(
4142 task_id,
4143 session,
4144 Some(project_b.path()),
4145 Some(storage.path()),
4146 1024,
4147 )
4148 .is_none());
4149 assert!(actor_b.task_for_session(task_id, session).is_none());
4150
4151 let actor_a = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
4152 let snapshot = actor_a
4153 .status(
4154 task_id,
4155 session,
4156 Some(project_a.path()),
4157 Some(storage.path()),
4158 1024,
4159 )
4160 .expect("owning project should replay its task");
4161 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
4162 }
4163
4164 #[cfg(unix)]
4165 #[test]
4166 fn multiline_pipeline_stdout_persists_all_lines_after_terminal_status() {
4167 let cases = [
4168 (
4169 "long-first",
4170 "sleep 0.5; printf 'one\\n' | cat\nprintf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
4171 vec!["one", "1", "three"],
4172 ),
4173 (
4174 "short-first",
4175 "printf 'one\\n' | cat\nsleep 0.2; printf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
4176 vec!["one", "1", "three"],
4177 ),
4178 (
4179 "failing-middle",
4180 "sleep 0.2; printf 'one\\n' | cat\nfalse; printf 'after-false\\n' | cat\nprintf 'three\\n' | cat",
4181 vec!["one", "after-false", "three"],
4182 ),
4183 ];
4184
4185 for (name, command, expected_lines) in cases {
4186 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
4187 let dir = tempfile::tempdir().unwrap();
4188 let session_id = format!("session-{name}");
4189 let task_id = registry
4190 .spawn(
4191 command,
4192 session_id.clone(),
4193 dir.path().to_path_buf(),
4194 HashMap::new(),
4195 Some(Duration::from_secs(30)),
4196 dir.path().to_path_buf(),
4197 10,
4198 true,
4199 true,
4200 Some(dir.path().to_path_buf()),
4201 )
4202 .unwrap();
4203
4204 let snapshot = wait_for_terminal_snapshot(
4205 ®istry,
4206 &task_id,
4207 &session_id,
4208 dir.path(),
4209 dir.path(),
4210 );
4211 assert_eq!(
4212 snapshot.info.status,
4213 BgTaskStatus::Completed,
4214 "{name}: task should complete; snapshot={snapshot:?}"
4215 );
4216 assert_eq!(
4217 snapshot.exit_code,
4218 Some(0),
4219 "{name}: script should use the final command's exit code"
4220 );
4221
4222 let stdout_path = task_paths(dir.path(), &session_id, &task_id).stdout;
4223 let stdout = fs::read_to_string(&stdout_path).unwrap_or_else(|error| {
4224 panic!(
4225 "{name}: failed to read raw stdout file {}: {error}",
4226 stdout_path.display()
4227 )
4228 });
4229 let lines: Vec<&str> = stdout.lines().collect();
4230 assert_eq!(
4231 lines,
4232 expected_lines,
4233 "{name}: raw stdout file must include every newline-separated command's output; stdout_path={}",
4234 stdout_path.display()
4235 );
4236 }
4237 }
4238
4239 #[test]
4240 fn recognizes_all_recovery_marker_forms() {
4241 assert!(is_recovery_marker(
4242 "[truncated output; full output: read \"/tmp/out\"]"
4243 ));
4244 assert!(is_recovery_marker(
4245 "[omitted output; see remaining: tail -n +42 \"/tmp/out\"]"
4246 ));
4247 assert!(is_recovery_marker(
4248 "[truncated output; full output unavailable]"
4249 ));
4250 assert!(is_recovery_marker(
4251 r#"[truncated 123 bytes from saved output prefix; retained output: read "/tmp/out"]"#
4252 ));
4253 }
4254
4255 #[test]
4256 fn recovery_marker_reports_disk_prefix_truncation_as_retained_output() {
4257 let recovery = RecoveryContext {
4258 dropped_by_class: BTreeMap::new(),
4259 had_inner_drop: false,
4260 offset_hint_eligible: false,
4261 offset_start_line: None,
4262 byte_truncated: false,
4263 disk_truncated_prefix_bytes: 4096,
4264 output_path: Some("/tmp/stdout".to_string()),
4265 stderr_path: None,
4266 include_stderr_path: false,
4267 };
4268
4269 let marker = recovery_marker(&recovery).expect("disk truncation must emit marker");
4270
4271 assert!(marker.contains("truncated 4096 bytes from saved output prefix"));
4272 assert!(marker.contains(r#"retained output: read "/tmp/stdout""#));
4273 assert!(!marker.contains("full output: read"));
4274 }
4275
4276 #[test]
4277 fn killed_exit_marker_sets_nonzero_sentinel_exit_code() {
4278 let metadata = PersistedTask::starting(
4279 "task".to_string(),
4280 "session".to_string(),
4281 "cargo test".to_string(),
4282 PathBuf::from("/tmp"),
4283 None,
4284 None,
4285 true,
4286 true,
4287 );
4288
4289 let terminal = terminal_metadata_from_marker(metadata, ExitMarker::Killed, None);
4290
4291 assert_eq!(terminal.status, BgTaskStatus::Killed);
4292 assert_eq!(terminal.exit_code, Some(137));
4293 }
4294
4295 #[test]
4296 fn terminal_status_polls_use_cached_render_once_and_off_lock() {
4297 let registry = BgTaskRegistry::default();
4298 let dir = tempfile::tempdir().unwrap();
4299 let (_task_id, task) = insert_terminal_piped_task(
4300 ®istry,
4301 &dir,
4302 "custom-tool --verbose",
4303 &"stdout line\n".repeat(200_000),
4304 "",
4305 true,
4306 );
4307 let calls = Arc::new(AtomicUsize::new(0));
4308 let saw_unlocked_state = Arc::new(AtomicBool::new(false));
4309 let task_holder = Arc::new(Mutex::new(Some(Arc::clone(&task))));
4310 let calls_for_closure = Arc::clone(&calls);
4311 let unlocked_for_closure = Arc::clone(&saw_unlocked_state);
4312 let task_for_closure = Arc::clone(&task_holder);
4313 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
4314 calls_for_closure.fetch_add(1, Ordering::SeqCst);
4315 if let Some(task) = task_for_closure.lock().unwrap().as_ref() {
4316 if task.state.try_lock().is_ok() {
4317 unlocked_for_closure.store(true, Ordering::SeqCst);
4318 }
4319 }
4320 CompressionResult::new(format!("compressed {} bytes", output.len()))
4321 });
4322
4323 let first = registry
4324 .status(
4325 &task.task_id,
4326 "session",
4327 None,
4328 Some(dir.path()),
4329 RUNNING_OUTPUT_PREVIEW_BYTES,
4330 )
4331 .unwrap();
4332 let second = registry
4333 .status(
4334 &task.task_id,
4335 "session",
4336 None,
4337 Some(dir.path()),
4338 RUNNING_OUTPUT_PREVIEW_BYTES,
4339 )
4340 .unwrap();
4341 let listed = registry.list(RUNNING_OUTPUT_PREVIEW_BYTES);
4342
4343 assert_eq!(
4344 calls.load(Ordering::SeqCst),
4345 1,
4346 "terminal render must be cached"
4347 );
4348 assert!(
4349 saw_unlocked_state.load(Ordering::SeqCst),
4350 "compressor must run after releasing the task state lock"
4351 );
4352 assert!(first.output_preview.starts_with("compressed "));
4353 assert_eq!(second.output_preview, first.output_preview);
4354 assert_eq!(listed[0].output_preview, first.output_preview);
4355 }
4356
4357 #[test]
4358 fn completion_preview_success_keeps_tail_only() {
4359 let registry = BgTaskRegistry::default();
4364 let dir = tempfile::tempdir().unwrap();
4365 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
4366 let (_task_id, task) =
4367 insert_terminal_piped_task(®istry, &dir, "cat big.log", &output, "", false);
4368
4369 registry.post_terminal_transition(&task, true).unwrap();
4370 let completions = registry.drain_completions_for_session(Some("session"));
4371 assert_eq!(completions.len(), 1);
4372 let preview = &completions[0].output_preview;
4373 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
4374 assert!(!preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
4375 assert!(completions[0].output_truncated);
4376 }
4377
4378 #[test]
4379 fn completion_preview_failure_keeps_head_and_tail() {
4380 let registry = BgTaskRegistry::default();
4383 let dir = tempfile::tempdir().unwrap();
4384 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
4385 let task_id = format!("bash-test-{}", random_slug());
4386 let paths = task_paths(dir.path(), "session", &task_id);
4387 fs::create_dir_all(&paths.dir).unwrap();
4388 fs::write(&paths.stdout, &output).unwrap();
4389 fs::write(&paths.stderr, "").unwrap();
4390 let mut metadata = PersistedTask::starting(
4391 task_id.clone(),
4392 "session".to_string(),
4393 "cat big.log".to_string(),
4394 dir.path().to_path_buf(),
4395 Some(dir.path().to_path_buf()),
4396 Some(30_000),
4397 true,
4398 false,
4399 );
4400 metadata.mark_terminal(BgTaskStatus::Failed, Some(1), None);
4401 write_task(&paths.json, &metadata).unwrap();
4402 registry
4403 .insert_rehydrated_task(metadata, paths, true)
4404 .expect("insert terminal task");
4405 let task = registry.task_for_session(&task_id, "session").unwrap();
4406
4407 registry.post_terminal_transition(&task, true).unwrap();
4408 let completions = registry.drain_completions_for_session(Some("session"));
4409 assert_eq!(completions.len(), 1);
4410 let preview = &completions[0].output_preview;
4411 assert!(preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
4412 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
4413 }
4414
4415 #[test]
4416 fn has_completions_for_session_matches_pending_delivery() {
4417 let registry = BgTaskRegistry::default();
4418 assert!(!registry.has_completions_for_session(Some("session")));
4419 assert!(!registry.has_completions_for_session(None));
4420
4421 let dir = tempfile::tempdir().unwrap();
4422 let (_task_id, task) =
4423 insert_terminal_piped_task(®istry, &dir, QUICK_SUCCESS_COMMAND, "done\n", "", false);
4424 registry.post_terminal_transition(&task, true).unwrap();
4425
4426 assert!(registry.has_completions_for_session(Some("session")));
4427 assert!(registry.has_completions_for_session(None));
4428 assert!(!registry.has_completions_for_session(Some("other-session")));
4429
4430 let completions = registry.drain_completions_for_session(Some("session"));
4431 assert_eq!(completions.len(), 1);
4432 assert_eq!(completions[0].task_id, task.task_id);
4433 }
4434
4435 #[test]
4436 fn structured_gh_json_survives_intact_and_ignores_stderr() {
4437 let registry = BgTaskRegistry::default();
4438 let dir = tempfile::tempdir().unwrap();
4439 let calls = Arc::new(AtomicUsize::new(0));
4440 let calls_for_closure = Arc::clone(&calls);
4441 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
4442 calls_for_closure.fetch_add(1, Ordering::SeqCst);
4443 CompressionResult::new(output)
4444 });
4445 let (task_id, _task) = insert_terminal_piped_task(
4446 ®istry,
4447 &dir,
4448 "gh pr view 123 --json body",
4449 "{\"body\":\"hello\"}",
4450 "warning: stderr must not join json",
4451 true,
4452 );
4453
4454 let snapshot = registry
4455 .status(
4456 &task_id,
4457 "session",
4458 None,
4459 Some(dir.path()),
4460 RUNNING_OUTPUT_PREVIEW_BYTES,
4461 )
4462 .unwrap();
4463
4464 assert_eq!(snapshot.output_preview, "{\"body\":\"hello\"}");
4465 assert!(!snapshot.output_preview.contains("warning"));
4466 assert!(!snapshot.output_truncated);
4467 assert_eq!(
4468 calls.load(Ordering::SeqCst),
4469 0,
4470 "structured JSON bypasses compression"
4471 );
4472 }
4473
4474 #[test]
4475 fn registry_emits_single_recovery_marker_for_class_drops() {
4476 let registry = BgTaskRegistry::default();
4477 let dir = tempfile::tempdir().unwrap();
4478 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
4479 let mut dropped = BTreeMap::new();
4480 dropped.insert(DropClass::Error, 18);
4481 dropped.insert(DropClass::Warning, 6);
4482 CompressionResult::with_class_drops("kept diagnostic", dropped)
4483 });
4484 let (task_id, task) =
4485 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
4486
4487 let snapshot = registry
4488 .status(
4489 &task_id,
4490 "session",
4491 None,
4492 Some(dir.path()),
4493 RUNNING_OUTPUT_PREVIEW_BYTES,
4494 )
4495 .unwrap();
4496
4497 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
4498 assert!(snapshot.output_preview.contains("+18 more errors"));
4499 assert!(snapshot.output_preview.contains("+6 more warnings"));
4500 assert!(snapshot
4501 .output_preview
4502 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
4503 assert!(!snapshot.output_preview.contains("tail -n +"));
4504 assert!(snapshot.output_truncated);
4505 }
4506
4507 #[test]
4508 fn registry_marker_reports_semantic_and_byte_drops_once() {
4509 let registry = BgTaskRegistry::default();
4510 let dir = tempfile::tempdir().unwrap();
4511 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
4512 let mut dropped = BTreeMap::new();
4513 dropped.insert(DropClass::Error, 1);
4514 CompressionResult::with_class_drops(
4515 format!("HEAD-SIGNAL\n{}TAIL-SIGNAL", "middle\n".repeat(8_000)),
4516 dropped,
4517 )
4518 });
4519 let (task_id, _task) =
4520 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
4521
4522 let snapshot = registry
4523 .status(
4524 &task_id,
4525 "session",
4526 None,
4527 Some(dir.path()),
4528 RUNNING_OUTPUT_PREVIEW_BYTES,
4529 )
4530 .unwrap();
4531
4532 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
4533 assert!(snapshot.output_preview.contains("+1 more error"));
4534 assert!(snapshot.output_preview.contains("truncated output"));
4535 assert!(snapshot.output_preview.contains("HEAD-SIGNAL"));
4536 assert!(snapshot.output_preview.contains("TAIL-SIGNAL"));
4537 assert!(!snapshot.output_preview.contains("...<truncated"));
4538 assert!(snapshot.output_truncated);
4539 }
4540
4541 #[test]
4542 fn cargo_stderr_class_drops_name_both_capture_paths() {
4543 let registry = BgTaskRegistry::default();
4544 let dir = tempfile::tempdir().unwrap();
4545 let filter_registry = crate::compress::toml_filter::FilterRegistry::default();
4546 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
4547 crate::compress::compress_with_registry_exit_code(
4548 command,
4549 &output,
4550 exit_code,
4551 &filter_registry,
4552 )
4553 });
4554 let stderr = (0..22)
4555 .map(|index| {
4556 format!(
4557 "error: cargo failure {index}\n --> src/lib.rs:{}:1\n |\n{} | boom\n",
4558 index + 1,
4559 index + 1
4560 )
4561 })
4562 .collect::<Vec<_>>()
4563 .join("\n");
4564 let (task_id, task) = insert_terminal_piped_task(
4565 ®istry,
4566 &dir,
4567 "cargo check",
4568 "Finished dev [unoptimized] target(s) in 0.01s\n",
4569 &stderr,
4570 true,
4571 );
4572
4573 let snapshot = registry
4574 .status(
4575 &task_id,
4576 "session",
4577 None,
4578 Some(dir.path()),
4579 RUNNING_OUTPUT_PREVIEW_BYTES,
4580 )
4581 .unwrap();
4582
4583 assert!(snapshot.output_preview.contains("+2 more errors"));
4584 assert!(snapshot
4585 .output_preview
4586 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
4587 assert!(snapshot
4588 .output_preview
4589 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
4590 assert!(!snapshot.output_preview.contains("tail -n +"));
4591 }
4592
4593 #[test]
4594 fn over_ceiling_structured_json_uses_pointer_not_partial_json() {
4595 let registry = BgTaskRegistry::default();
4596 let dir = tempfile::tempdir().unwrap();
4597 let body = format!("{{\"body\":\"{}\"}}", "x".repeat(60 * 1024));
4598 let (task_id, task) = insert_terminal_piped_task(
4599 ®istry,
4600 &dir,
4601 "cd /repo && gh pr view 123 --json body",
4602 &body,
4603 "",
4604 true,
4605 );
4606
4607 let snapshot = registry
4608 .status(
4609 &task_id,
4610 "session",
4611 None,
4612 Some(dir.path()),
4613 RUNNING_OUTPUT_PREVIEW_BYTES,
4614 )
4615 .unwrap();
4616
4617 assert!(snapshot.output_preview.starts_with("[JSON output "));
4618 assert!(snapshot
4619 .output_preview
4620 .contains(&task.paths.stdout.display().to_string()));
4621 assert!(!snapshot.output_preview.contains(&"x".repeat(1024)));
4622 assert!(snapshot.output_truncated);
4623 }
4624
4625 #[test]
4626 fn toml_strip_tail_cap_uses_full_output_hint_not_offset_hint() {
4627 let registry = BgTaskRegistry::default();
4628 let dir = tempfile::tempdir().unwrap();
4629 let filter_registry = crate::compress::toml_filter::build_registry(
4630 crate::compress::builtin_filters::ALL,
4631 None,
4632 None,
4633 );
4634 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
4635 crate::compress::compress_with_registry_exit_code(
4636 command,
4637 &output,
4638 exit_code,
4639 &filter_registry,
4640 )
4641 });
4642 let stdout = format!(
4643 "make[1]: Entering directory `/tmp`\n{}",
4644 (0..100)
4645 .map(|index| format!("compile line {index}"))
4646 .collect::<Vec<_>>()
4647 .join("\n")
4648 );
4649 let (task_id, task) =
4650 insert_terminal_piped_task(®istry, &dir, "make all", &stdout, "", true);
4651
4652 let snapshot = registry
4653 .status(
4654 &task_id,
4655 "session",
4656 None,
4657 Some(dir.path()),
4658 RUNNING_OUTPUT_PREVIEW_BYTES,
4659 )
4660 .unwrap();
4661
4662 assert!(snapshot.output_preview.contains("compile line 99"));
4663 assert!(snapshot.output_preview.contains(&format!(
4664 "full output: read \"{}\"",
4665 task.paths.stdout.display()
4666 )));
4667 assert!(!snapshot
4668 .output_preview
4669 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
4670 assert!(!snapshot.output_preview.contains("tail -n +"));
4671 }
4672
4673 #[test]
4674 fn compressed_false_raw_passthrough_uses_wider_head_tail_cap() {
4675 let registry = BgTaskRegistry::default();
4676 let dir = tempfile::tempdir().unwrap();
4677 let output = format!("RAW-HEAD\n{}RAW-TAIL\n", "raw-middle\n".repeat(8_000));
4678 let (task_id, task) =
4679 insert_terminal_piped_task(®istry, &dir, "cat raw.log", &output, "RAW-ERR\n", false);
4680
4681 let snapshot = registry
4682 .status(
4683 &task_id,
4684 "session",
4685 None,
4686 Some(dir.path()),
4687 RUNNING_OUTPUT_PREVIEW_BYTES,
4688 )
4689 .unwrap();
4690
4691 assert!(snapshot.output_preview.contains("RAW-HEAD"));
4692 assert!(snapshot.output_preview.contains("RAW-TAIL"));
4693 assert!(snapshot.output_preview.contains("truncated output"));
4694 assert!(snapshot
4695 .output_preview
4696 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
4697 assert!(snapshot
4698 .output_preview
4699 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
4700 assert!(!snapshot.output_preview.contains("tail -n +"));
4701 assert!(snapshot.output_preview.len() > 16 * 1024);
4702 assert!(snapshot.output_truncated);
4703 }
4704
4705 #[test]
4706 fn pty_terminal_snapshot_bypasses_line_compression() {
4707 let registry = BgTaskRegistry::default();
4708 let dir = tempfile::tempdir().unwrap();
4709 let calls = Arc::new(AtomicUsize::new(0));
4710 let calls_for_closure = Arc::clone(&calls);
4711 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
4712 calls_for_closure.fetch_add(1, Ordering::SeqCst);
4713 CompressionResult::new(output)
4714 });
4715 let (task_id, _task) = insert_terminal_pty_task(®istry, &dir, "raw\u{1b}[31m pty bytes");
4716
4717 let snapshot = registry
4718 .status(
4719 &task_id,
4720 "session",
4721 None,
4722 Some(dir.path()),
4723 RUNNING_OUTPUT_PREVIEW_BYTES,
4724 )
4725 .unwrap();
4726
4727 assert_eq!(snapshot.info.mode, BgMode::Pty);
4728 assert_eq!(snapshot.output_preview, "");
4729 assert_eq!(calls.load(Ordering::SeqCst), 0);
4730 }
4731
4732 #[test]
4733 fn pty_dimensions_are_persisted_and_returned_in_snapshot() {
4734 let registry = BgTaskRegistry::default();
4735 let dir = tempfile::tempdir().unwrap();
4736 let task_id = registry
4737 .spawn_pty(
4738 QUICK_SUCCESS_COMMAND,
4739 "session".to_string(),
4740 dir.path().to_path_buf(),
4741 HashMap::new(),
4742 Some(Duration::from_secs(30)),
4743 dir.path().to_path_buf(),
4744 10,
4745 true,
4746 false,
4747 Some(dir.path().to_path_buf()),
4748 50,
4749 120,
4750 )
4751 .unwrap();
4752
4753 let paths = task_paths(dir.path(), "session", &task_id);
4754 let metadata = read_task(&paths.json).unwrap();
4755 assert_eq!(
4756 metadata.schema_version,
4757 crate::bash_background::persistence::SCHEMA_VERSION
4758 );
4759 assert_eq!(metadata.mode, BgMode::Pty);
4760 assert_eq!(metadata.pty_rows, Some(50));
4761 assert_eq!(metadata.pty_cols, Some(120));
4762
4763 let snapshot = registry
4764 .status(&task_id, "session", None, Some(dir.path()), 1024)
4765 .unwrap();
4766 assert_eq!(snapshot.pty_rows, Some(50));
4767 assert_eq!(snapshot.pty_cols, Some(120));
4768 }
4769
4770 fn spawn_dead_child() -> std::process::Child {
4775 #[cfg(unix)]
4776 let mut cmd = std::process::Command::new("true");
4777 #[cfg(windows)]
4778 let mut cmd = {
4779 let mut c = std::process::Command::new("cmd");
4780 c.args(["/c", "exit", "0"]);
4781 c
4782 };
4783 cmd.stdin(std::process::Stdio::null());
4784 cmd.stdout(std::process::Stdio::null());
4785 cmd.stderr(std::process::Stdio::null());
4786 let mut child = cmd.spawn().expect("spawn replacement child for reap test");
4787 let started = Instant::now();
4796 loop {
4797 match child.try_wait() {
4798 Ok(Some(_)) => break,
4799 Ok(None) => {
4800 if started.elapsed() > Duration::from_secs(5) {
4801 panic!("dead-child stand-in did not exit within 5s");
4802 }
4803 std::thread::sleep(Duration::from_millis(10));
4804 }
4805 Err(error) => panic!("dead-child try_wait failed: {error}"),
4806 }
4807 }
4808 child
4809 }
4810
4811 #[test]
4812 fn ack_marks_delivered_even_when_completion_was_already_consumed_locally() {
4813 let registry = BgTaskRegistry::default();
4814 let dir = tempfile::tempdir().unwrap();
4815 let task_id = registry
4816 .spawn(
4817 LONG_RUNNING_COMMAND,
4818 "session".to_string(),
4819 dir.path().to_path_buf(),
4820 HashMap::new(),
4821 Some(Duration::from_secs(30)),
4822 dir.path().to_path_buf(),
4823 10,
4824 true,
4825 false,
4826 Some(dir.path().to_path_buf()),
4827 )
4828 .unwrap();
4829 registry
4830 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
4831 .unwrap();
4832 assert_eq!(
4833 registry
4834 .drain_completions_for_session(Some("session"))
4835 .len(),
4836 1
4837 );
4838
4839 registry.inner.completions.lock().unwrap().clear();
4842
4843 assert_eq!(
4844 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
4845 vec![task_id.clone()]
4846 );
4847 assert!(registry
4848 .drain_completions_for_session(Some("session"))
4849 .is_empty());
4850
4851 let paths = task_paths(dir.path(), "session", &task_id);
4852 let metadata = read_task(&paths.json).unwrap();
4853 assert!(metadata.completion_delivered);
4854
4855 let replayed = BgTaskRegistry::default();
4856 replayed
4857 .replay_session_inner(dir.path(), "session", None)
4858 .unwrap();
4859 assert!(replayed
4860 .drain_completions_for_session(Some("session"))
4861 .is_empty());
4862 }
4863
4864 #[test]
4865 fn register_watch_rejects_unknown_task() {
4866 let registry = BgTaskRegistry::default();
4867
4868 let result = registry.register_watch(
4869 "missing-task".to_string(),
4870 WatchPattern::Substring("READY".into()),
4871 true,
4872 );
4873
4874 assert_eq!(result, Err("task_not_found"));
4875 }
4876
4877 #[test]
4878 fn register_watch_on_terminal_task_scans_existing_output() {
4879 let frames = Arc::new(Mutex::new(Vec::new()));
4880 let captured = Arc::clone(&frames);
4881 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
4882 captured.lock().unwrap().push(frame);
4883 })
4884 as Box<dyn Fn(PushFrame) + Send + Sync>);
4885 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
4886 let dir = tempfile::tempdir().unwrap();
4887 let task_id = registry
4888 .spawn(
4889 LONG_RUNNING_COMMAND,
4890 "session".to_string(),
4891 dir.path().to_path_buf(),
4892 HashMap::new(),
4893 Some(Duration::from_secs(30)),
4894 dir.path().to_path_buf(),
4895 10,
4896 true,
4897 false,
4898 Some(dir.path().to_path_buf()),
4899 )
4900 .unwrap();
4901 registry
4902 .inner
4903 .shutdown
4904 .store(true, std::sync::atomic::Ordering::SeqCst);
4905 let task = registry.task_for_session(&task_id, "session").unwrap();
4906 std::fs::write(&task.paths.stdout, "READY\n").unwrap();
4907 registry
4908 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
4909 .unwrap();
4910 frames.lock().unwrap().clear();
4911 registry.inner.completions.lock().unwrap().clear();
4912
4913 registry
4914 .register_watch(
4915 task_id.clone(),
4916 WatchPattern::Substring("READY".into()),
4917 true,
4918 )
4919 .unwrap();
4920
4921 let frames = frames.lock().unwrap();
4922 let frame = frames
4923 .iter()
4924 .find_map(|frame| match frame {
4925 PushFrame::BashPatternMatch(frame) => Some(frame),
4926 _ => None,
4927 })
4928 .expect("terminal watch registration should emit pattern frame");
4929 assert_eq!(frame.reason, "pattern_match");
4930 assert_eq!(frame.task_id, task_id);
4931 assert_eq!(frame.session_id, "session");
4932 assert_eq!(frame.match_text, "READY");
4933 assert_eq!(frame.match_offset, 0);
4934 assert_eq!(registry.active_watch_count(&frame.task_id), 0);
4935 let metadata = read_task(&task.paths.json).unwrap();
4936 assert!(metadata.completion_delivered);
4937 }
4938
4939 #[test]
4940 fn cleanup_finished_removes_terminal_tasks_older_than_threshold() {
4941 let registry = BgTaskRegistry::default();
4942 let dir = tempfile::tempdir().unwrap();
4943 let task_id = registry
4944 .spawn(
4945 QUICK_SUCCESS_COMMAND,
4946 "session".to_string(),
4947 dir.path().to_path_buf(),
4948 HashMap::new(),
4949 Some(Duration::from_secs(30)),
4950 dir.path().to_path_buf(),
4951 10,
4952 true,
4953 false,
4954 Some(dir.path().to_path_buf()),
4955 )
4956 .unwrap();
4957 registry
4958 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
4959 .unwrap();
4960 let completions = registry.drain_completions_for_session(Some("session"));
4961 assert_eq!(completions.len(), 1);
4962 assert_eq!(
4963 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
4964 vec![task_id.clone()]
4965 );
4966
4967 registry.cleanup_finished(Duration::ZERO);
4968
4969 assert!(registry.inner.tasks.lock().unwrap().is_empty());
4970 }
4971
4972 #[test]
4973 fn cleanup_finished_retains_undelivered_terminals() {
4974 let registry = BgTaskRegistry::default();
4975 let dir = tempfile::tempdir().unwrap();
4976 let task_id = registry
4977 .spawn(
4978 QUICK_SUCCESS_COMMAND,
4979 "session".to_string(),
4980 dir.path().to_path_buf(),
4981 HashMap::new(),
4982 Some(Duration::from_secs(30)),
4983 dir.path().to_path_buf(),
4984 10,
4985 true,
4986 false,
4987 Some(dir.path().to_path_buf()),
4988 )
4989 .unwrap();
4990 registry
4991 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
4992 .unwrap();
4993
4994 registry.cleanup_finished(Duration::ZERO);
4995
4996 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
4997 }
4998
4999 #[test]
5007 fn reap_child_marks_failed_when_child_exits_without_exit_marker() {
5008 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5009 let dir = tempfile::tempdir().unwrap();
5010 let task_id = registry
5011 .spawn(
5012 QUICK_SUCCESS_COMMAND,
5013 "session".to_string(),
5014 dir.path().to_path_buf(),
5015 HashMap::new(),
5016 Some(Duration::from_secs(30)),
5017 dir.path().to_path_buf(),
5018 10,
5019 true,
5020 false,
5021 Some(dir.path().to_path_buf()),
5022 )
5023 .unwrap();
5024
5025 let task = registry.task_for_session(&task_id, "session").unwrap();
5026
5027 let started = Instant::now();
5032 loop {
5033 let exited = {
5034 let mut state = task.state.lock().unwrap();
5035 match &mut state.runtime {
5036 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
5037 _ => true,
5038 }
5039 };
5040 if exited {
5041 break;
5042 }
5043 assert!(
5044 started.elapsed() < Duration::from_secs(5),
5045 "child should exit quickly"
5046 );
5047 std::thread::sleep(Duration::from_millis(20));
5048 }
5049
5050 registry
5058 .inner
5059 .shutdown
5060 .store(true, std::sync::atomic::Ordering::SeqCst);
5061 std::thread::sleep(Duration::from_millis(550));
5065
5066 let _ = std::fs::remove_file(&task.paths.exit);
5069
5070 {
5085 let mut state = task.state.lock().unwrap();
5086 state.metadata.status = BgTaskStatus::Running;
5087 state.metadata.status_reason = None;
5088 state.metadata.exit_code = None;
5089 state.metadata.finished_at = None;
5090 state.metadata.duration_ms = None;
5091 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
5094 .expect("persist reset Running metadata for reap_child test");
5095 if matches!(state.runtime, TaskRuntime::Piped(None)) {
5099 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
5100 }
5101 }
5102 *task.terminal_at.lock().unwrap() = None;
5105
5106 assert!(
5109 task.is_running(),
5110 "precondition: metadata.status == Running"
5111 );
5112 assert!(
5113 !task.paths.exit.exists(),
5114 "precondition: exit marker absent"
5115 );
5116
5117 registry.reap_child(&task);
5122
5123 {
5124 let state = task.state.lock().unwrap();
5125 assert_eq!(
5126 state.metadata.status,
5127 BgTaskStatus::Running,
5128 "first reap must leave status Running while waiting one pass for marker"
5129 );
5130 assert_eq!(
5131 state.metadata.status_reason, None,
5132 "first reap must not record a failure reason"
5133 );
5134 assert!(
5135 matches!(state.runtime, TaskRuntime::Piped(None)),
5136 "child handle must be released after first reap"
5137 );
5138 assert!(
5139 state.detached,
5140 "task must be marked detached after first reap"
5141 );
5142 }
5143
5144 registry.reap_child(&task);
5148
5149 let state = task.state.lock().unwrap();
5150 assert!(
5151 state.metadata.status.is_terminal(),
5152 "second reap must transition to terminal when PID dead and no marker. Got status={:?}",
5153 state.metadata.status
5154 );
5155 assert_eq!(
5156 state.metadata.status,
5157 BgTaskStatus::Failed,
5158 "must specifically be Failed (not Killed): status={:?}",
5159 state.metadata.status
5160 );
5161 assert_eq!(
5162 state.metadata.status_reason.as_deref(),
5163 Some("process exited without exit marker"),
5164 "reason must match replay path's wording: {:?}",
5165 state.metadata.status_reason
5166 );
5167 assert!(
5168 matches!(state.runtime, TaskRuntime::Piped(None)),
5169 "child handle must stay released after second reap"
5170 );
5171 assert!(
5172 state.detached,
5173 "task must remain detached after second reap"
5174 );
5175 }
5176
5177 #[test]
5182 fn reap_child_preserves_running_when_exit_marker_exists() {
5183 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5184 let dir = tempfile::tempdir().unwrap();
5185 let task_id = registry
5186 .spawn(
5187 QUICK_SUCCESS_COMMAND,
5188 "session".to_string(),
5189 dir.path().to_path_buf(),
5190 HashMap::new(),
5191 Some(Duration::from_secs(30)),
5192 dir.path().to_path_buf(),
5193 10,
5194 true,
5195 false,
5196 Some(dir.path().to_path_buf()),
5197 )
5198 .unwrap();
5199
5200 let task = registry.task_for_session(&task_id, "session").unwrap();
5201
5202 let started = Instant::now();
5205 loop {
5206 let exited = {
5207 let mut state = task.state.lock().unwrap();
5208 match &mut state.runtime {
5209 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
5210 _ => true,
5211 }
5212 };
5213 if exited && task.paths.exit.exists() {
5214 break;
5215 }
5216 assert!(
5217 started.elapsed() < Duration::from_secs(5),
5218 "child should exit and write marker quickly"
5219 );
5220 std::thread::sleep(Duration::from_millis(20));
5221 }
5222
5223 registry
5229 .inner
5230 .shutdown
5231 .store(true, std::sync::atomic::Ordering::SeqCst);
5232 std::thread::sleep(Duration::from_millis(550));
5233
5234 {
5240 let mut state = task.state.lock().unwrap();
5241 state.metadata.status = BgTaskStatus::Running;
5242 state.metadata.status_reason = None;
5243 if matches!(state.runtime, TaskRuntime::Piped(None)) {
5244 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
5245 }
5246 }
5247 *task.terminal_at.lock().unwrap() = None;
5248 if !task.paths.exit.exists() {
5251 std::fs::write(&task.paths.exit, "0").expect("write replacement exit marker");
5252 }
5253
5254 registry.reap_child(&task);
5258
5259 let state = task.state.lock().unwrap();
5260 assert!(
5261 matches!(state.runtime, TaskRuntime::Piped(None)),
5262 "child handle still released even when marker exists"
5263 );
5264 assert!(
5265 state.detached,
5266 "task still marked detached even when marker exists"
5267 );
5268 assert_eq!(
5273 state.metadata.status,
5274 BgTaskStatus::Running,
5275 "reap_child must defer to poll_task when marker exists"
5276 );
5277 }
5278
5279 #[cfg(unix)]
5283 fn pid_stat(pid: u32) -> Option<String> {
5284 let output = std::process::Command::new("ps")
5285 .args(["-o", "stat=", "-p", &pid.to_string()])
5286 .output()
5287 .ok()?;
5288 if !output.status.success() {
5289 return None;
5290 }
5291 let stat = String::from_utf8_lossy(&output.stdout).trim().to_string();
5292 if stat.is_empty() {
5293 None
5294 } else {
5295 Some(stat)
5296 }
5297 }
5298
5299 #[cfg(unix)]
5301 fn is_zombie(pid: u32) -> bool {
5302 pid_stat(pid).is_some_and(|stat| stat.starts_with('Z'))
5303 }
5304
5305 #[cfg(unix)]
5311 fn spawn_unreaped_zombie() -> std::process::Child {
5312 let child = std::process::Command::new("true")
5313 .stdin(std::process::Stdio::null())
5314 .stdout(std::process::Stdio::null())
5315 .stderr(std::process::Stdio::null())
5316 .spawn()
5317 .expect("spawn zombie stand-in");
5318 let pid = child.id();
5319 let started = Instant::now();
5320 while !is_zombie(pid) {
5321 assert!(
5322 started.elapsed() < Duration::from_secs(5),
5323 "stand-in child should become a zombie within 5s"
5324 );
5325 std::thread::sleep(Duration::from_millis(10));
5326 }
5327 child
5329 }
5330
5331 #[cfg(unix)]
5341 #[test]
5342 fn finalize_from_marker_reaps_child_no_zombie() {
5343 use std::sync::atomic::Ordering;
5344
5345 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5346 let dir = tempfile::tempdir().unwrap();
5347 let task_id = registry
5348 .spawn(
5349 QUICK_SUCCESS_COMMAND,
5350 "session".to_string(),
5351 dir.path().to_path_buf(),
5352 HashMap::new(),
5353 Some(Duration::from_secs(30)),
5354 dir.path().to_path_buf(),
5355 10,
5356 true,
5357 false,
5358 Some(dir.path().to_path_buf()),
5359 )
5360 .unwrap();
5361
5362 registry.inner.shutdown.store(true, Ordering::SeqCst);
5366 std::thread::sleep(Duration::from_millis(550));
5367
5368 let task = registry.task_for_session(&task_id, "session").unwrap();
5369
5370 let started = Instant::now();
5374 while !task.paths.exit.exists() {
5375 assert!(
5376 started.elapsed() < Duration::from_secs(5),
5377 "exit marker should land quickly for `true`"
5378 );
5379 std::thread::sleep(Duration::from_millis(20));
5380 }
5381
5382 let zombie_pid;
5388 {
5389 let mut state = task.state.lock().unwrap();
5390 state.metadata.status = BgTaskStatus::Running;
5391 state.metadata.status_reason = None;
5392 state.metadata.exit_code = None;
5393 state.metadata.finished_at = None;
5394 state.metadata.duration_ms = None;
5395 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
5396 .expect("persist reset Running metadata");
5397 let zombie = spawn_unreaped_zombie();
5398 zombie_pid = zombie.id();
5399 state.runtime = TaskRuntime::Piped(Some(zombie));
5400 }
5401 *task.terminal_at.lock().unwrap() = None;
5402
5403 assert!(
5405 is_zombie(zombie_pid),
5406 "precondition: stand-in child {zombie_pid} must be a zombie before finalize"
5407 );
5408
5409 registry.poll_task(&task).unwrap();
5412
5413 {
5414 let state = task.state.lock().unwrap();
5415 assert!(
5416 matches!(state.runtime, TaskRuntime::Piped(None)),
5417 "child handle must be released after marker finalize"
5418 );
5419 assert!(
5420 state.metadata.status.is_terminal(),
5421 "task must be terminal after marker finalize: {:?}",
5422 state.metadata.status
5423 );
5424 }
5425
5426 assert!(
5429 !is_zombie(zombie_pid),
5430 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
5431 after the exit-marker terminal transition"
5432 );
5433 }
5434
5435 #[cfg(unix)]
5439 #[test]
5440 fn kill_with_existing_marker_reaps_child_no_zombie() {
5441 use std::sync::atomic::Ordering;
5442
5443 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5444 let dir = tempfile::tempdir().unwrap();
5445 let task_id = registry
5446 .spawn(
5447 QUICK_SUCCESS_COMMAND,
5448 "session".to_string(),
5449 dir.path().to_path_buf(),
5450 HashMap::new(),
5451 Some(Duration::from_secs(30)),
5452 dir.path().to_path_buf(),
5453 10,
5454 true,
5455 false,
5456 Some(dir.path().to_path_buf()),
5457 )
5458 .unwrap();
5459
5460 registry.inner.shutdown.store(true, Ordering::SeqCst);
5461 std::thread::sleep(Duration::from_millis(550));
5462
5463 let task = registry.task_for_session(&task_id, "session").unwrap();
5464
5465 let started = Instant::now();
5466 while !task.paths.exit.exists() {
5467 assert!(
5468 started.elapsed() < Duration::from_secs(5),
5469 "exit marker should land quickly for `true`"
5470 );
5471 std::thread::sleep(Duration::from_millis(20));
5472 }
5473
5474 let zombie_pid;
5475 {
5476 let mut state = task.state.lock().unwrap();
5477 state.metadata.status = BgTaskStatus::Running;
5478 state.metadata.status_reason = None;
5479 state.metadata.exit_code = None;
5480 state.metadata.finished_at = None;
5481 state.metadata.duration_ms = None;
5482 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
5483 .expect("persist reset Running metadata");
5484 let zombie = spawn_unreaped_zombie();
5485 zombie_pid = zombie.id();
5486 state.runtime = TaskRuntime::Piped(Some(zombie));
5487 }
5488 *task.terminal_at.lock().unwrap() = None;
5489
5490 assert!(
5491 is_zombie(zombie_pid),
5492 "precondition: stand-in child {zombie_pid} must be a zombie before kill"
5493 );
5494
5495 registry
5497 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
5498 .expect("kill should succeed");
5499
5500 {
5501 let state = task.state.lock().unwrap();
5502 assert!(
5503 matches!(state.runtime, TaskRuntime::Piped(None)),
5504 "child handle must be released after marker-aware kill"
5505 );
5506 assert!(state.metadata.status.is_terminal());
5507 }
5508
5509 assert!(
5510 !is_zombie(zombie_pid),
5511 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
5512 after a marker-aware kill"
5513 );
5514 }
5515
5516 #[test]
5517 fn cleanup_finished_keeps_running_tasks() {
5518 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5519 let dir = tempfile::tempdir().unwrap();
5520 let task_id = registry
5521 .spawn(
5522 LONG_RUNNING_COMMAND,
5523 "session".to_string(),
5524 dir.path().to_path_buf(),
5525 HashMap::new(),
5526 Some(Duration::from_secs(30)),
5527 dir.path().to_path_buf(),
5528 10,
5529 true,
5530 false,
5531 Some(dir.path().to_path_buf()),
5532 )
5533 .unwrap();
5534
5535 registry.cleanup_finished(Duration::ZERO);
5536
5537 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
5538 let _ = registry.kill(&task_id, "session");
5539 }
5540
5541 #[cfg(windows)]
5542 fn wait_for_file(path: &Path) -> String {
5543 let started = Instant::now();
5544 loop {
5545 if path.exists() {
5546 return fs::read_to_string(path).expect("read file");
5547 }
5548 assert!(
5549 started.elapsed() < Duration::from_secs(30),
5550 "timed out waiting for {}",
5551 path.display()
5552 );
5553 std::thread::sleep(Duration::from_millis(100));
5554 }
5555 }
5556
5557 #[cfg(windows)]
5558 fn spawn_windows_registry_command(
5559 command: &str,
5560 ) -> (BgTaskRegistry, tempfile::TempDir, String) {
5561 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5562 let dir = tempfile::tempdir().unwrap();
5563 let task_id = registry
5564 .spawn(
5565 command,
5566 "session".to_string(),
5567 dir.path().to_path_buf(),
5568 HashMap::new(),
5569 Some(Duration::from_secs(30)),
5570 dir.path().to_path_buf(),
5571 10,
5572 false,
5573 false,
5574 Some(dir.path().to_path_buf()),
5575 )
5576 .unwrap();
5577 (registry, dir, task_id)
5578 }
5579
5580 #[cfg(windows)]
5581 #[test]
5582 fn windows_spawn_writes_exit_marker_for_zero_exit() {
5583 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 0");
5584 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
5585
5586 let content = wait_for_file(&exit_path);
5587
5588 assert_eq!(content.trim(), "0");
5589 }
5590
5591 #[cfg(windows)]
5592 #[test]
5593 fn windows_spawn_writes_exit_marker_for_nonzero_exit() {
5594 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 42");
5595 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
5596
5597 let content = wait_for_file(&exit_path);
5598
5599 assert_eq!(content.trim(), "42");
5600 }
5601
5602 #[cfg(windows)]
5603 #[test]
5604 fn windows_spawn_captures_stdout_to_disk() {
5605 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c echo hello");
5606 let task = registry.task_for_session(&task_id, "session").unwrap();
5607 let stdout_path = task.paths.stdout.clone();
5608 let exit_path = task.paths.exit.clone();
5609
5610 let _ = wait_for_file(&exit_path);
5611 let stdout = fs::read_to_string(stdout_path).expect("read stdout");
5612
5613 assert!(stdout.contains("hello"), "stdout was {stdout:?}");
5614 }
5615
5616 #[cfg(windows)]
5617 #[test]
5618 fn windows_spawn_uses_pwsh_when_available() {
5619 let candidates = crate::windows_shell::shell_candidates_with(
5623 |binary| match binary {
5624 "pwsh.exe" => Some(std::path::PathBuf::from(r"C:\pwsh\pwsh.exe")),
5625 "powershell.exe" => Some(std::path::PathBuf::from(r"C:\ps\powershell.exe")),
5626 _ => None,
5627 },
5628 || None,
5629 );
5630 let shell = candidates.first().expect("at least one candidate").clone();
5631 assert_eq!(shell, crate::windows_shell::WindowsShell::Pwsh);
5632 assert_eq!(shell.binary().as_ref(), "pwsh.exe");
5633 }
5634
5635 #[cfg(windows)]
5642 #[test]
5643 fn windows_shell_cmd_wrapper_writes_exit_marker_with_move() {
5644 let exit_path = Path::new(r"C:\Temp\bash-test.exit");
5645 let script =
5646 crate::windows_shell::WindowsShell::Cmd.wrapper_script("cmd /c exit 42", exit_path);
5647
5648 assert!(
5652 script.contains("set CODE=%ERRORLEVEL%"),
5653 "wrapper must capture exit code into CODE: {script}"
5654 );
5655 assert!(
5656 script.contains("echo %CODE% >"),
5657 "wrapper must echo CODE to a temp marker file: {script}"
5658 );
5659 assert!(
5660 script.contains("move /Y"),
5661 "wrapper must use atomic move to write the marker: {script}"
5662 );
5663 assert!(
5666 script.contains("> nul"),
5667 "wrapper must redirect move output to nul: {script}"
5668 );
5669 assert!(
5671 script.contains("exit /B %CODE%"),
5672 "wrapper must propagate the captured exit code: {script}"
5673 );
5674 assert!(script.contains(r#""C:\Temp\bash-test.exit.tmp""#));
5675 assert!(script.contains(r#""C:\Temp\bash-test.exit""#));
5676 }
5677
5678 #[cfg(windows)]
5684 #[test]
5685 fn windows_shell_cmd_bg_command_uses_minimal_cmd_flags() {
5686 use crate::windows_shell::WindowsShell;
5687 let cmd = WindowsShell::Cmd.bg_command("echo wrapped");
5688 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
5689 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
5690 assert_eq!(
5691 args_strs,
5692 vec!["/D", "/S", "/C", "echo wrapped"],
5693 "Cmd::bg_command must prepend /D /S /C"
5694 );
5695 }
5696
5697 #[cfg(windows)]
5700 #[test]
5701 fn windows_shell_pwsh_bg_command_uses_standard_args() {
5702 use crate::windows_shell::WindowsShell;
5703 let cmd = WindowsShell::Pwsh.bg_command("Get-Date");
5704 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
5705 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
5706 assert!(
5707 args_strs.contains(&"-Command"),
5708 "Pwsh::bg_command must use -Command: {args_strs:?}"
5709 );
5710 assert!(
5711 args_strs.contains(&"Get-Date"),
5712 "Pwsh::bg_command must include the user command body"
5713 );
5714 }
5715}