1use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
2use std::fs;
3use std::io::{Read, Write};
4use std::path::{Path, PathBuf};
5use std::process::{Child, Command, Stdio};
6use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
7#[cfg(unix)]
8use std::sync::OnceLock;
9use std::sync::{Arc, Mutex, RwLock};
10use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
11
12use rusqlite::Connection;
13use serde::Serialize;
14
15use crate::bash_permissions::PermissionAsk;
16use crate::compress::caps::DropClass;
17use crate::compress::CompressionResult;
18use crate::context::SharedProgressSender;
19use crate::db::compression_events::CompressionAggregateCache;
20use crate::harness::Harness;
21use crate::protocol::{BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, PushFrame};
22use crate::sandbox_spawn::SpawnPlan;
23
24#[cfg(unix)]
25use std::os::unix::process::CommandExt;
26#[cfg(windows)]
27use std::os::windows::process::CommandExt;
28
29use super::buffer::{combine_streams, BgBuffer, DiskTruncation, StreamKind, TokenCountInput};
30use super::output::{
31 cap_completion_output, cap_completion_output_with_marker, cap_final_output,
32 cap_final_output_with_marker, completion_preview_threshold, json_output_pointer, quote_path,
33 retained_json_output_pointer, COMPRESS_INPUT_CAP_BYTES, COMPRESS_INPUT_HEAD_BYTES,
34 COMPRESS_INPUT_TAIL_BYTES, FINAL_OUTPUT_CAP_BYTES, RAW_PASSTHROUGH_CAP_BYTES,
35 RAW_PASSTHROUGH_HEAD_BYTES, RAW_PASSTHROUGH_TAIL_BYTES, RUNNING_OUTPUT_PREVIEW_BYTES,
36 STRUCTURED_OUTPUT_CAP_BYTES,
37};
38use super::persistence::{
39 create_capture_file, delete_task_bundle, read_exit_marker, read_task, session_tasks_dir,
40 task_paths, unix_millis, update_task, write_kill_marker_if_absent, write_task, BgMode,
41 ExitMarker, PersistedTask, TaskPaths,
42};
43use super::process::is_process_alive;
44#[cfg(unix)]
45use super::process::terminate_pgid;
46#[cfg(windows)]
47use super::process::terminate_pid;
48use super::pty_process::spawn_pty_for_command;
49use super::pty_runtime::PtyRuntime;
50use super::watches::{PatternMatch, WatchPattern, WatchRegistry};
51use super::{BgTaskInfo, BgTaskStatus};
52const DEFAULT_BG_TIMEOUT: Duration = Duration::from_secs(30 * 60);
55const STALE_RUNNING_AFTER: Duration = Duration::from_secs(24 * 60 * 60);
56const PERSISTED_GC_GRACE: Duration = Duration::from_secs(24 * 60 * 60);
57const QUARANTINE_GC_GRACE: Duration = Duration::from_secs(30 * 24 * 60 * 60);
58
59const TOKENIZE_CAP_BYTES_PER_STREAM: usize = 128 * 1024;
60
61#[derive(Debug, Clone, Serialize)]
62pub struct BgCompletion {
63 pub task_id: String,
64 #[serde(skip_serializing)]
67 pub session_id: String,
68 pub status: BgTaskStatus,
69 pub exit_code: Option<i32>,
70 pub command: String,
71 #[serde(default, skip_serializing_if = "String::is_empty")]
77 pub output_preview: String,
78 #[serde(default, skip_serializing_if = "is_false")]
83 pub output_truncated: bool,
84 #[serde(default, skip_serializing_if = "Option::is_none")]
87 pub original_tokens: Option<u32>,
88 #[serde(default, skip_serializing_if = "Option::is_none")]
91 pub compressed_tokens: Option<u32>,
92 #[serde(default, skip_serializing_if = "is_false")]
94 pub tokens_skipped: bool,
95}
96
97fn is_false(v: &bool) -> bool {
98 !*v
99}
100
101#[derive(Debug, Clone, Serialize)]
102pub struct BgTaskSnapshot {
103 #[serde(flatten)]
104 pub info: BgTaskInfo,
105 pub exit_code: Option<i32>,
106 pub child_pid: Option<u32>,
107 pub workdir: String,
108 pub output_preview: String,
109 pub output_truncated: bool,
110 pub output_path: Option<String>,
111 pub stderr_path: Option<String>,
112 #[serde(skip_serializing_if = "Option::is_none")]
113 pub pty_rows: Option<u16>,
114 #[serde(skip_serializing_if = "Option::is_none")]
115 pub pty_cols: Option<u16>,
116 #[serde(skip_serializing_if = "Option::is_none")]
117 pub pty_screen: Option<String>,
118 #[serde(default, skip_serializing_if = "Vec::is_empty")]
119 pub scanner_report: Vec<PermissionAsk>,
120 #[serde(default, skip_serializing_if = "is_false")]
121 pub sandbox_native: bool,
122 #[serde(default, skip_serializing_if = "is_false")]
123 pub sandbox_unavailable: bool,
124}
125
126#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
127pub struct BgTaskHealthCounts {
128 pub running: usize,
129 pub pending_completions: usize,
130}
131
132#[derive(Debug, Clone, Copy, PartialEq, Eq)]
133enum TerminalOutputKind {
134 Compressed,
135 Raw,
136 Structured,
137}
138
139#[derive(Debug, Clone, PartialEq, Eq)]
140struct TerminalOutputCache {
141 output_preview: String,
142 output_truncated: bool,
143 kind: TerminalOutputKind,
144 output_path: Option<String>,
145 stderr_path: Option<String>,
146 artifact_access: ArtifactRecoveryAccess,
147 recovery: Option<RecoveryContext>,
148}
149
150#[derive(Debug, Clone, PartialEq, Eq)]
151struct ArtifactRecoveryAccess {
152 task_id: String,
153 readable: bool,
154}
155
156#[derive(Debug, Clone, PartialEq, Eq)]
157struct RecoveryContext {
158 dropped_by_class: BTreeMap<DropClass, usize>,
159 had_inner_drop: bool,
160 offset_hint_eligible: bool,
161 offset_start_line: Option<usize>,
162 byte_truncated: bool,
163 disk_truncated_prefix_bytes: u64,
164 output_path: Option<String>,
165 stderr_path: Option<String>,
166 include_stderr_path: bool,
167 artifact_access: ArtifactRecoveryAccess,
168}
169
170fn optional_string_bytes(value: Option<&String>) -> u64 {
171 value
172 .map(|value| crate::memory::usize_to_u64(value.len()))
173 .unwrap_or(0)
174}
175
176fn terminal_output_cache_estimated_bytes(cache: &TerminalOutputCache) -> u64 {
177 let recovery_bytes = cache
178 .recovery
179 .as_ref()
180 .map(|recovery| {
181 crate::memory::usize_to_u64(recovery.dropped_by_class.len())
182 .saturating_mul(
183 (std::mem::size_of::<DropClass>() + std::mem::size_of::<usize>()) as u64,
184 )
185 .saturating_add(optional_string_bytes(recovery.output_path.as_ref()))
186 .saturating_add(optional_string_bytes(recovery.stderr_path.as_ref()))
187 .saturating_add(crate::memory::usize_to_u64(
188 recovery.artifact_access.task_id.len(),
189 ))
190 })
191 .unwrap_or(0);
192 (std::mem::size_of::<TerminalOutputCache>() as u64)
193 .saturating_add(crate::memory::usize_to_u64(cache.output_preview.len()))
194 .saturating_add(optional_string_bytes(cache.output_path.as_ref()))
195 .saturating_add(optional_string_bytes(cache.stderr_path.as_ref()))
196 .saturating_add(crate::memory::usize_to_u64(
197 cache.artifact_access.task_id.len(),
198 ))
199 .saturating_add(recovery_bytes)
200}
201
202fn completion_estimated_bytes(completion: &BgCompletion) -> u64 {
203 (std::mem::size_of::<BgCompletion>() as u64)
204 .saturating_add(crate::memory::usize_to_u64(completion.task_id.len()))
205 .saturating_add(crate::memory::usize_to_u64(completion.session_id.len()))
206 .saturating_add(crate::memory::usize_to_u64(completion.command.len()))
207 .saturating_add(crate::memory::usize_to_u64(completion.output_preview.len()))
208}
209
210impl RecoveryContext {
211 fn has_visible_drop(&self) -> bool {
212 self.byte_truncated
213 || self.disk_truncated_prefix_bytes > 0
214 || self.had_inner_drop
215 || !self.dropped_by_class.is_empty()
216 }
217}
218
219#[derive(Clone)]
220pub struct BgTaskRegistry {
221 pub(crate) inner: Arc<RegistryInner>,
222}
223
224pub(crate) struct RegistryInner {
225 pub(crate) tasks: Mutex<HashMap<String, Arc<BgTask>>>,
226 pub(crate) completions: Mutex<VecDeque<BgCompletion>>,
227 pub(crate) progress_sender: SharedProgressSender,
228 watchdog_started: AtomicBool,
229 pub(crate) shutdown: AtomicBool,
230 pub(crate) long_running_reminder_enabled: AtomicBool,
231 pub(crate) long_running_reminder_interval_ms: AtomicU64,
232 persisted_gc_started: AtomicBool,
233 #[cfg(test)]
234 persisted_gc_runs: AtomicU64,
235 pub(crate) compressor:
241 Mutex<Option<Box<dyn Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync>>>,
242 pub(crate) db_pool: RwLock<Option<Arc<Mutex<Connection>>>>,
243 pub(crate) db_harness: RwLock<Option<String>>,
244 pub(crate) compression_aggregates: Arc<CompressionAggregateCache>,
245 pub(crate) wake_tx: crossbeam_channel::Sender<()>,
246 pub(crate) wake_rx: crossbeam_channel::Receiver<()>,
247 pub(crate) watch_registry: Mutex<WatchRegistry>,
248 wait_detach_sessions: Mutex<HashSet<String>>,
249 active_wait_sessions: Mutex<HashMap<String, usize>>,
250}
251
252pub(crate) struct BgTask {
253 pub(crate) task_id: String,
254 pub(crate) session_id: String,
255 pub(crate) paths: TaskPaths,
256 artifact_root: PathBuf,
257 pub(crate) started: Instant,
258 pub(crate) last_reminder_at: Mutex<Option<Instant>>,
259 pub(crate) terminal_at: Mutex<Option<Instant>>,
260 pub(crate) state: Mutex<BgTaskState>,
261}
262
263pub(crate) enum TaskRuntime {
264 Piped(Option<Child>),
265 Pty(Option<PtyRuntime>),
266}
267
268pub(crate) struct BgTaskState {
269 pub(crate) metadata: PersistedTask,
270 pub(crate) runtime: TaskRuntime,
271 pub(crate) detached: bool,
272 pub(crate) child_exit_observed: bool,
283 pub(crate) buffer: BgBuffer,
284 terminal_output_cache: Option<TerminalOutputCache>,
285 pub(crate) pending_terminal_override: Option<BgTaskStatus>,
287}
288
289fn completion_matches_session(completion: &BgCompletion, session_id: Option<&str>) -> bool {
290 session_id
291 .map(|session_id| completion.session_id == session_id)
292 .unwrap_or(true)
293}
294
295impl BgTaskRegistry {
296 pub fn new(progress_sender: SharedProgressSender) -> Self {
297 let (wake_tx, wake_rx) = crossbeam_channel::bounded(1);
298 Self {
299 inner: Arc::new(RegistryInner {
300 tasks: Mutex::new(HashMap::new()),
301 completions: Mutex::new(VecDeque::new()),
302 progress_sender,
303 watchdog_started: AtomicBool::new(false),
304 shutdown: AtomicBool::new(false),
305 long_running_reminder_enabled: AtomicBool::new(true),
306 long_running_reminder_interval_ms: AtomicU64::new(600_000),
307 persisted_gc_started: AtomicBool::new(false),
308 #[cfg(test)]
309 persisted_gc_runs: AtomicU64::new(0),
310 compressor: Mutex::new(None),
311 db_pool: RwLock::new(None),
312 db_harness: RwLock::new(None),
313 compression_aggregates: Arc::new(CompressionAggregateCache::default()),
314 wake_tx,
315 wake_rx,
316 watch_registry: Mutex::new(WatchRegistry::default()),
317 wait_detach_sessions: Mutex::new(HashSet::new()),
318 active_wait_sessions: Mutex::new(HashMap::new()),
319 }),
320 }
321 }
322
323 pub fn is_session_owned_artifact_path(&self, session_id: &str, path: &Path) -> bool {
331 let Ok(requested) = fs::canonicalize(path) else {
332 return false;
333 };
334 let Ok(tasks) = self.inner.tasks.lock() else {
335 return false;
336 };
337
338 tasks.values().any(|task| {
339 task.session_id == session_id
340 && [
341 &task.paths.stdout,
342 &task.paths.stderr,
343 &task.paths.exit,
344 &task.paths.pty,
345 ]
346 .into_iter()
347 .filter_map(|known| known.file_name())
348 .any(|name| task.artifact_root.join(name) == requested)
349 })
350 }
351
352 pub fn set_harness(&self, harness: Harness) {
353 if let Ok(mut slot) = self.inner.db_harness.write() {
354 *slot = Some(harness.storage_segment());
355 }
356 }
357
358 pub fn set_db_pool(&self, conn: Arc<Mutex<Connection>>) {
359 if let Ok(mut slot) = self.inner.db_pool.write() {
360 *slot = Some(conn);
361 }
362 self.inner.compression_aggregates.clear();
363 }
364
365 pub fn clear_db_pool(&self) {
366 if let Ok(mut slot) = self.inner.db_pool.write() {
367 *slot = None;
368 }
369 self.inner.compression_aggregates.clear();
370 }
371
372 pub(crate) fn compression_aggregate_cache(&self) -> Arc<CompressionAggregateCache> {
373 Arc::clone(&self.inner.compression_aggregates)
374 }
375
376 pub fn begin_wait_mode_session(&self, session_id: &str) {
377 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
378 *active.entry(session_id.to_string()).or_insert(0) += 1;
379 }
380 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
381 detach.remove(session_id);
382 }
383 }
384
385 pub fn end_wait_mode_session(&self, session_id: &str) {
386 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
387 match active.get_mut(session_id) {
388 Some(count) if *count > 1 => *count -= 1,
389 Some(_) => {
390 active.remove(session_id);
391 }
392 None => {}
393 }
394 }
395 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
396 detach.remove(session_id);
397 }
398 }
399
400 pub fn signal_wait_mode_detach(&self, session_id: &str) -> bool {
401 let is_waiting = self
402 .inner
403 .active_wait_sessions
404 .lock()
405 .map(|active| active.get(session_id).copied().unwrap_or(0) > 0)
406 .unwrap_or(false);
407 if !is_waiting {
408 return false;
409 }
410 self.inner
411 .wait_detach_sessions
412 .lock()
413 .map(|mut detach| detach.insert(session_id.to_string()))
414 .unwrap_or(false)
415 }
416
417 pub fn active_wait_session_count(&self) -> usize {
420 self.inner
421 .active_wait_sessions
422 .lock()
423 .map(|active| active.len())
424 .unwrap_or(0)
425 }
426
427 pub fn take_wait_mode_detach(&self, session_id: &str) -> bool {
428 self.inner
429 .wait_detach_sessions
430 .lock()
431 .map(|mut detach| detach.remove(session_id))
432 .unwrap_or(false)
433 }
434
435 pub fn set_compressor<F>(&self, compressor: F)
440 where
441 F: Fn(&str, String) -> CompressionResult + Send + Sync + 'static,
442 {
443 self.set_compressor_with_exit_code(move |command, output, _exit_code| {
444 compressor(command, output)
445 });
446 }
447
448 pub fn set_compressor_with_exit_code<F>(&self, compressor: F)
449 where
450 F: Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync + 'static,
451 {
452 if let Ok(mut slot) = self.inner.compressor.lock() {
453 *slot = Some(Box::new(compressor));
454 }
455 }
456
457 pub(crate) fn compress_output(
460 &self,
461 command: &str,
462 output: String,
463 exit_code: Option<i32>,
464 ) -> CompressionResult {
465 let Ok(slot) = self.inner.compressor.lock() else {
466 return CompressionResult::new(output);
467 };
468 match slot.as_ref() {
469 Some(compressor) => compressor(command, output, exit_code),
470 None => CompressionResult::new(output),
471 }
472 }
473
474 fn ensure_terminal_output_cache(&self, task: &Arc<BgTask>) -> Option<TerminalOutputCache> {
475 let (metadata, buffer) = {
476 let state = task.state.lock().ok()?;
477 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
478 return None;
479 }
480 if let Some(cache) = state.terminal_output_cache.clone() {
481 return Some(cache);
482 }
483 (state.metadata.clone(), state.buffer.clone())
484 };
485
486 let mut cap_buffer = buffer.clone();
487 let disk_truncation = cap_buffer.enforce_terminal_cap();
488 let cache = self.render_terminal_output(&metadata, &cap_buffer, disk_truncation);
489 let mut state = task.state.lock().ok()?;
490 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
491 return None;
492 }
493 if let Some(existing) = state.terminal_output_cache.clone() {
494 return Some(existing);
495 }
496 state.terminal_output_cache = Some(cache.clone());
497 Some(cache)
498 }
499
500 fn render_terminal_output(
501 &self,
502 metadata: &PersistedTask,
503 buffer: &BgBuffer,
504 disk_truncation: DiskTruncation,
505 ) -> TerminalOutputCache {
506 let output_readable = buffer
507 .output_path()
508 .is_some_and(|path| self.is_session_owned_artifact_path(&metadata.session_id, &path));
509 let stderr_readable = buffer
510 .stderr_path()
511 .map(|path| self.is_session_owned_artifact_path(&metadata.session_id, path))
512 .unwrap_or(true);
513 let artifact_access = ArtifactRecoveryAccess {
514 task_id: metadata.task_id.clone(),
515 readable: output_readable && stderr_readable,
516 };
517
518 if metadata.mode == BgMode::Pty {
519 return TerminalOutputCache {
520 output_preview: String::new(),
521 output_truncated: false,
522 kind: TerminalOutputKind::Raw,
523 output_path: buffer.output_path().map(|path| path.display().to_string()),
524 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
525 artifact_access,
526 recovery: None,
527 };
528 }
529
530 if let Some(structured) = render_structured_output(
531 &metadata.command,
532 buffer,
533 disk_truncation,
534 artifact_access.clone(),
535 ) {
536 return structured;
537 }
538
539 if !metadata.compressed {
540 return render_raw_passthrough(buffer, disk_truncation, artifact_access);
541 }
542
543 let raw = buffer.read_combined_head_tail(
544 COMPRESS_INPUT_CAP_BYTES,
545 COMPRESS_INPUT_HEAD_BYTES,
546 COMPRESS_INPUT_TAIL_BYTES,
547 );
548 let compressed = self.compress_output(&metadata.command, raw.text, metadata.exit_code);
549 render_compressed_with_recovery(
550 buffer,
551 compressed,
552 raw.truncated,
553 disk_truncation,
554 artifact_access,
555 )
556 }
557
558 fn snapshot_with_terminal_cache(
559 &self,
560 task: &Arc<BgTask>,
561 preview_bytes: usize,
562 ) -> BgTaskSnapshot {
563 let mut snapshot = task.snapshot(preview_bytes);
564 self.maybe_compress_snapshot(task, &mut snapshot);
565 snapshot
566 }
567
568 fn post_terminal_transition(&self, task: &Arc<BgTask>, emit_frame: bool) -> Result<(), String> {
569 let (metadata, buffer) = {
570 let state = task
571 .state
572 .lock()
573 .map_err(|_| "background task lock poisoned".to_string())?;
574 if !state.metadata.status.is_terminal() {
575 return Ok(());
576 }
577 (state.metadata.clone(), state.buffer.clone())
578 };
579
580 let cache = self.ensure_terminal_output_cache(task);
581 self.enqueue_completion_from_parts(
582 &metadata,
583 Some(&buffer),
584 None,
585 emit_frame,
586 cache.as_ref(),
587 );
588 Ok(())
589 }
590
591 fn persist_task(&self, paths: &TaskPaths, metadata: &PersistedTask) -> std::io::Result<()> {
592 write_task(&paths.json, metadata)?;
593 self.dual_write_task(paths, metadata);
594 Ok(())
595 }
596
597 fn update_task_metadata<F>(
598 &self,
599 paths: &TaskPaths,
600 update: F,
601 ) -> std::io::Result<PersistedTask>
602 where
603 F: FnOnce(&mut PersistedTask),
604 {
605 let metadata = update_task(&paths.json, update)?;
606 self.dual_write_task(paths, &metadata);
607 Ok(metadata)
608 }
609
610 fn dual_write_task(&self, paths: &TaskPaths, metadata: &PersistedTask) {
611 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
612 let Some(pool) = pool else {
613 return;
614 };
615 let harness = self
616 .inner
617 .db_harness
618 .read()
619 .ok()
620 .and_then(|slot| slot.clone());
621 let Some(harness) = harness else {
622 crate::slog_warn!(
623 "dual-write bash_task to DB skipped for {}: harness not configured",
624 metadata.task_id
625 );
626 return;
627 };
628 let row = match metadata.to_bash_task_row(&harness, paths) {
629 Ok(row) => row,
630 Err(error) => {
631 crate::slog_warn!(
632 "dual-write bash_task to DB failed for {}: {}",
633 metadata.task_id,
634 error
635 );
636 return;
637 }
638 };
639 let conn = match pool.lock() {
640 Ok(conn) => conn,
641 Err(_) => {
642 crate::slog_warn!(
643 "dual-write bash_task to DB failed for {}: db mutex poisoned",
644 metadata.task_id
645 );
646 return;
647 }
648 };
649 if let Err(error) = crate::db::bash_tasks::upsert_bash_task(&conn, &row) {
650 crate::slog_warn!(
651 "dual-write bash_task to DB failed for {}: {}",
652 metadata.task_id,
653 error
654 );
655 }
656 }
657
658 fn delete_gc_task_from_db(&self, metadata: &PersistedTask) {
659 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
660 let Some(pool) = pool else {
661 return;
662 };
663 let harness = self
664 .inner
665 .db_harness
666 .read()
667 .ok()
668 .and_then(|slot| slot.clone());
669 let Some(harness) = harness else {
670 crate::slog_warn!(
671 "GC bash_task DB delete skipped for {}: harness not configured",
672 metadata.task_id
673 );
674 return;
675 };
676 let conn = match pool.lock() {
677 Ok(conn) => conn,
678 Err(_) => {
679 crate::slog_warn!(
680 "GC bash_task DB delete failed for {}: db mutex poisoned",
681 metadata.task_id
682 );
683 return;
684 }
685 };
686 if let Err(error) = crate::db::bash_tasks::delete_delivered_terminal_bash_task(
687 &conn,
688 &harness,
689 &metadata.session_id,
690 &metadata.task_id,
691 ) {
692 crate::slog_warn!(
693 "GC bash_task DB delete failed for {}: {}",
694 metadata.task_id,
695 error
696 );
697 }
698 }
699
700 pub fn record_scanner_report(
701 &self,
702 task_id: &str,
703 session_id: &str,
704 scanner_report: Vec<PermissionAsk>,
705 ) -> Result<(), String> {
706 if scanner_report.is_empty() {
707 return Ok(());
708 }
709 let task = self.task_for_session(task_id, session_id).ok_or_else(|| {
710 "background task not found while recording scanner report".to_string()
711 })?;
712 let metadata = {
713 let mut state = task
714 .state
715 .lock()
716 .map_err(|_| "background task lock poisoned".to_string())?;
717 state.metadata.scanner_report = scanner_report;
718 state.metadata.clone()
719 };
720 self.persist_task(&task.paths, &metadata)
721 .map_err(|error| format!("failed to persist scanner report: {error}"))
722 }
723
724 pub fn configure_long_running_reminders(&self, enabled: bool, interval_ms: u64) {
725 self.inner
726 .long_running_reminder_enabled
727 .store(enabled, Ordering::SeqCst);
728 self.inner
729 .long_running_reminder_interval_ms
730 .store(interval_ms, Ordering::SeqCst);
731 }
732
733 #[cfg(unix)]
734 #[allow(clippy::too_many_arguments)]
735 pub fn spawn(
736 &self,
737 spawn_plan: SpawnPlan,
738 command: &str,
739 session_id: String,
740 workdir: PathBuf,
741 env: HashMap<String, String>,
742 timeout: Option<Duration>,
743 storage_dir: PathBuf,
744 max_running: usize,
745 notify_on_completion: bool,
746 compressed: bool,
747 project_root: Option<PathBuf>,
748 ) -> Result<String, String> {
749 self.start_watchdog();
750
751 let running = self.running_count();
752 if running >= max_running {
753 return Err(format!(
754 "background bash task limit exceeded: {running} running (max {max_running})"
755 ));
756 }
757
758 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
759 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
760 let task_id = self.generate_unique_task_id()?;
761 let paths = task_paths(&storage_dir, &session_id, &task_id);
762 fs::create_dir_all(&paths.dir)
763 .map_err(|e| format!("failed to create background task dir: {e}"))?;
764
765 let mut metadata = PersistedTask::starting(
766 task_id.clone(),
767 session_id.clone(),
768 command.to_string(),
769 workdir.clone(),
770 project_root,
771 timeout_ms,
772 notify_on_completion,
773 compressed,
774 );
775 attach_sandbox_metadata(&mut metadata, &spawn_plan);
776 self.persist_task(&paths, &metadata)
777 .map_err(|e| format!("failed to persist background task metadata: {e}"))?;
778
779 create_capture_file(&paths.stdout)
783 .map_err(|e| format!("failed to create stdout capture file: {e}"))?;
784 create_capture_file(&paths.stderr)
785 .map_err(|e| format!("failed to create stderr capture file: {e}"))?;
786
787 let child = match spawn_detached_child(&spawn_plan, command, &paths, &workdir, &env) {
788 Ok(child) => child,
789 Err(error) => {
790 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
791 let _ = delete_task_bundle(&paths);
792 return Err(error);
793 }
794 };
795
796 let child_pid = child.id();
797 metadata.mark_running(child_pid, child_pid as i32);
798 self.persist_task(&paths, &metadata)
799 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
800
801 let task = Arc::new(BgTask {
802 task_id: task_id.clone(),
803 session_id,
804 paths: paths.clone(),
805 artifact_root: canonical_artifact_root(&paths),
806 started: Instant::now(),
807 last_reminder_at: Mutex::new(None),
808 terminal_at: Mutex::new(None),
809 state: Mutex::new(BgTaskState {
810 metadata,
811 runtime: TaskRuntime::Piped(Some(child)),
812 detached: false,
813 child_exit_observed: false,
814 buffer: BgBuffer::new(paths.stdout.clone(), paths.stderr.clone()),
815 terminal_output_cache: None,
816 pending_terminal_override: None,
817 }),
818 });
819
820 self.inner
821 .tasks
822 .lock()
823 .map_err(|_| "background task registry lock poisoned".to_string())?
824 .insert(task_id.clone(), task);
825
826 Ok(task_id)
827 }
828
829 #[allow(clippy::too_many_arguments)]
830 pub fn spawn_pty(
831 &self,
832 spawn_plan: SpawnPlan,
833 command: &str,
834 session_id: String,
835 workdir: PathBuf,
836 env: HashMap<String, String>,
837 timeout: Option<Duration>,
838 storage_dir: PathBuf,
839 max_running: usize,
840 notify_on_completion: bool,
841 compressed: bool,
842 project_root: Option<PathBuf>,
843 rows: u16,
844 cols: u16,
845 ) -> Result<String, String> {
846 self.start_watchdog();
847
848 let running = self.running_count();
849 if running >= max_running {
850 return Err(format!(
851 "background bash task limit exceeded: {running} running (max {max_running})"
852 ));
853 }
854
855 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
856 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
857 let task_id = self.generate_unique_task_id()?;
858 let paths = task_paths(&storage_dir, &session_id, &task_id);
859 fs::create_dir_all(&paths.dir)
860 .map_err(|e| format!("failed to create background task dir: {e}"))?;
861
862 let mut metadata = PersistedTask::starting(
863 task_id.clone(),
864 session_id.clone(),
865 command.to_string(),
866 workdir.clone(),
867 project_root,
868 timeout_ms,
869 notify_on_completion,
870 compressed,
871 );
872 attach_sandbox_metadata(&mut metadata, &spawn_plan);
873 metadata.mode = BgMode::Pty;
874 metadata.pty_rows = Some(rows);
875 metadata.pty_cols = Some(cols);
876 self.persist_task(&paths, &metadata)
877 .map_err(|e| format!("failed to persist background task metadata: {e}"))?;
878 create_capture_file(&paths.pty)
879 .map_err(|e| format!("failed to create PTY capture file: {e}"))?;
880
881 let runtime = match spawn_pty_for_command(
882 &spawn_plan,
883 &task_id,
884 &session_id,
885 command,
886 &paths,
887 &workdir,
888 &env,
889 rows,
890 cols,
891 self.inner.wake_tx.clone(),
892 ) {
893 Ok(runtime) => runtime,
894 Err(error) => {
895 crate::slog_warn!(
896 "failed to spawn PTY background bash task {task_id}; deleting partial bundle: {error}"
897 );
898 let _ = delete_task_bundle(&paths);
899 return Err(error);
900 }
901 };
902
903 if let Some(child_pid) = runtime.child_pid {
904 metadata.mark_running(child_pid, child_pid as i32);
905 } else {
906 metadata.status = BgTaskStatus::Running;
907 metadata.pgid = None;
908 }
909 self.persist_task(&paths, &metadata)
910 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
911
912 let task = Arc::new(BgTask {
913 task_id: task_id.clone(),
914 session_id,
915 paths: paths.clone(),
916 artifact_root: canonical_artifact_root(&paths),
917 started: Instant::now(),
918 last_reminder_at: Mutex::new(None),
919 terminal_at: Mutex::new(None),
920 state: Mutex::new(BgTaskState {
921 metadata,
922 runtime: TaskRuntime::Pty(Some(runtime)),
923 detached: false,
924 child_exit_observed: false,
925 buffer: BgBuffer::pty(paths.pty.clone()),
926 terminal_output_cache: None,
927 pending_terminal_override: None,
928 }),
929 });
930
931 self.inner
932 .tasks
933 .lock()
934 .map_err(|_| "background task registry lock poisoned".to_string())?
935 .insert(task_id.clone(), task);
936
937 Ok(task_id)
938 }
939
940 #[cfg(windows)]
941 #[allow(clippy::too_many_arguments)]
942 pub fn spawn(
943 &self,
944 spawn_plan: SpawnPlan,
945 command: &str,
946 session_id: String,
947 workdir: PathBuf,
948 env: HashMap<String, String>,
949 timeout: Option<Duration>,
950 storage_dir: PathBuf,
951 max_running: usize,
952 notify_on_completion: bool,
953 compressed: bool,
954 project_root: Option<PathBuf>,
955 ) -> Result<String, String> {
956 self.start_watchdog();
957
958 let running = self.running_count();
959 if running >= max_running {
960 return Err(format!(
961 "background bash task limit exceeded: {running} running (max {max_running})"
962 ));
963 }
964
965 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
966 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
967 let task_id = self.generate_unique_task_id()?;
968 let paths = task_paths(&storage_dir, &session_id, &task_id);
969 fs::create_dir_all(&paths.dir)
970 .map_err(|e| format!("failed to create background task dir: {e}"))?;
971
972 let mut metadata = PersistedTask::starting(
973 task_id.clone(),
974 session_id.clone(),
975 command.to_string(),
976 workdir.clone(),
977 project_root,
978 timeout_ms,
979 notify_on_completion,
980 compressed,
981 );
982 attach_sandbox_metadata(&mut metadata, &spawn_plan);
983 self.persist_task(&paths, &metadata)
984 .map_err(|e| format!("failed to persist background task metadata: {e}"))?;
985
986 create_capture_file(&paths.stdout)
992 .map_err(|e| format!("failed to create stdout capture file: {e}"))?;
993 create_capture_file(&paths.stderr)
994 .map_err(|e| format!("failed to create stderr capture file: {e}"))?;
995
996 let child = match spawn_detached_child(&spawn_plan, command, &paths, &workdir, &env) {
997 Ok(child) => child,
998 Err(error) => {
999 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
1000 let _ = delete_task_bundle(&paths);
1001 return Err(error);
1002 }
1003 };
1004
1005 let child_pid = child.id();
1006 metadata.status = BgTaskStatus::Running;
1007 metadata.child_pid = Some(child_pid);
1008 metadata.pgid = None;
1009 self.persist_task(&paths, &metadata)
1010 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1011
1012 let task = Arc::new(BgTask {
1013 task_id: task_id.clone(),
1014 session_id,
1015 paths: paths.clone(),
1016 artifact_root: canonical_artifact_root(&paths),
1017 started: Instant::now(),
1018 last_reminder_at: Mutex::new(None),
1019 terminal_at: Mutex::new(None),
1020 state: Mutex::new(BgTaskState {
1021 metadata,
1022 runtime: TaskRuntime::Piped(Some(child)),
1023 detached: false,
1024 child_exit_observed: false,
1025 buffer: BgBuffer::new(paths.stdout.clone(), paths.stderr.clone()),
1026 terminal_output_cache: None,
1027 pending_terminal_override: None,
1028 }),
1029 });
1030
1031 self.inner
1032 .tasks
1033 .lock()
1034 .map_err(|_| "background task registry lock poisoned".to_string())?
1035 .insert(task_id.clone(), task);
1036
1037 Ok(task_id)
1038 }
1039
1040 pub fn write_pty(
1041 &self,
1042 task_id: &str,
1043 session_id: &str,
1044 input: &[u8],
1045 ) -> Result<usize, String> {
1046 let task = self
1047 .task_for_session(task_id, session_id)
1048 .ok_or_else(|| "task_not_found".to_string())?;
1049
1050 let writer = {
1051 let state = task
1052 .state
1053 .lock()
1054 .map_err(|_| "background task lock poisoned".to_string())?;
1055 if state.metadata.mode != BgMode::Pty {
1056 return Err("task_not_pty".to_string());
1057 }
1058 if state.metadata.status.is_terminal() {
1059 return Err("task_exited".to_string());
1060 }
1061 match &state.runtime {
1062 TaskRuntime::Pty(Some(runtime)) => Arc::clone(&runtime.writer),
1063 TaskRuntime::Pty(None) => return Err("task_exited".to_string()),
1064 TaskRuntime::Piped(_) => return Err("task_not_pty".to_string()),
1065 }
1066 };
1067
1068 let mut writer = writer
1069 .lock()
1070 .map_err(|_| "PTY writer lock poisoned".to_string())?;
1071 writer
1072 .write_all(input)
1073 .map_err(|error| format!("failed to write to PTY: {error}"))?;
1074 writer
1075 .flush()
1076 .map_err(|error| format!("failed to flush PTY writer: {error}"))?;
1077 Ok(input.len())
1078 }
1079
1080 pub fn replay_session(&self, storage_dir: &Path, session_id: &str) -> Result<(), String> {
1081 self.replay_session_inner(storage_dir, session_id, None)
1082 }
1083
1084 pub fn replay_session_for_project(
1085 &self,
1086 storage_dir: &Path,
1087 session_id: &str,
1088 project_root: &Path,
1089 ) -> Result<(), String> {
1090 self.replay_session_inner(storage_dir, session_id, Some(project_root))
1091 }
1092
1093 fn replay_session_inner(
1094 &self,
1095 storage_dir: &Path,
1096 session_id: &str,
1097 project_root: Option<&Path>,
1098 ) -> Result<(), String> {
1099 self.start_watchdog();
1100 if !self.inner.persisted_gc_started.swap(true, Ordering::SeqCst) {
1101 if let Err(error) = self.maybe_gc_persisted(storage_dir) {
1102 crate::slog_warn!("failed to GC persisted background bash tasks: {error}");
1103 }
1104 }
1105
1106 let canonical_project = project_root.map(canonicalized_path);
1107 let tasks = match self.replay_session_from_db(session_id) {
1119 Some(Ok(tasks)) if !tasks.is_empty() => tasks,
1120 Some(Ok(_)) => {
1121 let disk_tasks = self.replay_session_from_disk(storage_dir, session_id)?;
1122 if !disk_tasks.is_empty() {
1123 crate::slog_info!(
1124 "bash task replay: 0 in DB for session {}, {} from disk fallback",
1125 session_id,
1126 disk_tasks.len()
1127 );
1128 }
1129 disk_tasks
1130 }
1131 Some(Err(error)) => {
1132 crate::slog_warn!(
1133 "bash task replay DB lookup failed for session {}; falling back to disk: {}",
1134 session_id,
1135 error
1136 );
1137 self.replay_session_from_disk(storage_dir, session_id)?
1138 }
1139 None => {
1140 self.replay_session_from_disk(storage_dir, session_id)?
1142 }
1143 };
1144
1145 for mut metadata in tasks {
1146 if metadata.session_id != session_id {
1147 continue;
1148 }
1149 if let Some(canonical_project) = canonical_project.as_deref() {
1150 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
1151 if metadata_project.as_deref() != Some(canonical_project) {
1152 continue;
1153 }
1154 }
1155
1156 let paths = task_paths(storage_dir, session_id, &metadata.task_id);
1157 match metadata.status {
1158 BgTaskStatus::Starting => {
1159 let completion_was_delivered = metadata.completion_delivered;
1160 metadata.mark_terminal(
1161 BgTaskStatus::Failed,
1162 None,
1163 Some("spawn aborted".to_string()),
1164 );
1165 metadata.completion_delivered |= completion_was_delivered;
1166 let _ = self.persist_task(&paths, &metadata);
1167 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1168 self.insert_rehydrated_task(metadata, paths, true)?;
1169 }
1170 BgTaskStatus::Running | BgTaskStatus::Killing => {
1171 if metadata.mode == BgMode::Pty {
1172 if let Ok(Some(marker)) = read_exit_marker(&paths.exit) {
1173 let completion_was_delivered = metadata.completion_delivered;
1174 metadata = terminal_metadata_from_marker(metadata, marker, None);
1175 metadata.completion_delivered |= completion_was_delivered;
1176 let _ = self.persist_task(&paths, &metadata);
1177 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1178 self.insert_rehydrated_task(metadata, paths, true)?;
1179 } else if metadata.status.is_terminal() {
1180 self.insert_rehydrated_task(metadata, paths, true)?;
1181 } else {
1182 let completion_was_delivered = metadata.completion_delivered;
1183 metadata.mark_terminal(
1184 BgTaskStatus::Killed,
1185 None,
1186 Some("pty_lost_on_bridge_restart".to_string()),
1187 );
1188 metadata.completion_delivered |= completion_was_delivered;
1189 let _ = self.persist_task(&paths, &metadata);
1190 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1191 self.insert_rehydrated_task(metadata, paths, true)?;
1192 }
1193 } else if self.running_metadata_is_stale(&metadata) {
1194 let completion_was_delivered = metadata.completion_delivered;
1195 metadata.mark_terminal(
1196 BgTaskStatus::Killed,
1197 None,
1198 Some("orphaned (>24h)".to_string()),
1199 );
1200 metadata.completion_delivered |= completion_was_delivered;
1201 if !paths.exit.exists() {
1202 let _ = write_kill_marker_if_absent(&paths.exit);
1203 }
1204 let _ = self.persist_task(&paths, &metadata);
1205 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1206 self.insert_rehydrated_task(metadata, paths, true)?;
1207 } else if let Ok(Some(marker)) = read_exit_marker(&paths.exit) {
1208 let reason = (metadata.status == BgTaskStatus::Killing).then(|| {
1209 "recovered from inconsistent killing state on replay".to_string()
1210 });
1211 if reason.is_some() {
1212 crate::slog_warn!("background task {} had killing state with exit marker; preferring marker",
1213 metadata.task_id);
1214 }
1215 let completion_was_delivered = metadata.completion_delivered;
1216 metadata = terminal_metadata_from_marker(metadata, marker, reason);
1217 metadata.completion_delivered |= completion_was_delivered;
1218 let _ = self.persist_task(&paths, &metadata);
1219 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1220 self.insert_rehydrated_task(metadata, paths, true)?;
1221 } else if metadata.status == BgTaskStatus::Killing {
1222 if !paths.exit.exists() {
1223 let _ = write_kill_marker_if_absent(&paths.exit);
1224 }
1225 let completion_was_delivered = metadata.completion_delivered;
1226 metadata.mark_terminal(
1227 BgTaskStatus::Killed,
1228 None,
1229 Some("recovered from inconsistent killing state on replay".to_string()),
1230 );
1231 metadata.completion_delivered |= completion_was_delivered;
1232 let _ = self.persist_task(&paths, &metadata);
1233 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1234 self.insert_rehydrated_task(metadata, paths, true)?;
1235 } else if metadata.child_pid.is_some_and(|pid| !is_process_alive(pid)) {
1236 let completion_was_delivered = metadata.completion_delivered;
1237 metadata.mark_terminal(
1238 BgTaskStatus::Failed,
1239 None,
1240 Some("process exited without exit marker".to_string()),
1241 );
1242 metadata.completion_delivered |= completion_was_delivered;
1243 let _ = self.persist_task(&paths, &metadata);
1244 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1245 self.insert_rehydrated_task(metadata, paths, true)?;
1246 } else {
1247 self.insert_rehydrated_task(metadata, paths, true)?;
1248 }
1249 }
1250 _ if metadata.status.is_terminal() => {
1251 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1257 self.insert_rehydrated_task(metadata, paths, true)?;
1258 }
1259 _ => {}
1260 }
1261 }
1262
1263 Ok(())
1264 }
1265
1266 fn replay_session_from_db(
1267 &self,
1268 session_id: &str,
1269 ) -> Option<Result<Vec<PersistedTask>, String>> {
1270 let pool = self
1271 .inner
1272 .db_pool
1273 .read()
1274 .ok()
1275 .and_then(|slot| slot.clone())?;
1276 let harness = self
1277 .inner
1278 .db_harness
1279 .read()
1280 .ok()
1281 .and_then(|slot| slot.clone())?;
1282 let conn = match pool.lock() {
1283 Ok(conn) => conn,
1284 Err(_) => return Some(Err("db mutex poisoned".to_string())),
1285 };
1286 Some(
1287 crate::db::bash_tasks::list_bash_tasks_for_session(&conn, &harness, session_id)
1288 .map(|rows| rows.into_iter().map(PersistedTask::from).collect())
1289 .map_err(|error| error.to_string()),
1290 )
1291 }
1292
1293 fn replay_session_from_disk(
1294 &self,
1295 storage_dir: &Path,
1296 session_id: &str,
1297 ) -> Result<Vec<PersistedTask>, String> {
1298 let dir = session_tasks_dir(storage_dir, session_id);
1299 if !dir.exists() {
1300 return Ok(Vec::new());
1301 }
1302
1303 let entries = fs::read_dir(&dir)
1304 .map_err(|e| format!("failed to read background task dir {}: {e}", dir.display()))?;
1305 let mut tasks = Vec::new();
1306 for entry in entries.flatten() {
1307 let path = entry.path();
1308 if path.extension().and_then(|extension| extension.to_str()) != Some("json") {
1309 continue;
1310 }
1311 match read_task(&path) {
1312 Ok(metadata) => tasks.push(metadata),
1313 Err(error) => {
1314 crate::slog_warn!(
1315 "quarantining invalid background task metadata {} during replay: {error}",
1316 path.display()
1317 );
1318 if let Err(quarantine_error) =
1319 quarantine_task_json(storage_dir, &dir, &path, QuarantineKind::Invalid)
1320 {
1321 crate::slog_warn!(
1322 "failed to quarantine invalid background task metadata {}: {quarantine_error}",
1323 path.display()
1324 );
1325 }
1326 }
1327 }
1328 }
1329 Ok(tasks)
1330 }
1331
1332 pub fn register_watch(
1333 &self,
1334 task_id: String,
1335 pattern: WatchPattern,
1336 once: bool,
1337 ) -> Result<String, &'static str> {
1338 let task = self.task(&task_id).ok_or("task_not_found")?;
1339 let (mode, terminal_at_registration, stdout, stderr, pty) = task
1340 .state
1341 .lock()
1342 .map(|state| {
1343 (
1344 state.metadata.mode.clone(),
1345 state.metadata.status.is_terminal(),
1346 task.paths.stdout.clone(),
1347 task.paths.stderr.clone(),
1348 task.paths.pty.clone(),
1349 )
1350 })
1351 .map_err(|_| "background_task_lock_poisoned")?;
1352
1353 let mut terminal_matches = Vec::new();
1354 let scanned_terminal = terminal_at_registration;
1355 let watch_id = {
1356 let mut registry = self
1357 .inner
1358 .watch_registry
1359 .lock()
1360 .map_err(|_| "watch_registry_poisoned")?;
1361 let watch_id = registry.register(task_id.clone(), pattern, once)?;
1362 match &mode {
1363 BgMode::Pipes => {
1364 let stdout_key = format!("{task_id}:stdout");
1365 let stderr_key = format!("{task_id}:stderr");
1366 if terminal_at_registration {
1367 registry.set_file_cursor(&stdout_key, 0);
1368 registry.set_file_cursor(&stderr_key, 0);
1369 terminal_matches.extend(registry.scan_file_new_bytes(
1370 &stdout_key,
1371 &task_id,
1372 &stdout,
1373 ));
1374 terminal_matches.extend(registry.scan_file_new_bytes(
1375 &stderr_key,
1376 &task_id,
1377 &stderr,
1378 ));
1379 } else {
1380 registry.prime_file_cursor(&stdout_key, &stdout);
1381 registry.prime_file_cursor(&stderr_key, &stderr);
1382 }
1383 }
1384 BgMode::Pty => {
1385 let pty_key = format!("{task_id}:pty");
1386 if terminal_at_registration {
1387 registry.set_file_cursor(&pty_key, 0);
1388 terminal_matches
1389 .extend(registry.scan_file_new_bytes(&pty_key, &task_id, &pty));
1390 } else {
1391 registry.prime_file_cursor(&pty_key, &pty);
1392 }
1393 }
1394 }
1395 watch_id
1396 };
1397
1398 if task.is_terminal() {
1399 if !scanned_terminal {
1400 terminal_matches = {
1401 let mut registry = self
1402 .inner
1403 .watch_registry
1404 .lock()
1405 .map_err(|_| "watch_registry_poisoned")?;
1406 match &mode {
1407 BgMode::Pipes => {
1408 let stdout_key = format!("{task_id}:stdout");
1409 let stderr_key = format!("{task_id}:stderr");
1410 registry.set_file_cursor(&stdout_key, 0);
1411 registry.set_file_cursor(&stderr_key, 0);
1412 let mut matches =
1413 registry.scan_file_new_bytes(&stdout_key, &task_id, &stdout);
1414 matches.extend(registry.scan_file_new_bytes(
1415 &stderr_key,
1416 &task_id,
1417 &stderr,
1418 ));
1419 matches
1420 }
1421 BgMode::Pty => {
1422 let pty_key = format!("{task_id}:pty");
1423 registry.set_file_cursor(&pty_key, 0);
1424 registry.scan_file_new_bytes(&pty_key, &task_id, &pty)
1425 }
1426 }
1427 };
1428 }
1429
1430 let (watch_controlled, watch_matched) = self.task_watch_state(&task_id);
1431 if terminal_matches.is_empty() && (!watch_controlled || watch_matched) {
1432 if watch_matched {
1433 let _ = task.set_completion_delivered(true, self);
1434 self.clear_task_watch_state(&task_id);
1435 }
1436 return Ok(watch_id);
1437 }
1438
1439 let completion = self
1440 .remove_pending_completion(&task_id)
1441 .or_else(|| self.completion_snapshot_for_task(&task));
1442 if terminal_matches.is_empty() {
1443 if let Some(completion) = completion.as_ref() {
1444 self.emit_bash_watch_exit(completion);
1445 }
1446 } else {
1447 for pattern_match in terminal_matches {
1448 self.emit_bash_pattern_match(&task.session_id, pattern_match);
1449 }
1450 }
1451 let _ = task.set_completion_delivered(true, self);
1452 self.clear_task_watch_state(&task_id);
1453 }
1454
1455 Ok(watch_id)
1456 }
1457
1458 pub fn unregister_watch(&self, task_id: &str, watch_id: &str) {
1459 if let Ok(mut registry) = self.inner.watch_registry.lock() {
1460 registry.unregister(task_id, watch_id);
1461 }
1462 }
1463
1464 pub fn active_watch_count(&self, task_id: &str) -> usize {
1465 self.inner
1466 .watch_registry
1467 .lock()
1468 .map(|registry| registry.active_count(task_id))
1469 .unwrap_or(0)
1470 }
1471
1472 fn task_watch_state(&self, task_id: &str) -> (bool, bool) {
1473 self.inner
1474 .watch_registry
1475 .lock()
1476 .map(|registry| {
1477 (
1478 registry.has_controlled_task(task_id),
1479 registry.has_matched_task(task_id),
1480 )
1481 })
1482 .unwrap_or((false, false))
1483 }
1484
1485 fn task_has_watch_control(&self, task_id: &str) -> bool {
1486 self.inner
1487 .watch_registry
1488 .lock()
1489 .map(|registry| registry.has_controlled_task(task_id))
1490 .unwrap_or(false)
1491 }
1492
1493 fn clear_task_watch_state(&self, task_id: &str) {
1494 if let Ok(mut registry) = self.inner.watch_registry.lock() {
1495 registry.clear_task(task_id);
1496 }
1497 }
1498
1499 pub(crate) fn scan_task_watch_output(&self, task: &Arc<BgTask>) {
1500 let (mode, stdout, stderr, pty) = match task.state.lock() {
1501 Ok(state) => (
1502 state.metadata.mode.clone(),
1503 task.paths.stdout.clone(),
1504 task.paths.stderr.clone(),
1505 task.paths.pty.clone(),
1506 ),
1507 Err(_) => return,
1508 };
1509 let mut matches = Vec::new();
1510 if let Ok(mut registry) = self.inner.watch_registry.lock() {
1511 match mode {
1512 BgMode::Pipes => {
1513 let stdout_key = format!("{}:stdout", task.task_id);
1514 let stderr_key = format!("{}:stderr", task.task_id);
1515 matches.extend(registry.scan_file_new_bytes(
1516 &stdout_key,
1517 &task.task_id,
1518 &stdout,
1519 ));
1520 matches.extend(registry.scan_file_new_bytes(
1521 &stderr_key,
1522 &task.task_id,
1523 &stderr,
1524 ));
1525 }
1526 BgMode::Pty => {
1527 let pty_key = format!("{}:pty", task.task_id);
1528 matches.extend(registry.scan_file_new_bytes(&pty_key, &task.task_id, &pty));
1529 }
1530 }
1531 }
1532 for pattern_match in matches {
1533 self.emit_bash_pattern_match(&task.session_id, pattern_match);
1534 }
1535 }
1536
1537 pub fn status(
1538 &self,
1539 task_id: &str,
1540 session_id: &str,
1541 project_root: Option<&Path>,
1542 storage_dir: Option<&Path>,
1543 preview_bytes: usize,
1544 ) -> Option<BgTaskSnapshot> {
1545 let mut task = self.task_for_session(task_id, session_id);
1546 if task.is_none() {
1547 if let Some(storage_dir) = storage_dir {
1548 let _ = if let Some(project_root) = project_root {
1549 self.replay_session_for_project(storage_dir, session_id, project_root)
1550 } else {
1551 self.replay_session(storage_dir, session_id)
1552 };
1553 task = self.task_for_session(task_id, session_id);
1554 }
1555 }
1556 let Some(task) = task else {
1557 return self.status_relaxed(
1558 task_id,
1559 session_id,
1560 project_root?,
1561 storage_dir?,
1562 preview_bytes,
1563 );
1564 };
1565 let _ = self.poll_task(&task);
1566 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
1567 }
1568
1569 fn status_relaxed_task(
1570 &self,
1571 task_id: &str,
1572 project_root: &Path,
1573 storage_dir: &Path,
1574 ) -> Option<Arc<BgTask>> {
1575 let canonical_project = canonicalized_path(project_root);
1576 match self.lookup_relaxed_task_from_db(task_id, project_root) {
1577 Some(Ok(Some(metadata))) => {
1578 if let Some(task) = self.task(task_id) {
1579 let matches_project = task
1580 .state
1581 .lock()
1582 .map(|state| {
1583 state
1584 .metadata
1585 .project_root
1586 .as_deref()
1587 .map(canonicalized_path)
1588 .as_deref()
1589 == Some(canonical_project.as_path())
1590 })
1591 .unwrap_or(false);
1592 return matches_project.then_some(task);
1593 }
1594 let paths = task_paths(storage_dir, &metadata.session_id, &metadata.task_id);
1595 if self.insert_rehydrated_task(metadata, paths, true).is_err() {
1596 return None;
1597 }
1598 return self.task(task_id);
1599 }
1600 Some(Ok(None)) => {
1601 crate::slog_info!(
1602 "bash task relaxed DB miss for {}; falling back to disk",
1603 task_id
1604 );
1605 }
1606 Some(Err(error)) => {
1607 crate::slog_warn!(
1608 "bash task relaxed DB lookup failed for {}; falling back to disk: {}",
1609 task_id,
1610 error
1611 );
1612 }
1613 None => {
1614 crate::slog_info!(
1615 "bash task relaxed DB unavailable for {}; falling back to disk",
1616 task_id
1617 );
1618 }
1619 }
1620 let root = storage_dir.join("bash-tasks");
1621 let entries = fs::read_dir(&root).ok()?;
1622 for entry in entries.flatten() {
1623 let dir = entry.path();
1624 if !dir.is_dir() {
1625 continue;
1626 }
1627 let path = dir.join(format!("{task_id}.json"));
1628 if !path.exists() {
1629 continue;
1630 }
1631 let metadata = match read_task(&path) {
1632 Ok(metadata) => metadata,
1633 Err(error) => {
1634 crate::slog_warn!(
1635 "quarantining invalid background task metadata {} during relaxed lookup: {error}",
1636 path.display()
1637 );
1638 if let Err(quarantine_error) =
1639 quarantine_task_json(storage_dir, &dir, &path, QuarantineKind::Invalid)
1640 {
1641 crate::slog_warn!(
1642 "failed to quarantine invalid background task metadata {}: {quarantine_error}",
1643 path.display()
1644 );
1645 }
1646 continue;
1647 }
1648 };
1649 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
1650 if metadata_project.as_deref() != Some(canonical_project.as_path()) {
1651 continue;
1652 }
1653 if let Some(task) = self.task(task_id) {
1654 let matches_project = task
1655 .state
1656 .lock()
1657 .map(|state| {
1658 state
1659 .metadata
1660 .project_root
1661 .as_deref()
1662 .map(canonicalized_path)
1663 .as_deref()
1664 == Some(canonical_project.as_path())
1665 })
1666 .unwrap_or(false);
1667 return matches_project.then_some(task);
1668 }
1669 let paths = task_paths(storage_dir, &metadata.session_id, &metadata.task_id);
1670 if self.insert_rehydrated_task(metadata, paths, true).is_err() {
1671 return None;
1672 }
1673 return self.task(task_id);
1674 }
1675 None
1676 }
1677
1678 fn lookup_relaxed_task_from_db(
1679 &self,
1680 task_id: &str,
1681 project_root: &Path,
1682 ) -> Option<Result<Option<PersistedTask>, String>> {
1683 let pool = self
1684 .inner
1685 .db_pool
1686 .read()
1687 .ok()
1688 .and_then(|slot| slot.clone())?;
1689 let harness = self
1690 .inner
1691 .db_harness
1692 .read()
1693 .ok()
1694 .and_then(|slot| slot.clone())?;
1695 let conn = match pool.lock() {
1696 Ok(conn) => conn,
1697 Err(_) => return Some(Err("db mutex poisoned".to_string())),
1698 };
1699 let project_key = crate::path_identity::project_scope_key(project_root);
1700 Some(
1701 crate::db::bash_tasks::find_bash_task_for_project(
1702 &conn,
1703 &harness,
1704 &project_key,
1705 task_id,
1706 )
1707 .map(|row| row.map(PersistedTask::from))
1708 .map_err(|error| error.to_string()),
1709 )
1710 }
1711
1712 pub(super) fn status_relaxed(
1713 &self,
1714 task_id: &str,
1715 _session_id: &str,
1716 project_root: &Path,
1717 storage_dir: &Path,
1718 preview_bytes: usize,
1719 ) -> Option<BgTaskSnapshot> {
1720 let task = self.status_relaxed_task(task_id, project_root, storage_dir)?;
1721 let _ = self.poll_task(&task);
1722 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
1723 }
1724
1725 pub fn kill_relaxed(
1726 &self,
1727 task_id: &str,
1728 project_root: &Path,
1729 storage_dir: &Path,
1730 ) -> Result<BgTaskSnapshot, String> {
1731 let task = self
1732 .status_relaxed_task(task_id, project_root, storage_dir)
1733 .ok_or_else(|| format!("background task not found: {task_id}"))?;
1734 self.kill_with_status(task_id, &task.session_id, BgTaskStatus::Killed)
1735 }
1736
1737 pub fn maybe_gc_persisted(&self, storage_dir: &Path) -> Result<usize, String> {
1738 #[cfg(test)]
1739 self.inner.persisted_gc_runs.fetch_add(1, Ordering::SeqCst);
1740
1741 let mut deleted = 0usize;
1742
1743 let root = storage_dir.join("bash-tasks");
1744 if root.exists() {
1745 let session_dirs = fs::read_dir(&root).map_err(|e| {
1746 format!(
1747 "failed to read background task root {}: {e}",
1748 root.display()
1749 )
1750 })?;
1751 for session_entry in session_dirs.flatten() {
1752 let session_dir = session_entry.path();
1753 if !session_dir.is_dir() {
1754 continue;
1755 }
1756 let task_entries = match fs::read_dir(&session_dir) {
1757 Ok(entries) => entries,
1758 Err(error) => {
1759 crate::slog_warn!(
1760 "failed to read background task session dir {}: {error}",
1761 session_dir.display()
1762 );
1763 continue;
1764 }
1765 };
1766 for task_entry in task_entries.flatten() {
1767 let json_path = task_entry.path();
1768 if json_path
1769 .extension()
1770 .and_then(|extension| extension.to_str())
1771 != Some("json")
1772 {
1773 continue;
1774 }
1775 if modified_within(&json_path, PERSISTED_GC_GRACE) {
1776 continue;
1777 }
1778 let metadata = match read_task(&json_path) {
1779 Ok(metadata) => metadata,
1780 Err(error) => {
1781 crate::slog_warn!(
1782 "quarantining corrupt background task metadata {}: {error}",
1783 json_path.display()
1784 );
1785 quarantine_task_json(
1786 storage_dir,
1787 &session_dir,
1788 &json_path,
1789 QuarantineKind::Corrupt,
1790 )?;
1791 continue;
1792 }
1793 };
1794 if !(metadata.status.is_terminal() && metadata.completion_delivered) {
1795 continue;
1796 }
1797 let paths = task_paths(storage_dir, &metadata.session_id, &metadata.task_id);
1798 match delete_task_bundle(&paths) {
1799 Ok(()) => {
1800 self.delete_gc_task_from_db(&metadata);
1801 deleted += 1;
1802 log::debug!(
1803 "deleted persisted background task bundle {}",
1804 metadata.task_id
1805 );
1806 }
1807 Err(error) => {
1808 crate::slog_warn!(
1809 "failed to delete background task bundle {}: {error}",
1810 metadata.task_id
1811 );
1812 continue;
1813 }
1814 }
1815 }
1816 }
1817 }
1818 gc_quarantine(storage_dir);
1819 Ok(deleted)
1820 }
1821
1822 pub fn list(&self, preview_bytes: usize) -> Vec<BgTaskSnapshot> {
1823 let tasks = self
1824 .inner
1825 .tasks
1826 .lock()
1827 .map(|tasks| tasks.values().cloned().collect::<Vec<_>>())
1828 .unwrap_or_default();
1829 tasks
1830 .into_iter()
1831 .map(|task| {
1832 let _ = self.poll_task(&task);
1833 self.snapshot_with_terminal_cache(&task, preview_bytes)
1834 })
1835 .collect()
1836 }
1837
1838 fn maybe_compress_snapshot(&self, task: &Arc<BgTask>, snapshot: &mut BgTaskSnapshot) {
1844 if !snapshot.info.status.is_terminal() || snapshot.info.mode == BgMode::Pty {
1845 return;
1846 }
1847 if let Some(cache) = self.ensure_terminal_output_cache(task) {
1848 snapshot.output_preview = cache.output_preview;
1849 snapshot.output_truncated = cache.output_truncated;
1850 }
1851 }
1852
1853 pub fn kill(&self, task_id: &str, session_id: &str) -> Result<BgTaskSnapshot, String> {
1854 self.kill_with_status(task_id, session_id, BgTaskStatus::Killed)
1855 }
1856
1857 pub fn promote(&self, task_id: &str, session_id: &str) -> Result<bool, String> {
1858 let task = self
1859 .task_for_session(task_id, session_id)
1860 .ok_or_else(|| format!("background task not found: {task_id}"))?;
1861 let terminal_after_promote = {
1862 let mut state = task
1863 .state
1864 .lock()
1865 .map_err(|_| "background task lock poisoned".to_string())?;
1866 let updated = self
1867 .update_task_metadata(&task.paths, |metadata| {
1868 metadata.notify_on_completion = true;
1869 metadata.completion_delivered = false;
1870 })
1871 .map_err(|e| format!("failed to promote background task: {e}"))?;
1872 state.metadata = updated;
1873 state.metadata.status.is_terminal()
1874 };
1875 if terminal_after_promote {
1876 self.post_terminal_transition(&task, true)?;
1877 }
1878 Ok(true)
1879 }
1880
1881 pub(crate) fn kill_for_timeout(&self, task_id: &str, session_id: &str) -> Result<(), String> {
1882 self.kill_with_status(task_id, session_id, BgTaskStatus::TimedOut)
1883 .map(|_| ())
1884 }
1885
1886 pub fn cleanup_finished(&self, older_than: Duration) {
1887 let cutoff = Instant::now().checked_sub(older_than);
1888 let removable_paths: Vec<(String, TaskPaths)> =
1889 if let Ok(mut tasks) = self.inner.tasks.lock() {
1890 let removable = tasks
1891 .iter()
1892 .filter_map(|(task_id, task)| {
1893 let delivered_terminal = task
1894 .state
1895 .lock()
1896 .map(|state| {
1897 state.metadata.status.is_terminal()
1898 && state.metadata.completion_delivered
1899 })
1900 .unwrap_or(false);
1901 if !delivered_terminal {
1902 return None;
1903 }
1904
1905 let terminal_at = task.terminal_at.lock().ok().and_then(|at| *at);
1906 let expired = match (terminal_at, cutoff) {
1907 (Some(terminal_at), Some(cutoff)) => terminal_at <= cutoff,
1908 (Some(_), None) => true,
1909 (None, _) => false,
1910 };
1911 expired.then(|| task_id.clone())
1912 })
1913 .collect::<Vec<_>>();
1914
1915 removable
1916 .into_iter()
1917 .filter_map(|task_id| {
1918 tasks
1919 .remove(&task_id)
1920 .map(|task| (task_id, task.paths.clone()))
1921 })
1922 .collect()
1923 } else {
1924 Vec::new()
1925 };
1926
1927 for (task_id, paths) in removable_paths {
1928 match delete_task_bundle(&paths) {
1929 Ok(()) => log::debug!("deleted persisted background task bundle {task_id}"),
1930 Err(error) => crate::slog_warn!(
1931 "failed to delete persisted background task bundle {task_id}: {error}"
1932 ),
1933 }
1934 }
1935 }
1936
1937 pub fn drain_completions(&self) -> Vec<BgCompletion> {
1938 self.drain_completions_for_session(None)
1939 }
1940
1941 pub fn drain_completions_for_session(&self, session_id: Option<&str>) -> Vec<BgCompletion> {
1942 let completions = match self.inner.completions.lock() {
1943 Ok(completions) => completions,
1944 Err(_) => return Vec::new(),
1945 };
1946
1947 completions
1948 .iter()
1949 .filter(|completion| completion_matches_session(completion, session_id))
1950 .cloned()
1951 .collect()
1952 }
1953
1954 pub fn has_completions_for_session(&self, session_id: Option<&str>) -> bool {
1955 match self.inner.completions.lock() {
1956 Ok(completions) => completions
1957 .iter()
1958 .any(|completion| completion_matches_session(completion, session_id)),
1959 Err(_) => true,
1963 }
1964 }
1965
1966 pub fn ack_completions_for_session(
1967 &self,
1968 session_id: Option<&str>,
1969 task_ids: &[String],
1970 ) -> Vec<String> {
1971 if task_ids.is_empty() {
1972 return Vec::new();
1973 }
1974 let requested_task_ids = task_ids.iter().map(String::as_str).collect::<HashSet<_>>();
1975 let mut completion_sessions = HashMap::new();
1976 if let Ok(mut completions) = self.inner.completions.lock() {
1977 completions.retain(|completion| {
1978 let session_matches = session_id
1979 .map(|session_id| completion.session_id == session_id)
1980 .unwrap_or(true);
1981 if session_matches && requested_task_ids.contains(completion.task_id.as_str()) {
1982 completion_sessions
1983 .insert(completion.task_id.clone(), completion.session_id.clone());
1984 false
1985 } else {
1986 true
1987 }
1988 });
1989 }
1990
1991 let mut delivered = Vec::new();
1992 for task_id in task_ids {
1993 let task = if let Some(session_id) = session_id {
1994 self.task_for_session(task_id, session_id)
1995 } else if let Some(completion_session_id) = completion_sessions.get(task_id) {
1996 self.task_for_session(task_id, completion_session_id)
1997 } else {
1998 self.task(task_id)
1999 };
2000 if let Some(task) = task {
2001 if task.set_completion_delivered(true, self).is_ok() {
2002 delivered.push(task_id.clone());
2003 }
2004 }
2005 }
2006
2007 delivered
2008 }
2009
2010 pub fn pending_completions_for_session(&self, session_id: &str) -> Vec<BgCompletion> {
2011 self.inner
2012 .completions
2013 .lock()
2014 .map(|completions| {
2015 completions
2016 .iter()
2017 .filter(|completion| completion.session_id == session_id)
2018 .cloned()
2019 .collect()
2020 })
2021 .unwrap_or_default()
2022 }
2023
2024 fn remove_pending_completion(&self, task_id: &str) -> Option<BgCompletion> {
2025 let mut completions = self.inner.completions.lock().ok()?;
2026 let idx = completions
2027 .iter()
2028 .position(|completion| completion.task_id == task_id)?;
2029 completions.remove(idx)
2030 }
2031
2032 fn completion_snapshot_for_task(&self, task: &Arc<BgTask>) -> Option<BgCompletion> {
2033 let snapshot = self.snapshot_with_terminal_cache(task, RUNNING_OUTPUT_PREVIEW_BYTES);
2034 if !snapshot.info.status.is_terminal() {
2035 return None;
2036 }
2037 let (output_preview, output_truncated) = if snapshot.info.mode == BgMode::Pty {
2038 (String::new(), false)
2039 } else {
2040 self.ensure_terminal_output_cache(task)
2041 .map(|cache| completion_preview_for_cache(&cache, snapshot.exit_code))
2042 .unwrap_or_else(|| (String::new(), false))
2043 };
2044 Some(BgCompletion {
2045 task_id: snapshot.info.task_id,
2046 session_id: task.session_id.clone(),
2047 status: snapshot.info.status,
2048 exit_code: snapshot.exit_code,
2049 command: snapshot.info.command,
2050 output_preview,
2051 output_truncated,
2052 original_tokens: None,
2053 compressed_tokens: None,
2054 tokens_skipped: false,
2055 })
2056 }
2057
2058 pub fn detach(&self) {
2059 self.inner.shutdown.store(true, Ordering::SeqCst);
2060 if let Ok(mut tasks) = self.inner.tasks.lock() {
2061 for task in tasks.values() {
2062 if let Ok(mut state) = task.state.lock() {
2063 match &mut state.runtime {
2064 TaskRuntime::Piped(child) => *child = None,
2065 TaskRuntime::Pty(runtime) => *runtime = None,
2066 }
2067 state.detached = true;
2068 }
2069 }
2070 tasks.clear();
2071 }
2072 }
2073
2074 pub fn shutdown(&self) {
2075 let tasks = self
2076 .inner
2077 .tasks
2078 .lock()
2079 .map(|tasks| {
2080 tasks
2081 .values()
2082 .map(|task| (task.task_id.clone(), task.session_id.clone()))
2083 .collect::<Vec<_>>()
2084 })
2085 .unwrap_or_default();
2086 for (task_id, session_id) in tasks {
2087 let _ = self.kill(&task_id, &session_id);
2088 }
2089 }
2090
2091 pub(crate) fn poll_task(&self, task: &Arc<BgTask>) -> Result<(), String> {
2092 if let Ok(state) = task.state.lock() {
2093 if let TaskRuntime::Pty(Some(pty)) = &state.runtime {
2094 if !pty.exit_observed.load(Ordering::SeqCst) {
2102 return Ok(());
2103 }
2104 }
2105 }
2106 let marker = match read_exit_marker(&task.paths.exit) {
2107 Ok(Some(marker)) => marker,
2108 Ok(None) => return Ok(()),
2109 Err(error) => return Err(format!("failed to read exit marker: {error}")),
2110 };
2111 self.finalize_from_marker(task, marker, None)
2112 }
2113
2114 pub(crate) fn reap_child(&self, task: &Arc<BgTask>) {
2115 let mut needs_completion = false;
2116 {
2117 let Ok(mut state) = task.state.lock() else {
2118 return;
2119 };
2120 match &mut state.runtime {
2121 TaskRuntime::Piped(child_slot) => {
2122 if let Some(child) = child_slot.as_mut() {
2123 if matches!(child.try_wait(), Ok(Some(_))) {
2124 *child_slot = None;
2125 state.detached = true;
2126 state.child_exit_observed = true;
2127 }
2128 } else if state.detached {
2129 let child_known_dead = state.child_exit_observed
2130 || state
2131 .metadata
2132 .child_pid
2133 .is_some_and(|pid| !is_process_alive(pid));
2134 if child_known_dead {
2135 needs_completion =
2136 self.fail_without_exit_marker_if_needed(task, &mut state);
2137 }
2138 }
2139 }
2140 TaskRuntime::Pty(Some(pty)) => {
2141 if pty.exit_observed.load(Ordering::SeqCst) {
2142 drop(state);
2143 let _ = self.poll_task(task);
2144 return;
2145 }
2146 }
2147 TaskRuntime::Pty(None) => {}
2148 }
2149 }
2150 if needs_completion {
2151 let _ = self.post_terminal_transition(task, true);
2152 }
2153 }
2154
2155 fn fail_without_exit_marker_if_needed(
2156 &self,
2157 task: &Arc<BgTask>,
2158 state: &mut BgTaskState,
2159 ) -> bool {
2160 if state.metadata.status.is_terminal() {
2161 return false;
2162 }
2163 if matches!(read_exit_marker(&task.paths.exit), Ok(Some(_))) {
2164 return false;
2165 }
2166 let watch_controlled = self.task_has_watch_control(&task.task_id);
2167 let updated = self.update_task_metadata(&task.paths, |metadata| {
2168 metadata.mark_terminal(
2169 BgTaskStatus::Failed,
2170 None,
2171 Some("process exited without exit marker".to_string()),
2172 );
2173 if watch_controlled {
2174 metadata.completion_delivered = true;
2175 }
2176 });
2177 if let Ok(metadata) = updated {
2178 state.pending_terminal_override = None;
2179 state.metadata = metadata;
2180 task.mark_terminal_now();
2181 return true;
2182 }
2183 false
2184 }
2185
2186 pub(crate) fn running_tasks(&self) -> Vec<Arc<BgTask>> {
2187 self.inner
2188 .tasks
2189 .lock()
2190 .map(|tasks| {
2191 tasks
2192 .values()
2193 .filter(|task| task.is_running())
2194 .cloned()
2195 .collect()
2196 })
2197 .unwrap_or_default()
2198 }
2199
2200 fn insert_rehydrated_task(
2201 &self,
2202 metadata: PersistedTask,
2203 paths: TaskPaths,
2204 detached: bool,
2205 ) -> Result<(), String> {
2206 let task_id = metadata.task_id.clone();
2207 let session_id = metadata.session_id.clone();
2208 let started = started_instant_from_unix_millis(metadata.started_at);
2209 let suppress_replayed_running_reminder = metadata.status == BgTaskStatus::Running;
2210 let mode = metadata.mode.clone();
2211 let task = Arc::new(BgTask {
2212 task_id: task_id.clone(),
2213 session_id,
2214 paths: paths.clone(),
2215 artifact_root: canonical_artifact_root(&paths),
2216 started,
2217 last_reminder_at: Mutex::new(suppress_replayed_running_reminder.then(Instant::now)),
2218 terminal_at: Mutex::new(metadata.status.is_terminal().then(Instant::now)),
2219 state: Mutex::new(BgTaskState {
2220 metadata,
2221 runtime: if mode == BgMode::Pty {
2222 TaskRuntime::Pty(None)
2223 } else {
2224 TaskRuntime::Piped(None)
2225 },
2226 detached,
2227 child_exit_observed: false,
2234 buffer: if mode == BgMode::Pty {
2235 BgBuffer::pty(paths.pty.clone())
2236 } else {
2237 BgBuffer::new(paths.stdout.clone(), paths.stderr.clone())
2238 },
2239 terminal_output_cache: None,
2240 pending_terminal_override: None,
2241 }),
2242 });
2243 self.inner
2244 .tasks
2245 .lock()
2246 .map_err(|_| "background task registry lock poisoned".to_string())?
2247 .insert(task_id, task);
2248 Ok(())
2249 }
2250
2251 fn kill_with_status(
2252 &self,
2253 task_id: &str,
2254 session_id: &str,
2255 terminal_status: BgTaskStatus,
2256 ) -> Result<BgTaskSnapshot, String> {
2257 let task = self
2258 .task_for_session(task_id, session_id)
2259 .ok_or_else(|| format!("background task not found: {task_id}"))?;
2260 let mut terminalized = false;
2261
2262 {
2263 let mut state = task
2264 .state
2265 .lock()
2266 .map_err(|_| "background task lock poisoned".to_string())?;
2267 if state.metadata.status.is_terminal() {
2268 state.pending_terminal_override = None;
2269 } else if let Ok(Some(marker)) = read_exit_marker(&task.paths.exit) {
2270 state.metadata =
2271 terminal_metadata_from_marker(state.metadata.clone(), marker, None);
2272 if self.task_has_watch_control(&task.task_id) {
2273 state.metadata.completion_delivered = true;
2274 }
2275 state.pending_terminal_override = None;
2276 task.mark_terminal_now();
2277 match &mut state.runtime {
2278 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
2285 TaskRuntime::Pty(runtime) => *runtime = None,
2286 }
2287 state.detached = true;
2288 self.persist_task(&task.paths, &state.metadata)
2289 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
2290 terminalized = true;
2291 } else {
2292 let was_already_killing = state.metadata.status == BgTaskStatus::Killing;
2293 if !was_already_killing {
2294 state.metadata.status = BgTaskStatus::Killing;
2295 self.persist_task(&task.paths, &state.metadata)
2296 .map_err(|e| format!("failed to persist killing state: {e}"))?;
2297 }
2298
2299 #[cfg(unix)]
2300 let pgid = state.metadata.pgid;
2301 #[cfg(windows)]
2302 let child_pid = state.metadata.child_pid;
2303 if !was_already_killing
2304 && state.metadata.mode == BgMode::Pty
2305 && terminal_status == BgTaskStatus::TimedOut
2306 {
2307 state.pending_terminal_override = Some(BgTaskStatus::TimedOut);
2308 }
2309
2310 #[cfg(windows)]
2311 let mut pty_forced_terminal_status: Option<BgTaskStatus> = None;
2312
2313 match &mut state.runtime {
2314 TaskRuntime::Piped(child_slot) => {
2315 #[cfg(unix)]
2316 if let Some(pgid) = pgid {
2317 terminate_pgid(pgid, child_slot.as_mut());
2318 }
2319 #[cfg(windows)]
2320 if let Some(child) = child_slot.as_mut() {
2321 super::process::terminate_process(child);
2322 } else if let Some(pid) = child_pid {
2323 terminate_pid(pid);
2324 }
2325 if let Some(child) = child_slot.as_mut() {
2326 let _ = child.wait();
2327 }
2328 *child_slot = None;
2329 state.detached = true;
2330
2331 if !task.paths.exit.exists() {
2332 write_kill_marker_if_absent(&task.paths.exit)
2333 .map_err(|e| format!("failed to write kill marker: {e}"))?;
2334 }
2335
2336 let exit_code = terminal_exit_code_for_status(&terminal_status);
2337 state
2338 .metadata
2339 .mark_terminal(terminal_status, exit_code, None);
2340 if self.task_has_watch_control(&task.task_id) {
2341 state.metadata.completion_delivered = true;
2342 }
2343 state.pending_terminal_override = None;
2344 task.mark_terminal_now();
2345 self.persist_task(&task.paths, &state.metadata)
2346 .map_err(|e| format!("failed to persist killed state: {e}"))?;
2347 terminalized = true;
2348 }
2349 TaskRuntime::Pty(Some(pty)) => {
2350 pty.was_killed.store(true, Ordering::SeqCst);
2351 if let Err(error) = pty.killer.kill() {
2352 crate::slog_warn!(
2353 "[pty-kill] {task_id} ChildKiller::kill failed: {error}"
2354 );
2355 }
2356 if let Some(pid) = pty.child_pid {
2357 #[cfg(unix)]
2358 terminate_pgid(pid as i32, None);
2359 #[cfg(windows)]
2360 terminate_pid(pid);
2361 }
2362 drop(pty.master.take());
2363
2364 #[cfg(windows)]
2365 {
2366 let default_status = if terminal_status == BgTaskStatus::TimedOut {
2367 BgTaskStatus::TimedOut
2368 } else {
2369 BgTaskStatus::Killed
2370 };
2371 pty_forced_terminal_status = Some(
2372 state
2373 .pending_terminal_override
2374 .take()
2375 .unwrap_or(default_status),
2376 );
2377 }
2378 }
2379 TaskRuntime::Pty(None) => {}
2380 }
2381
2382 #[cfg(windows)]
2383 if let Some(target_status) = pty_forced_terminal_status {
2384 if !task.paths.exit.exists() {
2385 write_kill_marker_if_absent(&task.paths.exit)
2386 .map_err(|e| format!("failed to write kill marker: {e}"))?;
2387 }
2388
2389 let exit_code = terminal_exit_code_for_status(&target_status);
2390 state.metadata.mark_terminal(target_status, exit_code, None);
2391 if self.task_has_watch_control(&task.task_id) {
2392 state.metadata.completion_delivered = true;
2393 }
2394 state.pending_terminal_override = None;
2395 task.mark_terminal_now();
2396 if let TaskRuntime::Pty(runtime) = &mut state.runtime {
2397 *runtime = None;
2398 }
2399 state.detached = true;
2400 self.persist_task(&task.paths, &state.metadata)
2401 .map_err(|e| format!("failed to persist killed PTY state: {e}"))?;
2402 terminalized = true;
2403 }
2404 }
2405 }
2406
2407 if terminalized {
2408 self.post_terminal_transition(&task, true)?;
2409 }
2410 Ok(self.snapshot_with_terminal_cache(&task, RUNNING_OUTPUT_PREVIEW_BYTES))
2411 }
2412
2413 fn finalize_from_marker(
2414 &self,
2415 task: &Arc<BgTask>,
2416 marker: ExitMarker,
2417 reason: Option<String>,
2418 ) -> Result<(), String> {
2419 let watch_controlled = self.task_has_watch_control(&task.task_id);
2420 let mut pty_reader_done = None;
2421 {
2422 let mut state = task
2423 .state
2424 .lock()
2425 .map_err(|_| "background task lock poisoned".to_string())?;
2426 if state.metadata.status.is_terminal() {
2427 state.pending_terminal_override = None;
2428 return Ok(());
2429 }
2430
2431 let pending_override = state.pending_terminal_override.take();
2432 let is_pty = state.metadata.mode == BgMode::Pty;
2433 let updated = self
2434 .update_task_metadata(&task.paths, |metadata| {
2435 let mut new_metadata = if is_pty && marker == ExitMarker::Killed {
2436 let mut metadata = metadata.clone();
2437 let target_status = pending_override.unwrap_or(BgTaskStatus::Killed);
2438 let exit_code = terminal_exit_code_for_status(&target_status);
2439 metadata.mark_terminal(target_status, exit_code, reason);
2440 metadata
2441 } else {
2442 terminal_metadata_from_marker(metadata.clone(), marker, reason)
2443 };
2444 if watch_controlled {
2445 new_metadata.completion_delivered = true;
2446 }
2447 *metadata = new_metadata;
2448 })
2449 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
2450 state.metadata = updated;
2451 task.mark_terminal_now();
2452 match &mut state.runtime {
2453 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
2458 TaskRuntime::Pty(runtime) => {
2459 pty_reader_done = runtime
2460 .as_ref()
2461 .map(|runtime| Arc::clone(&runtime.reader_done));
2462 *runtime = None;
2463 }
2464 }
2465 state.detached = true;
2466 }
2467
2468 if let Some(reader_done) = pty_reader_done {
2469 let deadline = Instant::now() + Duration::from_millis(200);
2470 while !reader_done.load(Ordering::SeqCst) && Instant::now() < deadline {
2471 std::thread::sleep(Duration::from_millis(10));
2472 }
2473 }
2474
2475 self.scan_task_watch_output(task);
2478
2479 self.post_terminal_transition(task, true)
2480 }
2481
2482 fn enqueue_completion_if_needed(
2483 &self,
2484 metadata: &PersistedTask,
2485 paths: Option<&TaskPaths>,
2486 emit_frame: bool,
2487 ) {
2488 if metadata.status.is_terminal() && !metadata.completion_delivered {
2489 let cache =
2490 paths.and_then(|paths| self.render_terminal_output_from_paths(metadata, paths));
2491 self.enqueue_completion_from_parts(metadata, None, paths, emit_frame, cache.as_ref());
2492 }
2493 }
2494
2495 fn render_terminal_output_from_paths(
2496 &self,
2497 metadata: &PersistedTask,
2498 paths: &TaskPaths,
2499 ) -> Option<TerminalOutputCache> {
2500 if metadata.mode == BgMode::Pty {
2501 return None;
2502 }
2503 let mut buffer = BgBuffer::new(paths.stdout.clone(), paths.stderr.clone());
2504 let disk_truncation = buffer.enforce_terminal_cap();
2505 Some(self.render_terminal_output(metadata, &buffer, disk_truncation))
2506 }
2507
2508 fn enqueue_completion_from_parts(
2509 &self,
2510 metadata: &PersistedTask,
2511 buffer: Option<&BgBuffer>,
2512 paths: Option<&TaskPaths>,
2513 emit_frame: bool,
2514 terminal_render: Option<&TerminalOutputCache>,
2515 ) {
2516 if !metadata.status.is_terminal() {
2527 return;
2528 }
2529
2530 let owned_buffer = if buffer.is_none() && metadata.mode != BgMode::Pty {
2531 paths.map(|paths| BgBuffer::new(paths.stdout.clone(), paths.stderr.clone()))
2532 } else {
2533 None
2534 };
2535 let render_buffer = buffer.or(owned_buffer.as_ref());
2536 let owned_render = if terminal_render.is_none() {
2537 render_buffer.map(|buffer| {
2538 let mut capped_buffer = buffer.clone();
2539 let disk_truncation = capped_buffer.enforce_terminal_cap();
2540 self.render_terminal_output(metadata, &capped_buffer, disk_truncation)
2541 })
2542 } else {
2543 None
2544 };
2545 let render = terminal_render.or(owned_render.as_ref());
2546
2547 let (output_preview, output_truncated) = render
2551 .map(|cache| completion_preview_for_cache(cache, metadata.exit_code))
2552 .unwrap_or_else(|| (String::new(), false));
2553
2554 let token_counts = self.completion_token_counts(
2555 metadata,
2556 buffer,
2557 paths,
2558 render.map(|render| render.output_preview.as_str()),
2559 );
2560 let completion = BgCompletion {
2561 task_id: metadata.task_id.clone(),
2562 session_id: metadata.session_id.clone(),
2563 status: metadata.status.clone(),
2564 exit_code: metadata.exit_code,
2565 command: metadata.command.clone(),
2566 output_preview,
2567 output_truncated,
2568 original_tokens: token_counts.original_tokens,
2569 compressed_tokens: token_counts.compressed_tokens,
2570 tokens_skipped: token_counts.tokens_skipped,
2571 };
2572
2573 self.record_compression_event_if_applicable(metadata, &token_counts);
2584
2585 let (watch_controlled, watch_matched) = self.task_watch_state(&metadata.task_id);
2586 if watch_controlled {
2587 if emit_frame && !watch_matched {
2588 self.emit_bash_watch_exit(&completion);
2589 }
2590 self.clear_task_watch_state(&metadata.task_id);
2591 return;
2592 }
2593
2594 if metadata.completion_delivered {
2604 return;
2605 }
2606
2607 let pushed = if let Ok(mut completions) = self.inner.completions.lock() {
2610 if completions
2611 .iter()
2612 .any(|existing| existing.task_id == metadata.task_id)
2613 {
2614 false
2615 } else {
2616 completions.push_back(completion.clone());
2617 true
2618 }
2619 } else {
2620 false
2621 };
2622
2623 if pushed && emit_frame {
2624 self.emit_bash_completed(completion);
2625 }
2626 }
2627
2628 fn record_compression_event_if_applicable(
2629 &self,
2630 metadata: &PersistedTask,
2631 token_counts: &CompletionTokenCounts,
2632 ) {
2633 if metadata.mode == BgMode::Pty {
2634 return;
2635 }
2636
2637 let (original_tokens, compressed_tokens, original_bytes, compressed_bytes) = match (
2638 token_counts.original_tokens,
2639 token_counts.compressed_tokens,
2640 token_counts.original_bytes,
2641 token_counts.compressed_bytes,
2642 ) {
2643 (
2644 Some(original_tokens),
2645 Some(compressed_tokens),
2646 Some(original_bytes),
2647 Some(compressed_bytes),
2648 ) => (
2649 original_tokens,
2650 compressed_tokens,
2651 original_bytes,
2652 compressed_bytes,
2653 ),
2654 _ => {
2655 crate::slog_warn!(
2656 "compression event skipped for {}: token counts unavailable (likely spill file missing or unreadable)",
2657 metadata.task_id
2658 );
2659 return;
2660 }
2661 };
2662
2663 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
2664 let Some(pool) = pool else {
2665 crate::slog_warn!(
2666 "compression event skipped for {}: db_pool not initialized — was configure run?",
2667 metadata.task_id
2668 );
2669 return;
2670 };
2671 let harness = self
2672 .inner
2673 .db_harness
2674 .read()
2675 .ok()
2676 .and_then(|slot| slot.clone());
2677 let Some(harness) = harness else {
2678 crate::slog_warn!(
2679 "compression event insert skipped for {}: harness not configured",
2680 metadata.task_id
2681 );
2682 return;
2683 };
2684
2685 let project_root = metadata
2686 .project_root
2687 .as_deref()
2688 .unwrap_or(&metadata.workdir);
2689 let project_key = crate::path_identity::project_scope_key(project_root);
2690 let row = crate::db::compression_events::CompressionEventRow {
2691 harness: &harness,
2692 session_id: Some(&metadata.session_id),
2693 project_key: &project_key,
2694 tool: "bash",
2695 task_id: Some(&metadata.task_id),
2696 command: Some(&metadata.command),
2697 compressor: if metadata.compressed {
2698 "registry"
2699 } else {
2700 "none"
2701 },
2702 original_bytes,
2703 compressed_bytes,
2704 original_tokens,
2705 compressed_tokens,
2706 created_at: unix_millis() as i64,
2707 };
2708
2709 let conn = match pool.lock() {
2710 Ok(conn) => conn,
2711 Err(_) => {
2712 crate::slog_warn!(
2713 "compression event insert failed for {}: db mutex poisoned",
2714 metadata.task_id
2715 );
2716 return;
2717 }
2718 };
2719 match crate::db::compression_events::insert_compression_event(&conn, &row) {
2720 Ok(Some(row_id)) => {
2721 self.inner
2725 .compression_aggregates
2726 .record_successful_insert(&conn, &row, row_id);
2727 crate::slog_debug!(
2731 "compression event recorded for {} (project={}, session={}, {} → {} tokens)",
2732 metadata.task_id,
2733 project_key,
2734 metadata.session_id,
2735 original_tokens,
2736 compressed_tokens
2737 );
2738 }
2739 Ok(None) => {
2740 crate::slog_debug!(
2741 "duplicate compression event ignored for {} (project={}, session={})",
2742 metadata.task_id,
2743 project_key,
2744 metadata.session_id
2745 );
2746 }
2747 Err(error) => {
2748 crate::slog_warn!(
2749 "compression event insert failed for {}: {}",
2750 metadata.task_id,
2751 error
2752 );
2753 }
2754 }
2755 }
2756
2757 fn emit_bash_pattern_match(&self, session_id: &str, pattern_match: PatternMatch) {
2758 let Ok(progress_sender) = self
2759 .inner
2760 .progress_sender
2761 .lock()
2762 .map(|sender| sender.clone())
2763 else {
2764 return;
2765 };
2766 if let Some(sender) = progress_sender.as_ref() {
2767 sender(PushFrame::BashPatternMatch(BashPatternMatchFrame::new(
2768 pattern_match.task_id,
2769 session_id.to_string(),
2770 pattern_match.watch_id,
2771 pattern_match.match_text,
2772 pattern_match.match_offset,
2773 pattern_match.context,
2774 pattern_match.once,
2775 )));
2776 }
2777 }
2778
2779 fn emit_bash_watch_exit(&self, completion: &BgCompletion) {
2780 let Ok(progress_sender) = self
2781 .inner
2782 .progress_sender
2783 .lock()
2784 .map(|sender| sender.clone())
2785 else {
2786 return;
2787 };
2788 let Some(sender) = progress_sender.as_ref() else {
2789 return;
2790 };
2791 let status = completion_status_text(&completion.status, completion.exit_code);
2792 let preview = completion.output_preview.trim_end();
2793 let context = if preview.is_empty() {
2794 format!("task {} exited ({status})", completion.task_id)
2795 } else {
2796 format!(
2797 "task {} exited ({status})
2798{preview}",
2799 completion.task_id
2800 )
2801 };
2802 sender(PushFrame::BashPatternMatch(
2803 BashPatternMatchFrame::task_exit(
2804 completion.task_id.clone(),
2805 completion.session_id.clone(),
2806 format!("exited ({status})"),
2807 context,
2808 ),
2809 ));
2810 }
2811
2812 fn emit_bash_completed(&self, completion: BgCompletion) {
2813 let Ok(progress_sender) = self
2814 .inner
2815 .progress_sender
2816 .lock()
2817 .map(|sender| sender.clone())
2818 else {
2819 return;
2820 };
2821 let Some(sender) = progress_sender.as_ref() else {
2822 return;
2823 };
2824 sender(PushFrame::BashCompleted(BashCompletedFrame::new(
2832 completion.task_id,
2833 completion.session_id,
2834 completion.status,
2835 completion.exit_code,
2836 completion.command,
2837 completion.output_preview,
2838 completion.output_truncated,
2839 completion.original_tokens,
2840 completion.compressed_tokens,
2841 completion.tokens_skipped,
2842 )));
2843 }
2844
2845 fn completion_token_counts(
2846 &self,
2847 metadata: &PersistedTask,
2848 buffer: Option<&BgBuffer>,
2849 paths: Option<&TaskPaths>,
2850 rendered_output: Option<&str>,
2851 ) -> CompletionTokenCounts {
2852 if metadata.mode == BgMode::Pty {
2853 return CompletionTokenCounts::skipped();
2854 }
2855
2856 let raw = match buffer {
2857 Some(buffer) => buffer.read_for_token_count(TOKENIZE_CAP_BYTES_PER_STREAM),
2858 None => paths
2859 .map(|paths| {
2860 read_for_token_count_from_disk(metadata, paths, TOKENIZE_CAP_BYTES_PER_STREAM)
2861 })
2862 .unwrap_or(TokenCountInput::Skipped),
2863 };
2864
2865 let TokenCountInput::Text(raw_output) = raw else {
2866 return CompletionTokenCounts::skipped();
2867 };
2868
2869 let original_tokens = token_count_u32(&raw_output);
2870 let original_bytes = raw_output.len() as i64;
2871 let compressed_output = rendered_output.unwrap_or(&raw_output);
2872 let compressed_tokens = token_count_u32(compressed_output);
2873 let compressed_bytes = compressed_output.len() as i64;
2874 CompletionTokenCounts {
2875 original_tokens: Some(original_tokens),
2876 compressed_tokens: Some(compressed_tokens),
2877 original_bytes: Some(original_bytes),
2878 compressed_bytes: Some(compressed_bytes),
2879 tokens_skipped: false,
2880 }
2881 }
2882
2883 pub(crate) fn maybe_emit_long_running_reminder(&self, task: &Arc<BgTask>) {
2884 if !self
2885 .inner
2886 .long_running_reminder_enabled
2887 .load(Ordering::SeqCst)
2888 {
2889 return;
2890 }
2891 let interval_ms = self
2892 .inner
2893 .long_running_reminder_interval_ms
2894 .load(Ordering::SeqCst);
2895 if interval_ms == 0 {
2896 return;
2897 }
2898 let interval = Duration::from_millis(interval_ms);
2899 let now = Instant::now();
2900 let Ok(mut last_reminder_at) = task.last_reminder_at.lock() else {
2901 return;
2902 };
2903 let since = last_reminder_at.unwrap_or(task.started);
2904 if now.duration_since(since) < interval {
2905 return;
2906 }
2907 let command = task
2908 .state
2909 .lock()
2910 .map(|state| state.metadata.command.clone())
2911 .unwrap_or_default();
2912 *last_reminder_at = Some(now);
2913 self.emit_bash_long_running(BashLongRunningFrame::new(
2914 task.task_id.clone(),
2915 task.session_id.clone(),
2916 command,
2917 task.started.elapsed().as_millis() as u64,
2918 ));
2919 }
2920
2921 fn emit_bash_long_running(&self, frame: BashLongRunningFrame) {
2922 let Ok(progress_sender) = self
2923 .inner
2924 .progress_sender
2925 .lock()
2926 .map(|sender| sender.clone())
2927 else {
2928 return;
2929 };
2930 if let Some(sender) = progress_sender.as_ref() {
2931 sender(PushFrame::BashLongRunning(frame));
2932 }
2933 }
2934
2935 fn task(&self, task_id: &str) -> Option<Arc<BgTask>> {
2936 self.inner
2937 .tasks
2938 .lock()
2939 .ok()
2940 .and_then(|tasks| tasks.get(task_id).cloned())
2941 }
2942
2943 fn task_for_session(&self, task_id: &str, session_id: &str) -> Option<Arc<BgTask>> {
2944 self.task(task_id)
2945 .filter(|task| task.session_id == session_id)
2946 }
2947
2948 pub fn try_health_counts(&self) -> Option<BgTaskHealthCounts> {
2949 let running = self
2950 .inner
2951 .tasks
2952 .try_lock()
2953 .ok()
2954 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())?;
2955 let pending_completions = self.inner.completions.try_lock().ok().map(|q| q.len())?;
2956 Some(BgTaskHealthCounts {
2957 running,
2958 pending_completions,
2959 })
2960 }
2961
2962 pub fn estimated_memory(&self) -> crate::memory::MemoryEstimate {
2966 let tasks = match self.inner.tasks.try_lock() {
2967 Ok(tasks) => tasks.values().cloned().collect::<Vec<_>>(),
2968 Err(_) => return crate::memory::MemoryEstimate::busy(),
2969 };
2970 let mut bytes = 0u64;
2971 let mut terminal_output_caches = 0usize;
2972 let mut sessions = HashSet::new();
2973 for task in &tasks {
2974 sessions.insert(task.session_id.clone());
2975 let state = match task.state.try_lock() {
2976 Ok(state) => state,
2977 Err(_) => return crate::memory::MemoryEstimate::busy(),
2978 };
2979 if let Some(cache) = state.terminal_output_cache.as_ref() {
2980 terminal_output_caches = terminal_output_caches.saturating_add(1);
2981 bytes = bytes.saturating_add(terminal_output_cache_estimated_bytes(cache));
2982 }
2983 }
2984 let completion_count = match self.inner.completions.try_lock() {
2985 Ok(completions) => {
2986 for completion in completions.iter() {
2987 sessions.insert(completion.session_id.clone());
2988 bytes = bytes.saturating_add(completion_estimated_bytes(completion));
2989 }
2990 completions.len()
2991 }
2992 Err(_) => return crate::memory::MemoryEstimate::busy(),
2993 };
2994
2995 crate::memory::MemoryEstimate::estimated(bytes)
2996 .count("tasks", tasks.len())
2997 .count("sessions", sessions.len())
2998 .count("terminal_output_caches", terminal_output_caches)
2999 .count("completion_caches", completion_count)
3000 .count_u64("output_ring_bytes", 0)
3001 }
3002
3003 fn running_count(&self) -> usize {
3004 self.inner
3005 .tasks
3006 .lock()
3007 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())
3008 .unwrap_or(0)
3009 }
3010
3011 fn start_watchdog(&self) {
3012 if !self.inner.watchdog_started.swap(true, Ordering::SeqCst) {
3013 super::watchdog::start(self.clone());
3014 }
3015 }
3016
3017 fn running_metadata_is_stale(&self, metadata: &PersistedTask) -> bool {
3018 unix_millis().saturating_sub(metadata.started_at) > STALE_RUNNING_AFTER.as_millis() as u64
3019 }
3020
3021 #[cfg(test)]
3022 pub fn task_json_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
3023 self.task_for_session(task_id, session_id)
3024 .map(|task| task.paths.json.clone())
3025 }
3026
3027 #[cfg(test)]
3028 pub fn task_exit_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
3029 self.task_for_session(task_id, session_id)
3030 .map(|task| task.paths.exit.clone())
3031 }
3032
3033 fn generate_unique_task_id(&self) -> Result<String, String> {
3035 for _ in 0..32 {
3036 let candidate = random_slug();
3037 let tasks = self
3038 .inner
3039 .tasks
3040 .lock()
3041 .map_err(|_| "background task registry lock poisoned".to_string())?;
3042 if tasks.contains_key(&candidate) {
3043 continue;
3044 }
3045 let completions = self
3046 .inner
3047 .completions
3048 .lock()
3049 .map_err(|_| "background completions lock poisoned".to_string())?;
3050 if completions
3051 .iter()
3052 .any(|completion| completion.task_id == candidate)
3053 {
3054 continue;
3055 }
3056 return Ok(candidate);
3057 }
3058 Err("failed to allocate unique background task id after 32 attempts".to_string())
3059 }
3060}
3061
3062fn canonical_artifact_root(paths: &TaskPaths) -> PathBuf {
3063 fs::canonicalize(&paths.dir).unwrap_or_else(|_| paths.dir.clone())
3064}
3065
3066fn render_compressed_with_recovery(
3067 buffer: &BgBuffer,
3068 mut compressed: CompressionResult,
3069 input_truncated: bool,
3070 disk_truncation: DiskTruncation,
3071 artifact_access: ArtifactRecoveryAccess,
3072) -> TerminalOutputCache {
3073 let had_trailing_newline = compressed.text.ends_with('\n');
3081 let mut text = strip_plain_truncation_marker_lines(&compressed.text)
3082 .trim_end()
3083 .to_string();
3084 if had_trailing_newline && !text.is_empty() {
3085 text.push('\n');
3086 }
3087 compressed.text = text;
3088
3089 let output_path = buffer.output_path().map(|path| path.display().to_string());
3090 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
3091 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
3092 let mut recovery = RecoveryContext {
3093 dropped_by_class: compressed.dropped_by_class,
3094 had_inner_drop: compressed.had_inner_drop,
3095 offset_hint_eligible: compressed.offset_hint_eligible,
3096 offset_start_line: compressed.offset_start_line,
3097 byte_truncated: input_truncated,
3098 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
3099 output_path: output_path.clone(),
3100 stderr_path: stderr_path.clone(),
3101 include_stderr_path,
3102 artifact_access: artifact_access.clone(),
3103 };
3104
3105 let (output_preview, output_truncated) =
3106 render_body_with_recovery_marker(&compressed.text, &mut recovery);
3107 TerminalOutputCache {
3108 output_preview,
3109 output_truncated,
3110 kind: TerminalOutputKind::Compressed,
3111 output_path,
3112 stderr_path,
3113 artifact_access,
3114 recovery: Some(recovery),
3115 }
3116}
3117
3118fn render_body_with_recovery_marker(body: &str, recovery: &mut RecoveryContext) -> (String, bool) {
3119 render_body_with_recovery_marker_at_cap(
3120 body,
3121 recovery,
3122 FINAL_OUTPUT_CAP_BYTES,
3123 cap_final_output,
3124 cap_final_output_with_marker,
3125 )
3126}
3127
3128fn render_raw_body_with_recovery_marker(
3129 body: &str,
3130 recovery: &mut RecoveryContext,
3131) -> (String, bool) {
3132 render_body_with_recovery_marker_at_cap(
3133 body,
3134 recovery,
3135 RAW_PASSTHROUGH_CAP_BYTES,
3136 |input| {
3137 super::output::cap_head_tail(
3138 input,
3139 RAW_PASSTHROUGH_CAP_BYTES,
3140 RAW_PASSTHROUGH_HEAD_BYTES,
3141 RAW_PASSTHROUGH_TAIL_BYTES,
3142 )
3143 },
3144 |input, marker| {
3145 super::output::cap_head_tail_with_marker(
3146 input,
3147 RAW_PASSTHROUGH_CAP_BYTES,
3148 RAW_PASSTHROUGH_HEAD_BYTES,
3149 RAW_PASSTHROUGH_TAIL_BYTES,
3150 marker,
3151 )
3152 },
3153 )
3154}
3155
3156fn render_body_with_recovery_marker_at_cap<F, G>(
3157 body: &str,
3158 recovery: &mut RecoveryContext,
3159 cap_bytes: usize,
3160 cap_plain: F,
3161 cap_with_marker: G,
3162) -> (String, bool)
3163where
3164 F: Fn(&str) -> super::output::CappedText,
3165 G: Fn(&str, &str) -> super::output::CappedText,
3166{
3167 let needs_marker = recovery.has_visible_drop();
3168 if body.len() > cap_bytes {
3169 recovery.byte_truncated = true;
3170 if let Some(marker) = recovery_marker(recovery) {
3171 let capped = cap_with_marker(body, &marker);
3172 return (capped.text, true);
3173 }
3174 let capped = cap_plain(body);
3175 return (capped.text, capped.truncated || needs_marker);
3176 }
3177
3178 if !needs_marker {
3179 return (body.to_string(), false);
3180 }
3181
3182 let Some(marker) = recovery_marker(recovery) else {
3183 return (body.to_string(), true);
3184 };
3185 let with_marker = append_recovery_marker(body, &marker);
3186 if with_marker.len() <= cap_bytes {
3187 return (with_marker, true);
3188 }
3189
3190 recovery.byte_truncated = true;
3191 let marker = recovery_marker(recovery).unwrap_or(marker);
3192 let capped = cap_with_marker(body, &marker);
3193 (capped.text, true)
3194}
3195
3196fn append_recovery_marker(body: &str, marker: &str) -> String {
3197 if body.is_empty() {
3198 return marker.to_string();
3199 }
3200 let mut output = body.trim_end().to_string();
3201 output.push('\n');
3202 output.push_str(marker);
3203 output
3204}
3205
3206fn recovery_marker(recovery: &RecoveryContext) -> Option<String> {
3207 let mut parts = Vec::new();
3208 for (class, count) in &recovery.dropped_by_class {
3209 let label = if *count == 1 {
3210 class.singular()
3211 } else {
3212 class.plural()
3213 };
3214 parts.push(format!("+{count} more {label}"));
3215 }
3216 if recovery.byte_truncated {
3217 parts.push("truncated output".to_string());
3218 }
3219 let disk_truncated_prefix_bytes = recovery.disk_truncated_prefix_bytes;
3220 if disk_truncated_prefix_bytes > 0 {
3221 parts.push(format!(
3222 "truncated {disk_truncated_prefix_bytes} bytes from saved output prefix"
3223 ));
3224 } else if recovery.had_inner_drop && parts.is_empty() {
3225 parts.push("omitted output".to_string());
3226 }
3227
3228 if parts.is_empty() {
3229 return None;
3230 }
3231
3232 let hint = recovery_hint(recovery);
3233 Some(format!("[{}; {hint}]", parts.join(", ")))
3234}
3235
3236fn bash_status_recovery_hint(access: &ArtifactRecoveryAccess) -> String {
3237 let task_id = serde_json::to_string(&access.task_id)
3238 .unwrap_or_else(|_| format!("\"{}\"", access.task_id));
3239 format!("use bash_status({{taskId: {task_id}}})")
3240}
3241
3242fn recovery_hint(recovery: &RecoveryContext) -> String {
3243 if !recovery.artifact_access.readable {
3244 return bash_status_recovery_hint(&recovery.artifact_access);
3245 }
3246
3247 if recovery.offset_hint_eligible
3251 && !recovery.byte_truncated
3252 && recovery.dropped_by_class.is_empty()
3253 && !recovery.include_stderr_path
3254 {
3255 if let (Some(path), Some(line)) =
3256 (recovery.output_path.as_deref(), recovery.offset_start_line)
3257 {
3258 return format!("see remaining: tail -n +{line} {}", quote_path(path));
3259 }
3260 }
3261
3262 let mut paths = Vec::new();
3263 if let Some(path) = recovery.output_path.as_deref() {
3264 paths.push(path);
3265 }
3266 if recovery.include_stderr_path {
3267 if let Some(path) = recovery.stderr_path.as_deref() {
3268 if !paths.contains(&path) {
3269 paths.push(path);
3270 }
3271 }
3272 }
3273
3274 if paths.is_empty() {
3275 return "full output unavailable".to_string();
3276 }
3277
3278 let reads = paths
3279 .into_iter()
3280 .map(|path| format!("read {}", quote_path(path)))
3281 .collect::<Vec<_>>()
3282 .join(" and ");
3283 if recovery.disk_truncated_prefix_bytes > 0 {
3284 format!("retained output: {reads}")
3285 } else {
3286 format!("full output: {reads}")
3287 }
3288}
3289
3290fn strip_plain_truncation_marker_lines(input: &str) -> String {
3291 input
3292 .lines()
3293 .filter(|line| !is_plain_truncation_marker(line.trim()))
3294 .collect::<Vec<_>>()
3295 .join("\n")
3296}
3297
3298fn strip_recovery_marker_lines(input: &str) -> String {
3299 input
3300 .lines()
3301 .filter(|line| !is_recovery_marker(line.trim()))
3302 .collect::<Vec<_>>()
3303 .join("\n")
3304}
3305
3306fn is_plain_truncation_marker(line: &str) -> bool {
3307 let Some(rest) = line.strip_prefix("...<truncated ") else {
3308 return false;
3309 };
3310 let Some(bytes) = rest.strip_suffix(" bytes>...") else {
3311 return false;
3312 };
3313 !bytes.is_empty() && bytes.chars().all(|ch| ch.is_ascii_digit())
3314}
3315
3316fn is_recovery_marker(line: &str) -> bool {
3317 line.starts_with('[')
3318 && line.ends_with(']')
3319 && (line.contains("full output: read ")
3320 || line.contains("retained output: read ")
3321 || line.contains("see remaining: tail -n +")
3322 || line.contains("use bash_status({taskId:")
3323 || line.contains("full output unavailable"))
3324}
3325
3326fn structured_output_pointer(
3327 total_bytes: u64,
3328 output_path: &str,
3329 truncated_prefix_bytes: u64,
3330 artifact_access: &ArtifactRecoveryAccess,
3331) -> String {
3332 if artifact_access.readable {
3333 return if truncated_prefix_bytes > 0 {
3334 retained_json_output_pointer(total_bytes, output_path, truncated_prefix_bytes)
3335 } else {
3336 json_output_pointer(total_bytes, output_path)
3337 };
3338 }
3339
3340 let kb = total_bytes.div_ceil(1024);
3341 let hint = bash_status_recovery_hint(artifact_access);
3342 if truncated_prefix_bytes > 0 {
3343 format!(
3344 "[JSON output {kb} KB; truncated {truncated_prefix_bytes} bytes from saved output prefix; retained output: {hint}]"
3345 )
3346 } else {
3347 format!("[JSON output {kb} KB; full output: {hint}]")
3348 }
3349}
3350
3351fn render_structured_output(
3352 command: &str,
3353 buffer: &BgBuffer,
3354 disk_truncation: DiskTruncation,
3355 artifact_access: ArtifactRecoveryAccess,
3356) -> Option<TerminalOutputCache> {
3357 if !is_gh_structured_command(command) {
3358 return None;
3359 }
3360
3361 let output_path = buffer
3362 .output_path()
3363 .map(|path| path.display().to_string())?;
3364 let stdout_bytes = buffer.stream_len(StreamKind::Stdout);
3365 if stdout_bytes == 0 {
3366 return None;
3367 }
3368
3369 if stdout_bytes > STRUCTURED_OUTPUT_CAP_BYTES as u64 {
3370 if !stream_starts_like_json(buffer, StreamKind::Stdout) {
3371 return None;
3372 }
3373 let output_preview = structured_output_pointer(
3374 stdout_bytes,
3375 &output_path,
3376 disk_truncation.total_prefix_bytes(),
3377 &artifact_access,
3378 );
3379 return Some(TerminalOutputCache {
3380 output_preview,
3381 output_truncated: true,
3382 kind: TerminalOutputKind::Structured,
3383 output_path: Some(output_path),
3384 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
3385 artifact_access,
3386 recovery: None,
3387 });
3388 }
3389
3390 let stdout = buffer.read_stream_bounded(StreamKind::Stdout, STRUCTURED_OUTPUT_CAP_BYTES);
3391 if stdout.truncated || !is_structured_body(&stdout.text) {
3392 return None;
3393 }
3394
3395 Some(TerminalOutputCache {
3396 output_preview: stdout.text,
3397 output_truncated: false,
3398 kind: TerminalOutputKind::Structured,
3399 output_path: Some(output_path),
3400 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
3401 artifact_access,
3402 recovery: None,
3403 })
3404}
3405
3406fn render_raw_passthrough(
3407 buffer: &BgBuffer,
3408 disk_truncation: DiskTruncation,
3409 artifact_access: ArtifactRecoveryAccess,
3410) -> TerminalOutputCache {
3411 let raw = buffer.read_combined_head_tail(
3412 RAW_PASSTHROUGH_CAP_BYTES,
3413 RAW_PASSTHROUGH_HEAD_BYTES,
3414 RAW_PASSTHROUGH_TAIL_BYTES,
3415 );
3416 let output_path = buffer.output_path().map(|path| path.display().to_string());
3417 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
3418 if !raw.truncated && disk_truncation.total_prefix_bytes() == 0 {
3419 return TerminalOutputCache {
3420 output_preview: raw.text,
3421 output_truncated: false,
3422 kind: TerminalOutputKind::Raw,
3423 output_path,
3424 stderr_path,
3425 artifact_access,
3426 recovery: None,
3427 };
3428 }
3429
3430 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
3431 let mut recovery = RecoveryContext {
3432 dropped_by_class: BTreeMap::new(),
3433 had_inner_drop: false,
3434 offset_hint_eligible: false,
3435 offset_start_line: None,
3436 byte_truncated: raw.truncated,
3437 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
3438 output_path: output_path.clone(),
3439 stderr_path: stderr_path.clone(),
3440 include_stderr_path,
3441 artifact_access: artifact_access.clone(),
3442 };
3443 let (output_preview, output_truncated) =
3444 render_raw_body_with_recovery_marker(&raw.text, &mut recovery);
3445 TerminalOutputCache {
3446 output_preview,
3447 output_truncated,
3448 kind: TerminalOutputKind::Raw,
3449 output_path,
3450 stderr_path,
3451 artifact_access,
3452 recovery: Some(recovery),
3453 }
3454}
3455
3456fn completion_preview_for_cache(
3457 cache: &TerminalOutputCache,
3458 exit_code: Option<i32>,
3459) -> (String, bool) {
3460 let exit_ok = exit_code == Some(0);
3463 let threshold = completion_preview_threshold(exit_ok);
3464 if cache.kind == TerminalOutputKind::Structured && cache.output_preview.len() > threshold {
3465 if let Some(path) = cache.output_path.as_deref() {
3466 return (
3467 structured_output_pointer(
3468 cache.output_preview.len() as u64,
3469 path,
3470 0,
3471 &cache.artifact_access,
3472 ),
3473 true,
3474 );
3475 }
3476 return (cache.output_preview.clone(), cache.output_truncated);
3477 }
3478
3479 if let Some(recovery) = cache.recovery.as_ref() {
3480 if cache.output_preview.len() <= threshold {
3481 return (cache.output_preview.clone(), cache.output_truncated);
3482 }
3483 let body = strip_recovery_marker_lines(&cache.output_preview);
3484 let mut completion_recovery = recovery.clone();
3485 completion_recovery.byte_truncated = true;
3486 if let Some(marker) = recovery_marker(&completion_recovery) {
3487 let capped = cap_completion_output_with_marker(&body, &marker, exit_ok);
3488 return (capped.text, true);
3489 }
3490 }
3491
3492 let capped = cap_completion_output(&cache.output_preview, exit_ok);
3493 (capped.text, cache.output_truncated || capped.truncated)
3494}
3495
3496fn is_gh_structured_command(command: &str) -> bool {
3497 let Some(normalized) = crate::compress::plain_command_for_structured_output(command) else {
3498 return false;
3499 };
3500 let tokens = shell_words_for_flags(&normalized);
3501 let Some(head) = tokens.first() else {
3502 return false;
3503 };
3504 let head_name = Path::new(head)
3505 .file_name()
3506 .and_then(|name| name.to_str())
3507 .unwrap_or(head);
3508 if !(head_name == "gh" || head_name.eq_ignore_ascii_case("gh.exe")) {
3509 return false;
3510 }
3511 tokens.iter().any(|token| {
3512 matches!(token.as_str(), "--json" | "--jq" | "--template")
3513 || token.starts_with("--json=")
3514 || token.starts_with("--jq=")
3515 || token.starts_with("--template=")
3516 })
3517}
3518
3519fn shell_words_for_flags(command: &str) -> Vec<String> {
3520 let mut words = Vec::new();
3521 let mut current = String::new();
3522 let mut in_single = false;
3523 let mut in_double = false;
3524 let mut escaped = false;
3525
3526 for ch in command.chars() {
3527 if escaped {
3528 current.push(ch);
3529 escaped = false;
3530 continue;
3531 }
3532 if ch == '\\' && !in_single {
3533 escaped = true;
3534 continue;
3535 }
3536 if ch == '\'' && !in_double {
3537 in_single = !in_single;
3538 continue;
3539 }
3540 if ch == '"' && !in_single {
3541 in_double = !in_double;
3542 continue;
3543 }
3544 if ch.is_whitespace() && !in_single && !in_double {
3545 if !current.is_empty() {
3546 words.push(std::mem::take(&mut current));
3547 }
3548 continue;
3549 }
3550 if matches!(ch, ';' | '&' | '|') && !in_single && !in_double {
3551 if !current.is_empty() {
3552 words.push(std::mem::take(&mut current));
3553 }
3554 continue;
3555 }
3556 current.push(ch);
3557 }
3558 if !current.is_empty() {
3559 words.push(current);
3560 }
3561 words
3562}
3563
3564fn is_structured_body(body: &str) -> bool {
3565 let trimmed = body.trim();
3566 if trimmed.is_empty() {
3567 return false;
3568 }
3569 if serde_json::from_str::<serde_json::Value>(trimmed).is_ok() {
3570 return true;
3571 }
3572
3573 let mut saw_line = false;
3574 for line in trimmed
3575 .lines()
3576 .map(str::trim)
3577 .filter(|line| !line.is_empty())
3578 {
3579 saw_line = true;
3580 if serde_json::from_str::<serde_json::Value>(line).is_err() {
3581 return false;
3582 }
3583 }
3584 saw_line
3585}
3586
3587fn stream_starts_like_json(buffer: &BgBuffer, stream: StreamKind) -> bool {
3588 let path = match (buffer, stream) {
3589 (BgBuffer::Pipes { stdout_path, .. }, StreamKind::Stdout) => Some(stdout_path),
3590 (BgBuffer::Pipes { stderr_path, .. }, StreamKind::Stderr) => Some(stderr_path),
3591 (BgBuffer::Pty { combined_path }, _) => Some(combined_path),
3592 };
3593 let Some(path) = path else {
3594 return false;
3595 };
3596 let Ok(file) = std::fs::File::open(path) else {
3597 return false;
3598 };
3599 let mut limited = file.take(512);
3600 let mut bytes = Vec::new();
3601 if limited.read_to_end(&mut bytes).is_err() {
3602 return false;
3603 }
3604 String::from_utf8_lossy(&bytes)
3605 .chars()
3606 .find(|ch| !ch.is_whitespace())
3607 .is_some_and(|ch| matches!(ch, '{' | '[' | '"' | '-' | '0'..='9' | 't' | 'f' | 'n'))
3608}
3609
3610struct CompletionTokenCounts {
3611 original_tokens: Option<u32>,
3612 compressed_tokens: Option<u32>,
3613 original_bytes: Option<i64>,
3614 compressed_bytes: Option<i64>,
3615 tokens_skipped: bool,
3616}
3617
3618impl CompletionTokenCounts {
3619 fn skipped() -> Self {
3620 Self {
3621 original_tokens: None,
3622 compressed_tokens: None,
3623 original_bytes: None,
3624 compressed_bytes: None,
3625 tokens_skipped: true,
3626 }
3627 }
3628}
3629
3630fn completion_status_text(status: &BgTaskStatus, exit_code: Option<i32>) -> String {
3631 match status {
3632 BgTaskStatus::TimedOut => "timed out".to_string(),
3633 BgTaskStatus::Killed => "killed".to_string(),
3634 _ => exit_code
3635 .map(|code| format!("exit {code}"))
3636 .unwrap_or_else(|| format!("{status:?}").to_lowercase()),
3637 }
3638}
3639
3640fn token_count_u32(text: &str) -> u32 {
3641 aft_tokenizer::count_tokens(text)
3642 .try_into()
3643 .unwrap_or(u32::MAX)
3644}
3645
3646impl Default for BgTaskRegistry {
3647 fn default() -> Self {
3648 Self::new(Arc::new(Mutex::new(None)))
3649 }
3650}
3651
3652fn modified_within(path: &Path, grace: Duration) -> bool {
3653 fs::metadata(path)
3654 .and_then(|metadata| metadata.modified())
3655 .ok()
3656 .and_then(|modified| SystemTime::now().duration_since(modified).ok())
3657 .map(|age| age < grace)
3658 .unwrap_or(false)
3659}
3660
3661fn canonicalized_path(path: &Path) -> PathBuf {
3662 fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf())
3663}
3664
3665fn started_instant_from_unix_millis(started_at: u64) -> Instant {
3666 let now_ms = SystemTime::now()
3667 .duration_since(UNIX_EPOCH)
3668 .ok()
3669 .map(|duration| duration.as_millis() as u64)
3670 .unwrap_or(started_at);
3671 let elapsed_ms = now_ms.saturating_sub(started_at);
3672 Instant::now()
3673 .checked_sub(Duration::from_millis(elapsed_ms))
3674 .unwrap_or_else(Instant::now)
3675}
3676
3677fn gc_quarantine(storage_dir: &Path) {
3678 let quarantine_root = storage_dir.join("bash-tasks-quarantine");
3679 let Ok(session_dirs) = fs::read_dir(&quarantine_root) else {
3680 return;
3681 };
3682 for session_entry in session_dirs.flatten() {
3683 let session_quarantine_dir = session_entry.path();
3684 if !session_quarantine_dir.is_dir() {
3685 continue;
3686 }
3687 let entries = match fs::read_dir(&session_quarantine_dir) {
3688 Ok(entries) => entries,
3689 Err(error) => {
3690 crate::slog_warn!(
3691 "failed to read background task quarantine dir {}: {error}",
3692 session_quarantine_dir.display()
3693 );
3694 continue;
3695 }
3696 };
3697 for entry in entries.flatten() {
3698 let path = entry.path();
3699 if modified_within(&path, QUARANTINE_GC_GRACE) {
3700 continue;
3701 }
3702 let result = if path.is_dir() {
3703 fs::remove_dir_all(&path)
3704 } else {
3705 fs::remove_file(&path)
3706 };
3707 match result {
3708 Ok(()) => log::debug!(
3709 "deleted old background task quarantine entry {}",
3710 path.display()
3711 ),
3712 Err(error) => crate::slog_warn!(
3713 "failed to delete old background task quarantine entry {}: {error}",
3714 path.display()
3715 ),
3716 }
3717 }
3718 let _ = fs::remove_dir(&session_quarantine_dir);
3719 }
3720 let _ = fs::remove_dir(&quarantine_root);
3721}
3722
3723enum QuarantineKind {
3724 Corrupt,
3725 Invalid,
3726}
3727
3728fn quarantine_task_json(
3729 storage_dir: &Path,
3730 session_dir: &Path,
3731 json_path: &Path,
3732 kind: QuarantineKind,
3733) -> Result<(), String> {
3734 let session_hash = session_dir
3735 .file_name()
3736 .and_then(|name| name.to_str())
3737 .ok_or_else(|| {
3738 format!(
3739 "invalid background task session dir: {}",
3740 session_dir.display()
3741 )
3742 })?;
3743 let task_name = json_path
3744 .file_name()
3745 .and_then(|name| name.to_str())
3746 .ok_or_else(|| format!("invalid background task json path: {}", json_path.display()))?;
3747 let unix_ts = SystemTime::now()
3748 .duration_since(UNIX_EPOCH)
3749 .map(|duration| duration.as_secs())
3750 .unwrap_or(0);
3751 let quarantine_dir = storage_dir.join("bash-tasks-quarantine").join(session_hash);
3752 fs::create_dir_all(&quarantine_dir).map_err(|e| {
3753 format!(
3754 "failed to create background task quarantine dir {}: {e}",
3755 quarantine_dir.display()
3756 )
3757 })?;
3758 let target_name = quarantine_name(task_name, unix_ts, &kind);
3759 let target = quarantine_dir.join(target_name);
3760 fs::rename(json_path, &target).map_err(|e| {
3761 format!(
3762 "failed to quarantine background task metadata {} to {}: {e}",
3763 json_path.display(),
3764 target.display()
3765 )
3766 })?;
3767
3768 for sibling in task_sibling_paths(json_path) {
3769 if !sibling.exists() {
3770 continue;
3771 }
3772 let Some(sibling_name) = sibling.file_name().and_then(|name| name.to_str()) else {
3773 crate::slog_warn!(
3774 "skipping background task sibling with invalid name during quarantine: {}",
3775 sibling.display()
3776 );
3777 continue;
3778 };
3779 let sibling_target = quarantine_dir.join(quarantine_name(sibling_name, unix_ts, &kind));
3780 if let Err(error) = fs::rename(&sibling, &sibling_target) {
3781 crate::slog_warn!(
3782 "failed to quarantine background task sibling {} to {}: {error}",
3783 sibling.display(),
3784 sibling_target.display()
3785 );
3786 }
3787 }
3788
3789 let _ = fs::remove_dir(session_dir);
3790 Ok(())
3791}
3792
3793fn quarantine_name(file_name: &str, unix_ts: u64, kind: &QuarantineKind) -> String {
3794 match kind {
3795 QuarantineKind::Corrupt => format!("{file_name}.corrupt-{unix_ts}"),
3796 QuarantineKind::Invalid => {
3797 let path = Path::new(file_name);
3798 let stem = path.file_stem().and_then(|stem| stem.to_str());
3799 let extension = path.extension().and_then(|extension| extension.to_str());
3800 match (stem, extension) {
3801 (Some(stem), Some(extension)) => format!("{stem}.invalid.{unix_ts}.{extension}"),
3802 _ => format!("{file_name}.invalid.{unix_ts}"),
3803 }
3804 }
3805 }
3806}
3807
3808fn task_sibling_paths(json_path: &Path) -> Vec<PathBuf> {
3809 let Some(parent) = json_path.parent() else {
3810 return Vec::new();
3811 };
3812 let Some(stem) = json_path.file_stem().and_then(|stem| stem.to_str()) else {
3813 return Vec::new();
3814 };
3815 ["stdout", "stderr", "exit", "pty", "ps1", "bat", "sh"]
3816 .into_iter()
3817 .map(|extension| parent.join(format!("{stem}.{extension}")))
3818 .collect()
3819}
3820
3821fn read_for_token_count_from_disk(
3822 metadata: &PersistedTask,
3823 paths: &TaskPaths,
3824 max_bytes_per_stream: usize,
3825) -> TokenCountInput {
3826 if metadata.mode == BgMode::Pty {
3827 return TokenCountInput::Skipped;
3828 }
3829 let stdout = read_file_tail_capped(&paths.stdout, max_bytes_per_stream);
3836 let stderr = read_file_tail_capped(&paths.stderr, max_bytes_per_stream);
3837 match (stdout, stderr) {
3838 (Ok(stdout), Ok(stderr)) => TokenCountInput::Text(combine_streams(
3839 String::from_utf8_lossy(&stdout).as_ref(),
3840 String::from_utf8_lossy(&stderr).as_ref(),
3841 )),
3842 (Ok(stdout), Err(_)) => TokenCountInput::Text(combine_streams(
3843 String::from_utf8_lossy(&stdout).as_ref(),
3844 "",
3845 )),
3846 (Err(_), Ok(stderr)) => TokenCountInput::Text(combine_streams(
3847 "",
3848 String::from_utf8_lossy(&stderr).as_ref(),
3849 )),
3850 (Err(_), Err(_)) => TokenCountInput::Skipped,
3851 }
3852}
3853
3854fn read_file_tail_capped(path: &Path, max_bytes: usize) -> std::io::Result<Vec<u8>> {
3859 use std::io::{Read, Seek, SeekFrom};
3860 let mut file = std::fs::File::open(path)?;
3861 let len = file.metadata()?.len();
3862 let read_len = len.min(max_bytes as u64);
3863 if read_len > 0 && len > max_bytes as u64 {
3864 file.seek(SeekFrom::End(-(read_len as i64)))?;
3865 }
3866 let mut bytes = Vec::with_capacity(read_len as usize);
3867 file.read_to_end(&mut bytes)?;
3868 Ok(bytes)
3869}
3870
3871impl BgTask {
3872 fn snapshot(&self, preview_bytes: usize) -> BgTaskSnapshot {
3873 let state = self
3874 .state
3875 .lock()
3876 .unwrap_or_else(|poison| poison.into_inner());
3877 self.snapshot_locked(&state, preview_bytes)
3878 }
3879
3880 fn snapshot_locked(&self, state: &BgTaskState, preview_bytes: usize) -> BgTaskSnapshot {
3881 let metadata = &state.metadata;
3882 let duration_ms = metadata.duration_ms.or_else(|| {
3883 metadata
3884 .status
3885 .is_terminal()
3886 .then(|| self.started.elapsed().as_millis() as u64)
3887 });
3888 let (output_preview, output_truncated) = if metadata.mode == BgMode::Pty {
3889 (String::new(), false)
3890 } else if metadata.status.is_terminal() {
3891 state
3892 .terminal_output_cache
3893 .as_ref()
3894 .map(|cache| (cache.output_preview.clone(), cache.output_truncated))
3895 .unwrap_or_else(|| (String::new(), false))
3896 } else if preview_bytes == 0 {
3897 (String::new(), false)
3898 } else {
3899 state.buffer.read_tail(preview_bytes)
3900 };
3901 BgTaskSnapshot {
3902 info: BgTaskInfo {
3903 task_id: self.task_id.clone(),
3904 status: metadata.status.clone(),
3905 command: metadata.command.clone(),
3906 mode: metadata.mode.clone(),
3907 started_at: metadata.started_at,
3908 duration_ms,
3909 },
3910 exit_code: metadata.exit_code,
3911 child_pid: metadata.child_pid,
3912 workdir: metadata.workdir.display().to_string(),
3913 output_preview,
3914 output_truncated,
3915 output_path: state
3916 .buffer
3917 .output_path()
3918 .map(|path| path.display().to_string()),
3919 stderr_path: state
3920 .buffer
3921 .stderr_path()
3922 .map(|path| path.display().to_string()),
3923 pty_rows: (metadata.mode == BgMode::Pty).then_some(metadata.pty_rows.unwrap_or(24)),
3924 pty_cols: (metadata.mode == BgMode::Pty).then_some(metadata.pty_cols.unwrap_or(80)),
3925 pty_screen: None,
3926 scanner_report: metadata.scanner_report.clone(),
3927 sandbox_native: metadata.sandbox_native,
3928 sandbox_unavailable: metadata.sandbox_native
3929 && self.paths.sandbox_unavailable.is_file(),
3930 }
3931 }
3932
3933 pub(crate) fn is_running(&self) -> bool {
3934 self.state
3935 .lock()
3936 .map(|state| {
3937 state.metadata.status == BgTaskStatus::Running
3938 || (state.metadata.mode == BgMode::Pty
3939 && state.metadata.status == BgTaskStatus::Killing)
3940 })
3941 .unwrap_or(false)
3942 }
3943
3944 fn is_terminal(&self) -> bool {
3945 self.state
3946 .lock()
3947 .map(|state| state.metadata.status.is_terminal())
3948 .unwrap_or(false)
3949 }
3950
3951 fn mark_terminal_now(&self) {
3952 if let Ok(mut terminal_at) = self.terminal_at.lock() {
3953 if terminal_at.is_none() {
3954 *terminal_at = Some(Instant::now());
3955 }
3956 }
3957 }
3958
3959 fn set_completion_delivered(
3960 &self,
3961 delivered: bool,
3962 registry: &BgTaskRegistry,
3963 ) -> Result<(), String> {
3964 let mut state = self
3965 .state
3966 .lock()
3967 .map_err(|_| "background task lock poisoned".to_string())?;
3968 let updated = registry
3969 .update_task_metadata(&self.paths, |metadata| {
3970 metadata.completion_delivered = delivered;
3971 })
3972 .map_err(|e| format!("failed to update completion delivery: {e}"))?;
3973 state.metadata = updated;
3974 Ok(())
3975 }
3976}
3977
3978#[cfg(unix)]
3999fn reap_piped_child(child_slot: &mut Option<Child>) {
4000 if let Some(mut child) = child_slot.take() {
4001 if matches!(child.try_wait(), Ok(None)) {
4002 let _ = child.wait();
4003 }
4004 }
4005}
4006
4007#[cfg(windows)]
4012fn reap_piped_child(child_slot: &mut Option<Child>) {
4013 *child_slot = None;
4014}
4015
4016fn terminal_metadata_from_marker(
4017 mut metadata: PersistedTask,
4018 marker: ExitMarker,
4019 reason: Option<String>,
4020) -> PersistedTask {
4021 match marker {
4022 ExitMarker::Code(code) => {
4023 let status = if code == 0 {
4024 BgTaskStatus::Completed
4025 } else {
4026 BgTaskStatus::Failed
4027 };
4028 metadata.mark_terminal(status, Some(code), reason);
4029 }
4030 ExitMarker::Killed => metadata.mark_terminal(
4031 BgTaskStatus::Killed,
4032 terminal_exit_code_for_status(&BgTaskStatus::Killed),
4033 reason,
4034 ),
4035 }
4036 metadata
4037}
4038
4039fn terminal_exit_code_for_status(status: &BgTaskStatus) -> Option<i32> {
4040 match status {
4041 BgTaskStatus::TimedOut => Some(124),
4042 BgTaskStatus::Killed => Some(137),
4043 _ => None,
4044 }
4045}
4046
4047fn attach_sandbox_metadata(metadata: &mut PersistedTask, spawn_plan: &SpawnPlan) {
4048 metadata.sandbox_native = spawn_plan.is_native_launcher();
4049 metadata.sandbox_temp_dir = spawn_plan.temp_dir().map(Path::to_path_buf);
4050}
4051
4052#[cfg(unix)]
4053fn write_unix_command_script(command: &str, paths: &TaskPaths) -> Result<PathBuf, String> {
4054 let stem = paths
4055 .json
4056 .file_stem()
4057 .and_then(|stem| stem.to_str())
4058 .unwrap_or("wrapper");
4059 let script_path = paths.dir.join(format!("{stem}.sh"));
4060 fs::write(&script_path, command)
4061 .map_err(|e| format!("failed to write background bash command script: {e}"))?;
4062 Ok(script_path)
4063}
4064
4065#[cfg(unix)]
4066fn detached_shell_command(
4067 spawn_plan: &SpawnPlan,
4068 command_script: &Path,
4069 exit_path: &Path,
4070 paths: &TaskPaths,
4071) -> Result<Command, String> {
4072 let shell = spawn_plan
4073 .host_shell_path()
4074 .map(Path::to_path_buf)
4075 .unwrap_or_else(resolve_posix_shell);
4076 let args = vec![
4082 "-c".into(),
4083 r#""$0" "$1"; code=$?; printf "%s" "$code" > "$2.tmp.$$"; mv -f "$2.tmp.$$" "$2""#.into(),
4084 shell.as_os_str().to_os_string(),
4085 command_script.as_os_str().to_os_string(),
4086 exit_path.as_os_str().to_os_string(),
4087 ];
4088 let mut cmd = crate::sandbox_spawn::detached_command_for_plan(
4089 spawn_plan,
4090 shell.as_os_str(),
4091 &args,
4092 &paths.json,
4093 exit_path,
4094 )?;
4095 unsafe {
4096 cmd.pre_exec(|| {
4097 if libc::setsid() == -1 {
4098 return Err(std::io::Error::last_os_error());
4099 }
4100 Ok(())
4101 });
4102 }
4103 Ok(cmd)
4104}
4105
4106#[cfg(unix)]
4107pub(crate) fn resolve_posix_shell() -> PathBuf {
4108 static POSIX_SHELL: OnceLock<PathBuf> = OnceLock::new();
4109 POSIX_SHELL
4110 .get_or_init(|| {
4111 std::env::var_os("BASH")
4112 .filter(|value| !value.is_empty())
4113 .map(PathBuf::from)
4114 .filter(|path| path.exists())
4115 .or_else(|| which::which("bash").ok())
4116 .or_else(|| which::which("zsh").ok())
4117 .unwrap_or_else(|| PathBuf::from("/bin/sh"))
4118 })
4119 .clone()
4120}
4121
4122#[cfg(windows)]
4123fn detached_shell_command_for(
4124 shell: crate::windows_shell::WindowsShell,
4125 command: &str,
4126 exit_path: &Path,
4127 paths: &TaskPaths,
4128 creation_flags: u32,
4129) -> Result<Command, String> {
4130 use crate::windows_shell::WindowsShell;
4131 let wrapper_body = shell.wrapper_script_bytes(command, exit_path);
4144 let wrapper_ext = match shell {
4145 WindowsShell::Pwsh | WindowsShell::Powershell => "ps1",
4146 WindowsShell::Cmd => "bat",
4147 WindowsShell::Posix(_) => "sh",
4151 };
4152 let wrapper_path = paths.dir.join(format!(
4153 "{}.{}",
4154 paths
4155 .json
4156 .file_stem()
4157 .and_then(|s| s.to_str())
4158 .unwrap_or("wrapper"),
4159 wrapper_ext
4160 ));
4161 fs::write(&wrapper_path, wrapper_body)
4162 .map_err(|e| format!("failed to write background bash wrapper script: {e}"))?;
4163
4164 let mut cmd = Command::new(shell.binary().as_ref());
4165 match shell {
4166 WindowsShell::Pwsh | WindowsShell::Powershell => {
4167 cmd.args([
4170 "-NoLogo",
4171 "-NoProfile",
4172 "-NonInteractive",
4173 "-ExecutionPolicy",
4174 "Bypass",
4175 "-File",
4176 ]);
4177 cmd.arg(&wrapper_path);
4178 }
4179 WindowsShell::Cmd => {
4180 cmd.args(["/D", "/C"]);
4187 cmd.arg(&wrapper_path);
4188 }
4189 WindowsShell::Posix(_) => {
4190 cmd.arg(&wrapper_path);
4195 }
4196 }
4197
4198 cmd.creation_flags(creation_flags);
4202 Ok(cmd)
4203}
4204
4205fn spawn_detached_child(
4221 spawn_plan: &SpawnPlan,
4222 command: &str,
4223 paths: &TaskPaths,
4224 workdir: &Path,
4225 env: &HashMap<String, String>,
4226) -> Result<std::process::Child, String> {
4227 #[cfg(not(windows))]
4228 {
4229 let stdout = create_capture_file(&paths.stdout)
4230 .map_err(|e| format!("failed to open stdout capture file: {e}"))?;
4231 let stderr = create_capture_file(&paths.stderr)
4232 .map_err(|e| format!("failed to open stderr capture file: {e}"))?;
4233 let command_script = write_unix_command_script(command, paths)?;
4234 let mut command = detached_shell_command(spawn_plan, &command_script, &paths.exit, paths)?;
4235 command
4236 .current_dir(workdir)
4237 .envs(env)
4238 .stdin(Stdio::null())
4239 .stdout(Stdio::from(stdout))
4240 .stderr(Stdio::from(stderr));
4241 crate::sandbox_spawn::apply_sandbox_environment(spawn_plan, &mut command);
4242 command
4243 .spawn()
4244 .map_err(|e| format!("failed to spawn background bash command: {e}"))
4245 }
4246 #[cfg(windows)]
4247 {
4248 use crate::windows_shell::shell_candidates;
4249 match spawn_plan {
4250 SpawnPlan::Unsandboxed | SpawnPlan::Host { .. } => {}
4251 SpawnPlan::Refused { code, .. } => return Err((*code).to_string()),
4252 SpawnPlan::Launcher { .. } => return Err("sandbox_unavailable".to_string()),
4253 }
4254 let candidates: Vec<crate::windows_shell::WindowsShell> = shell_candidates();
4265 const FLAG_CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200;
4286 const FLAG_CREATE_BREAKAWAY_FROM_JOB: u32 = 0x0100_0000;
4287 const FLAG_CREATE_NO_WINDOW: u32 = 0x0800_0000;
4288 let with_breakaway =
4289 FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP | FLAG_CREATE_BREAKAWAY_FROM_JOB;
4290 let without_breakaway = FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP;
4291 let mut last_error: Option<String> = None;
4292 for (idx, shell) in candidates.iter().enumerate() {
4293 for &flags in &[with_breakaway, without_breakaway] {
4297 let stdout = create_capture_file(&paths.stdout)
4299 .map_err(|e| format!("failed to open stdout capture file: {e}"))?;
4300 let stderr = create_capture_file(&paths.stderr)
4301 .map_err(|e| format!("failed to open stderr capture file: {e}"))?;
4302 let mut cmd =
4303 detached_shell_command_for(shell.clone(), command, &paths.exit, paths, flags)?;
4304 cmd.current_dir(workdir)
4305 .envs(env)
4306 .stdin(Stdio::null())
4307 .stdout(Stdio::from(stdout))
4308 .stderr(Stdio::from(stderr));
4309 match cmd.spawn() {
4310 Ok(child) => {
4311 if idx > 0 {
4312 crate::slog_warn!("background bash spawn fell back to {} after {} earlier candidate(s) failed; \
4313 the cached PATH probe disagreed with runtime spawn — likely PATH \
4314 inheritance, antivirus / AppLocker / Defender ASR, or sandbox policy.",
4315 shell.binary(),
4316 idx);
4317 }
4318 if flags == without_breakaway {
4319 crate::slog_warn!(
4320 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected \
4321 (likely a restrictive Job Object — CI sandbox or MDM policy). \
4322 Spawned without breakaway; the bg task will be torn down if the \
4323 AFT process group is killed."
4324 );
4325 }
4326 return Ok(child);
4327 }
4328 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
4329 crate::slog_warn!("background bash spawn: {} returned NotFound at runtime — trying next candidate",
4330 shell.binary());
4331 last_error = Some(format!("{}: {e}", shell.binary()));
4332 break;
4335 }
4336 Err(e) if flags == with_breakaway && e.raw_os_error() == Some(5) => {
4337 crate::slog_warn!(
4339 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected with \
4340 Access Denied — retrying {} without breakaway",
4341 shell.binary()
4342 );
4343 last_error = Some(format!("{}: {e}", shell.binary()));
4344 continue;
4345 }
4346 Err(e) => {
4347 return Err(format!(
4348 "failed to spawn background bash command via {}: {e}",
4349 shell.binary()
4350 ));
4351 }
4352 }
4353 }
4354 }
4355 Err(format!(
4356 "failed to spawn background bash command: no Windows shell could be spawned. \
4357 Last error: {}. PATH-probed candidates: {:?}",
4358 last_error.unwrap_or_else(|| "no candidates were attempted".to_string()),
4359 candidates.iter().map(|s| s.binary()).collect::<Vec<_>>()
4360 ))
4361 }
4362}
4363
4364fn random_slug() -> String {
4365 let mut bytes = [0u8; 8];
4373 getrandom::fill(&mut bytes).unwrap_or_else(|_| {
4375 let t = SystemTime::now()
4377 .duration_since(UNIX_EPOCH)
4378 .map(|d| d.as_nanos() as u64)
4379 .unwrap_or(0);
4380 let p = u64::from(std::process::id());
4381 bytes.copy_from_slice(&(t ^ p.rotate_left(32)).to_le_bytes());
4382 });
4383 let hex: String = bytes.iter().map(|b| format!("{b:02x}")).collect();
4385 format!("bash-{hex}")
4386}
4387
4388#[cfg(test)]
4389mod tests {
4390 use std::collections::HashMap;
4391 use std::fs;
4392 use std::sync::atomic::{AtomicBool, AtomicUsize};
4393 use std::sync::{Arc, Mutex};
4394 use std::time::Duration;
4395 #[cfg(windows)]
4396 use std::time::Instant;
4397
4398 use super::*;
4399
4400 #[cfg(unix)]
4401 const QUICK_SUCCESS_COMMAND: &str = "true";
4402 #[cfg(windows)]
4403 const QUICK_SUCCESS_COMMAND: &str = "cmd /c exit 0";
4404
4405 #[cfg(unix)]
4406 const LONG_RUNNING_COMMAND: &str = "sleep 5";
4407 #[cfg(windows)]
4408 const LONG_RUNNING_COMMAND: &str = "cmd /c timeout /t 5 /nobreak > nul";
4409
4410 #[test]
4411 fn bash_memory_estimate_is_zero_when_empty_and_nonzero_for_completion_cache() {
4412 let registry = BgTaskRegistry::default();
4413 assert_eq!(registry.estimated_memory().estimated_bytes, Some(0));
4414 registry
4415 .inner
4416 .completions
4417 .lock()
4418 .unwrap()
4419 .push_back(BgCompletion {
4420 task_id: "bash-memory".to_string(),
4421 session_id: "session-memory".to_string(),
4422 status: BgTaskStatus::Completed,
4423 exit_code: Some(0),
4424 command: "printf memory".to_string(),
4425 output_preview: "resident completion output".to_string(),
4426 output_truncated: false,
4427 original_tokens: None,
4428 compressed_tokens: None,
4429 tokens_skipped: false,
4430 });
4431 let estimate = registry.estimated_memory();
4432 assert!(estimate.estimated_bytes.unwrap() > 0);
4433 assert_eq!(estimate.counts["completion_caches"], 1);
4434 assert_eq!(estimate.counts["sessions"], 1);
4435 }
4436
4437 #[test]
4438 fn gh_structured_detection_rejects_piped_commands() {
4439 assert!(is_gh_structured_command(
4440 "gh issue list --json number,title"
4441 ));
4442 assert!(is_gh_structured_command(
4443 "cd repo && gh issue list --json number,title"
4444 ));
4445
4446 assert!(!is_gh_structured_command(
4447 "gh issue list --json number,title | jq '.[]'"
4448 ));
4449 assert!(!is_gh_structured_command(
4450 "gh issue list --json number,title |"
4451 ));
4452 }
4453
4454 fn insert_terminal_piped_task(
4455 registry: &BgTaskRegistry,
4456 dir: &tempfile::TempDir,
4457 command: &str,
4458 stdout: &str,
4459 stderr: &str,
4460 compressed: bool,
4461 ) -> (String, Arc<BgTask>) {
4462 let task_id = format!("bash-test-{}", random_slug());
4463 let paths = task_paths(dir.path(), "session", &task_id);
4464 fs::create_dir_all(&paths.dir).unwrap();
4465 fs::write(&paths.stdout, stdout).unwrap();
4466 fs::write(&paths.stderr, stderr).unwrap();
4467 let mut metadata = PersistedTask::starting(
4468 task_id.clone(),
4469 "session".to_string(),
4470 command.to_string(),
4471 dir.path().to_path_buf(),
4472 Some(dir.path().to_path_buf()),
4473 Some(30_000),
4474 true,
4475 compressed,
4476 );
4477 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
4478 write_task(&paths.json, &metadata).unwrap();
4479 registry
4480 .insert_rehydrated_task(metadata, paths, true)
4481 .expect("insert terminal task");
4482 let task = registry.task_for_session(&task_id, "session").unwrap();
4483 (task_id, task)
4484 }
4485
4486 #[test]
4487 fn bash_zero_preview_running_status_skips_output_read_while_explicit_preview_reads() {
4488 let registry = BgTaskRegistry::default();
4489 let dir = tempfile::tempdir().unwrap();
4490 let task_id = format!("bash-test-{}", random_slug());
4491 let paths = task_paths(dir.path(), "session", &task_id);
4492 fs::create_dir_all(&paths.dir).unwrap();
4493 fs::write(&paths.stdout, "live output\n").unwrap();
4494 fs::write(&paths.stderr, "").unwrap();
4495 let stdout_path = paths.stdout.clone();
4496 let mut metadata = PersistedTask::starting(
4497 task_id.clone(),
4498 "session".to_string(),
4499 "sleep 60".to_string(),
4500 dir.path().to_path_buf(),
4501 Some(dir.path().to_path_buf()),
4502 Some(30_000),
4503 true,
4504 false,
4505 );
4506 metadata.status = BgTaskStatus::Running;
4507 write_task(&paths.json, &metadata).unwrap();
4508 registry
4509 .insert_rehydrated_task(metadata, paths, false)
4510 .expect("insert running task");
4511
4512 crate::bash_background::buffer::reset_tail_read_count(&stdout_path);
4513 for _ in 0..5 {
4514 let snapshot = registry
4515 .status(&task_id, "session", Some(dir.path()), Some(dir.path()), 0)
4516 .expect("running snapshot");
4517 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
4518 assert!(snapshot.output_preview.is_empty());
4519 }
4520 assert_eq!(
4521 crate::bash_background::buffer::tail_read_count(&stdout_path),
4522 0
4523 );
4524
4525 let snapshot = registry
4526 .status(
4527 &task_id,
4528 "session",
4529 Some(dir.path()),
4530 Some(dir.path()),
4531 RUNNING_OUTPUT_PREVIEW_BYTES,
4532 )
4533 .expect("explicit running snapshot");
4534 assert_eq!(snapshot.output_preview, "live output\n");
4535 assert_eq!(
4536 crate::bash_background::buffer::tail_read_count(&stdout_path),
4537 1
4538 );
4539 }
4540
4541 #[test]
4542 fn artifact_read_capability_requires_exact_canonical_path_and_session() {
4543 let registry = BgTaskRegistry::default();
4544 let dir = tempfile::tempdir().unwrap();
4545 let (_task_id, task) = insert_terminal_piped_task(
4546 ®istry,
4547 &dir,
4548 "printf output",
4549 "stdout\n",
4550 "stderr\n",
4551 true,
4552 );
4553 fs::write(&task.paths.exit, "0\n").unwrap();
4554
4555 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stdout));
4556 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stderr));
4557 assert!(registry.is_session_owned_artifact_path("session", &task.paths.exit));
4558 assert!(!registry.is_session_owned_artifact_path("different-session", &task.paths.stdout));
4559 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.json));
4560
4561 let unregistered = task.paths.dir.join("unregistered-output");
4562 fs::write(&unregistered, "not a task artifact\n").unwrap();
4563 assert!(!registry.is_session_owned_artifact_path("session", &unregistered));
4564 }
4565
4566 #[cfg(unix)]
4567 #[test]
4568 fn artifact_directory_symlink_does_not_create_a_prefix_exception() {
4569 let registry = BgTaskRegistry::default();
4570 let dir = tempfile::tempdir().unwrap();
4571 let project = dir.path().join("project");
4572 fs::create_dir_all(&project).unwrap();
4573 let (_task_id, task) =
4574 insert_terminal_piped_task(®istry, &dir, "printf output", "stdout\n", "", true);
4575 let link = project.join("task-artifacts");
4576 std::os::unix::fs::symlink(&task.paths.dir, &link).unwrap();
4577 let unregistered = task.paths.dir.join("unregistered-output");
4578 fs::write(&unregistered, "not registered\n").unwrap();
4579
4580 assert!(!registry.is_session_owned_artifact_path("session", &link));
4581 assert!(
4582 !registry.is_session_owned_artifact_path("session", &link.join("unregistered-output"))
4583 );
4584 assert!(registry.is_session_owned_artifact_path(
4585 "session",
4586 &link.join(task.paths.stdout.file_name().unwrap())
4587 ));
4588
4589 let outside = dir.path().join("outside-secret");
4590 fs::write(&outside, "must stay private\n").unwrap();
4591 fs::remove_file(&task.paths.stdout).unwrap();
4592 std::os::unix::fs::symlink(&outside, &task.paths.stdout).unwrap();
4593 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.stdout));
4594 }
4595
4596 #[test]
4597 fn recovery_footer_uses_bash_status_when_artifact_is_not_registered() {
4598 let registry = BgTaskRegistry::default();
4599 let dir = tempfile::tempdir().unwrap();
4600 let task_id = "bash-unregistered-footer";
4601 let paths = task_paths(dir.path(), "session", task_id);
4602 fs::create_dir_all(&paths.dir).unwrap();
4603 fs::write(
4604 &paths.stdout,
4605 format!("{}tail\n", "output-line\n".repeat(2_000)),
4606 )
4607 .unwrap();
4608 fs::write(&paths.stderr, "").unwrap();
4609 let mut metadata = PersistedTask::starting(
4610 task_id.to_string(),
4611 "session".to_string(),
4612 "printf output".to_string(),
4613 dir.path().to_path_buf(),
4614 Some(dir.path().to_path_buf()),
4615 Some(30_000),
4616 true,
4617 true,
4618 );
4619 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
4620
4621 let cache = registry
4622 .render_terminal_output_from_paths(&metadata, &paths)
4623 .expect("terminal render");
4624
4625 assert!(cache
4626 .output_preview
4627 .contains("use bash_status({taskId: \"bash-unregistered-footer\"})"));
4628 assert!(!cache.output_preview.contains("full output: read "));
4629 }
4630
4631 fn insert_terminal_pty_task(
4632 registry: &BgTaskRegistry,
4633 dir: &tempfile::TempDir,
4634 pty_output: &str,
4635 ) -> (String, Arc<BgTask>) {
4636 let task_id = format!("bash-test-{}", random_slug());
4637 let paths = task_paths(dir.path(), "session", &task_id);
4638 fs::create_dir_all(&paths.dir).unwrap();
4639 fs::write(&paths.pty, pty_output).unwrap();
4640 let mut metadata = PersistedTask::starting(
4641 task_id.clone(),
4642 "session".to_string(),
4643 "python".to_string(),
4644 dir.path().to_path_buf(),
4645 Some(dir.path().to_path_buf()),
4646 Some(30_000),
4647 true,
4648 true,
4649 );
4650 metadata.mode = BgMode::Pty;
4651 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
4652 write_task(&paths.json, &metadata).unwrap();
4653 registry
4654 .insert_rehydrated_task(metadata, paths, true)
4655 .expect("insert terminal pty task");
4656 let task = registry.task_for_session(&task_id, "session").unwrap();
4657 (task_id, task)
4658 }
4659
4660 #[cfg(unix)]
4661 fn wait_for_terminal_snapshot(
4662 registry: &BgTaskRegistry,
4663 task_id: &str,
4664 session_id: &str,
4665 project: &Path,
4666 storage: &Path,
4667 ) -> BgTaskSnapshot {
4668 let started = Instant::now();
4669 loop {
4670 let snapshot = registry
4671 .status(task_id, session_id, Some(project), Some(storage), 4096)
4672 .expect("spawned task should be visible to status");
4673 if snapshot.info.status.is_terminal() {
4674 return snapshot;
4675 }
4676 assert!(
4677 started.elapsed() < Duration::from_secs(10),
4678 "timed out waiting for task {task_id} to finish; last status={:?}",
4679 snapshot.info.status
4680 );
4681 std::thread::sleep(Duration::from_millis(50));
4682 }
4683 }
4684
4685 fn write_running_project_task(storage: &Path, project: &Path, session: &str, task_id: &str) {
4686 let paths = task_paths(storage, session, task_id);
4687 let mut metadata = PersistedTask::starting(
4688 task_id.to_string(),
4689 session.to_string(),
4690 "sleep 60".to_string(),
4691 project.to_path_buf(),
4692 Some(project.to_path_buf()),
4693 Some(30_000),
4694 true,
4695 true,
4696 );
4697 metadata.status = BgTaskStatus::Running;
4698 write_task(&paths.json, &metadata).unwrap();
4699 fs::write(&paths.stdout, "still running\n").unwrap();
4700 fs::write(&paths.stderr, "").unwrap();
4701 }
4702
4703 #[test]
4704 fn status_replay_filters_same_session_by_project_root() {
4705 let project_a = tempfile::tempdir().unwrap();
4706 let project_b = tempfile::tempdir().unwrap();
4707 let storage = tempfile::tempdir().unwrap();
4708 let session = "shared-session";
4709 let task_id = "bash-project-a";
4710 write_running_project_task(storage.path(), project_a.path(), session, task_id);
4711
4712 let actor_b = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
4713 assert!(actor_b
4714 .status(
4715 task_id,
4716 session,
4717 Some(project_b.path()),
4718 Some(storage.path()),
4719 1024,
4720 )
4721 .is_none());
4722 assert!(actor_b.task_for_session(task_id, session).is_none());
4723
4724 let actor_a = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
4725 let snapshot = actor_a
4726 .status(
4727 task_id,
4728 session,
4729 Some(project_a.path()),
4730 Some(storage.path()),
4731 1024,
4732 )
4733 .expect("owning project should replay its task");
4734 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
4735 }
4736
4737 #[cfg(unix)]
4738 #[test]
4739 fn multiline_pipeline_stdout_persists_all_lines_after_terminal_status() {
4740 let cases = [
4741 (
4742 "long-first",
4743 "sleep 0.5; printf 'one\\n' | cat\nprintf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
4744 vec!["one", "1", "three"],
4745 ),
4746 (
4747 "short-first",
4748 "printf 'one\\n' | cat\nsleep 0.2; printf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
4749 vec!["one", "1", "three"],
4750 ),
4751 (
4752 "failing-middle",
4753 "sleep 0.2; printf 'one\\n' | cat\nfalse; printf 'after-false\\n' | cat\nprintf 'three\\n' | cat",
4754 vec!["one", "after-false", "three"],
4755 ),
4756 ];
4757
4758 for (name, command, expected_lines) in cases {
4759 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
4760 let dir = tempfile::tempdir().unwrap();
4761 let session_id = format!("session-{name}");
4762 let task_id = registry
4763 .spawn(
4764 SpawnPlan::Unsandboxed,
4765 command,
4766 session_id.clone(),
4767 dir.path().to_path_buf(),
4768 HashMap::new(),
4769 Some(Duration::from_secs(30)),
4770 dir.path().to_path_buf(),
4771 10,
4772 true,
4773 true,
4774 Some(dir.path().to_path_buf()),
4775 )
4776 .unwrap();
4777
4778 let snapshot = wait_for_terminal_snapshot(
4779 ®istry,
4780 &task_id,
4781 &session_id,
4782 dir.path(),
4783 dir.path(),
4784 );
4785 assert_eq!(
4786 snapshot.info.status,
4787 BgTaskStatus::Completed,
4788 "{name}: task should complete; snapshot={snapshot:?}"
4789 );
4790 assert_eq!(
4791 snapshot.exit_code,
4792 Some(0),
4793 "{name}: script should use the final command's exit code"
4794 );
4795
4796 let stdout_path = task_paths(dir.path(), &session_id, &task_id).stdout;
4797 let stdout = fs::read_to_string(&stdout_path).unwrap_or_else(|error| {
4798 panic!(
4799 "{name}: failed to read raw stdout file {}: {error}",
4800 stdout_path.display()
4801 )
4802 });
4803 let lines: Vec<&str> = stdout.lines().collect();
4804 assert_eq!(
4805 lines,
4806 expected_lines,
4807 "{name}: raw stdout file must include every newline-separated command's output; stdout_path={}",
4808 stdout_path.display()
4809 );
4810 }
4811 }
4812
4813 #[test]
4814 fn recognizes_all_recovery_marker_forms() {
4815 assert!(is_recovery_marker(
4816 "[truncated output; full output: read \"/tmp/out\"]"
4817 ));
4818 assert!(is_recovery_marker(
4819 "[omitted output; see remaining: tail -n +42 \"/tmp/out\"]"
4820 ));
4821 assert!(is_recovery_marker(
4822 "[truncated output; full output unavailable]"
4823 ));
4824 assert!(is_recovery_marker(
4825 r#"[truncated 123 bytes from saved output prefix; retained output: read "/tmp/out"]"#
4826 ));
4827 }
4828
4829 #[test]
4830 fn recovery_marker_reports_disk_prefix_truncation_as_retained_output() {
4831 let recovery = RecoveryContext {
4832 dropped_by_class: BTreeMap::new(),
4833 had_inner_drop: false,
4834 offset_hint_eligible: false,
4835 offset_start_line: None,
4836 byte_truncated: false,
4837 disk_truncated_prefix_bytes: 4096,
4838 output_path: Some("/tmp/stdout".to_string()),
4839 stderr_path: None,
4840 include_stderr_path: false,
4841 artifact_access: ArtifactRecoveryAccess {
4842 task_id: "bash-test".to_string(),
4843 readable: true,
4844 },
4845 };
4846
4847 let marker = recovery_marker(&recovery).expect("disk truncation must emit marker");
4848
4849 assert!(marker.contains("truncated 4096 bytes from saved output prefix"));
4850 assert!(marker.contains(r#"retained output: read "/tmp/stdout""#));
4851 assert!(!marker.contains("full output: read"));
4852 }
4853
4854 #[test]
4855 fn killed_exit_marker_sets_nonzero_sentinel_exit_code() {
4856 let metadata = PersistedTask::starting(
4857 "task".to_string(),
4858 "session".to_string(),
4859 "cargo test".to_string(),
4860 PathBuf::from("/tmp"),
4861 None,
4862 None,
4863 true,
4864 true,
4865 );
4866
4867 let terminal = terminal_metadata_from_marker(metadata, ExitMarker::Killed, None);
4868
4869 assert_eq!(terminal.status, BgTaskStatus::Killed);
4870 assert_eq!(terminal.exit_code, Some(137));
4871 }
4872
4873 #[test]
4874 fn terminal_status_polls_use_cached_render_once_and_off_lock() {
4875 let registry = BgTaskRegistry::default();
4876 let dir = tempfile::tempdir().unwrap();
4877 let (_task_id, task) = insert_terminal_piped_task(
4878 ®istry,
4879 &dir,
4880 "custom-tool --verbose",
4881 &"stdout line\n".repeat(200_000),
4882 "",
4883 true,
4884 );
4885 let calls = Arc::new(AtomicUsize::new(0));
4886 let saw_unlocked_state = Arc::new(AtomicBool::new(false));
4887 let task_holder = Arc::new(Mutex::new(Some(Arc::clone(&task))));
4888 let calls_for_closure = Arc::clone(&calls);
4889 let unlocked_for_closure = Arc::clone(&saw_unlocked_state);
4890 let task_for_closure = Arc::clone(&task_holder);
4891 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
4892 calls_for_closure.fetch_add(1, Ordering::SeqCst);
4893 if let Some(task) = task_for_closure.lock().unwrap().as_ref() {
4894 if task.state.try_lock().is_ok() {
4895 unlocked_for_closure.store(true, Ordering::SeqCst);
4896 }
4897 }
4898 CompressionResult::new(format!("compressed {} bytes", output.len()))
4899 });
4900
4901 let first = registry
4902 .status(
4903 &task.task_id,
4904 "session",
4905 None,
4906 Some(dir.path()),
4907 RUNNING_OUTPUT_PREVIEW_BYTES,
4908 )
4909 .unwrap();
4910 let second = registry
4911 .status(
4912 &task.task_id,
4913 "session",
4914 None,
4915 Some(dir.path()),
4916 RUNNING_OUTPUT_PREVIEW_BYTES,
4917 )
4918 .unwrap();
4919 let listed = registry.list(RUNNING_OUTPUT_PREVIEW_BYTES);
4920
4921 assert_eq!(
4922 calls.load(Ordering::SeqCst),
4923 1,
4924 "terminal render must be cached"
4925 );
4926 assert!(
4927 saw_unlocked_state.load(Ordering::SeqCst),
4928 "compressor must run after releasing the task state lock"
4929 );
4930 assert!(first.output_preview.starts_with("compressed "));
4931 assert_eq!(second.output_preview, first.output_preview);
4932 assert_eq!(listed[0].output_preview, first.output_preview);
4933 }
4934
4935 #[test]
4936 fn completion_preview_success_keeps_tail_only() {
4937 let registry = BgTaskRegistry::default();
4942 let dir = tempfile::tempdir().unwrap();
4943 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
4944 let (_task_id, task) =
4945 insert_terminal_piped_task(®istry, &dir, "cat big.log", &output, "", false);
4946
4947 registry.post_terminal_transition(&task, true).unwrap();
4948 let completions = registry.drain_completions_for_session(Some("session"));
4949 assert_eq!(completions.len(), 1);
4950 let preview = &completions[0].output_preview;
4951 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
4952 assert!(!preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
4953 assert!(completions[0].output_truncated);
4954 }
4955
4956 #[test]
4957 fn completion_preview_failure_keeps_head_and_tail() {
4958 let registry = BgTaskRegistry::default();
4961 let dir = tempfile::tempdir().unwrap();
4962 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
4963 let task_id = format!("bash-test-{}", random_slug());
4964 let paths = task_paths(dir.path(), "session", &task_id);
4965 fs::create_dir_all(&paths.dir).unwrap();
4966 fs::write(&paths.stdout, &output).unwrap();
4967 fs::write(&paths.stderr, "").unwrap();
4968 let mut metadata = PersistedTask::starting(
4969 task_id.clone(),
4970 "session".to_string(),
4971 "cat big.log".to_string(),
4972 dir.path().to_path_buf(),
4973 Some(dir.path().to_path_buf()),
4974 Some(30_000),
4975 true,
4976 false,
4977 );
4978 metadata.mark_terminal(BgTaskStatus::Failed, Some(1), None);
4979 write_task(&paths.json, &metadata).unwrap();
4980 registry
4981 .insert_rehydrated_task(metadata, paths, true)
4982 .expect("insert terminal task");
4983 let task = registry.task_for_session(&task_id, "session").unwrap();
4984
4985 registry.post_terminal_transition(&task, true).unwrap();
4986 let completions = registry.drain_completions_for_session(Some("session"));
4987 assert_eq!(completions.len(), 1);
4988 let preview = &completions[0].output_preview;
4989 assert!(preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
4990 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
4991 }
4992
4993 #[test]
4994 fn has_completions_for_session_matches_pending_delivery() {
4995 let registry = BgTaskRegistry::default();
4996 assert!(!registry.has_completions_for_session(Some("session")));
4997 assert!(!registry.has_completions_for_session(None));
4998
4999 let dir = tempfile::tempdir().unwrap();
5000 let (_task_id, task) =
5001 insert_terminal_piped_task(®istry, &dir, QUICK_SUCCESS_COMMAND, "done\n", "", false);
5002 registry.post_terminal_transition(&task, true).unwrap();
5003
5004 assert!(registry.has_completions_for_session(Some("session")));
5005 assert!(registry.has_completions_for_session(None));
5006 assert!(!registry.has_completions_for_session(Some("other-session")));
5007
5008 let completions = registry.drain_completions_for_session(Some("session"));
5009 assert_eq!(completions.len(), 1);
5010 assert_eq!(completions[0].task_id, task.task_id);
5011 }
5012
5013 #[test]
5014 fn structured_gh_json_survives_intact_and_ignores_stderr() {
5015 let registry = BgTaskRegistry::default();
5016 let dir = tempfile::tempdir().unwrap();
5017 let calls = Arc::new(AtomicUsize::new(0));
5018 let calls_for_closure = Arc::clone(&calls);
5019 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
5020 calls_for_closure.fetch_add(1, Ordering::SeqCst);
5021 CompressionResult::new(output)
5022 });
5023 let (task_id, _task) = insert_terminal_piped_task(
5024 ®istry,
5025 &dir,
5026 "gh pr view 123 --json body",
5027 "{\"body\":\"hello\"}",
5028 "warning: stderr must not join json",
5029 true,
5030 );
5031
5032 let snapshot = registry
5033 .status(
5034 &task_id,
5035 "session",
5036 None,
5037 Some(dir.path()),
5038 RUNNING_OUTPUT_PREVIEW_BYTES,
5039 )
5040 .unwrap();
5041
5042 assert_eq!(snapshot.output_preview, "{\"body\":\"hello\"}");
5043 assert!(!snapshot.output_preview.contains("warning"));
5044 assert!(!snapshot.output_truncated);
5045 assert_eq!(
5046 calls.load(Ordering::SeqCst),
5047 0,
5048 "structured JSON bypasses compression"
5049 );
5050 }
5051
5052 #[test]
5053 fn registry_emits_single_recovery_marker_for_class_drops() {
5054 let registry = BgTaskRegistry::default();
5055 let dir = tempfile::tempdir().unwrap();
5056 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
5057 let mut dropped = BTreeMap::new();
5058 dropped.insert(DropClass::Error, 18);
5059 dropped.insert(DropClass::Warning, 6);
5060 CompressionResult::with_class_drops("kept diagnostic", dropped)
5061 });
5062 let (task_id, task) =
5063 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
5064
5065 let snapshot = registry
5066 .status(
5067 &task_id,
5068 "session",
5069 None,
5070 Some(dir.path()),
5071 RUNNING_OUTPUT_PREVIEW_BYTES,
5072 )
5073 .unwrap();
5074
5075 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
5076 assert!(snapshot.output_preview.contains("+18 more errors"));
5077 assert!(snapshot.output_preview.contains("+6 more warnings"));
5078 assert!(snapshot
5079 .output_preview
5080 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
5081 assert!(!snapshot.output_preview.contains("tail -n +"));
5082 assert!(snapshot.output_truncated);
5083 }
5084
5085 #[test]
5086 fn registry_marker_reports_semantic_and_byte_drops_once() {
5087 let registry = BgTaskRegistry::default();
5088 let dir = tempfile::tempdir().unwrap();
5089 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
5090 let mut dropped = BTreeMap::new();
5091 dropped.insert(DropClass::Error, 1);
5092 CompressionResult::with_class_drops(
5093 format!("HEAD-SIGNAL\n{}TAIL-SIGNAL", "middle\n".repeat(8_000)),
5094 dropped,
5095 )
5096 });
5097 let (task_id, _task) =
5098 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
5099
5100 let snapshot = registry
5101 .status(
5102 &task_id,
5103 "session",
5104 None,
5105 Some(dir.path()),
5106 RUNNING_OUTPUT_PREVIEW_BYTES,
5107 )
5108 .unwrap();
5109
5110 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
5111 assert!(snapshot.output_preview.contains("+1 more error"));
5112 assert!(snapshot.output_preview.contains("truncated output"));
5113 assert!(snapshot.output_preview.contains("HEAD-SIGNAL"));
5114 assert!(snapshot.output_preview.contains("TAIL-SIGNAL"));
5115 assert!(!snapshot.output_preview.contains("...<truncated"));
5116 assert!(snapshot.output_truncated);
5117 }
5118
5119 #[test]
5120 fn cargo_stderr_class_drops_name_both_capture_paths() {
5121 let registry = BgTaskRegistry::default();
5122 let dir = tempfile::tempdir().unwrap();
5123 let filter_registry = crate::compress::toml_filter::FilterRegistry::default();
5124 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
5125 crate::compress::compress_with_registry_exit_code(
5126 command,
5127 &output,
5128 exit_code,
5129 &filter_registry,
5130 )
5131 });
5132 let stderr = (0..22)
5133 .map(|index| {
5134 format!(
5135 "error: cargo failure {index}\n --> src/lib.rs:{}:1\n |\n{} | boom\n",
5136 index + 1,
5137 index + 1
5138 )
5139 })
5140 .collect::<Vec<_>>()
5141 .join("\n");
5142 let (task_id, task) = insert_terminal_piped_task(
5143 ®istry,
5144 &dir,
5145 "cargo check",
5146 "Finished dev [unoptimized] target(s) in 0.01s\n",
5147 &stderr,
5148 true,
5149 );
5150
5151 let snapshot = registry
5152 .status(
5153 &task_id,
5154 "session",
5155 None,
5156 Some(dir.path()),
5157 RUNNING_OUTPUT_PREVIEW_BYTES,
5158 )
5159 .unwrap();
5160
5161 assert!(snapshot.output_preview.contains("+2 more errors"));
5162 assert!(snapshot
5163 .output_preview
5164 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
5165 assert!(snapshot
5166 .output_preview
5167 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
5168 assert!(!snapshot.output_preview.contains("tail -n +"));
5169 }
5170
5171 #[test]
5172 fn over_ceiling_structured_json_uses_pointer_not_partial_json() {
5173 let registry = BgTaskRegistry::default();
5174 let dir = tempfile::tempdir().unwrap();
5175 let body = format!("{{\"body\":\"{}\"}}", "x".repeat(60 * 1024));
5176 let (task_id, task) = insert_terminal_piped_task(
5177 ®istry,
5178 &dir,
5179 "cd /repo && gh pr view 123 --json body",
5180 &body,
5181 "",
5182 true,
5183 );
5184
5185 let snapshot = registry
5186 .status(
5187 &task_id,
5188 "session",
5189 None,
5190 Some(dir.path()),
5191 RUNNING_OUTPUT_PREVIEW_BYTES,
5192 )
5193 .unwrap();
5194
5195 assert!(snapshot.output_preview.starts_with("[JSON output "));
5196 assert!(snapshot
5197 .output_preview
5198 .contains(&task.paths.stdout.display().to_string()));
5199 assert!(!snapshot.output_preview.contains(&"x".repeat(1024)));
5200 assert!(snapshot.output_truncated);
5201 }
5202
5203 #[test]
5204 fn toml_strip_tail_cap_uses_full_output_hint_not_offset_hint() {
5205 let registry = BgTaskRegistry::default();
5206 let dir = tempfile::tempdir().unwrap();
5207 let filter_registry = crate::compress::toml_filter::build_registry(
5208 crate::compress::builtin_filters::ALL,
5209 None,
5210 None,
5211 );
5212 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
5213 crate::compress::compress_with_registry_exit_code(
5214 command,
5215 &output,
5216 exit_code,
5217 &filter_registry,
5218 )
5219 });
5220 let stdout = format!(
5221 "make[1]: Entering directory `/tmp`\n{}",
5222 (0..100)
5223 .map(|index| format!("compile line {index}"))
5224 .collect::<Vec<_>>()
5225 .join("\n")
5226 );
5227 let (task_id, task) =
5228 insert_terminal_piped_task(®istry, &dir, "make all", &stdout, "", true);
5229
5230 let snapshot = registry
5231 .status(
5232 &task_id,
5233 "session",
5234 None,
5235 Some(dir.path()),
5236 RUNNING_OUTPUT_PREVIEW_BYTES,
5237 )
5238 .unwrap();
5239
5240 assert!(snapshot.output_preview.contains("compile line 99"));
5241 assert!(snapshot.output_preview.contains(&format!(
5242 "full output: read \"{}\"",
5243 task.paths.stdout.display()
5244 )));
5245 assert!(!snapshot
5246 .output_preview
5247 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
5248 assert!(!snapshot.output_preview.contains("tail -n +"));
5249 }
5250
5251 #[test]
5252 fn compressed_false_raw_passthrough_uses_wider_head_tail_cap() {
5253 let registry = BgTaskRegistry::default();
5254 let dir = tempfile::tempdir().unwrap();
5255 let output = format!("RAW-HEAD\n{}RAW-TAIL\n", "raw-middle\n".repeat(8_000));
5256 let (task_id, task) =
5257 insert_terminal_piped_task(®istry, &dir, "cat raw.log", &output, "RAW-ERR\n", false);
5258
5259 let snapshot = registry
5260 .status(
5261 &task_id,
5262 "session",
5263 None,
5264 Some(dir.path()),
5265 RUNNING_OUTPUT_PREVIEW_BYTES,
5266 )
5267 .unwrap();
5268
5269 assert!(snapshot.output_preview.contains("RAW-HEAD"));
5270 assert!(snapshot.output_preview.contains("RAW-TAIL"));
5271 assert!(snapshot.output_preview.contains("truncated output"));
5272 assert!(snapshot
5273 .output_preview
5274 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
5275 assert!(snapshot
5276 .output_preview
5277 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
5278 assert!(!snapshot.output_preview.contains("tail -n +"));
5279 assert!(snapshot.output_preview.len() > 16 * 1024);
5280 assert!(snapshot.output_truncated);
5281 }
5282
5283 #[test]
5284 fn pty_terminal_snapshot_bypasses_line_compression() {
5285 let registry = BgTaskRegistry::default();
5286 let dir = tempfile::tempdir().unwrap();
5287 let calls = Arc::new(AtomicUsize::new(0));
5288 let calls_for_closure = Arc::clone(&calls);
5289 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
5290 calls_for_closure.fetch_add(1, Ordering::SeqCst);
5291 CompressionResult::new(output)
5292 });
5293 let (task_id, _task) = insert_terminal_pty_task(®istry, &dir, "raw\u{1b}[31m pty bytes");
5294
5295 let snapshot = registry
5296 .status(
5297 &task_id,
5298 "session",
5299 None,
5300 Some(dir.path()),
5301 RUNNING_OUTPUT_PREVIEW_BYTES,
5302 )
5303 .unwrap();
5304
5305 assert_eq!(snapshot.info.mode, BgMode::Pty);
5306 assert_eq!(snapshot.output_preview, "");
5307 assert_eq!(calls.load(Ordering::SeqCst), 0);
5308 }
5309
5310 #[test]
5311 fn pty_dimensions_are_persisted_and_returned_in_snapshot() {
5312 let registry = BgTaskRegistry::default();
5313 let dir = tempfile::tempdir().unwrap();
5314 let task_id = registry
5315 .spawn_pty(
5316 SpawnPlan::Unsandboxed,
5317 QUICK_SUCCESS_COMMAND,
5318 "session".to_string(),
5319 dir.path().to_path_buf(),
5320 HashMap::new(),
5321 Some(Duration::from_secs(30)),
5322 dir.path().to_path_buf(),
5323 10,
5324 true,
5325 false,
5326 Some(dir.path().to_path_buf()),
5327 50,
5328 120,
5329 )
5330 .unwrap();
5331
5332 let paths = task_paths(dir.path(), "session", &task_id);
5333 let metadata = read_task(&paths.json).unwrap();
5334 assert_eq!(
5335 metadata.schema_version,
5336 crate::bash_background::persistence::SCHEMA_VERSION
5337 );
5338 assert_eq!(metadata.mode, BgMode::Pty);
5339 assert_eq!(metadata.pty_rows, Some(50));
5340 assert_eq!(metadata.pty_cols, Some(120));
5341
5342 let snapshot = registry
5343 .status(&task_id, "session", None, Some(dir.path()), 1024)
5344 .unwrap();
5345 assert_eq!(snapshot.pty_rows, Some(50));
5346 assert_eq!(snapshot.pty_cols, Some(120));
5347 }
5348
5349 fn spawn_dead_child() -> std::process::Child {
5354 #[cfg(unix)]
5355 let mut cmd = std::process::Command::new("true");
5356 #[cfg(windows)]
5357 let mut cmd = {
5358 let mut c = std::process::Command::new("cmd");
5359 c.args(["/c", "exit", "0"]);
5360 c
5361 };
5362 cmd.stdin(std::process::Stdio::null());
5363 cmd.stdout(std::process::Stdio::null());
5364 cmd.stderr(std::process::Stdio::null());
5365 let mut child = cmd.spawn().expect("spawn replacement child for reap test");
5366 let started = Instant::now();
5375 loop {
5376 match child.try_wait() {
5377 Ok(Some(_)) => break,
5378 Ok(None) => {
5379 if started.elapsed() > Duration::from_secs(5) {
5380 panic!("dead-child stand-in did not exit within 5s");
5381 }
5382 std::thread::sleep(Duration::from_millis(10));
5383 }
5384 Err(error) => panic!("dead-child try_wait failed: {error}"),
5385 }
5386 }
5387 child
5388 }
5389
5390 #[test]
5391 fn ack_marks_delivered_even_when_completion_was_already_consumed_locally() {
5392 let registry = BgTaskRegistry::default();
5393 let dir = tempfile::tempdir().unwrap();
5394 let task_id = registry
5395 .spawn(
5396 SpawnPlan::Unsandboxed,
5397 LONG_RUNNING_COMMAND,
5398 "session".to_string(),
5399 dir.path().to_path_buf(),
5400 HashMap::new(),
5401 Some(Duration::from_secs(30)),
5402 dir.path().to_path_buf(),
5403 10,
5404 true,
5405 false,
5406 Some(dir.path().to_path_buf()),
5407 )
5408 .unwrap();
5409 registry
5410 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
5411 .unwrap();
5412 assert_eq!(
5413 registry
5414 .drain_completions_for_session(Some("session"))
5415 .len(),
5416 1
5417 );
5418
5419 registry.inner.completions.lock().unwrap().clear();
5422
5423 assert_eq!(
5424 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
5425 vec![task_id.clone()]
5426 );
5427 assert!(registry
5428 .drain_completions_for_session(Some("session"))
5429 .is_empty());
5430
5431 let paths = task_paths(dir.path(), "session", &task_id);
5432 let metadata = read_task(&paths.json).unwrap();
5433 assert!(metadata.completion_delivered);
5434
5435 let replayed = BgTaskRegistry::default();
5436 replayed
5437 .replay_session_inner(dir.path(), "session", None)
5438 .unwrap();
5439 assert!(replayed
5440 .drain_completions_for_session(Some("session"))
5441 .is_empty());
5442 }
5443
5444 #[test]
5445 fn register_watch_rejects_unknown_task() {
5446 let registry = BgTaskRegistry::default();
5447
5448 let result = registry.register_watch(
5449 "missing-task".to_string(),
5450 WatchPattern::Substring("READY".into()),
5451 true,
5452 );
5453
5454 assert_eq!(result, Err("task_not_found"));
5455 }
5456
5457 #[test]
5458 fn register_watch_on_terminal_task_scans_existing_output() {
5459 let frames = Arc::new(Mutex::new(Vec::new()));
5460 let captured = Arc::clone(&frames);
5461 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
5462 captured.lock().unwrap().push(frame);
5463 })
5464 as Box<dyn Fn(PushFrame) + Send + Sync>);
5465 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
5466 let dir = tempfile::tempdir().unwrap();
5467 let task_id = registry
5468 .spawn(
5469 SpawnPlan::Unsandboxed,
5470 LONG_RUNNING_COMMAND,
5471 "session".to_string(),
5472 dir.path().to_path_buf(),
5473 HashMap::new(),
5474 Some(Duration::from_secs(30)),
5475 dir.path().to_path_buf(),
5476 10,
5477 true,
5478 false,
5479 Some(dir.path().to_path_buf()),
5480 )
5481 .unwrap();
5482 registry
5483 .inner
5484 .shutdown
5485 .store(true, std::sync::atomic::Ordering::SeqCst);
5486 let task = registry.task_for_session(&task_id, "session").unwrap();
5487 std::fs::write(&task.paths.stdout, "READY\n").unwrap();
5488 registry
5489 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
5490 .unwrap();
5491 frames.lock().unwrap().clear();
5492 registry.inner.completions.lock().unwrap().clear();
5493
5494 registry
5495 .register_watch(
5496 task_id.clone(),
5497 WatchPattern::Substring("READY".into()),
5498 true,
5499 )
5500 .unwrap();
5501
5502 let frames = frames.lock().unwrap();
5503 let frame = frames
5504 .iter()
5505 .find_map(|frame| match frame {
5506 PushFrame::BashPatternMatch(frame) => Some(frame),
5507 _ => None,
5508 })
5509 .expect("terminal watch registration should emit pattern frame");
5510 assert_eq!(frame.reason, "pattern_match");
5511 assert_eq!(frame.task_id, task_id);
5512 assert_eq!(frame.session_id, "session");
5513 assert_eq!(frame.match_text, "READY");
5514 assert_eq!(frame.match_offset, 0);
5515 assert_eq!(registry.active_watch_count(&frame.task_id), 0);
5516 let metadata = read_task(&task.paths.json).unwrap();
5517 assert!(metadata.completion_delivered);
5518 }
5519
5520 #[test]
5521 fn cleanup_finished_removes_terminal_tasks_older_than_threshold() {
5522 let registry = BgTaskRegistry::default();
5523 let dir = tempfile::tempdir().unwrap();
5524 let task_id = registry
5525 .spawn(
5526 SpawnPlan::Unsandboxed,
5527 QUICK_SUCCESS_COMMAND,
5528 "session".to_string(),
5529 dir.path().to_path_buf(),
5530 HashMap::new(),
5531 Some(Duration::from_secs(30)),
5532 dir.path().to_path_buf(),
5533 10,
5534 true,
5535 false,
5536 Some(dir.path().to_path_buf()),
5537 )
5538 .unwrap();
5539 registry
5540 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
5541 .unwrap();
5542 let completions = registry.drain_completions_for_session(Some("session"));
5543 assert_eq!(completions.len(), 1);
5544 assert_eq!(
5545 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
5546 vec![task_id.clone()]
5547 );
5548
5549 registry.cleanup_finished(Duration::ZERO);
5550
5551 assert!(registry.inner.tasks.lock().unwrap().is_empty());
5552 }
5553
5554 #[test]
5555 fn cleanup_finished_retains_undelivered_terminals() {
5556 let registry = BgTaskRegistry::default();
5557 let dir = tempfile::tempdir().unwrap();
5558 let task_id = registry
5559 .spawn(
5560 SpawnPlan::Unsandboxed,
5561 QUICK_SUCCESS_COMMAND,
5562 "session".to_string(),
5563 dir.path().to_path_buf(),
5564 HashMap::new(),
5565 Some(Duration::from_secs(30)),
5566 dir.path().to_path_buf(),
5567 10,
5568 true,
5569 false,
5570 Some(dir.path().to_path_buf()),
5571 )
5572 .unwrap();
5573 registry
5574 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
5575 .unwrap();
5576
5577 registry.cleanup_finished(Duration::ZERO);
5578
5579 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
5580 }
5581
5582 #[test]
5590 fn reap_child_marks_failed_when_child_exits_without_exit_marker() {
5591 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5592 let dir = tempfile::tempdir().unwrap();
5593 let task_id = registry
5594 .spawn(
5595 SpawnPlan::Unsandboxed,
5596 QUICK_SUCCESS_COMMAND,
5597 "session".to_string(),
5598 dir.path().to_path_buf(),
5599 HashMap::new(),
5600 Some(Duration::from_secs(30)),
5601 dir.path().to_path_buf(),
5602 10,
5603 true,
5604 false,
5605 Some(dir.path().to_path_buf()),
5606 )
5607 .unwrap();
5608
5609 let task = registry.task_for_session(&task_id, "session").unwrap();
5610
5611 let started = Instant::now();
5616 loop {
5617 let exited = {
5618 let mut state = task.state.lock().unwrap();
5619 match &mut state.runtime {
5620 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
5621 _ => true,
5622 }
5623 };
5624 if exited {
5625 break;
5626 }
5627 assert!(
5628 started.elapsed() < Duration::from_secs(5),
5629 "child should exit quickly"
5630 );
5631 std::thread::sleep(Duration::from_millis(20));
5632 }
5633
5634 registry
5642 .inner
5643 .shutdown
5644 .store(true, std::sync::atomic::Ordering::SeqCst);
5645 std::thread::sleep(Duration::from_millis(550));
5649
5650 let _ = std::fs::remove_file(&task.paths.exit);
5653
5654 {
5669 let mut state = task.state.lock().unwrap();
5670 state.metadata.status = BgTaskStatus::Running;
5671 state.metadata.status_reason = None;
5672 state.metadata.exit_code = None;
5673 state.metadata.finished_at = None;
5674 state.metadata.duration_ms = None;
5675 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
5678 .expect("persist reset Running metadata for reap_child test");
5679 if matches!(state.runtime, TaskRuntime::Piped(None)) {
5683 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
5684 }
5685 }
5686 *task.terminal_at.lock().unwrap() = None;
5689
5690 assert!(
5693 task.is_running(),
5694 "precondition: metadata.status == Running"
5695 );
5696 assert!(
5697 !task.paths.exit.exists(),
5698 "precondition: exit marker absent"
5699 );
5700
5701 registry.reap_child(&task);
5706
5707 {
5708 let state = task.state.lock().unwrap();
5709 assert_eq!(
5710 state.metadata.status,
5711 BgTaskStatus::Running,
5712 "first reap must leave status Running while waiting one pass for marker"
5713 );
5714 assert_eq!(
5715 state.metadata.status_reason, None,
5716 "first reap must not record a failure reason"
5717 );
5718 assert!(
5719 matches!(state.runtime, TaskRuntime::Piped(None)),
5720 "child handle must be released after first reap"
5721 );
5722 assert!(
5723 state.detached,
5724 "task must be marked detached after first reap"
5725 );
5726 }
5727
5728 registry.reap_child(&task);
5732
5733 let state = task.state.lock().unwrap();
5734 assert!(
5735 state.metadata.status.is_terminal(),
5736 "second reap must transition to terminal when PID dead and no marker. Got status={:?}",
5737 state.metadata.status
5738 );
5739 assert_eq!(
5740 state.metadata.status,
5741 BgTaskStatus::Failed,
5742 "must specifically be Failed (not Killed): status={:?}",
5743 state.metadata.status
5744 );
5745 assert_eq!(
5746 state.metadata.status_reason.as_deref(),
5747 Some("process exited without exit marker"),
5748 "reason must match replay path's wording: {:?}",
5749 state.metadata.status_reason
5750 );
5751 assert!(
5752 matches!(state.runtime, TaskRuntime::Piped(None)),
5753 "child handle must stay released after second reap"
5754 );
5755 assert!(
5756 state.detached,
5757 "task must remain detached after second reap"
5758 );
5759 }
5760
5761 #[test]
5766 fn reap_child_preserves_running_when_exit_marker_exists() {
5767 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5768 let dir = tempfile::tempdir().unwrap();
5769 let task_id = registry
5770 .spawn(
5771 SpawnPlan::Unsandboxed,
5772 QUICK_SUCCESS_COMMAND,
5773 "session".to_string(),
5774 dir.path().to_path_buf(),
5775 HashMap::new(),
5776 Some(Duration::from_secs(30)),
5777 dir.path().to_path_buf(),
5778 10,
5779 true,
5780 false,
5781 Some(dir.path().to_path_buf()),
5782 )
5783 .unwrap();
5784
5785 let task = registry.task_for_session(&task_id, "session").unwrap();
5786
5787 let started = Instant::now();
5790 loop {
5791 let exited = {
5792 let mut state = task.state.lock().unwrap();
5793 match &mut state.runtime {
5794 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
5795 _ => true,
5796 }
5797 };
5798 if exited && task.paths.exit.exists() {
5799 break;
5800 }
5801 assert!(
5802 started.elapsed() < Duration::from_secs(5),
5803 "child should exit and write marker quickly"
5804 );
5805 std::thread::sleep(Duration::from_millis(20));
5806 }
5807
5808 registry
5814 .inner
5815 .shutdown
5816 .store(true, std::sync::atomic::Ordering::SeqCst);
5817 std::thread::sleep(Duration::from_millis(550));
5818
5819 {
5825 let mut state = task.state.lock().unwrap();
5826 state.metadata.status = BgTaskStatus::Running;
5827 state.metadata.status_reason = None;
5828 if matches!(state.runtime, TaskRuntime::Piped(None)) {
5829 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
5830 }
5831 }
5832 *task.terminal_at.lock().unwrap() = None;
5833 if !task.paths.exit.exists() {
5836 std::fs::write(&task.paths.exit, "0").expect("write replacement exit marker");
5837 }
5838
5839 registry.reap_child(&task);
5843
5844 let state = task.state.lock().unwrap();
5845 assert!(
5846 matches!(state.runtime, TaskRuntime::Piped(None)),
5847 "child handle still released even when marker exists"
5848 );
5849 assert!(
5850 state.detached,
5851 "task still marked detached even when marker exists"
5852 );
5853 assert_eq!(
5858 state.metadata.status,
5859 BgTaskStatus::Running,
5860 "reap_child must defer to poll_task when marker exists"
5861 );
5862 }
5863
5864 #[cfg(unix)]
5868 fn pid_stat(pid: u32) -> Option<String> {
5869 let output = std::process::Command::new("ps")
5870 .args(["-o", "stat=", "-p", &pid.to_string()])
5871 .output()
5872 .ok()?;
5873 if !output.status.success() {
5874 return None;
5875 }
5876 let stat = String::from_utf8_lossy(&output.stdout).trim().to_string();
5877 if stat.is_empty() {
5878 None
5879 } else {
5880 Some(stat)
5881 }
5882 }
5883
5884 #[cfg(unix)]
5886 fn is_zombie(pid: u32) -> bool {
5887 pid_stat(pid).is_some_and(|stat| stat.starts_with('Z'))
5888 }
5889
5890 #[cfg(unix)]
5896 fn spawn_unreaped_zombie() -> std::process::Child {
5897 let child = std::process::Command::new("true")
5898 .stdin(std::process::Stdio::null())
5899 .stdout(std::process::Stdio::null())
5900 .stderr(std::process::Stdio::null())
5901 .spawn()
5902 .expect("spawn zombie stand-in");
5903 let pid = child.id();
5904 let started = Instant::now();
5905 while !is_zombie(pid) {
5906 assert!(
5907 started.elapsed() < Duration::from_secs(5),
5908 "stand-in child should become a zombie within 5s"
5909 );
5910 std::thread::sleep(Duration::from_millis(10));
5911 }
5912 child
5914 }
5915
5916 #[cfg(unix)]
5926 #[test]
5927 fn finalize_from_marker_reaps_child_no_zombie() {
5928 use std::sync::atomic::Ordering;
5929
5930 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5931 let dir = tempfile::tempdir().unwrap();
5932 let task_id = registry
5933 .spawn(
5934 SpawnPlan::Unsandboxed,
5935 QUICK_SUCCESS_COMMAND,
5936 "session".to_string(),
5937 dir.path().to_path_buf(),
5938 HashMap::new(),
5939 Some(Duration::from_secs(30)),
5940 dir.path().to_path_buf(),
5941 10,
5942 true,
5943 false,
5944 Some(dir.path().to_path_buf()),
5945 )
5946 .unwrap();
5947
5948 registry.inner.shutdown.store(true, Ordering::SeqCst);
5952 std::thread::sleep(Duration::from_millis(550));
5953
5954 let task = registry.task_for_session(&task_id, "session").unwrap();
5955
5956 let started = Instant::now();
5960 while !task.paths.exit.exists() {
5961 assert!(
5962 started.elapsed() < Duration::from_secs(5),
5963 "exit marker should land quickly for `true`"
5964 );
5965 std::thread::sleep(Duration::from_millis(20));
5966 }
5967
5968 let zombie_pid;
5974 {
5975 let mut state = task.state.lock().unwrap();
5976 state.metadata.status = BgTaskStatus::Running;
5977 state.metadata.status_reason = None;
5978 state.metadata.exit_code = None;
5979 state.metadata.finished_at = None;
5980 state.metadata.duration_ms = None;
5981 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
5982 .expect("persist reset Running metadata");
5983 let zombie = spawn_unreaped_zombie();
5984 zombie_pid = zombie.id();
5985 state.runtime = TaskRuntime::Piped(Some(zombie));
5986 }
5987 *task.terminal_at.lock().unwrap() = None;
5988
5989 assert!(
5991 is_zombie(zombie_pid),
5992 "precondition: stand-in child {zombie_pid} must be a zombie before finalize"
5993 );
5994
5995 registry.poll_task(&task).unwrap();
5998
5999 {
6000 let state = task.state.lock().unwrap();
6001 assert!(
6002 matches!(state.runtime, TaskRuntime::Piped(None)),
6003 "child handle must be released after marker finalize"
6004 );
6005 assert!(
6006 state.metadata.status.is_terminal(),
6007 "task must be terminal after marker finalize: {:?}",
6008 state.metadata.status
6009 );
6010 }
6011
6012 assert!(
6015 !is_zombie(zombie_pid),
6016 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
6017 after the exit-marker terminal transition"
6018 );
6019 }
6020
6021 #[cfg(unix)]
6025 #[test]
6026 fn kill_with_existing_marker_reaps_child_no_zombie() {
6027 use std::sync::atomic::Ordering;
6028
6029 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6030 let dir = tempfile::tempdir().unwrap();
6031 let task_id = registry
6032 .spawn(
6033 SpawnPlan::Unsandboxed,
6034 QUICK_SUCCESS_COMMAND,
6035 "session".to_string(),
6036 dir.path().to_path_buf(),
6037 HashMap::new(),
6038 Some(Duration::from_secs(30)),
6039 dir.path().to_path_buf(),
6040 10,
6041 true,
6042 false,
6043 Some(dir.path().to_path_buf()),
6044 )
6045 .unwrap();
6046
6047 registry.inner.shutdown.store(true, Ordering::SeqCst);
6048 std::thread::sleep(Duration::from_millis(550));
6049
6050 let task = registry.task_for_session(&task_id, "session").unwrap();
6051
6052 let started = Instant::now();
6053 while !task.paths.exit.exists() {
6054 assert!(
6055 started.elapsed() < Duration::from_secs(5),
6056 "exit marker should land quickly for `true`"
6057 );
6058 std::thread::sleep(Duration::from_millis(20));
6059 }
6060
6061 let zombie_pid;
6062 {
6063 let mut state = task.state.lock().unwrap();
6064 state.metadata.status = BgTaskStatus::Running;
6065 state.metadata.status_reason = None;
6066 state.metadata.exit_code = None;
6067 state.metadata.finished_at = None;
6068 state.metadata.duration_ms = None;
6069 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
6070 .expect("persist reset Running metadata");
6071 let zombie = spawn_unreaped_zombie();
6072 zombie_pid = zombie.id();
6073 state.runtime = TaskRuntime::Piped(Some(zombie));
6074 }
6075 *task.terminal_at.lock().unwrap() = None;
6076
6077 assert!(
6078 is_zombie(zombie_pid),
6079 "precondition: stand-in child {zombie_pid} must be a zombie before kill"
6080 );
6081
6082 registry
6084 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6085 .expect("kill should succeed");
6086
6087 {
6088 let state = task.state.lock().unwrap();
6089 assert!(
6090 matches!(state.runtime, TaskRuntime::Piped(None)),
6091 "child handle must be released after marker-aware kill"
6092 );
6093 assert!(state.metadata.status.is_terminal());
6094 }
6095
6096 assert!(
6097 !is_zombie(zombie_pid),
6098 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
6099 after a marker-aware kill"
6100 );
6101 }
6102
6103 #[test]
6104 fn cleanup_finished_keeps_running_tasks() {
6105 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6106 let dir = tempfile::tempdir().unwrap();
6107 let task_id = registry
6108 .spawn(
6109 SpawnPlan::Unsandboxed,
6110 LONG_RUNNING_COMMAND,
6111 "session".to_string(),
6112 dir.path().to_path_buf(),
6113 HashMap::new(),
6114 Some(Duration::from_secs(30)),
6115 dir.path().to_path_buf(),
6116 10,
6117 true,
6118 false,
6119 Some(dir.path().to_path_buf()),
6120 )
6121 .unwrap();
6122
6123 registry.cleanup_finished(Duration::ZERO);
6124
6125 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
6126 let _ = registry.kill(&task_id, "session");
6127 }
6128
6129 #[cfg(windows)]
6130 fn wait_for_file(path: &Path) -> String {
6131 let started = Instant::now();
6132 loop {
6133 if path.exists() {
6134 return fs::read_to_string(path).expect("read file");
6135 }
6136 assert!(
6137 started.elapsed() < Duration::from_secs(30),
6138 "timed out waiting for {}",
6139 path.display()
6140 );
6141 std::thread::sleep(Duration::from_millis(100));
6142 }
6143 }
6144
6145 #[cfg(windows)]
6146 fn spawn_windows_registry_command(
6147 command: &str,
6148 ) -> (BgTaskRegistry, tempfile::TempDir, String) {
6149 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6150 let dir = tempfile::tempdir().unwrap();
6151 let task_id = registry
6152 .spawn(
6153 SpawnPlan::Unsandboxed,
6154 command,
6155 "session".to_string(),
6156 dir.path().to_path_buf(),
6157 HashMap::new(),
6158 Some(Duration::from_secs(30)),
6159 dir.path().to_path_buf(),
6160 10,
6161 false,
6162 false,
6163 Some(dir.path().to_path_buf()),
6164 )
6165 .unwrap();
6166 (registry, dir, task_id)
6167 }
6168
6169 #[cfg(windows)]
6170 #[test]
6171 fn windows_spawn_writes_exit_marker_for_zero_exit() {
6172 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 0");
6173 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
6174
6175 let content = wait_for_file(&exit_path);
6176
6177 assert_eq!(content.trim(), "0");
6178 }
6179
6180 #[cfg(windows)]
6181 #[test]
6182 fn windows_spawn_writes_exit_marker_for_nonzero_exit() {
6183 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 42");
6184 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
6185
6186 let content = wait_for_file(&exit_path);
6187
6188 assert_eq!(content.trim(), "42");
6189 }
6190
6191 #[cfg(windows)]
6192 #[test]
6193 fn windows_spawn_captures_stdout_to_disk() {
6194 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c echo hello");
6195 let task = registry.task_for_session(&task_id, "session").unwrap();
6196 let stdout_path = task.paths.stdout.clone();
6197 let exit_path = task.paths.exit.clone();
6198
6199 let _ = wait_for_file(&exit_path);
6200 let stdout = fs::read_to_string(stdout_path).expect("read stdout");
6201
6202 assert!(stdout.contains("hello"), "stdout was {stdout:?}");
6203 }
6204
6205 #[cfg(windows)]
6206 #[test]
6207 fn windows_spawn_uses_pwsh_when_available() {
6208 let candidates = crate::windows_shell::shell_candidates_with(
6212 |binary| match binary {
6213 "pwsh.exe" => Some(std::path::PathBuf::from(r"C:\pwsh\pwsh.exe")),
6214 "powershell.exe" => Some(std::path::PathBuf::from(r"C:\ps\powershell.exe")),
6215 _ => None,
6216 },
6217 || None,
6218 );
6219 let shell = candidates.first().expect("at least one candidate").clone();
6220 assert_eq!(shell, crate::windows_shell::WindowsShell::Pwsh);
6221 assert_eq!(shell.binary().as_ref(), "pwsh.exe");
6222 }
6223
6224 #[cfg(windows)]
6231 #[test]
6232 fn windows_shell_cmd_wrapper_writes_exit_marker_with_move() {
6233 let exit_path = Path::new(r"C:\Temp\bash-test.exit");
6234 let script =
6235 crate::windows_shell::WindowsShell::Cmd.wrapper_script("cmd /c exit 42", exit_path);
6236
6237 assert!(
6241 script.contains("set CODE=%ERRORLEVEL%"),
6242 "wrapper must capture exit code into CODE: {script}"
6243 );
6244 assert!(
6245 script.contains("echo %CODE% >"),
6246 "wrapper must echo CODE to a temp marker file: {script}"
6247 );
6248 assert!(
6249 script.contains("move /Y"),
6250 "wrapper must use atomic move to write the marker: {script}"
6251 );
6252 assert!(
6255 script.contains("> nul"),
6256 "wrapper must redirect move output to nul: {script}"
6257 );
6258 assert!(
6260 script.contains("exit /B %CODE%"),
6261 "wrapper must propagate the captured exit code: {script}"
6262 );
6263 assert!(script.contains(r#""C:\Temp\bash-test.exit.tmp""#));
6264 assert!(script.contains(r#""C:\Temp\bash-test.exit""#));
6265 }
6266
6267 #[cfg(windows)]
6273 #[test]
6274 fn windows_shell_cmd_bg_command_uses_minimal_cmd_flags() {
6275 use crate::windows_shell::WindowsShell;
6276 let cmd = WindowsShell::Cmd.bg_command("echo wrapped");
6277 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
6278 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
6279 assert_eq!(
6280 args_strs,
6281 vec!["/D", "/S", "/C", "echo wrapped"],
6282 "Cmd::bg_command must prepend /D /S /C"
6283 );
6284 }
6285
6286 #[cfg(windows)]
6289 #[test]
6290 fn windows_shell_pwsh_bg_command_uses_standard_args() {
6291 use crate::windows_shell::WindowsShell;
6292 let cmd = WindowsShell::Pwsh.bg_command("Get-Date");
6293 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
6294 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
6295 assert!(
6296 args_strs.contains(&"-Command"),
6297 "Pwsh::bg_command must use -Command: {args_strs:?}"
6298 );
6299 assert!(
6300 args_strs.contains(&"Get-Date"),
6301 "Pwsh::bg_command must include the user command body"
6302 );
6303 }
6304}