1use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
2#[cfg(not(windows))]
3use std::ffi::OsString;
4use std::fs;
5use std::io::Write;
6use std::path::{Path, PathBuf};
7#[cfg(windows)]
8use std::process::Command;
9use std::process::{Child, Stdio};
10use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
11#[cfg(unix)]
12use std::sync::OnceLock;
13use std::sync::{Arc, Mutex, RwLock};
14use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
15
16use rusqlite::Connection;
17use serde::Serialize;
18
19use crate::bash_permissions::PermissionAsk;
20use crate::compress::caps::DropClass;
21#[cfg(unix)]
22use crate::compress::single_top_level_pipeline;
23use crate::compress::CompressionResult;
24use crate::context::SharedProgressSender;
25use crate::db::compression_events::CompressionAggregateCache;
26use crate::harness::Harness;
27use crate::protocol::{BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, PushFrame};
28use crate::sandbox_spawn::SpawnPlan;
29
30#[cfg(windows)]
31use std::os::windows::process::CommandExt;
32
33use super::buffer::{combine_streams, BgBuffer, DiskTruncation, StreamKind, TokenCountInput};
34use super::output::{
35 cap_completion_output, cap_completion_output_with_marker, cap_final_output,
36 cap_final_output_with_marker, completion_preview_threshold, json_output_pointer, quote_path,
37 retained_json_output_pointer, COMPRESS_INPUT_CAP_BYTES, COMPRESS_INPUT_HEAD_BYTES,
38 COMPRESS_INPUT_TAIL_BYTES, FINAL_OUTPUT_CAP_BYTES, RAW_PASSTHROUGH_CAP_BYTES,
39 RAW_PASSTHROUGH_HEAD_BYTES, RAW_PASSTHROUGH_TAIL_BYTES, RUNNING_OUTPUT_PREVIEW_BYTES,
40 STRUCTURED_OUTPUT_CAP_BYTES,
41};
42use super::persistence::{
43 allocate_task_layout, delete_resolved_task, delete_task_bundle, discover_task_ids,
44 open_task_artifact, quarantine_invalid_entry, quarantine_task_layout, read_exit_marker,
45 read_task_at, resolve_task_layout, session_tasks_dir, uninitialized_layout_is_recent,
46 unix_millis, update_task_at, validate_task_id, write_kill_marker_if_absent, write_task_at,
47 BgMode, ExitMarker, PersistedTask, TaskArtifact, TaskIoHandles, TaskPaths,
48};
49use super::process::is_process_alive;
50#[cfg(unix)]
51use super::process::terminate_pgid;
52#[cfg(windows)]
53use super::process::terminate_pid;
54use super::pty_process::spawn_pty_for_command;
55use super::pty_runtime::PtyRuntime;
56use super::watches::{PatternMatch, WatchPattern, WatchRegistry};
57use super::{BgTaskInfo, BgTaskStatus};
58use crate::db::bash_watches::BashPatternWatchRow;
59const DEFAULT_BG_TIMEOUT: Duration = Duration::from_secs(30 * 60);
62const STALE_RUNNING_AFTER: Duration = Duration::from_secs(24 * 60 * 60);
63const PERSISTED_GC_GRACE: Duration = Duration::from_secs(24 * 60 * 60);
64const QUARANTINE_GC_GRACE: Duration = Duration::from_secs(30 * 24 * 60 * 60);
65
66const TOKENIZE_CAP_BYTES_PER_STREAM: usize = 128 * 1024;
67pub const ROOT_RECLAIMED_REASON: &str = "root_reclaimed";
68
69#[derive(Debug, Clone, Serialize)]
70pub struct BgCompletion {
71 pub task_id: String,
72 #[serde(skip_serializing)]
75 pub session_id: String,
76 pub status: BgTaskStatus,
77 pub exit_code: Option<i32>,
78 pub command: String,
79 #[serde(default, skip_serializing_if = "String::is_empty")]
85 pub output_preview: String,
86 #[serde(default, skip_serializing_if = "is_false")]
91 pub output_truncated: bool,
92 #[serde(default, skip_serializing_if = "Option::is_none")]
95 pub original_tokens: Option<u32>,
96 #[serde(default, skip_serializing_if = "Option::is_none")]
99 pub compressed_tokens: Option<u32>,
100 #[serde(default, skip_serializing_if = "is_false")]
102 pub tokens_skipped: bool,
103 #[serde(default, skip_serializing_if = "Option::is_none")]
104 pub status_reason: Option<String>,
105}
106
107fn is_false(v: &bool) -> bool {
108 !*v
109}
110
111#[derive(Debug, Clone, Serialize)]
112pub struct BgTaskSnapshot {
113 #[serde(flatten)]
114 pub info: BgTaskInfo,
115 pub exit_code: Option<i32>,
116 pub child_pid: Option<u32>,
117 pub workdir: String,
118 pub output_preview: String,
119 pub output_truncated: bool,
120 pub output_path: Option<String>,
121 pub stderr_path: Option<String>,
122 #[serde(skip_serializing_if = "Option::is_none")]
123 pub pty_rows: Option<u16>,
124 #[serde(skip_serializing_if = "Option::is_none")]
125 pub pty_cols: Option<u16>,
126 #[serde(skip_serializing_if = "Option::is_none")]
127 pub pty_screen: Option<String>,
128 #[serde(default, skip_serializing_if = "Vec::is_empty")]
129 pub scanner_report: Vec<PermissionAsk>,
130 #[serde(default, skip_serializing_if = "is_false")]
131 pub sandbox_native: bool,
132 #[serde(default, skip_serializing_if = "is_false")]
133 pub sandbox_unavailable: bool,
134}
135
136#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
137pub struct BgTaskHealthCounts {
138 pub running: usize,
139 pub pending_completions: usize,
140}
141
142#[derive(Debug, Clone, Copy, PartialEq, Eq)]
143enum TerminalOutputKind {
144 Compressed,
145 Raw,
146 Structured,
147}
148
149#[derive(Debug, Clone, PartialEq, Eq)]
150struct TerminalOutputCache {
151 output_preview: String,
152 output_truncated: bool,
153 kind: TerminalOutputKind,
154 output_path: Option<String>,
155 stderr_path: Option<String>,
156 artifact_access: ArtifactRecoveryAccess,
157 recovery: Option<RecoveryContext>,
158}
159
160#[derive(Debug, Clone, PartialEq, Eq)]
161struct ArtifactRecoveryAccess {
162 task_id: String,
163 readable: bool,
164}
165
166#[derive(Debug, Clone, PartialEq, Eq)]
167struct RecoveryContext {
168 dropped_by_class: BTreeMap<DropClass, usize>,
169 had_inner_drop: bool,
170 offset_hint_eligible: bool,
171 offset_start_line: Option<usize>,
172 byte_truncated: bool,
173 disk_truncated_prefix_bytes: u64,
174 output_path: Option<String>,
175 stderr_path: Option<String>,
176 include_stderr_path: bool,
177 artifact_access: ArtifactRecoveryAccess,
178}
179
180fn optional_string_bytes(value: Option<&String>) -> u64 {
181 value
182 .map(|value| crate::memory::usize_to_u64(value.len()))
183 .unwrap_or(0)
184}
185
186fn terminal_output_cache_estimated_bytes(cache: &TerminalOutputCache) -> u64 {
187 let recovery_bytes = cache
188 .recovery
189 .as_ref()
190 .map(|recovery| {
191 crate::memory::usize_to_u64(recovery.dropped_by_class.len())
192 .saturating_mul(
193 (std::mem::size_of::<DropClass>() + std::mem::size_of::<usize>()) as u64,
194 )
195 .saturating_add(optional_string_bytes(recovery.output_path.as_ref()))
196 .saturating_add(optional_string_bytes(recovery.stderr_path.as_ref()))
197 .saturating_add(crate::memory::usize_to_u64(
198 recovery.artifact_access.task_id.len(),
199 ))
200 })
201 .unwrap_or(0);
202 (std::mem::size_of::<TerminalOutputCache>() as u64)
203 .saturating_add(crate::memory::usize_to_u64(cache.output_preview.len()))
204 .saturating_add(optional_string_bytes(cache.output_path.as_ref()))
205 .saturating_add(optional_string_bytes(cache.stderr_path.as_ref()))
206 .saturating_add(crate::memory::usize_to_u64(
207 cache.artifact_access.task_id.len(),
208 ))
209 .saturating_add(recovery_bytes)
210}
211
212fn completion_estimated_bytes(completion: &BgCompletion) -> u64 {
213 (std::mem::size_of::<BgCompletion>() as u64)
214 .saturating_add(crate::memory::usize_to_u64(completion.task_id.len()))
215 .saturating_add(crate::memory::usize_to_u64(completion.session_id.len()))
216 .saturating_add(crate::memory::usize_to_u64(completion.command.len()))
217 .saturating_add(crate::memory::usize_to_u64(completion.output_preview.len()))
218}
219
220impl RecoveryContext {
221 fn has_visible_drop(&self) -> bool {
222 self.byte_truncated
223 || self.disk_truncated_prefix_bytes > 0
224 || self.had_inner_drop
225 || !self.dropped_by_class.is_empty()
226 }
227}
228
229#[derive(Clone)]
230pub struct BgTaskRegistry {
231 pub(crate) inner: Arc<RegistryInner>,
232}
233
234pub(crate) struct RegistryInner {
235 pub(crate) tasks: Mutex<HashMap<String, Arc<BgTask>>>,
236 pub(crate) completions: Mutex<VecDeque<BgCompletion>>,
237 pub(crate) progress_sender: SharedProgressSender,
238 watchdog_started: AtomicBool,
239 pub(crate) shutdown: AtomicBool,
240 pub(crate) long_running_reminder_enabled: AtomicBool,
241 pub(crate) long_running_reminder_interval_ms: AtomicU64,
242 persisted_gc_started: AtomicBool,
243 #[cfg(test)]
244 persisted_gc_runs: AtomicU64,
245 pub(crate) compressor:
251 Mutex<Option<Box<dyn Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync>>>,
252 pub(crate) db_pool: RwLock<Option<Arc<Mutex<Connection>>>>,
253 pub(crate) db_harness: RwLock<Option<String>>,
254 pub(crate) compression_aggregates: Arc<CompressionAggregateCache>,
255 pub(crate) wake_tx: crossbeam_channel::Sender<()>,
256 pub(crate) wake_rx: crossbeam_channel::Receiver<()>,
257 pub(crate) watch_registry: Mutex<WatchRegistry>,
258 wait_detach_sessions: Mutex<HashSet<String>>,
259 active_wait_sessions: Mutex<HashMap<String, usize>>,
260 wait_registered_tasks: Mutex<HashMap<String, HashSet<String>>>,
261}
262
263pub(crate) struct BgTask {
264 pub(crate) task_id: String,
265 pub(crate) session_id: String,
266 pub(crate) paths: TaskPaths,
267 artifact_root: PathBuf,
268 pub(crate) started: Instant,
269 pub(crate) last_reminder_at: Mutex<Option<Instant>>,
270 pub(crate) terminal_at: Mutex<Option<Instant>>,
271 pub(crate) state: Mutex<BgTaskState>,
272}
273
274pub(crate) enum TaskRuntime {
275 Piped(Option<Child>),
276 Pty(Option<PtyRuntime>),
277}
278
279pub(crate) struct BgTaskState {
280 pub(crate) metadata: PersistedTask,
281 pub(crate) runtime: TaskRuntime,
282 pub(crate) io_handles: Option<TaskIoHandles>,
285 pub(crate) detached: bool,
286 pub(crate) child_exit_observed: bool,
297 pub(crate) buffer: BgBuffer,
298 terminal_output_cache: Option<TerminalOutputCache>,
299 pub(crate) pending_terminal_override: Option<BgTaskStatus>,
301}
302
303fn completion_matches_session(completion: &BgCompletion, session_id: Option<&str>) -> bool {
304 session_id
305 .map(|session_id| completion.session_id == session_id)
306 .unwrap_or(true)
307}
308
309impl BgTaskRegistry {
310 pub fn new(progress_sender: SharedProgressSender) -> Self {
311 let (wake_tx, wake_rx) = crossbeam_channel::bounded(1);
312 Self {
313 inner: Arc::new(RegistryInner {
314 tasks: Mutex::new(HashMap::new()),
315 completions: Mutex::new(VecDeque::new()),
316 progress_sender,
317 watchdog_started: AtomicBool::new(false),
318 shutdown: AtomicBool::new(false),
319 long_running_reminder_enabled: AtomicBool::new(true),
320 long_running_reminder_interval_ms: AtomicU64::new(600_000),
321 persisted_gc_started: AtomicBool::new(false),
322 #[cfg(test)]
323 persisted_gc_runs: AtomicU64::new(0),
324 compressor: Mutex::new(None),
325 db_pool: RwLock::new(None),
326 db_harness: RwLock::new(None),
327 compression_aggregates: Arc::new(CompressionAggregateCache::default()),
328 wake_tx,
329 wake_rx,
330 watch_registry: Mutex::new(WatchRegistry::default()),
331 wait_detach_sessions: Mutex::new(HashSet::new()),
332 active_wait_sessions: Mutex::new(HashMap::new()),
333 wait_registered_tasks: Mutex::new(HashMap::new()),
334 }),
335 }
336 }
337
338 pub fn is_session_owned_artifact_path(&self, session_id: &str, path: &Path) -> bool {
346 let Ok(requested) = fs::canonicalize(path) else {
347 return false;
348 };
349 let Ok(tasks) = self.inner.tasks.lock() else {
350 return false;
351 };
352
353 tasks.values().any(|task| {
354 task.session_id == session_id
355 && [
356 &task.paths.stdout,
357 &task.paths.stderr,
358 &task.paths.exit,
359 &task.paths.pty,
360 ]
361 .into_iter()
362 .filter_map(|known| known.file_name())
363 .any(|name| task.artifact_root.join(name) == requested)
364 })
365 }
366
367 pub fn read_artifact_path(
368 &self,
369 session_id: &str,
370 path: &Path,
371 ) -> Option<Result<Vec<u8>, String>> {
372 let requested = fs::canonicalize(path).ok()?;
373 let tasks = self.inner.tasks.lock().ok()?;
374 let (task, artifact) = tasks.values().find_map(|task| {
375 if task.session_id != session_id {
376 return None;
377 }
378 TaskArtifact::ALL.into_iter().find_map(|artifact| {
379 let expected = task
380 .paths
381 .artifact_path(artifact)
382 .file_name()
383 .map(|name| task.artifact_root.join(name));
384 (expected.as_deref() == Some(requested.as_path()))
385 .then(|| (Arc::clone(task), artifact))
386 })
387 })?;
388 drop(tasks);
389 Some(self.read_artifact(&task.task_id, session_id, artifact))
390 }
391
392 pub fn read_artifact_range(
393 &self,
394 task_id: &str,
395 session_id: &str,
396 artifact: TaskArtifact,
397 offset: u64,
398 ) -> Result<(Vec<u8>, u64), String> {
399 validate_task_id(task_id).map_err(|error| error.to_string())?;
400 let task = self
401 .task_for_session(task_id, session_id)
402 .ok_or_else(|| "task_not_found".to_string())?;
403 let mut file = open_task_artifact(&task.paths, artifact)
404 .map_err(|error| format!("artifact_refused: {error}"))?;
405 let len = file
406 .len()
407 .map_err(|error| format!("artifact_refused: {error}"))?;
408 let start = offset.min(len);
409 let bytes = file
410 .read_range(start, len.saturating_sub(start))
411 .map_err(|error| format!("artifact_refused: {error}"))?;
412 Ok((bytes, len))
413 }
414
415 pub fn read_artifact(
416 &self,
417 task_id: &str,
418 session_id: &str,
419 artifact: TaskArtifact,
420 ) -> Result<Vec<u8>, String> {
421 validate_task_id(task_id).map_err(|error| error.to_string())?;
422 let task = self
423 .task_for_session(task_id, session_id)
424 .ok_or_else(|| "task_not_found".to_string())?;
425 let mut file = open_task_artifact(&task.paths, artifact)
426 .map_err(|error| format!("artifact_refused: {error}"))?;
427 file.read_all()
428 .map_err(|error| format!("artifact_refused: {error}"))
429 }
430
431 pub fn set_harness(&self, harness: Harness) {
432 if let Ok(mut slot) = self.inner.db_harness.write() {
433 *slot = Some(harness.storage_segment());
434 }
435 }
436
437 pub fn set_db_pool(&self, conn: Arc<Mutex<Connection>>) {
438 if let Ok(mut slot) = self.inner.db_pool.write() {
439 *slot = Some(conn);
440 }
441 self.inner.compression_aggregates.clear();
442 }
443
444 pub fn clear_db_pool(&self) {
445 if let Ok(mut slot) = self.inner.db_pool.write() {
446 *slot = None;
447 }
448 self.inner.compression_aggregates.clear();
449 }
450
451 pub(crate) fn compression_aggregate_cache(&self) -> Arc<CompressionAggregateCache> {
452 Arc::clone(&self.inner.compression_aggregates)
453 }
454
455 pub fn register_foreground_task(&self, session_id: &str, task_id: &str) {
456 if let Ok(mut tasks) = self.inner.wait_registered_tasks.lock() {
457 tasks
458 .entry(session_id.to_string())
459 .or_default()
460 .insert(task_id.to_string());
461 }
462 }
463
464 pub fn begin_wait_mode_session(&self, session_id: &str, task_id: &str) {
465 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
466 *active.entry(session_id.to_string()).or_insert(0) += 1;
467 }
468 self.register_foreground_task(session_id, task_id);
469 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
470 detach.remove(session_id);
471 }
472 }
473
474 pub fn unregister_foreground_task(&self, session_id: &str, task_id: &str) {
475 if let Ok(mut tasks) = self.inner.wait_registered_tasks.lock() {
476 if let Some(session_tasks) = tasks.get_mut(session_id) {
477 session_tasks.remove(task_id);
478 if session_tasks.is_empty() {
479 tasks.remove(session_id);
480 }
481 }
482 }
483 }
484
485 pub fn end_wait_mode_session(&self, session_id: &str, task_id: &str) {
486 let no_active_wait = if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
487 match active.get_mut(session_id) {
488 Some(count) if *count > 1 => *count -= 1,
489 Some(_) => {
490 active.remove(session_id);
491 }
492 None => {}
493 }
494 !active.contains_key(session_id)
495 } else {
496 false
497 };
498 self.unregister_foreground_task(session_id, task_id);
499 if no_active_wait {
500 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
501 detach.remove(session_id);
502 }
503 }
504 }
505
506 pub fn abort_inflight(&self, session_id: &str) -> Result<usize, String> {
510 let task_ids = self
511 .inner
512 .wait_registered_tasks
513 .lock()
514 .map(|mut tasks| tasks.remove(session_id).unwrap_or_default())
515 .map_err(|_| "wait registration lock poisoned".to_string())?;
516 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
517 active.remove(session_id);
518 }
519 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
520 detach.remove(session_id);
521 }
522
523 let mut killed = 0;
524 for task_id in task_ids {
525 let Some(task) = self.task_for_session(&task_id, session_id) else {
526 continue;
527 };
528 let is_terminal = task
529 .state
530 .lock()
531 .map(|state| state.metadata.status.is_terminal())
532 .map_err(|_| "background task lock poisoned".to_string())?;
533 if is_terminal {
534 continue;
535 }
536 let snapshot = self.kill_with_status_reason(
537 &task_id,
538 session_id,
539 BgTaskStatus::Killed,
540 Some("call_aborted".to_string()),
541 )?;
542 if snapshot.info.status == BgTaskStatus::Killed
543 && snapshot.info.status_reason.as_deref() == Some("call_aborted")
544 {
545 killed += 1;
546 }
547 }
548 Ok(killed)
549 }
550
551 pub fn signal_wait_mode_detach(&self, session_id: &str) -> bool {
552 let is_waiting = self
553 .inner
554 .active_wait_sessions
555 .lock()
556 .map(|active| active.get(session_id).copied().unwrap_or(0) > 0)
557 .unwrap_or(false);
558 if !is_waiting {
559 return false;
560 }
561 self.inner
562 .wait_detach_sessions
563 .lock()
564 .map(|mut detach| detach.insert(session_id.to_string()))
565 .unwrap_or(false)
566 }
567
568 pub fn active_wait_session_count(&self) -> usize {
571 self.inner
572 .active_wait_sessions
573 .lock()
574 .map(|active| active.len())
575 .unwrap_or(0)
576 }
577
578 pub fn take_wait_mode_detach(&self, session_id: &str) -> bool {
579 self.inner
580 .wait_detach_sessions
581 .lock()
582 .map(|mut detach| detach.remove(session_id))
583 .unwrap_or(false)
584 }
585
586 pub fn set_compressor<F>(&self, compressor: F)
591 where
592 F: Fn(&str, String) -> CompressionResult + Send + Sync + 'static,
593 {
594 self.set_compressor_with_exit_code(move |command, output, _exit_code| {
595 compressor(command, output)
596 });
597 }
598
599 pub fn set_compressor_with_exit_code<F>(&self, compressor: F)
600 where
601 F: Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync + 'static,
602 {
603 if let Ok(mut slot) = self.inner.compressor.lock() {
604 *slot = Some(Box::new(compressor));
605 }
606 }
607
608 pub(crate) fn compress_output(
611 &self,
612 command: &str,
613 output: String,
614 exit_code: Option<i32>,
615 ) -> CompressionResult {
616 let Ok(slot) = self.inner.compressor.lock() else {
617 return CompressionResult::new(output);
618 };
619 match slot.as_ref() {
620 Some(compressor) => compressor(command, output, exit_code),
621 None => CompressionResult::new(output),
622 }
623 }
624
625 fn ensure_terminal_output_cache(&self, task: &Arc<BgTask>) -> Option<TerminalOutputCache> {
626 let (metadata, buffer) = {
627 let state = task.state.lock().ok()?;
628 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
629 return None;
630 }
631 if let Some(cache) = state.terminal_output_cache.clone() {
632 return Some(cache);
633 }
634 (state.metadata.clone(), state.buffer.clone())
635 };
636
637 let mut cap_buffer = buffer.clone();
638 let disk_truncation = cap_buffer.enforce_terminal_cap();
639 let cache =
640 self.render_terminal_output(&metadata, &cap_buffer, disk_truncation, Some(&task.paths));
641 let mut state = task.state.lock().ok()?;
642 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
643 return None;
644 }
645 if let Some(existing) = state.terminal_output_cache.clone() {
646 return Some(existing);
647 }
648 state.terminal_output_cache = Some(cache.clone());
649 Some(cache)
650 }
651
652 fn render_terminal_output(
653 &self,
654 metadata: &PersistedTask,
655 buffer: &BgBuffer,
656 disk_truncation: DiskTruncation,
657 paths: Option<&TaskPaths>,
658 ) -> TerminalOutputCache {
659 let output_readable = buffer
660 .output_path()
661 .is_some_and(|path| self.is_session_owned_artifact_path(&metadata.session_id, &path));
662 let stderr_readable = buffer
663 .stderr_path()
664 .map(|path| self.is_session_owned_artifact_path(&metadata.session_id, path))
665 .unwrap_or(true);
666 let artifact_access = ArtifactRecoveryAccess {
667 task_id: metadata.task_id.clone(),
668 readable: output_readable && stderr_readable,
669 };
670
671 if metadata.mode == BgMode::Pty {
672 return TerminalOutputCache {
673 output_preview: String::new(),
674 output_truncated: false,
675 kind: TerminalOutputKind::Raw,
676 output_path: buffer.output_path().map(|path| path.display().to_string()),
677 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
678 artifact_access,
679 recovery: None,
680 };
681 }
682
683 let mut rendered = if let Some(structured) = render_structured_output(
684 &metadata.command,
685 buffer,
686 disk_truncation,
687 artifact_access.clone(),
688 ) {
689 structured
690 } else if !metadata.compressed {
691 render_raw_passthrough(buffer, disk_truncation, artifact_access)
692 } else {
693 let raw = buffer.read_combined_head_tail(
694 COMPRESS_INPUT_CAP_BYTES,
695 COMPRESS_INPUT_HEAD_BYTES,
696 COMPRESS_INPUT_TAIL_BYTES,
697 );
698 let compressed = self.compress_output(&metadata.command, raw.text, metadata.exit_code);
699 render_compressed_with_recovery(
700 buffer,
701 compressed,
702 raw.truncated,
703 disk_truncation,
704 artifact_access,
705 )
706 };
707 normalize_piped_display_output(&mut rendered.output_preview);
708 append_pipeline_warning(&mut rendered, metadata, paths);
709 rendered
710 }
711
712 fn snapshot_with_terminal_cache(
713 &self,
714 task: &Arc<BgTask>,
715 preview_bytes: usize,
716 ) -> BgTaskSnapshot {
717 let mut snapshot = task.snapshot(preview_bytes);
718 self.maybe_compress_snapshot(task, &mut snapshot);
719 snapshot
720 }
721
722 fn post_terminal_transition(&self, task: &Arc<BgTask>, emit_frame: bool) -> Result<(), String> {
723 let (metadata, buffer) = {
724 let state = task
725 .state
726 .lock()
727 .map_err(|_| "background task lock poisoned".to_string())?;
728 if !state.metadata.status.is_terminal() {
729 return Ok(());
730 }
731 (state.metadata.clone(), state.buffer.clone())
732 };
733
734 let cache = self.ensure_terminal_output_cache(task);
735 self.enqueue_completion_from_parts(
736 &metadata,
737 Some(&buffer),
738 None,
739 emit_frame,
740 cache.as_ref(),
741 );
742 Ok(())
743 }
744
745 fn persist_task(&self, paths: &TaskPaths, metadata: &PersistedTask) -> std::io::Result<()> {
746 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
747 write_task_at(&task, metadata)?;
748 self.dual_write_task(paths, metadata);
749 Ok(())
750 }
751
752 fn update_task_metadata<F>(
753 &self,
754 paths: &TaskPaths,
755 update: F,
756 ) -> std::io::Result<PersistedTask>
757 where
758 F: FnOnce(&mut PersistedTask),
759 {
760 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
761 let metadata = update_task_at(&task, update)?;
762 self.dual_write_task(paths, &metadata);
763 Ok(metadata)
764 }
765
766 fn dual_write_task(&self, paths: &TaskPaths, metadata: &PersistedTask) {
767 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
768 let Some(pool) = pool else {
769 return;
770 };
771 let harness = self
772 .inner
773 .db_harness
774 .read()
775 .ok()
776 .and_then(|slot| slot.clone());
777 let Some(harness) = harness else {
778 crate::slog_warn!(
779 "dual-write bash_task to DB skipped for {}: harness not configured",
780 metadata.task_id
781 );
782 return;
783 };
784 let row = match metadata.to_bash_task_row(&harness, paths) {
785 Ok(row) => row,
786 Err(error) => {
787 crate::slog_warn!(
788 "dual-write bash_task to DB failed for {}: {}",
789 metadata.task_id,
790 error
791 );
792 return;
793 }
794 };
795 let conn = match pool.lock() {
796 Ok(conn) => conn,
797 Err(_) => {
798 crate::slog_warn!(
799 "dual-write bash_task to DB failed for {}: db mutex poisoned",
800 metadata.task_id
801 );
802 return;
803 }
804 };
805 if let Err(error) = crate::db::bash_tasks::upsert_bash_task(&conn, &row) {
806 crate::slog_warn!(
807 "dual-write bash_task to DB failed for {}: {}",
808 metadata.task_id,
809 error
810 );
811 }
812 }
813
814 fn delete_gc_task_from_db(&self, metadata: &PersistedTask) {
815 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
816 let Some(pool) = pool else {
817 return;
818 };
819 let harness = self
820 .inner
821 .db_harness
822 .read()
823 .ok()
824 .and_then(|slot| slot.clone());
825 let Some(harness) = harness else {
826 crate::slog_warn!(
827 "GC bash_task DB delete skipped for {}: harness not configured",
828 metadata.task_id
829 );
830 return;
831 };
832 let conn = match pool.lock() {
833 Ok(conn) => conn,
834 Err(_) => {
835 crate::slog_warn!(
836 "GC bash_task DB delete failed for {}: db mutex poisoned",
837 metadata.task_id
838 );
839 return;
840 }
841 };
842 if let Err(error) = crate::db::bash_tasks::delete_delivered_terminal_bash_task(
843 &conn,
844 &harness,
845 &metadata.session_id,
846 &metadata.task_id,
847 ) {
848 crate::slog_warn!(
849 "GC bash_task DB delete failed for {}: {}",
850 metadata.task_id,
851 error
852 );
853 }
854 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
856 &conn,
857 &harness,
858 &metadata.session_id,
859 &metadata.task_id,
860 ) {
861 crate::slog_warn!(
862 "GC bash_pattern_watches DB delete failed for {}: {}",
863 metadata.task_id,
864 error
865 );
866 }
867 }
868
869 fn db_harness_and_pool(&self) -> Option<(String, Arc<Mutex<Connection>>)> {
870 let pool = self
871 .inner
872 .db_pool
873 .read()
874 .ok()
875 .and_then(|slot| slot.clone())?;
876 let harness = self
877 .inner
878 .db_harness
879 .read()
880 .ok()
881 .and_then(|slot| slot.clone())?;
882 Some((harness, pool))
883 }
884
885 fn persist_watch_registration(
886 &self,
887 session_id: &str,
888 task_id: &str,
889 watch_id: &str,
890 pattern: &WatchPattern,
891 once: bool,
892 stdout_offset: u64,
893 stderr_offset: u64,
894 pty_offset: u64,
895 ) {
896 let Some((harness, pool)) = self.db_harness_and_pool() else {
897 return;
898 };
899 let Ok(conn) = pool.lock() else {
900 return;
901 };
902 let row = BashPatternWatchRow {
903 harness,
904 session_id: session_id.to_string(),
905 task_id: task_id.to_string(),
906 watch_id: watch_id.to_string(),
907 pattern_kind: pattern.kind_name().to_string(),
908 pattern: pattern.pattern_text().to_string(),
909 once,
910 created_at: unix_millis() as i64,
911 stdout_offset: stdout_offset as i64,
912 stderr_offset: stderr_offset as i64,
913 pty_offset: pty_offset as i64,
914 scanning: true,
915 pending_match: false,
916 match_text: None,
917 match_offset: None,
918 match_context: None,
919 };
920 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
921 crate::slog_warn!(
922 "persist bash_pattern_watch failed for {task_id}/{watch_id}: {error}"
923 );
924 }
925 }
926
927 fn delete_persisted_watch(&self, session_id: &str, task_id: &str, watch_id: &str) {
928 let Some((harness, pool)) = self.db_harness_and_pool() else {
929 return;
930 };
931 let Ok(conn) = pool.lock() else {
932 return;
933 };
934 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watch(
935 &conn, &harness, session_id, task_id, watch_id,
936 ) {
937 crate::slog_warn!("delete bash_pattern_watch failed for {task_id}/{watch_id}: {error}");
938 }
939 }
940
941 fn delete_persisted_watches_for_task(&self, session_id: &str, task_id: &str) {
942 let Some((harness, pool)) = self.db_harness_and_pool() else {
943 return;
944 };
945 let Ok(conn) = pool.lock() else {
946 return;
947 };
948 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
949 &conn, &harness, session_id, task_id,
950 ) {
951 crate::slog_warn!("delete bash_pattern_watches for {task_id} failed: {error}");
952 }
953 }
954
955 fn persist_watch_match(
956 &self,
957 session_id: &str,
958 task_id: &str,
959 pattern_match: &PatternMatch,
960 stdout_offset: u64,
961 stderr_offset: u64,
962 pty_offset: u64,
963 ) {
964 let Some((harness, pool)) = self.db_harness_and_pool() else {
965 return;
966 };
967 let Ok(conn) = pool.lock() else {
968 return;
969 };
970 let Ok(Some(mut row)) = crate::db::bash_watches::get_bash_pattern_watch(
971 &conn,
972 &harness,
973 session_id,
974 task_id,
975 &pattern_match.watch_id,
976 ) else {
977 return;
978 };
979 row.stdout_offset = stdout_offset as i64;
980 row.stderr_offset = stderr_offset as i64;
981 row.pty_offset = pty_offset as i64;
982 row.pending_match = true;
983 row.match_text = Some(pattern_match.match_text.clone());
984 row.match_offset = Some(pattern_match.match_offset as i64);
985 row.match_context = Some(pattern_match.context.clone());
986 if pattern_match.once {
987 row.scanning = false;
990 }
991 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
992 crate::slog_warn!(
993 "persist bash_pattern_watch match failed for {}/{}: {error}",
994 task_id,
995 pattern_match.watch_id
996 );
997 }
998 }
999
1000 fn persist_task_watch_cursors(
1001 &self,
1002 session_id: &str,
1003 task_id: &str,
1004 stdout_offset: u64,
1005 stderr_offset: u64,
1006 pty_offset: u64,
1007 ) {
1008 let Some((harness, pool)) = self.db_harness_and_pool() else {
1009 return;
1010 };
1011 let Ok(conn) = pool.lock() else {
1012 return;
1013 };
1014 if let Err(error) = crate::db::bash_watches::update_watch_offsets_for_task(
1015 &conn,
1016 &harness,
1017 session_id,
1018 task_id,
1019 stdout_offset as i64,
1020 stderr_offset as i64,
1021 pty_offset as i64,
1022 ) {
1023 crate::slog_warn!("persist bash_pattern_watch cursors failed for {task_id}: {error}");
1024 }
1025 }
1026
1027 fn watch_stream_cursors(&self, task_id: &str) -> (u64, u64, u64) {
1028 let Ok(registry) = self.inner.watch_registry.lock() else {
1029 return (0, 0, 0);
1030 };
1031 let stdout = registry
1032 .file_cursor(&format!("{task_id}:stdout"))
1033 .unwrap_or(0);
1034 let stderr = registry
1035 .file_cursor(&format!("{task_id}:stderr"))
1036 .unwrap_or(0);
1037 let pty = registry.file_cursor(&format!("{task_id}:pty")).unwrap_or(0);
1038 (stdout, stderr, pty)
1039 }
1040
1041 fn ack_persisted_watches_for_task(&self, session_id: &str, task_id: &str, task_terminal: bool) {
1044 let Some((harness, pool)) = self.db_harness_and_pool() else {
1045 return;
1046 };
1047 let Ok(conn) = pool.lock() else {
1048 return;
1049 };
1050 if task_terminal {
1051 let _ = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
1052 &conn, &harness, session_id, task_id,
1053 );
1054 return;
1055 }
1056 let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
1057 &conn, &harness, session_id, task_id,
1058 ) else {
1059 return;
1060 };
1061 for mut row in rows {
1062 if row.once && (!row.scanning || row.pending_match) {
1063 let _ = crate::db::bash_watches::delete_bash_pattern_watch(
1064 &conn,
1065 &harness,
1066 session_id,
1067 task_id,
1068 &row.watch_id,
1069 );
1070 continue;
1071 }
1072 if row.pending_match {
1073 row.pending_match = false;
1074 row.match_text = None;
1075 row.match_offset = None;
1076 row.match_context = None;
1077 let _ = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row);
1078 }
1079 }
1080 }
1081
1082 pub fn record_scanner_report(
1083 &self,
1084 task_id: &str,
1085 session_id: &str,
1086 scanner_report: Vec<PermissionAsk>,
1087 ) -> Result<(), String> {
1088 if scanner_report.is_empty() {
1089 return Ok(());
1090 }
1091 let task = self.task_for_session(task_id, session_id).ok_or_else(|| {
1092 "background task not found while recording scanner report".to_string()
1093 })?;
1094 let metadata = {
1095 let mut state = task
1096 .state
1097 .lock()
1098 .map_err(|_| "background task lock poisoned".to_string())?;
1099 state.metadata.scanner_report = scanner_report;
1100 state.metadata.clone()
1101 };
1102 self.persist_task(&task.paths, &metadata)
1103 .map_err(|error| format!("failed to persist scanner report: {error}"))
1104 }
1105
1106 pub fn configure_long_running_reminders(&self, enabled: bool, interval_ms: u64) {
1107 self.inner
1108 .long_running_reminder_enabled
1109 .store(enabled, Ordering::SeqCst);
1110 self.inner
1111 .long_running_reminder_interval_ms
1112 .store(interval_ms, Ordering::SeqCst);
1113 }
1114
1115 #[cfg(unix)]
1116 #[allow(clippy::too_many_arguments)]
1117 pub fn spawn(
1118 &self,
1119 spawn_plan: SpawnPlan,
1120 command: &str,
1121 session_id: String,
1122 workdir: PathBuf,
1123 env: HashMap<String, String>,
1124 timeout: Option<Duration>,
1125 storage_dir: PathBuf,
1126 max_running: usize,
1127 notify_on_completion: bool,
1128 compressed: bool,
1129 project_root: Option<PathBuf>,
1130 ) -> Result<String, String> {
1131 self.start_watchdog();
1132
1133 let running = self.running_count();
1134 if running >= max_running {
1135 #[cfg(unix)]
1136 if let Some(prepared) = spawn_plan.prepared_task() {
1137 let _ = delete_resolved_task(&prepared.resolved_task());
1138 }
1139 return Err(format!(
1140 "background bash task limit exceeded: {running} running (max {max_running})"
1141 ));
1142 }
1143
1144 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1145 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1146 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1147 (spawn_plan.clone(), prepared.resolved_task())
1148 } else {
1149 let task = allocate_task_layout(&storage_dir, &session_id)
1150 .map_err(|error| format!("failed to create background task layout: {error}"))?;
1151 let shell = resolve_posix_shell();
1152 let root = project_root.as_deref().unwrap_or(&workdir);
1153 let environment =
1154 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1155 let prepared = match crate::sandbox_spawn::prepare_task_payload(
1156 &task,
1157 command.as_bytes(),
1158 root,
1159 &workdir,
1160 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1161 &shell,
1162 &environment,
1163 ) {
1164 Ok(prepared) => prepared,
1165 Err(error) => {
1166 let _ = delete_resolved_task(&task);
1167 return Err(error);
1168 }
1169 };
1170 let task = prepared.resolved_task();
1171 (spawn_plan.with_prepared_task(prepared), task)
1172 };
1173 let task_id = task_layout.paths.task_id.clone();
1174 let paths = task_layout.paths.clone();
1175
1176 if self.task(&task_id).is_some() {
1177 let _ = delete_resolved_task(&task_layout);
1178 return Err("background task id collided with a live task".to_string());
1179 }
1180
1181 let mut metadata = PersistedTask::starting(
1182 task_id.clone(),
1183 session_id.clone(),
1184 command.to_string(),
1185 workdir.clone(),
1186 project_root,
1187 timeout_ms,
1188 notify_on_completion,
1189 compressed,
1190 );
1191 #[cfg(unix)]
1195 let capture_pipeline_status = {
1196 let shell = spawn_plan
1197 .host_shell_path()
1198 .map(Path::to_path_buf)
1199 .unwrap_or_else(resolve_posix_shell);
1200 let pipeline = single_top_level_pipeline(command);
1201 let capture = should_capture_pipeline_status(&spawn_plan, pipeline.is_some(), &shell);
1202 if capture {
1203 metadata.pipeline_segments = pipeline
1204 .as_ref()
1205 .map(|pipeline| {
1206 pipeline
1207 .segments
1208 .iter()
1209 .map(|segment| segment.label.clone())
1210 .collect()
1211 })
1212 .unwrap_or_default();
1213 }
1214 capture
1215 };
1216 #[cfg(windows)]
1217 let capture_pipeline_status = false;
1218 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1219 if let Err(error) = write_task_at(&task_layout, &metadata) {
1220 let _ = delete_resolved_task(&task_layout);
1221 return Err(format!(
1222 "failed to persist background task metadata: {error}"
1223 ));
1224 }
1225 self.dual_write_task(&paths, &metadata);
1226
1227 let mut io_handles =
1228 TaskIoHandles::create(&task_layout, BgMode::Pipes, capture_pipeline_status)
1229 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
1230 let child = match spawn_detached_child(
1231 &spawn_plan,
1232 command,
1233 &paths,
1234 &workdir,
1235 &env,
1236 &mut io_handles,
1237 capture_pipeline_status,
1238 ) {
1239 Ok(child) => child,
1240 Err(error) => {
1241 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
1242 let _ = delete_task_bundle(&paths);
1243 return Err(error);
1244 }
1245 };
1246
1247 let child_pid = child.id();
1248 metadata.mark_running(child_pid, child_pid as i32);
1249 self.persist_task(&paths, &metadata)
1250 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1251
1252 let task = Arc::new(BgTask {
1253 task_id: task_id.clone(),
1254 session_id,
1255 paths: paths.clone(),
1256 artifact_root: canonical_artifact_root(&paths),
1257 started: Instant::now(),
1258 last_reminder_at: Mutex::new(None),
1259 terminal_at: Mutex::new(None),
1260 state: Mutex::new(BgTaskState {
1261 metadata,
1262 runtime: TaskRuntime::Piped(Some(child)),
1263 io_handles: Some(io_handles),
1264 detached: false,
1265 child_exit_observed: false,
1266 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
1267 terminal_output_cache: None,
1268 pending_terminal_override: None,
1269 }),
1270 });
1271
1272 self.inner
1273 .tasks
1274 .lock()
1275 .map_err(|_| "background task registry lock poisoned".to_string())?
1276 .insert(task_id.clone(), task);
1277
1278 Ok(task_id)
1279 }
1280
1281 #[allow(clippy::too_many_arguments)]
1282 pub fn spawn_pty(
1283 &self,
1284 spawn_plan: SpawnPlan,
1285 command: &str,
1286 session_id: String,
1287 workdir: PathBuf,
1288 env: HashMap<String, String>,
1289 timeout: Option<Duration>,
1290 storage_dir: PathBuf,
1291 max_running: usize,
1292 notify_on_completion: bool,
1293 compressed: bool,
1294 project_root: Option<PathBuf>,
1295 rows: u16,
1296 cols: u16,
1297 ) -> Result<String, String> {
1298 self.start_watchdog();
1299
1300 let running = self.running_count();
1301 if running >= max_running {
1302 #[cfg(unix)]
1303 if let Some(prepared) = spawn_plan.prepared_task() {
1304 let _ = delete_resolved_task(&prepared.resolved_task());
1305 }
1306 return Err(format!(
1307 "background bash task limit exceeded: {running} running (max {max_running})"
1308 ));
1309 }
1310
1311 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1312 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1313 #[cfg(unix)]
1314 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1315 (spawn_plan.clone(), prepared.resolved_task())
1316 } else {
1317 let task = allocate_task_layout(&storage_dir, &session_id)
1318 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1319 let shell = super::resolved_shell_path(true);
1320 let root = project_root.as_deref().unwrap_or(&workdir);
1321 let environment =
1322 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1323 let prepared = match crate::sandbox_spawn::prepare_task_payload(
1324 &task,
1325 command.as_bytes(),
1326 root,
1327 &workdir,
1328 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1329 &shell,
1330 &environment,
1331 ) {
1332 Ok(prepared) => prepared,
1333 Err(error) => {
1334 let _ = delete_resolved_task(&task);
1335 return Err(error);
1336 }
1337 };
1338 let task = prepared.resolved_task();
1339 (spawn_plan.with_prepared_task(prepared), task)
1340 };
1341 #[cfg(windows)]
1342 let task_layout = allocate_task_layout(&storage_dir, &session_id)
1343 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1344 let task_id = task_layout.paths.task_id.clone();
1345 let paths = task_layout.paths.clone();
1346
1347 let mut metadata = PersistedTask::starting(
1348 task_id.clone(),
1349 session_id.clone(),
1350 command.to_string(),
1351 workdir.clone(),
1352 project_root,
1353 timeout_ms,
1354 notify_on_completion,
1355 compressed,
1356 );
1357 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1358 metadata.mode = BgMode::Pty;
1359 metadata.pty_rows = Some(rows);
1360 metadata.pty_cols = Some(cols);
1361 if let Err(error) = write_task_at(&task_layout, &metadata) {
1362 let _ = delete_resolved_task(&task_layout);
1363 return Err(format!(
1364 "failed to persist background task metadata: {error}"
1365 ));
1366 }
1367 self.dual_write_task(&paths, &metadata);
1368 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pty, false)
1369 .map_err(|error| format!("failed to pre-open PTY output handles: {error}"))?;
1370
1371 let runtime = match spawn_pty_for_command(
1372 &spawn_plan,
1373 &task_id,
1374 &session_id,
1375 command,
1376 &paths,
1377 &workdir,
1378 &env,
1379 rows,
1380 cols,
1381 self.inner.wake_tx.clone(),
1382 &mut io_handles,
1383 ) {
1384 Ok(runtime) => runtime,
1385 Err(error) => {
1386 crate::slog_warn!(
1387 "failed to spawn PTY background bash task {task_id}; deleting partial bundle: {error}"
1388 );
1389 let _ = delete_task_bundle(&paths);
1390 return Err(error);
1391 }
1392 };
1393
1394 if let Some(child_pid) = runtime.child_pid {
1395 metadata.mark_running(child_pid, child_pid as i32);
1396 } else {
1397 metadata.status = BgTaskStatus::Running;
1398 metadata.pgid = None;
1399 }
1400 self.persist_task(&paths, &metadata)
1401 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1402
1403 let task = Arc::new(BgTask {
1404 task_id: task_id.clone(),
1405 session_id,
1406 paths: paths.clone(),
1407 artifact_root: canonical_artifact_root(&paths),
1408 started: Instant::now(),
1409 last_reminder_at: Mutex::new(None),
1410 terminal_at: Mutex::new(None),
1411 state: Mutex::new(BgTaskState {
1412 metadata,
1413 runtime: TaskRuntime::Pty(Some(runtime)),
1414 io_handles: Some(io_handles),
1415 detached: false,
1416 child_exit_observed: false,
1417 buffer: BgBuffer::registered(&paths, BgMode::Pty),
1418 terminal_output_cache: None,
1419 pending_terminal_override: None,
1420 }),
1421 });
1422
1423 self.inner
1424 .tasks
1425 .lock()
1426 .map_err(|_| "background task registry lock poisoned".to_string())?
1427 .insert(task_id.clone(), task);
1428
1429 Ok(task_id)
1430 }
1431
1432 #[cfg(windows)]
1433 #[allow(clippy::too_many_arguments)]
1434 pub fn spawn(
1435 &self,
1436 spawn_plan: SpawnPlan,
1437 command: &str,
1438 session_id: String,
1439 workdir: PathBuf,
1440 env: HashMap<String, String>,
1441 timeout: Option<Duration>,
1442 storage_dir: PathBuf,
1443 max_running: usize,
1444 notify_on_completion: bool,
1445 compressed: bool,
1446 project_root: Option<PathBuf>,
1447 ) -> Result<String, String> {
1448 self.start_watchdog();
1449
1450 let running = self.running_count();
1451 if running >= max_running {
1452 #[cfg(unix)]
1453 if let Some(prepared) = spawn_plan.prepared_task() {
1454 let _ = delete_resolved_task(&prepared.resolved_task());
1455 }
1456 return Err(format!(
1457 "background bash task limit exceeded: {running} running (max {max_running})"
1458 ));
1459 }
1460
1461 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1462 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1463 let task_layout = allocate_task_layout(&storage_dir, &session_id)
1464 .map_err(|error| format!("failed to create background task layout: {error}"))?;
1465 let task_id = task_layout.paths.task_id.clone();
1466 let paths = task_layout.paths.clone();
1467
1468 let mut metadata = PersistedTask::starting(
1469 task_id.clone(),
1470 session_id.clone(),
1471 command.to_string(),
1472 workdir.clone(),
1473 project_root,
1474 timeout_ms,
1475 notify_on_completion,
1476 compressed,
1477 );
1478 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1479 if let Err(error) = write_task_at(&task_layout, &metadata) {
1480 let _ = delete_resolved_task(&task_layout);
1481 return Err(format!(
1482 "failed to persist background task metadata: {error}"
1483 ));
1484 }
1485 self.dual_write_task(&paths, &metadata);
1486 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pipes, false)
1487 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
1488
1489 let child = match spawn_detached_child(
1490 &spawn_plan,
1491 command,
1492 &paths,
1493 &workdir,
1494 &env,
1495 &mut io_handles,
1496 false,
1497 ) {
1498 Ok(child) => child,
1499 Err(error) => {
1500 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
1501 let _ = delete_task_bundle(&paths);
1502 return Err(error);
1503 }
1504 };
1505
1506 let child_pid = child.id();
1507 metadata.status = BgTaskStatus::Running;
1508 metadata.child_pid = Some(child_pid);
1509 metadata.pgid = None;
1510 self.persist_task(&paths, &metadata)
1511 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1512
1513 let task = Arc::new(BgTask {
1514 task_id: task_id.clone(),
1515 session_id,
1516 paths: paths.clone(),
1517 artifact_root: canonical_artifact_root(&paths),
1518 started: Instant::now(),
1519 last_reminder_at: Mutex::new(None),
1520 terminal_at: Mutex::new(None),
1521 state: Mutex::new(BgTaskState {
1522 metadata,
1523 runtime: TaskRuntime::Piped(Some(child)),
1524 io_handles: Some(io_handles),
1525 detached: false,
1526 child_exit_observed: false,
1527 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
1528 terminal_output_cache: None,
1529 pending_terminal_override: None,
1530 }),
1531 });
1532
1533 self.inner
1534 .tasks
1535 .lock()
1536 .map_err(|_| "background task registry lock poisoned".to_string())?
1537 .insert(task_id.clone(), task);
1538
1539 Ok(task_id)
1540 }
1541
1542 pub fn write_pty(
1543 &self,
1544 task_id: &str,
1545 session_id: &str,
1546 input: &[u8],
1547 ) -> Result<usize, String> {
1548 let task = self
1549 .task_for_session(task_id, session_id)
1550 .ok_or_else(|| "task_not_found".to_string())?;
1551
1552 let writer = {
1553 let state = task
1554 .state
1555 .lock()
1556 .map_err(|_| "background task lock poisoned".to_string())?;
1557 if state.metadata.mode != BgMode::Pty {
1558 return Err("task_not_pty".to_string());
1559 }
1560 if state.metadata.status.is_terminal() {
1561 return Err("task_exited".to_string());
1562 }
1563 match &state.runtime {
1564 TaskRuntime::Pty(Some(runtime)) => Arc::clone(&runtime.writer),
1565 TaskRuntime::Pty(None) => return Err("task_exited".to_string()),
1566 TaskRuntime::Piped(_) => return Err("task_not_pty".to_string()),
1567 }
1568 };
1569
1570 let mut writer = writer
1571 .lock()
1572 .map_err(|_| "PTY writer lock poisoned".to_string())?;
1573 writer
1574 .write_all(input)
1575 .map_err(|error| format!("failed to write to PTY: {error}"))?;
1576 writer
1577 .flush()
1578 .map_err(|error| format!("failed to flush PTY writer: {error}"))?;
1579 Ok(input.len())
1580 }
1581
1582 pub fn replay_session(&self, storage_dir: &Path, session_id: &str) -> Result<(), String> {
1583 self.replay_session_inner(storage_dir, session_id, None)
1584 }
1585
1586 pub fn replay_session_for_project(
1587 &self,
1588 storage_dir: &Path,
1589 session_id: &str,
1590 project_root: &Path,
1591 ) -> Result<(), String> {
1592 self.replay_session_inner(storage_dir, session_id, Some(project_root))
1593 }
1594
1595 fn replay_session_inner(
1596 &self,
1597 storage_dir: &Path,
1598 session_id: &str,
1599 project_root: Option<&Path>,
1600 ) -> Result<(), String> {
1601 self.start_watchdog();
1602 if !self.inner.persisted_gc_started.swap(true, Ordering::SeqCst) {
1603 if let Err(error) = self.maybe_gc_persisted(storage_dir) {
1604 crate::slog_warn!("failed to GC persisted background bash tasks: {error}");
1605 }
1606 }
1607
1608 let canonical_project = project_root.map(canonicalized_path);
1609 let tasks = match self.replay_session_from_db(session_id) {
1621 Some(Ok(tasks)) if !tasks.is_empty() => tasks,
1622 Some(Ok(_)) => {
1623 let disk_tasks = self.replay_session_from_disk(storage_dir, session_id)?;
1624 if !disk_tasks.is_empty() {
1625 crate::slog_info!(
1626 "bash task replay: 0 in DB for session {}, {} from disk fallback",
1627 session_id,
1628 disk_tasks.len()
1629 );
1630 }
1631 disk_tasks
1632 }
1633 Some(Err(error)) => {
1634 crate::slog_warn!(
1635 "bash task replay DB lookup failed for session {}; falling back to disk: {}",
1636 session_id,
1637 error
1638 );
1639 self.replay_session_from_disk(storage_dir, session_id)?
1640 }
1641 None => {
1642 self.replay_session_from_disk(storage_dir, session_id)?
1644 }
1645 };
1646
1647 for mut metadata in tasks {
1648 if metadata.session_id != session_id {
1649 continue;
1650 }
1651 if let Some(canonical_project) = canonical_project.as_deref() {
1652 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
1653 if metadata_project.as_deref() != Some(canonical_project) {
1654 continue;
1655 }
1656 }
1657
1658 if validate_task_id(&metadata.task_id).is_err() {
1659 crate::slog_warn!(
1660 "ignoring persisted background task with invalid id {:?}",
1661 metadata.task_id
1662 );
1663 continue;
1664 }
1665 let session_dir = session_tasks_dir(storage_dir, session_id);
1666 let resolved = match resolve_task_layout(&session_dir, &metadata.task_id) {
1667 Ok(task) => task,
1668 Err(error) => {
1669 crate::slog_warn!(
1670 "quarantining unresolved background task {}: {error}",
1671 metadata.task_id
1672 );
1673 let _ = quarantine_task_layout(
1674 storage_dir,
1675 &session_dir,
1676 &metadata.task_id,
1677 "invalid",
1678 );
1679 continue;
1680 }
1681 };
1682 match read_task_at(&resolved) {
1683 Ok(disk) if disk.task_id == metadata.task_id && disk.session_id == session_id => {}
1684 Ok(_) | Err(_) => {
1685 let _ = quarantine_task_layout(
1686 storage_dir,
1687 &session_dir,
1688 &metadata.task_id,
1689 "mismatch",
1690 );
1691 continue;
1692 }
1693 }
1694 let paths = resolved.paths;
1695 match metadata.status {
1696 BgTaskStatus::Starting => {
1697 let completion_was_delivered = metadata.completion_delivered;
1698 metadata.mark_terminal(
1699 BgTaskStatus::Failed,
1700 None,
1701 Some("spawn aborted".to_string()),
1702 );
1703 metadata.completion_delivered |= completion_was_delivered;
1704 let _ = self.persist_task(&paths, &metadata);
1705 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1706 self.insert_rehydrated_task(metadata, paths, true)?;
1707 }
1708 BgTaskStatus::Running | BgTaskStatus::Killing => {
1709 if metadata.mode == BgMode::Pty {
1710 if let Ok(Some(marker)) = read_exit_marker(&paths) {
1711 let completion_was_delivered = metadata.completion_delivered;
1712 metadata = terminal_metadata_from_marker(metadata, marker, None);
1713 metadata.completion_delivered |= completion_was_delivered;
1714 let _ = self.persist_task(&paths, &metadata);
1715 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1716 self.insert_rehydrated_task(metadata, paths, true)?;
1717 } else if metadata.status.is_terminal() {
1718 self.insert_rehydrated_task(metadata, paths, true)?;
1719 } else {
1720 let completion_was_delivered = metadata.completion_delivered;
1721 metadata.mark_terminal(
1722 BgTaskStatus::Killed,
1723 None,
1724 Some("pty_lost_on_bridge_restart".to_string()),
1725 );
1726 metadata.completion_delivered |= completion_was_delivered;
1727 let _ = self.persist_task(&paths, &metadata);
1728 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1729 self.insert_rehydrated_task(metadata, paths, true)?;
1730 }
1731 } else if self.running_metadata_is_stale(&metadata) {
1732 let completion_was_delivered = metadata.completion_delivered;
1733 metadata.mark_terminal(
1734 BgTaskStatus::Killed,
1735 None,
1736 Some("orphaned (>24h)".to_string()),
1737 );
1738 metadata.completion_delivered |= completion_was_delivered;
1739 let _ = write_kill_marker_if_absent(&paths);
1740 let _ = self.persist_task(&paths, &metadata);
1741 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1742 self.insert_rehydrated_task(metadata, paths, true)?;
1743 } else if let Ok(Some(marker)) = read_exit_marker(&paths) {
1744 let reason = (metadata.status == BgTaskStatus::Killing).then(|| {
1745 "recovered from inconsistent killing state on replay".to_string()
1746 });
1747 if reason.is_some() {
1748 crate::slog_warn!("background task {} had killing state with exit marker; preferring marker",
1749 metadata.task_id);
1750 }
1751 let completion_was_delivered = metadata.completion_delivered;
1752 metadata = terminal_metadata_from_marker(metadata, marker, reason);
1753 metadata.completion_delivered |= completion_was_delivered;
1754 let _ = self.persist_task(&paths, &metadata);
1755 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1756 self.insert_rehydrated_task(metadata, paths, true)?;
1757 } else if metadata.status == BgTaskStatus::Killing {
1758 let _ = write_kill_marker_if_absent(&paths);
1759 let completion_was_delivered = metadata.completion_delivered;
1760 metadata.mark_terminal(
1761 BgTaskStatus::Killed,
1762 None,
1763 Some("recovered from inconsistent killing state on replay".to_string()),
1764 );
1765 metadata.completion_delivered |= completion_was_delivered;
1766 let _ = self.persist_task(&paths, &metadata);
1767 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1768 self.insert_rehydrated_task(metadata, paths, true)?;
1769 } else if metadata.child_pid.is_some_and(|pid| !is_process_alive(pid)) {
1770 let completion_was_delivered = metadata.completion_delivered;
1771 metadata.mark_terminal(
1772 BgTaskStatus::Failed,
1773 None,
1774 Some("process exited without exit marker".to_string()),
1775 );
1776 metadata.completion_delivered |= completion_was_delivered;
1777 let _ = self.persist_task(&paths, &metadata);
1778 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1779 self.insert_rehydrated_task(metadata, paths, true)?;
1780 } else {
1781 self.insert_rehydrated_task(metadata, paths, true)?;
1782 }
1783 }
1784 _ if metadata.status.is_terminal() => {
1785 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1791 self.insert_rehydrated_task(metadata, paths, true)?;
1792 }
1793 _ => {}
1794 }
1795 }
1796
1797 Ok(())
1798 }
1799
1800 fn replay_session_from_db(
1801 &self,
1802 session_id: &str,
1803 ) -> Option<Result<Vec<PersistedTask>, String>> {
1804 let pool = self
1805 .inner
1806 .db_pool
1807 .read()
1808 .ok()
1809 .and_then(|slot| slot.clone())?;
1810 let harness = self
1811 .inner
1812 .db_harness
1813 .read()
1814 .ok()
1815 .and_then(|slot| slot.clone())?;
1816 let conn = match pool.lock() {
1817 Ok(conn) => conn,
1818 Err(_) => return Some(Err("db mutex poisoned".to_string())),
1819 };
1820 Some(
1821 crate::db::bash_tasks::list_bash_tasks_for_session(&conn, &harness, session_id)
1822 .map(|rows| rows.into_iter().map(PersistedTask::from).collect())
1823 .map_err(|error| error.to_string()),
1824 )
1825 }
1826
1827 fn replay_session_from_disk(
1828 &self,
1829 storage_dir: &Path,
1830 session_id: &str,
1831 ) -> Result<Vec<PersistedTask>, String> {
1832 let dir = session_tasks_dir(storage_dir, session_id);
1833 if !dir.exists() {
1834 return Ok(Vec::new());
1835 }
1836
1837 let (task_ids, invalid_entries) = discover_task_ids(&dir)
1838 .map_err(|error| format!("failed to discover background task layouts: {error}"))?;
1839 for entry in invalid_entries {
1840 if let Err(error) = quarantine_invalid_entry(storage_dir, &dir, &entry) {
1841 crate::slog_warn!(
1842 "failed to quarantine invalid background task entry {:?}: {error}",
1843 entry
1844 );
1845 }
1846 }
1847
1848 let mut tasks = Vec::new();
1849 for task_id in task_ids {
1850 let task = match resolve_task_layout(&dir, &task_id) {
1851 Ok(task) => task,
1852 Err(error)
1853 if error.kind() == std::io::ErrorKind::NotFound
1854 && uninitialized_layout_is_recent(
1855 &dir,
1856 &task_id,
1857 Duration::from_secs(5 * 60),
1858 )
1859 .unwrap_or(false) =>
1860 {
1861 continue;
1862 }
1863 Err(error) => {
1864 crate::slog_warn!(
1865 "quarantining unresolved background task {task_id} during replay: {error}"
1866 );
1867 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
1868 continue;
1869 }
1870 };
1871 match read_task_at(&task) {
1872 Ok(metadata) if metadata.session_id == session_id => tasks.push(metadata),
1873 Ok(_) => {
1874 crate::slog_warn!(
1875 "quarantining background task {task_id} with mismatched session metadata"
1876 );
1877 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "mismatch");
1878 }
1879 Err(error) => {
1880 crate::slog_warn!(
1881 "quarantining invalid background task metadata {task_id} during replay: {error}"
1882 );
1883 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
1884 }
1885 }
1886 }
1887 Ok(tasks)
1888 }
1889
1890 pub fn register_watch(
1891 &self,
1892 task_id: String,
1893 pattern: WatchPattern,
1894 once: bool,
1895 ) -> Result<String, &'static str> {
1896 let task = self.task(&task_id).ok_or("task_not_found")?;
1897 validate_task_id(&task_id).map_err(|_| "invalid_task_id")?;
1898 let (mode, terminal_at_registration) = task
1899 .state
1900 .lock()
1901 .map(|state| {
1902 (
1903 state.metadata.mode.clone(),
1904 state.metadata.status.is_terminal(),
1905 )
1906 })
1907 .map_err(|_| "background_task_lock_poisoned")?;
1908 let mut stdout = (mode == BgMode::Pipes)
1909 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
1910 .transpose()
1911 .map_err(|_| "artifact_refused")?;
1912 let mut stderr = (mode == BgMode::Pipes)
1913 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
1914 .transpose()
1915 .map_err(|_| "artifact_refused")?;
1916 let mut pty = (mode == BgMode::Pty)
1917 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
1918 .transpose()
1919 .map_err(|_| "artifact_refused")?;
1920
1921 let mut terminal_matches = Vec::new();
1922 let scanned_terminal = terminal_at_registration;
1923 let watch_id = {
1924 let mut registry = self
1925 .inner
1926 .watch_registry
1927 .lock()
1928 .map_err(|_| "watch_registry_poisoned")?;
1929 let watch_id = registry.register(task_id.clone(), pattern.clone(), once)?;
1930 match &mode {
1931 BgMode::Pipes => {
1932 let stdout_key = format!("{task_id}:stdout");
1933 let stderr_key = format!("{task_id}:stderr");
1934 if terminal_at_registration {
1935 registry.set_file_cursor(&stdout_key, 0);
1936 registry.set_file_cursor(&stderr_key, 0);
1937 terminal_matches.extend(registry.scan_file_new_bytes(
1938 &stdout_key,
1939 &task_id,
1940 stdout.as_mut().expect("pipe stdout opened"),
1941 ));
1942 terminal_matches.extend(registry.scan_file_new_bytes(
1943 &stderr_key,
1944 &task_id,
1945 stderr.as_mut().expect("pipe stderr opened"),
1946 ));
1947 } else {
1948 registry.prime_file_cursor(
1949 &stdout_key,
1950 stdout.as_ref().expect("pipe stdout opened"),
1951 );
1952 registry.prime_file_cursor(
1953 &stderr_key,
1954 stderr.as_ref().expect("pipe stderr opened"),
1955 );
1956 }
1957 }
1958 BgMode::Pty => {
1959 let pty_key = format!("{task_id}:pty");
1960 if terminal_at_registration {
1961 registry.set_file_cursor(&pty_key, 0);
1962 terminal_matches.extend(registry.scan_file_new_bytes(
1963 &pty_key,
1964 &task_id,
1965 pty.as_mut().expect("PTY artifact opened"),
1966 ));
1967 } else {
1968 registry.prime_file_cursor(
1969 &pty_key,
1970 pty.as_ref().expect("PTY artifact opened"),
1971 );
1972 }
1973 }
1974 }
1975 watch_id
1976 };
1977
1978 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
1979 self.persist_watch_registration(
1980 &task.session_id,
1981 &task_id,
1982 &watch_id,
1983 &pattern,
1984 once,
1985 stdout_offset,
1986 stderr_offset,
1987 pty_offset,
1988 );
1989
1990 if task.is_terminal() {
1991 if !scanned_terminal {
1992 terminal_matches = {
1993 let mut registry = self
1994 .inner
1995 .watch_registry
1996 .lock()
1997 .map_err(|_| "watch_registry_poisoned")?;
1998 match &mode {
1999 BgMode::Pipes => {
2000 let stdout_key = format!("{task_id}:stdout");
2001 let stderr_key = format!("{task_id}:stderr");
2002 registry.set_file_cursor(&stdout_key, 0);
2003 registry.set_file_cursor(&stderr_key, 0);
2004 let mut matches = registry.scan_file_new_bytes(
2005 &stdout_key,
2006 &task_id,
2007 stdout.as_mut().expect("pipe stdout opened"),
2008 );
2009 matches.extend(registry.scan_file_new_bytes(
2010 &stderr_key,
2011 &task_id,
2012 stderr.as_mut().expect("pipe stderr opened"),
2013 ));
2014 matches
2015 }
2016 BgMode::Pty => {
2017 let pty_key = format!("{task_id}:pty");
2018 registry.set_file_cursor(&pty_key, 0);
2019 registry.scan_file_new_bytes(
2020 &pty_key,
2021 &task_id,
2022 pty.as_mut().expect("PTY artifact opened"),
2023 )
2024 }
2025 }
2026 };
2027 }
2028
2029 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
2030 let (watch_controlled, watch_matched) = self.task_watch_state(&task_id);
2031 if terminal_matches.is_empty() && (!watch_controlled || watch_matched) {
2032 if watch_matched {
2033 let _ = task.set_completion_delivered(true, self);
2034 self.clear_task_watch_state(&task_id);
2035 self.delete_persisted_watches_for_task(&task.session_id, &task_id);
2037 }
2038 return Ok(watch_id);
2039 }
2040
2041 let completion = self
2042 .remove_pending_completion(&task_id)
2043 .or_else(|| self.completion_snapshot_for_task(&task));
2044 if terminal_matches.is_empty() {
2045 if let Some(completion) = completion.as_ref() {
2046 self.emit_bash_watch_exit(completion);
2047 }
2048 } else {
2049 for pattern_match in &terminal_matches {
2050 self.persist_watch_match(
2051 &task.session_id,
2052 &task_id,
2053 pattern_match,
2054 stdout_offset,
2055 stderr_offset,
2056 pty_offset,
2057 );
2058 self.emit_bash_pattern_match(&task.session_id, pattern_match.clone());
2059 }
2060 }
2061 let _ = task.set_completion_delivered(true, self);
2062 self.clear_task_watch_state(&task_id);
2063 self.delete_persisted_watches_for_task(&task.session_id, &task_id);
2065 }
2066
2067 Ok(watch_id)
2068 }
2069
2070 pub fn unregister_watch(&self, task_id: &str, watch_id: &str) {
2071 let session_id = self.task(task_id).map(|task| task.session_id.clone());
2072 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2073 registry.unregister(task_id, watch_id);
2074 }
2075 if let Some(session_id) = session_id {
2076 self.delete_persisted_watch(&session_id, task_id, watch_id);
2077 }
2078 }
2079
2080 pub fn active_watch_count(&self, task_id: &str) -> usize {
2081 self.inner
2082 .watch_registry
2083 .lock()
2084 .map(|registry| registry.active_count(task_id))
2085 .unwrap_or(0)
2086 }
2087
2088 fn task_watch_state(&self, task_id: &str) -> (bool, bool) {
2089 self.inner
2090 .watch_registry
2091 .lock()
2092 .map(|registry| {
2093 (
2094 registry.has_controlled_task(task_id),
2095 registry.has_matched_task(task_id),
2096 )
2097 })
2098 .unwrap_or((false, false))
2099 }
2100
2101 fn task_has_watch_control(&self, task_id: &str) -> bool {
2102 self.inner
2103 .watch_registry
2104 .lock()
2105 .map(|registry| registry.has_controlled_task(task_id))
2106 .unwrap_or(false)
2107 }
2108
2109 fn clear_task_watch_state(&self, task_id: &str) {
2110 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2111 registry.clear_task(task_id);
2112 }
2113 }
2114
2115 pub(crate) fn scan_task_watch_output(&self, task: &Arc<BgTask>) {
2116 let mode = match task.state.lock() {
2117 Ok(state) => state.metadata.mode.clone(),
2118 Err(_) => return,
2119 };
2120 let mut stdout = (mode == BgMode::Pipes)
2121 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
2122 .transpose()
2123 .ok()
2124 .flatten();
2125 let mut stderr = (mode == BgMode::Pipes)
2126 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
2127 .transpose()
2128 .ok()
2129 .flatten();
2130 let mut pty = (mode == BgMode::Pty)
2131 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
2132 .transpose()
2133 .ok()
2134 .flatten();
2135 let mut matches = Vec::new();
2136 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2137 match mode {
2138 BgMode::Pipes => {
2139 let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) else {
2140 return;
2141 };
2142 let stdout_key = format!("{}:stdout", task.task_id);
2143 let stderr_key = format!("{}:stderr", task.task_id);
2144 matches.extend(registry.scan_file_new_bytes(
2145 &stdout_key,
2146 &task.task_id,
2147 stdout,
2148 ));
2149 matches.extend(registry.scan_file_new_bytes(
2150 &stderr_key,
2151 &task.task_id,
2152 stderr,
2153 ));
2154 }
2155 BgMode::Pty => {
2156 let Some(pty) = pty.as_mut() else {
2157 return;
2158 };
2159 let pty_key = format!("{}:pty", task.task_id);
2160 matches.extend(registry.scan_file_new_bytes(&pty_key, &task.task_id, pty));
2161 }
2162 }
2163 }
2164 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
2165 if matches.is_empty() {
2166 if self.task_has_watch_control(&task.task_id) {
2169 self.persist_task_watch_cursors(
2170 &task.session_id,
2171 &task.task_id,
2172 stdout_offset,
2173 stderr_offset,
2174 pty_offset,
2175 );
2176 }
2177 return;
2178 }
2179 for pattern_match in matches {
2180 self.persist_watch_match(
2181 &task.session_id,
2182 &task.task_id,
2183 &pattern_match,
2184 stdout_offset,
2185 stderr_offset,
2186 pty_offset,
2187 );
2188 self.emit_bash_pattern_match(&task.session_id, pattern_match);
2189 }
2190 self.persist_task_watch_cursors(
2191 &task.session_id,
2192 &task.task_id,
2193 stdout_offset,
2194 stderr_offset,
2195 pty_offset,
2196 );
2197 }
2198
2199 pub fn status(
2200 &self,
2201 task_id: &str,
2202 session_id: &str,
2203 project_root: Option<&Path>,
2204 storage_dir: Option<&Path>,
2205 preview_bytes: usize,
2206 ) -> Option<BgTaskSnapshot> {
2207 validate_task_id(task_id).ok()?;
2208 let mut task = self.task_for_session(task_id, session_id);
2209 if task.is_none() {
2210 if let Some(storage_dir) = storage_dir {
2211 let _ = if let Some(project_root) = project_root {
2212 self.replay_session_for_project(storage_dir, session_id, project_root)
2213 } else {
2214 self.replay_session(storage_dir, session_id)
2215 };
2216 task = self.task_for_session(task_id, session_id);
2217 }
2218 }
2219 let Some(task) = task else {
2220 return self.status_relaxed(
2221 task_id,
2222 session_id,
2223 project_root?,
2224 storage_dir?,
2225 preview_bytes,
2226 );
2227 };
2228 let _ = self.poll_task(&task);
2229 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
2230 }
2231
2232 fn status_relaxed_task(
2233 &self,
2234 task_id: &str,
2235 project_root: &Path,
2236 storage_dir: &Path,
2237 ) -> Option<Arc<BgTask>> {
2238 validate_task_id(task_id).ok()?;
2239 let canonical_project = canonicalized_path(project_root);
2240 match self.lookup_relaxed_task_from_db(task_id, project_root) {
2241 Some(Ok(Some(metadata))) => {
2242 if let Some(task) = self.task(task_id) {
2243 let matches_project = task
2244 .state
2245 .lock()
2246 .map(|state| {
2247 state
2248 .metadata
2249 .project_root
2250 .as_deref()
2251 .map(canonicalized_path)
2252 .as_deref()
2253 == Some(canonical_project.as_path())
2254 })
2255 .unwrap_or(false);
2256 return matches_project.then_some(task);
2257 }
2258 let resolved = resolve_task_layout(
2259 &session_tasks_dir(storage_dir, &metadata.session_id),
2260 &metadata.task_id,
2261 )
2262 .ok()?;
2263 let disk = read_task_at(&resolved).ok()?;
2264 if disk.task_id != metadata.task_id || disk.session_id != metadata.session_id {
2265 return None;
2266 }
2267 if self
2268 .insert_rehydrated_task(metadata, resolved.paths, true)
2269 .is_err()
2270 {
2271 return None;
2272 }
2273 return self.task(task_id);
2274 }
2275 Some(Ok(None)) => {
2276 crate::slog_info!(
2277 "bash task relaxed DB miss for {}; falling back to disk",
2278 task_id
2279 );
2280 }
2281 Some(Err(error)) => {
2282 crate::slog_warn!(
2283 "bash task relaxed DB lookup failed for {}; falling back to disk: {}",
2284 task_id,
2285 error
2286 );
2287 }
2288 None => {
2289 crate::slog_info!(
2290 "bash task relaxed DB unavailable for {}; falling back to disk",
2291 task_id
2292 );
2293 }
2294 }
2295 let root = storage_dir.join("bash-tasks");
2296 let entries = fs::read_dir(&root).ok()?;
2297 for entry in entries.flatten() {
2298 let dir = entry.path();
2299 if !dir.is_dir() {
2300 continue;
2301 }
2302 let resolved = match resolve_task_layout(&dir, task_id) {
2303 Ok(task) => task,
2304 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
2305 Err(error) => {
2306 crate::slog_warn!(
2307 "quarantining unresolved background task {task_id} during relaxed lookup: {error}"
2308 );
2309 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
2310 continue;
2311 }
2312 };
2313 let metadata = match read_task_at(&resolved) {
2314 Ok(metadata) => metadata,
2315 Err(error) => {
2316 crate::slog_warn!(
2317 "quarantining invalid background task metadata {task_id} during relaxed lookup: {error}"
2318 );
2319 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
2320 continue;
2321 }
2322 };
2323 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
2324 if metadata_project.as_deref() != Some(canonical_project.as_path()) {
2325 continue;
2326 }
2327 if let Some(task) = self.task(task_id) {
2328 let matches_project = task
2329 .state
2330 .lock()
2331 .map(|state| {
2332 state
2333 .metadata
2334 .project_root
2335 .as_deref()
2336 .map(canonicalized_path)
2337 .as_deref()
2338 == Some(canonical_project.as_path())
2339 })
2340 .unwrap_or(false);
2341 return matches_project.then_some(task);
2342 }
2343 if self
2344 .insert_rehydrated_task(metadata, resolved.paths, true)
2345 .is_err()
2346 {
2347 return None;
2348 }
2349 return self.task(task_id);
2350 }
2351 None
2352 }
2353
2354 fn lookup_relaxed_task_from_db(
2355 &self,
2356 task_id: &str,
2357 project_root: &Path,
2358 ) -> Option<Result<Option<PersistedTask>, String>> {
2359 let pool = self
2360 .inner
2361 .db_pool
2362 .read()
2363 .ok()
2364 .and_then(|slot| slot.clone())?;
2365 let harness = self
2366 .inner
2367 .db_harness
2368 .read()
2369 .ok()
2370 .and_then(|slot| slot.clone())?;
2371 let conn = match pool.lock() {
2372 Ok(conn) => conn,
2373 Err(_) => return Some(Err("db mutex poisoned".to_string())),
2374 };
2375 let project_key = crate::path_identity::project_scope_key(project_root);
2376 Some(
2377 crate::db::bash_tasks::find_bash_task_for_project(
2378 &conn,
2379 &harness,
2380 &project_key,
2381 task_id,
2382 )
2383 .map(|row| row.map(PersistedTask::from))
2384 .map_err(|error| error.to_string()),
2385 )
2386 }
2387
2388 pub(super) fn status_relaxed(
2389 &self,
2390 task_id: &str,
2391 _session_id: &str,
2392 project_root: &Path,
2393 storage_dir: &Path,
2394 preview_bytes: usize,
2395 ) -> Option<BgTaskSnapshot> {
2396 let task = self.status_relaxed_task(task_id, project_root, storage_dir)?;
2397 let _ = self.poll_task(&task);
2398 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
2399 }
2400
2401 pub fn kill_relaxed(
2402 &self,
2403 task_id: &str,
2404 project_root: &Path,
2405 storage_dir: &Path,
2406 ) -> Result<BgTaskSnapshot, String> {
2407 let task = self
2408 .status_relaxed_task(task_id, project_root, storage_dir)
2409 .ok_or_else(|| format!("background task not found: {task_id}"))?;
2410 self.kill_with_status(task_id, &task.session_id, BgTaskStatus::Killed)
2411 }
2412
2413 pub fn maybe_gc_persisted(&self, storage_dir: &Path) -> Result<usize, String> {
2414 #[cfg(test)]
2415 self.inner.persisted_gc_runs.fetch_add(1, Ordering::SeqCst);
2416
2417 let mut deleted = 0usize;
2418
2419 let root = storage_dir.join("bash-tasks");
2420 if root.exists() {
2421 let session_dirs = fs::read_dir(&root).map_err(|e| {
2422 format!(
2423 "failed to read background task root {}: {e}",
2424 root.display()
2425 )
2426 })?;
2427 for session_entry in session_dirs.flatten() {
2428 let session_dir = session_entry.path();
2429 if !session_dir.is_dir() {
2430 continue;
2431 }
2432 let (task_ids, invalid_entries) = match discover_task_ids(&session_dir) {
2433 Ok(discovery) => discovery,
2434 Err(error) => {
2435 crate::slog_warn!(
2436 "failed to discover background task session {}: {error}",
2437 session_dir.display()
2438 );
2439 continue;
2440 }
2441 };
2442 for entry in invalid_entries {
2443 let _ = quarantine_invalid_entry(storage_dir, &session_dir, &entry);
2444 }
2445 for task_id in task_ids {
2446 let resolved = match resolve_task_layout(&session_dir, &task_id) {
2447 Ok(task) => task,
2448 Err(error)
2449 if error.kind() == std::io::ErrorKind::NotFound
2450 && uninitialized_layout_is_recent(
2451 &session_dir,
2452 &task_id,
2453 Duration::from_secs(5 * 60),
2454 )
2455 .unwrap_or(false) =>
2456 {
2457 continue;
2458 }
2459 Err(error) => {
2460 crate::slog_warn!(
2461 "quarantining unresolved background task {task_id}: {error}"
2462 );
2463 quarantine_task_layout(storage_dir, &session_dir, &task_id, "invalid")
2464 .map_err(|error| error.to_string())?;
2465 continue;
2466 }
2467 };
2468 if modified_within(&resolved.paths.json, PERSISTED_GC_GRACE) {
2469 continue;
2470 }
2471 let metadata = match read_task_at(&resolved) {
2472 Ok(metadata) => metadata,
2473 Err(error) => {
2474 crate::slog_warn!(
2475 "quarantining corrupt background task metadata {task_id}: {error}"
2476 );
2477 quarantine_task_layout(storage_dir, &session_dir, &task_id, "corrupt")
2478 .map_err(|error| error.to_string())?;
2479 continue;
2480 }
2481 };
2482 if !(metadata.status.is_terminal() && metadata.completion_delivered) {
2483 continue;
2484 }
2485 match delete_task_bundle(&resolved.paths) {
2486 Ok(()) => {
2487 self.delete_gc_task_from_db(&metadata);
2488 deleted += 1;
2489 log::debug!(
2490 "deleted persisted background task bundle {}",
2491 metadata.task_id
2492 );
2493 }
2494 Err(error) => {
2495 crate::slog_warn!(
2496 "failed to delete background task bundle {}: {error}",
2497 metadata.task_id
2498 );
2499 }
2500 }
2501 }
2502 }
2503 }
2504 gc_quarantine(storage_dir);
2505 Ok(deleted)
2506 }
2507
2508 pub fn list(&self, preview_bytes: usize) -> Vec<BgTaskSnapshot> {
2509 let tasks = self
2510 .inner
2511 .tasks
2512 .lock()
2513 .map(|tasks| tasks.values().cloned().collect::<Vec<_>>())
2514 .unwrap_or_default();
2515 tasks
2516 .into_iter()
2517 .map(|task| {
2518 let _ = self.poll_task(&task);
2519 self.snapshot_with_terminal_cache(&task, preview_bytes)
2520 })
2521 .collect()
2522 }
2523
2524 fn maybe_compress_snapshot(&self, task: &Arc<BgTask>, snapshot: &mut BgTaskSnapshot) {
2530 if !snapshot.info.status.is_terminal() || snapshot.info.mode == BgMode::Pty {
2531 return;
2532 }
2533 if let Some(cache) = self.ensure_terminal_output_cache(task) {
2534 snapshot.output_preview = cache.output_preview;
2535 snapshot.output_truncated = cache.output_truncated;
2536 }
2537 }
2538
2539 pub fn kill(&self, task_id: &str, session_id: &str) -> Result<BgTaskSnapshot, String> {
2540 self.kill_with_status(task_id, session_id, BgTaskStatus::Killed)
2541 }
2542
2543 pub fn kill_running_tasks_for_root(&self, project_root: &Path) -> usize {
2553 let canonical_root = canonicalized_path(project_root);
2554 let targets = self
2555 .inner
2556 .tasks
2557 .lock()
2558 .map(|tasks| {
2559 tasks
2560 .values()
2561 .filter_map(|task| {
2562 let state = task.state.lock().ok()?;
2563 let status = &state.metadata.status;
2564 let running = matches!(status, BgTaskStatus::Running)
2565 || (state.metadata.mode == BgMode::Pty
2566 && matches!(status, BgTaskStatus::Killing));
2567 if !running {
2568 return None;
2569 }
2570 let task_root = state
2571 .metadata
2572 .project_root
2573 .as_deref()
2574 .unwrap_or(&state.metadata.workdir);
2575 (canonicalized_path(task_root) == canonical_root)
2576 .then(|| (task.task_id.clone(), task.session_id.clone()))
2577 })
2578 .collect::<Vec<_>>()
2579 })
2580 .unwrap_or_default();
2581
2582 let mut killed = 0;
2583 for (task_id, session_id) in targets {
2584 match self.kill_with_status_reason(
2585 &task_id,
2586 &session_id,
2587 BgTaskStatus::Killed,
2588 Some(ROOT_RECLAIMED_REASON.to_string()),
2589 ) {
2590 Ok(_) => killed += 1,
2591 Err(error) => crate::slog_warn!(
2592 "failed to terminate background task {task_id} for reclaimed root {}: {error}",
2593 project_root.display()
2594 ),
2595 }
2596 }
2597 killed
2598 }
2599
2600 pub fn promote(&self, task_id: &str, session_id: &str) -> Result<bool, String> {
2601 let task = self
2602 .task_for_session(task_id, session_id)
2603 .ok_or_else(|| format!("background task not found: {task_id}"))?;
2604 let terminal_after_promote = {
2605 let mut state = task
2606 .state
2607 .lock()
2608 .map_err(|_| "background task lock poisoned".to_string())?;
2609 let updated = self
2610 .update_task_metadata(&task.paths, |metadata| {
2611 metadata.notify_on_completion = true;
2612 metadata.completion_delivered = false;
2613 })
2614 .map_err(|e| format!("failed to promote background task: {e}"))?;
2615 state.metadata = updated;
2616 state.metadata.status.is_terminal()
2617 };
2618 if terminal_after_promote {
2619 self.post_terminal_transition(&task, true)?;
2620 }
2621 Ok(true)
2622 }
2623
2624 pub(crate) fn kill_for_timeout(&self, task_id: &str, session_id: &str) -> Result<(), String> {
2625 self.kill_with_status(task_id, session_id, BgTaskStatus::TimedOut)
2626 .map(|_| ())
2627 }
2628
2629 pub fn cleanup_finished(&self, older_than: Duration) {
2630 let cutoff = Instant::now().checked_sub(older_than);
2631 let removable_paths: Vec<(String, TaskPaths)> =
2632 if let Ok(mut tasks) = self.inner.tasks.lock() {
2633 let removable = tasks
2634 .iter()
2635 .filter_map(|(task_id, task)| {
2636 let delivered_terminal = task
2637 .state
2638 .lock()
2639 .map(|state| {
2640 state.metadata.status.is_terminal()
2641 && state.metadata.completion_delivered
2642 })
2643 .unwrap_or(false);
2644 if !delivered_terminal {
2645 return None;
2646 }
2647
2648 let terminal_at = task.terminal_at.lock().ok().and_then(|at| *at);
2649 let expired = match (terminal_at, cutoff) {
2650 (Some(terminal_at), Some(cutoff)) => terminal_at <= cutoff,
2651 (Some(_), None) => true,
2652 (None, _) => false,
2653 };
2654 expired.then(|| task_id.clone())
2655 })
2656 .collect::<Vec<_>>();
2657
2658 removable
2659 .into_iter()
2660 .filter_map(|task_id| {
2661 tasks
2662 .remove(&task_id)
2663 .map(|task| (task_id, task.paths.clone()))
2664 })
2665 .collect()
2666 } else {
2667 Vec::new()
2668 };
2669
2670 for (task_id, paths) in removable_paths {
2671 match delete_task_bundle(&paths) {
2672 Ok(()) => log::debug!("deleted persisted background task bundle {task_id}"),
2673 Err(error) => crate::slog_warn!(
2674 "failed to delete persisted background task bundle {task_id}: {error}"
2675 ),
2676 }
2677 }
2678 }
2679
2680 pub fn drain_completions(&self) -> Vec<BgCompletion> {
2681 self.drain_completions_for_session(None)
2682 }
2683
2684 pub fn drain_completions_for_session(&self, session_id: Option<&str>) -> Vec<BgCompletion> {
2685 let completions = match self.inner.completions.lock() {
2686 Ok(completions) => completions,
2687 Err(_) => return Vec::new(),
2688 };
2689
2690 completions
2691 .iter()
2692 .filter(|completion| completion_matches_session(completion, session_id))
2693 .cloned()
2694 .collect()
2695 }
2696
2697 pub fn has_completions_for_session(&self, session_id: Option<&str>) -> bool {
2698 match self.inner.completions.lock() {
2699 Ok(completions) => completions
2700 .iter()
2701 .any(|completion| completion_matches_session(completion, session_id)),
2702 Err(_) => true,
2706 }
2707 }
2708
2709 pub fn ack_completions_for_session(
2710 &self,
2711 session_id: Option<&str>,
2712 task_ids: &[String],
2713 ) -> Vec<String> {
2714 if task_ids.is_empty() {
2715 return Vec::new();
2716 }
2717 let requested_task_ids = task_ids.iter().map(String::as_str).collect::<HashSet<_>>();
2718 let mut completion_sessions = HashMap::new();
2719 if let Ok(mut completions) = self.inner.completions.lock() {
2720 completions.retain(|completion| {
2721 let session_matches = session_id
2722 .map(|session_id| completion.session_id == session_id)
2723 .unwrap_or(true);
2724 if session_matches && requested_task_ids.contains(completion.task_id.as_str()) {
2725 completion_sessions
2726 .insert(completion.task_id.clone(), completion.session_id.clone());
2727 false
2728 } else {
2729 true
2730 }
2731 });
2732 }
2733
2734 let mut delivered = Vec::new();
2735 for task_id in task_ids {
2736 let task = if let Some(session_id) = session_id {
2737 self.task_for_session(task_id, session_id)
2738 } else if let Some(completion_session_id) = completion_sessions.get(task_id) {
2739 self.task_for_session(task_id, completion_session_id)
2740 } else {
2741 self.task(task_id)
2742 };
2743 if let Some(task) = task {
2744 let terminal = task
2745 .state
2746 .lock()
2747 .map(|state| state.metadata.status.is_terminal())
2748 .unwrap_or(false);
2749 self.ack_persisted_watches_for_task(&task.session_id, task_id, terminal);
2753 if terminal {
2754 self.clear_task_watch_state(task_id);
2755 if task.set_completion_delivered(true, self).is_ok() {
2756 delivered.push(task_id.clone());
2757 }
2758 } else {
2759 self.sync_memory_watches_after_ack(task_id);
2762 delivered.push(task_id.clone());
2763 }
2764 } else if let Some(session_id) = session_id {
2765 self.ack_persisted_watches_for_task(session_id, task_id, true);
2767 delivered.push(task_id.clone());
2768 }
2769 }
2770
2771 delivered
2772 }
2773
2774 fn sync_memory_watches_after_ack(&self, task_id: &str) {
2775 let Some((harness, pool)) = self.db_harness_and_pool() else {
2776 return;
2777 };
2778 let session_id = match self.task(task_id) {
2779 Some(task) => task.session_id.clone(),
2780 None => return,
2781 };
2782 let Ok(conn) = pool.lock() else {
2783 return;
2784 };
2785 let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
2786 &conn,
2787 &harness,
2788 &session_id,
2789 task_id,
2790 ) else {
2791 return;
2792 };
2793 let remaining: HashSet<String> = rows.into_iter().map(|row| row.watch_id).collect();
2794 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2795 registry.retain_watch_ids(task_id, &remaining);
2796 }
2797 }
2798
2799 pub fn pending_completions_for_session(&self, session_id: &str) -> Vec<BgCompletion> {
2800 self.inner
2801 .completions
2802 .lock()
2803 .map(|completions| {
2804 completions
2805 .iter()
2806 .filter(|completion| completion.session_id == session_id)
2807 .cloned()
2808 .collect()
2809 })
2810 .unwrap_or_default()
2811 }
2812
2813 fn remove_pending_completion(&self, task_id: &str) -> Option<BgCompletion> {
2814 let mut completions = self.inner.completions.lock().ok()?;
2815 let idx = completions
2816 .iter()
2817 .position(|completion| completion.task_id == task_id)?;
2818 completions.remove(idx)
2819 }
2820
2821 fn completion_snapshot_for_task(&self, task: &Arc<BgTask>) -> Option<BgCompletion> {
2822 let snapshot = self.snapshot_with_terminal_cache(task, RUNNING_OUTPUT_PREVIEW_BYTES);
2823 if !snapshot.info.status.is_terminal() {
2824 return None;
2825 }
2826 let (output_preview, output_truncated) = if snapshot.info.mode == BgMode::Pty {
2827 (String::new(), false)
2828 } else {
2829 self.ensure_terminal_output_cache(task)
2830 .map(|cache| completion_preview_for_cache(&cache, snapshot.exit_code))
2831 .unwrap_or_else(|| (String::new(), false))
2832 };
2833 Some(BgCompletion {
2834 task_id: snapshot.info.task_id,
2835 session_id: task.session_id.clone(),
2836 status: snapshot.info.status,
2837 exit_code: snapshot.exit_code,
2838 command: snapshot.info.command,
2839 output_preview,
2840 output_truncated,
2841 original_tokens: None,
2842 compressed_tokens: None,
2843 tokens_skipped: false,
2844 status_reason: snapshot.info.status_reason,
2845 })
2846 }
2847
2848 pub fn detach(&self) {
2849 self.inner.shutdown.store(true, Ordering::SeqCst);
2850 if let Ok(mut tasks) = self.inner.tasks.lock() {
2851 for task in tasks.values() {
2852 if let Ok(mut state) = task.state.lock() {
2853 match &mut state.runtime {
2854 TaskRuntime::Piped(child) => *child = None,
2855 TaskRuntime::Pty(runtime) => *runtime = None,
2856 }
2857 state.detached = true;
2858 }
2859 }
2860 tasks.clear();
2861 }
2862 }
2863
2864 pub fn shutdown(&self) {
2865 let tasks = self
2866 .inner
2867 .tasks
2868 .lock()
2869 .map(|tasks| {
2870 tasks
2871 .values()
2872 .map(|task| (task.task_id.clone(), task.session_id.clone()))
2873 .collect::<Vec<_>>()
2874 })
2875 .unwrap_or_default();
2876 for (task_id, session_id) in tasks {
2877 let _ = self.kill(&task_id, &session_id);
2878 }
2879 }
2880
2881 pub(crate) fn poll_task(&self, task: &Arc<BgTask>) -> Result<(), String> {
2882 if let Ok(state) = task.state.lock() {
2883 if let TaskRuntime::Pty(Some(pty)) = &state.runtime {
2884 if !pty.exit_observed.load(Ordering::SeqCst) {
2892 return Ok(());
2893 }
2894 }
2895 }
2896 let marker = match read_exit_marker(&task.paths) {
2897 Ok(Some(marker)) => marker,
2898 Ok(None) => return Ok(()),
2899 Err(error) => return Err(format!("failed to read exit marker: {error}")),
2900 };
2901 self.finalize_from_marker(task, marker, None)
2902 }
2903
2904 pub(crate) fn reap_child(&self, task: &Arc<BgTask>) {
2905 let mut needs_completion = false;
2906 {
2907 let Ok(mut state) = task.state.lock() else {
2908 return;
2909 };
2910 match &mut state.runtime {
2911 TaskRuntime::Piped(child_slot) => {
2912 if let Some(child) = child_slot.as_mut() {
2913 if let Ok(Some(status)) = child.try_wait() {
2914 *child_slot = None;
2915 state.detached = true;
2916 state.child_exit_observed = true;
2917 if let Some(handles) = state.io_handles.as_mut() {
2918 if handles.artifact_len(TaskArtifact::Exit).unwrap_or(1) == 0 {
2919 let marker = status
2920 .code()
2921 .map(|code| code.to_string())
2922 .unwrap_or_else(|| "1".to_string());
2923 let _ = handles.write(TaskArtifact::Exit, marker.as_bytes());
2924 }
2925 }
2926 }
2927 } else if state.detached {
2928 let child_known_dead = state.child_exit_observed
2929 || state
2930 .metadata
2931 .child_pid
2932 .is_some_and(|pid| !is_process_alive(pid));
2933 if child_known_dead {
2934 needs_completion =
2935 self.fail_without_exit_marker_if_needed(task, &mut state);
2936 }
2937 }
2938 }
2939 TaskRuntime::Pty(Some(pty)) => {
2940 if pty.exit_observed.load(Ordering::SeqCst) {
2941 drop(state);
2942 let _ = self.poll_task(task);
2943 return;
2944 }
2945 }
2946 TaskRuntime::Pty(None) => {}
2947 }
2948 }
2949 if needs_completion {
2950 let _ = self.post_terminal_transition(task, true);
2951 }
2952 }
2953
2954 fn fail_without_exit_marker_if_needed(
2955 &self,
2956 task: &Arc<BgTask>,
2957 state: &mut BgTaskState,
2958 ) -> bool {
2959 if state.metadata.status.is_terminal() {
2960 return false;
2961 }
2962 if matches!(read_exit_marker(&task.paths), Ok(Some(_))) {
2963 return false;
2964 }
2965 let watch_controlled = self.task_has_watch_control(&task.task_id);
2966 let updated = self.update_task_metadata(&task.paths, |metadata| {
2967 metadata.mark_terminal(
2968 BgTaskStatus::Failed,
2969 None,
2970 Some("process exited without exit marker".to_string()),
2971 );
2972 if watch_controlled {
2973 metadata.completion_delivered = true;
2974 }
2975 });
2976 if let Ok(metadata) = updated {
2977 state.pending_terminal_override = None;
2978 state.metadata = metadata;
2979 task.mark_terminal_now();
2980 return true;
2981 }
2982 false
2983 }
2984
2985 pub(crate) fn running_tasks(&self) -> Vec<Arc<BgTask>> {
2986 self.inner
2987 .tasks
2988 .lock()
2989 .map(|tasks| {
2990 tasks
2991 .values()
2992 .filter(|task| task.is_running())
2993 .cloned()
2994 .collect()
2995 })
2996 .unwrap_or_default()
2997 }
2998
2999 fn insert_rehydrated_task(
3000 &self,
3001 metadata: PersistedTask,
3002 paths: TaskPaths,
3003 detached: bool,
3004 ) -> Result<(), String> {
3005 let task_id = metadata.task_id.clone();
3006 let session_id = metadata.session_id.clone();
3007 let started = started_instant_from_unix_millis(metadata.started_at);
3008 let suppress_replayed_running_reminder = metadata.status == BgTaskStatus::Running;
3009 let mode = metadata.mode.clone();
3010 let task = Arc::new(BgTask {
3011 task_id: task_id.clone(),
3012 session_id,
3013 paths: paths.clone(),
3014 artifact_root: canonical_artifact_root(&paths),
3015 started,
3016 last_reminder_at: Mutex::new(suppress_replayed_running_reminder.then(Instant::now)),
3017 terminal_at: Mutex::new(metadata.status.is_terminal().then(Instant::now)),
3018 state: Mutex::new(BgTaskState {
3019 metadata,
3020 runtime: if mode == BgMode::Pty {
3021 TaskRuntime::Pty(None)
3022 } else {
3023 TaskRuntime::Piped(None)
3024 },
3025 io_handles: None,
3026 detached,
3027 child_exit_observed: false,
3034 buffer: BgBuffer::registered(&paths, mode.clone()),
3035 terminal_output_cache: None,
3036 pending_terminal_override: None,
3037 }),
3038 });
3039 self.inner
3040 .tasks
3041 .lock()
3042 .map_err(|_| "background task registry lock poisoned".to_string())?
3043 .insert(task_id.clone(), Arc::clone(&task));
3044 self.rearm_persisted_watches(&task);
3048 Ok(())
3049 }
3050
3051 fn rearm_persisted_watches(&self, task: &Arc<BgTask>) {
3052 let Some((harness, pool)) = self.db_harness_and_pool() else {
3053 return;
3054 };
3055 let rows = {
3056 let Ok(conn) = pool.lock() else {
3057 return;
3058 };
3059 match crate::db::bash_watches::list_bash_pattern_watches_for_task(
3060 &conn,
3061 &harness,
3062 &task.session_id,
3063 &task.task_id,
3064 ) {
3065 Ok(rows) if !rows.is_empty() => rows,
3066 _ => return,
3067 }
3068 };
3069
3070 let mode = match task.state.lock() {
3071 Ok(state) => state.metadata.mode.clone(),
3072 Err(_) => return,
3073 };
3074 let terminal = task
3075 .state
3076 .lock()
3077 .map(|state| state.metadata.status.is_terminal())
3078 .unwrap_or(false);
3079 let completion_delivered = task
3080 .state
3081 .lock()
3082 .map(|state| state.metadata.completion_delivered)
3083 .unwrap_or(true);
3084
3085 let mut stdout = (mode == BgMode::Pipes)
3086 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
3087 .transpose()
3088 .ok()
3089 .flatten();
3090 let mut stderr = (mode == BgMode::Pipes)
3091 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
3092 .transpose()
3093 .ok()
3094 .flatten();
3095 let mut pty = (mode == BgMode::Pty)
3096 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
3097 .transpose()
3098 .ok()
3099 .flatten();
3100
3101 let mut pending_to_emit = Vec::new();
3102 let mut gap_matches = Vec::new();
3103 {
3104 let Ok(mut registry) = self.inner.watch_registry.lock() else {
3105 return;
3106 };
3107 let stdout_key = format!("{}:stdout", task.task_id);
3108 let stderr_key = format!("{}:stderr", task.task_id);
3109 let pty_key = format!("{}:pty", task.task_id);
3110
3111 let first = &rows[0];
3113 match mode {
3114 BgMode::Pipes => {
3115 registry.set_file_cursor(&stdout_key, first.stdout_offset.max(0) as u64);
3116 registry.set_file_cursor(&stderr_key, first.stderr_offset.max(0) as u64);
3117 }
3118 BgMode::Pty => {
3119 registry.set_file_cursor(&pty_key, first.pty_offset.max(0) as u64);
3120 }
3121 }
3122
3123 for row in &rows {
3124 let Ok(pattern) = WatchPattern::from_persisted(&row.pattern_kind, &row.pattern)
3125 else {
3126 crate::slog_warn!(
3127 "skipping unreadable persisted watch {}/{}",
3128 row.task_id,
3129 row.watch_id
3130 );
3131 continue;
3132 };
3133 if let Err(error) = registry.restore(
3134 row.watch_id.clone(),
3135 row.task_id.clone(),
3136 pattern,
3137 row.once,
3138 row.scanning,
3139 ) {
3140 crate::slog_warn!(
3141 "failed to restore watch {}/{}: {error}",
3142 row.task_id,
3143 row.watch_id
3144 );
3145 continue;
3146 }
3147 if row.pending_match {
3148 if let (Some(match_text), Some(match_offset), Some(context)) = (
3149 row.match_text.clone(),
3150 row.match_offset,
3151 row.match_context.clone(),
3152 ) {
3153 pending_to_emit.push(PatternMatch {
3154 watch_id: row.watch_id.clone(),
3155 task_id: row.task_id.clone(),
3156 match_text,
3157 match_offset: match_offset.max(0) as u64,
3158 context,
3159 once: row.once,
3160 });
3161 }
3162 }
3163 }
3164
3165 let should_gap_scan =
3169 rows.iter().any(|row| row.scanning) && !pending_to_emit.iter().any(|m| m.once);
3170 if should_gap_scan {
3171 match mode {
3172 BgMode::Pipes => {
3173 if let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) {
3174 gap_matches.extend(registry.scan_file_new_bytes(
3175 &stdout_key,
3176 &task.task_id,
3177 stdout,
3178 ));
3179 gap_matches.extend(registry.scan_file_new_bytes(
3180 &stderr_key,
3181 &task.task_id,
3182 stderr,
3183 ));
3184 }
3185 }
3186 BgMode::Pty => {
3187 if let Some(pty) = pty.as_mut() {
3188 gap_matches.extend(registry.scan_file_new_bytes(
3189 &pty_key,
3190 &task.task_id,
3191 pty,
3192 ));
3193 }
3194 }
3195 }
3196 }
3197 }
3198
3199 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
3200 for pattern_match in &gap_matches {
3201 self.persist_watch_match(
3202 &task.session_id,
3203 &task.task_id,
3204 pattern_match,
3205 stdout_offset,
3206 stderr_offset,
3207 pty_offset,
3208 );
3209 }
3210 if !gap_matches.is_empty() || rows.iter().any(|row| row.scanning) {
3211 self.persist_task_watch_cursors(
3212 &task.session_id,
3213 &task.task_id,
3214 stdout_offset,
3215 stderr_offset,
3216 pty_offset,
3217 );
3218 }
3219
3220 let emitted_pending = !pending_to_emit.is_empty();
3222 let to_emit = if emitted_pending {
3223 pending_to_emit
3224 } else {
3225 gap_matches
3226 };
3227 for pattern_match in to_emit {
3228 self.emit_bash_pattern_match(&task.session_id, pattern_match);
3229 }
3230
3231 if !terminal {
3232 return;
3233 }
3234
3235 let _ = self.remove_pending_completion(&task.task_id);
3238 let (watch_controlled, watch_matched) = self.task_watch_state(&task.task_id);
3239 if !watch_controlled {
3240 return;
3241 }
3242 if watch_matched {
3243 return;
3245 }
3246 if completion_delivered {
3247 self.clear_task_watch_state(&task.task_id);
3249 self.delete_persisted_watches_for_task(&task.session_id, &task.task_id);
3250 return;
3251 }
3252 if let Some(completion) = self.completion_snapshot_for_task(task) {
3253 self.emit_bash_watch_exit(&completion);
3254 }
3255 self.clear_task_watch_state(&task.task_id);
3257 }
3258
3259 fn kill_with_status(
3260 &self,
3261 task_id: &str,
3262 session_id: &str,
3263 terminal_status: BgTaskStatus,
3264 ) -> Result<BgTaskSnapshot, String> {
3265 self.kill_with_status_reason(task_id, session_id, terminal_status, None)
3266 }
3267
3268 fn kill_with_status_reason(
3269 &self,
3270 task_id: &str,
3271 session_id: &str,
3272 terminal_status: BgTaskStatus,
3273 reason: Option<String>,
3274 ) -> Result<BgTaskSnapshot, String> {
3275 let task = self
3276 .task_for_session(task_id, session_id)
3277 .ok_or_else(|| format!("background task not found: {task_id}"))?;
3278 let mut terminalized = false;
3279
3280 {
3281 let mut state = task
3282 .state
3283 .lock()
3284 .map_err(|_| "background task lock poisoned".to_string())?;
3285 if state.metadata.status.is_terminal() {
3286 state.pending_terminal_override = None;
3287 } else if let Ok(Some(marker)) = read_exit_marker(&task.paths) {
3288 state.metadata =
3289 terminal_metadata_from_marker(state.metadata.clone(), marker, reason.clone());
3290 if self.task_has_watch_control(&task.task_id) {
3291 state.metadata.completion_delivered = true;
3292 }
3293 state.pending_terminal_override = None;
3294 task.mark_terminal_now();
3295 match &mut state.runtime {
3296 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
3303 TaskRuntime::Pty(runtime) => *runtime = None,
3304 }
3305 state.detached = true;
3306 self.persist_task(&task.paths, &state.metadata)
3307 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
3308 terminalized = true;
3309 } else {
3310 let was_already_killing = state.metadata.status == BgTaskStatus::Killing;
3311 if !was_already_killing {
3312 state.metadata.status = BgTaskStatus::Killing;
3313 }
3314 if reason.is_some() {
3315 state.metadata.status_reason = reason.clone();
3316 }
3317 if !was_already_killing || reason.is_some() {
3318 self.persist_task(&task.paths, &state.metadata)
3319 .map_err(|e| format!("failed to persist killing state: {e}"))?;
3320 }
3321
3322 #[cfg(unix)]
3323 let pgid = state.metadata.pgid;
3324 #[cfg(windows)]
3325 let child_pid = state.metadata.child_pid;
3326 if !was_already_killing
3327 && state.metadata.mode == BgMode::Pty
3328 && terminal_status == BgTaskStatus::TimedOut
3329 {
3330 state.pending_terminal_override = Some(BgTaskStatus::TimedOut);
3331 }
3332
3333 #[cfg(windows)]
3334 let mut pty_forced_terminal_status: Option<BgTaskStatus> = None;
3335
3336 match &mut state.runtime {
3337 TaskRuntime::Piped(child_slot) => {
3338 #[cfg(unix)]
3339 if let Some(pgid) = pgid {
3340 terminate_pgid(pgid, child_slot.as_mut());
3341 }
3342 #[cfg(windows)]
3343 if let Some(child) = child_slot.as_mut() {
3344 super::process::terminate_process(child);
3345 } else if let Some(pid) = child_pid {
3346 terminate_pid(pid);
3347 }
3348 if let Some(child) = child_slot.as_mut() {
3349 let _ = child.wait();
3350 }
3351 *child_slot = None;
3352 state.detached = true;
3353
3354 if let Some(handles) = state.io_handles.as_mut() {
3355 handles.write(TaskArtifact::Exit, b"killed").map_err(|e| {
3356 format!("failed to write retained kill marker: {e}")
3357 })?;
3358 } else {
3359 write_kill_marker_if_absent(&task.paths)
3360 .map_err(|e| format!("failed to write kill marker: {e}"))?;
3361 }
3362
3363 let exit_code = terminal_exit_code_for_status(&terminal_status);
3364 state
3365 .metadata
3366 .mark_terminal(terminal_status, exit_code, reason.clone());
3367 if self.task_has_watch_control(&task.task_id) {
3368 state.metadata.completion_delivered = true;
3369 }
3370 state.pending_terminal_override = None;
3371 task.mark_terminal_now();
3372 self.persist_task(&task.paths, &state.metadata)
3373 .map_err(|e| format!("failed to persist killed state: {e}"))?;
3374 terminalized = true;
3375 }
3376 TaskRuntime::Pty(Some(pty)) => {
3377 pty.was_killed.store(true, Ordering::SeqCst);
3378 if let Err(error) = pty.killer.kill() {
3379 crate::slog_warn!(
3380 "[pty-kill] {task_id} ChildKiller::kill failed: {error}"
3381 );
3382 }
3383 if let Some(pid) = pty.child_pid {
3384 #[cfg(unix)]
3385 terminate_pgid(pid as i32, None);
3386 #[cfg(windows)]
3387 terminate_pid(pid);
3388 }
3389 drop(pty.master.take());
3390
3391 #[cfg(windows)]
3392 {
3393 let default_status = if terminal_status == BgTaskStatus::TimedOut {
3394 BgTaskStatus::TimedOut
3395 } else {
3396 BgTaskStatus::Killed
3397 };
3398 pty_forced_terminal_status = Some(
3399 state
3400 .pending_terminal_override
3401 .take()
3402 .unwrap_or(default_status),
3403 );
3404 }
3405 }
3406 TaskRuntime::Pty(None) => {}
3407 }
3408
3409 #[cfg(windows)]
3410 if let Some(target_status) = pty_forced_terminal_status {
3411 if !task.paths.exit.exists() {
3412 write_kill_marker_if_absent(&task.paths)
3413 .map_err(|e| format!("failed to write kill marker: {e}"))?;
3414 }
3415
3416 let exit_code = terminal_exit_code_for_status(&target_status);
3417 state
3418 .metadata
3419 .mark_terminal(target_status, exit_code, reason.clone());
3420 if self.task_has_watch_control(&task.task_id) {
3421 state.metadata.completion_delivered = true;
3422 }
3423 state.pending_terminal_override = None;
3424 task.mark_terminal_now();
3425 if let TaskRuntime::Pty(runtime) = &mut state.runtime {
3426 *runtime = None;
3427 }
3428 state.detached = true;
3429 self.persist_task(&task.paths, &state.metadata)
3430 .map_err(|e| format!("failed to persist killed PTY state: {e}"))?;
3431 terminalized = true;
3432 }
3433 }
3434 }
3435
3436 if terminalized {
3437 self.post_terminal_transition(&task, true)?;
3438 }
3439 Ok(self.snapshot_with_terminal_cache(&task, RUNNING_OUTPUT_PREVIEW_BYTES))
3440 }
3441
3442 fn finalize_from_marker(
3443 &self,
3444 task: &Arc<BgTask>,
3445 marker: ExitMarker,
3446 reason: Option<String>,
3447 ) -> Result<(), String> {
3448 let watch_controlled = self.task_has_watch_control(&task.task_id);
3449 let mut pty_reader_done = None;
3450 {
3451 let mut state = task
3452 .state
3453 .lock()
3454 .map_err(|_| "background task lock poisoned".to_string())?;
3455 if state.metadata.status.is_terminal() {
3456 state.pending_terminal_override = None;
3457 return Ok(());
3458 }
3459
3460 let pending_override = state.pending_terminal_override.take();
3461 let is_pty = state.metadata.mode == BgMode::Pty;
3462 let reason = reason.or_else(|| state.metadata.status_reason.clone());
3463 let updated = self
3464 .update_task_metadata(&task.paths, |metadata| {
3465 let mut new_metadata = if is_pty && marker == ExitMarker::Killed {
3466 let mut metadata = metadata.clone();
3467 let target_status = pending_override.unwrap_or(BgTaskStatus::Killed);
3468 let exit_code = terminal_exit_code_for_status(&target_status);
3469 metadata.mark_terminal(target_status, exit_code, reason.clone());
3470 metadata
3471 } else {
3472 terminal_metadata_from_marker(metadata.clone(), marker, reason.clone())
3473 };
3474 if watch_controlled {
3475 new_metadata.completion_delivered = true;
3476 }
3477 *metadata = new_metadata;
3478 })
3479 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
3480 state.metadata = updated;
3481 task.mark_terminal_now();
3482 match &mut state.runtime {
3483 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
3488 TaskRuntime::Pty(runtime) => {
3489 pty_reader_done = runtime
3490 .as_ref()
3491 .map(|runtime| Arc::clone(&runtime.reader_done));
3492 *runtime = None;
3493 }
3494 }
3495 state.detached = true;
3496 }
3497
3498 if let Some(reader_done) = pty_reader_done {
3499 let deadline = Instant::now() + Duration::from_millis(200);
3500 while !reader_done.load(Ordering::SeqCst) && Instant::now() < deadline {
3501 std::thread::sleep(Duration::from_millis(10));
3502 }
3503 }
3504
3505 self.scan_task_watch_output(task);
3508
3509 self.post_terminal_transition(task, true)
3510 }
3511
3512 fn enqueue_completion_if_needed(
3513 &self,
3514 metadata: &PersistedTask,
3515 paths: Option<&TaskPaths>,
3516 emit_frame: bool,
3517 ) {
3518 if metadata.status.is_terminal() && !metadata.completion_delivered {
3519 let cache =
3520 paths.and_then(|paths| self.render_terminal_output_from_paths(metadata, paths));
3521 self.enqueue_completion_from_parts(metadata, None, paths, emit_frame, cache.as_ref());
3522 }
3523 }
3524
3525 fn render_terminal_output_from_paths(
3526 &self,
3527 metadata: &PersistedTask,
3528 paths: &TaskPaths,
3529 ) -> Option<TerminalOutputCache> {
3530 if metadata.mode == BgMode::Pty {
3531 return None;
3532 }
3533 let mut buffer = BgBuffer::registered(paths, BgMode::Pipes);
3534 let disk_truncation = buffer.enforce_terminal_cap();
3535 Some(self.render_terminal_output(metadata, &buffer, disk_truncation, Some(paths)))
3536 }
3537
3538 fn enqueue_completion_from_parts(
3539 &self,
3540 metadata: &PersistedTask,
3541 buffer: Option<&BgBuffer>,
3542 paths: Option<&TaskPaths>,
3543 emit_frame: bool,
3544 terminal_render: Option<&TerminalOutputCache>,
3545 ) {
3546 if !metadata.status.is_terminal() {
3557 return;
3558 }
3559
3560 let owned_buffer = if buffer.is_none() && metadata.mode != BgMode::Pty {
3561 paths.map(|paths| BgBuffer::registered(paths, BgMode::Pipes))
3562 } else {
3563 None
3564 };
3565 let render_buffer = buffer.or(owned_buffer.as_ref());
3566 let owned_render = if terminal_render.is_none() {
3567 render_buffer.map(|buffer| {
3568 let mut capped_buffer = buffer.clone();
3569 let disk_truncation = capped_buffer.enforce_terminal_cap();
3570 self.render_terminal_output(metadata, &capped_buffer, disk_truncation, paths)
3571 })
3572 } else {
3573 None
3574 };
3575 let render = terminal_render.or(owned_render.as_ref());
3576
3577 let (output_preview, output_truncated) = render
3581 .map(|cache| completion_preview_for_cache(cache, metadata.exit_code))
3582 .unwrap_or_else(|| (String::new(), false));
3583
3584 let token_counts = self.completion_token_counts(
3585 metadata,
3586 buffer,
3587 paths,
3588 render.map(|render| render.output_preview.as_str()),
3589 );
3590 let completion = BgCompletion {
3591 task_id: metadata.task_id.clone(),
3592 session_id: metadata.session_id.clone(),
3593 status: metadata.status.clone(),
3594 exit_code: metadata.exit_code,
3595 command: metadata.command.clone(),
3596 output_preview,
3597 output_truncated,
3598 original_tokens: token_counts.original_tokens,
3599 compressed_tokens: token_counts.compressed_tokens,
3600 tokens_skipped: token_counts.tokens_skipped,
3601 status_reason: metadata.status_reason.clone(),
3602 };
3603
3604 self.record_compression_event_if_applicable(metadata, &token_counts);
3615
3616 let (watch_controlled, watch_matched) = self.task_watch_state(&metadata.task_id);
3617 if watch_controlled {
3618 if emit_frame && !watch_matched {
3619 self.emit_bash_watch_exit(&completion);
3620 } else if watch_matched {
3621 if let Some(task) = self.task(&metadata.task_id) {
3626 let _ = task.set_completion_delivered(true, self);
3627 }
3628 }
3629 self.clear_task_watch_state(&metadata.task_id);
3631 return;
3632 }
3633
3634 if metadata.completion_delivered {
3644 return;
3645 }
3646
3647 let pushed = if let Ok(mut completions) = self.inner.completions.lock() {
3650 if completions
3651 .iter()
3652 .any(|existing| existing.task_id == metadata.task_id)
3653 {
3654 false
3655 } else {
3656 completions.push_back(completion.clone());
3657 true
3658 }
3659 } else {
3660 false
3661 };
3662
3663 if pushed && emit_frame {
3664 self.emit_bash_completed(completion);
3665 }
3666 }
3667
3668 fn record_compression_event_if_applicable(
3669 &self,
3670 metadata: &PersistedTask,
3671 token_counts: &CompletionTokenCounts,
3672 ) {
3673 if metadata.mode == BgMode::Pty {
3674 return;
3675 }
3676
3677 let (original_tokens, compressed_tokens, original_bytes, compressed_bytes) = match (
3678 token_counts.original_tokens,
3679 token_counts.compressed_tokens,
3680 token_counts.original_bytes,
3681 token_counts.compressed_bytes,
3682 ) {
3683 (
3684 Some(original_tokens),
3685 Some(compressed_tokens),
3686 Some(original_bytes),
3687 Some(compressed_bytes),
3688 ) => (
3689 original_tokens,
3690 compressed_tokens,
3691 original_bytes,
3692 compressed_bytes,
3693 ),
3694 _ => {
3695 crate::slog_warn!(
3696 "compression event skipped for {}: token counts unavailable (likely spill file missing or unreadable)",
3697 metadata.task_id
3698 );
3699 return;
3700 }
3701 };
3702
3703 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
3704 let Some(pool) = pool else {
3705 crate::slog_warn!(
3706 "compression event skipped for {}: db_pool not initialized — was configure run?",
3707 metadata.task_id
3708 );
3709 return;
3710 };
3711 let harness = self
3712 .inner
3713 .db_harness
3714 .read()
3715 .ok()
3716 .and_then(|slot| slot.clone());
3717 let Some(harness) = harness else {
3718 crate::slog_warn!(
3719 "compression event insert skipped for {}: harness not configured",
3720 metadata.task_id
3721 );
3722 return;
3723 };
3724
3725 let project_root = metadata
3726 .project_root
3727 .as_deref()
3728 .unwrap_or(&metadata.workdir);
3729 let project_key = crate::path_identity::project_scope_key(project_root);
3730 let row = crate::db::compression_events::CompressionEventRow {
3731 harness: &harness,
3732 session_id: Some(&metadata.session_id),
3733 project_key: &project_key,
3734 tool: "bash",
3735 task_id: Some(&metadata.task_id),
3736 command: Some(&metadata.command),
3737 compressor: if metadata.compressed {
3738 "registry"
3739 } else {
3740 "none"
3741 },
3742 original_bytes,
3743 compressed_bytes,
3744 original_tokens,
3745 compressed_tokens,
3746 created_at: unix_millis() as i64,
3747 };
3748
3749 let conn = match pool.lock() {
3750 Ok(conn) => conn,
3751 Err(_) => {
3752 crate::slog_warn!(
3753 "compression event insert failed for {}: db mutex poisoned",
3754 metadata.task_id
3755 );
3756 return;
3757 }
3758 };
3759 match crate::db::compression_events::insert_compression_event(&conn, &row) {
3760 Ok(Some(row_id)) => {
3761 self.inner
3765 .compression_aggregates
3766 .record_successful_insert(&conn, &row, row_id);
3767 crate::slog_debug!(
3771 "compression event recorded for {} (project={}, session={}, {} → {} tokens)",
3772 metadata.task_id,
3773 project_key,
3774 metadata.session_id,
3775 original_tokens,
3776 compressed_tokens
3777 );
3778 }
3779 Ok(None) => {
3780 crate::slog_debug!(
3781 "duplicate compression event ignored for {} (project={}, session={})",
3782 metadata.task_id,
3783 project_key,
3784 metadata.session_id
3785 );
3786 }
3787 Err(error) => {
3788 crate::slog_warn!(
3789 "compression event insert failed for {}: {}",
3790 metadata.task_id,
3791 error
3792 );
3793 }
3794 }
3795 }
3796
3797 fn emit_bash_pattern_match(&self, session_id: &str, pattern_match: PatternMatch) {
3798 let Ok(progress_sender) = self
3799 .inner
3800 .progress_sender
3801 .lock()
3802 .map(|sender| sender.clone())
3803 else {
3804 return;
3805 };
3806 if let Some(sender) = progress_sender.as_ref() {
3807 sender(PushFrame::BashPatternMatch(BashPatternMatchFrame::new(
3808 pattern_match.task_id,
3809 session_id.to_string(),
3810 pattern_match.watch_id,
3811 pattern_match.match_text,
3812 pattern_match.match_offset,
3813 pattern_match.context,
3814 pattern_match.once,
3815 )));
3816 }
3817 }
3818
3819 fn emit_bash_watch_exit(&self, completion: &BgCompletion) {
3820 let Ok(progress_sender) = self
3821 .inner
3822 .progress_sender
3823 .lock()
3824 .map(|sender| sender.clone())
3825 else {
3826 return;
3827 };
3828 let Some(sender) = progress_sender.as_ref() else {
3829 return;
3830 };
3831 let status = completion_status_text(&completion.status, completion.exit_code);
3832 let preview = completion.output_preview.trim_end();
3833 let context = if preview.is_empty() {
3834 format!("task {} exited ({status})", completion.task_id)
3835 } else {
3836 format!(
3837 "task {} exited ({status})
3838{preview}",
3839 completion.task_id
3840 )
3841 };
3842 sender(PushFrame::BashPatternMatch(
3843 BashPatternMatchFrame::task_exit(
3844 completion.task_id.clone(),
3845 completion.session_id.clone(),
3846 format!("exited ({status})"),
3847 context,
3848 ),
3849 ));
3850 }
3851
3852 fn emit_bash_completed(&self, completion: BgCompletion) {
3853 let Ok(progress_sender) = self
3854 .inner
3855 .progress_sender
3856 .lock()
3857 .map(|sender| sender.clone())
3858 else {
3859 return;
3860 };
3861 let Some(sender) = progress_sender.as_ref() else {
3862 return;
3863 };
3864 let mut frame = BashCompletedFrame::new(
3872 completion.task_id,
3873 completion.session_id,
3874 completion.status,
3875 completion.exit_code,
3876 completion.command,
3877 completion.output_preview,
3878 completion.output_truncated,
3879 completion.original_tokens,
3880 completion.compressed_tokens,
3881 completion.tokens_skipped,
3882 );
3883 frame.status_reason = completion.status_reason;
3884 sender(PushFrame::BashCompleted(frame));
3885 }
3886
3887 fn completion_token_counts(
3888 &self,
3889 metadata: &PersistedTask,
3890 buffer: Option<&BgBuffer>,
3891 paths: Option<&TaskPaths>,
3892 rendered_output: Option<&str>,
3893 ) -> CompletionTokenCounts {
3894 if metadata.mode == BgMode::Pty {
3895 return CompletionTokenCounts::skipped();
3896 }
3897
3898 let raw = match buffer {
3899 Some(buffer) => buffer.read_for_token_count(TOKENIZE_CAP_BYTES_PER_STREAM),
3900 None => paths
3901 .map(|paths| {
3902 read_for_token_count_from_disk(metadata, paths, TOKENIZE_CAP_BYTES_PER_STREAM)
3903 })
3904 .unwrap_or(TokenCountInput::Skipped),
3905 };
3906
3907 let TokenCountInput::Text(raw_output) = raw else {
3908 return CompletionTokenCounts::skipped();
3909 };
3910
3911 let original_tokens = token_count_u32(&raw_output);
3912 let original_bytes = raw_output.len() as i64;
3913 let compressed_output = rendered_output.unwrap_or(&raw_output);
3914 let compressed_tokens = token_count_u32(compressed_output);
3915 let compressed_bytes = compressed_output.len() as i64;
3916 CompletionTokenCounts {
3917 original_tokens: Some(original_tokens),
3918 compressed_tokens: Some(compressed_tokens),
3919 original_bytes: Some(original_bytes),
3920 compressed_bytes: Some(compressed_bytes),
3921 tokens_skipped: false,
3922 }
3923 }
3924
3925 pub(crate) fn maybe_emit_long_running_reminder(&self, task: &Arc<BgTask>) {
3926 if !self
3927 .inner
3928 .long_running_reminder_enabled
3929 .load(Ordering::SeqCst)
3930 {
3931 return;
3932 }
3933 let interval_ms = self
3934 .inner
3935 .long_running_reminder_interval_ms
3936 .load(Ordering::SeqCst);
3937 if interval_ms == 0 {
3938 return;
3939 }
3940 let interval = Duration::from_millis(interval_ms);
3941 let now = Instant::now();
3942 let Ok(mut last_reminder_at) = task.last_reminder_at.lock() else {
3943 return;
3944 };
3945 let since = last_reminder_at.unwrap_or(task.started);
3946 if now.duration_since(since) < interval {
3947 return;
3948 }
3949 let command = task
3950 .state
3951 .lock()
3952 .map(|state| state.metadata.command.clone())
3953 .unwrap_or_default();
3954 *last_reminder_at = Some(now);
3955 self.emit_bash_long_running(BashLongRunningFrame::new(
3956 task.task_id.clone(),
3957 task.session_id.clone(),
3958 command,
3959 task.started.elapsed().as_millis() as u64,
3960 ));
3961 }
3962
3963 fn emit_bash_long_running(&self, frame: BashLongRunningFrame) {
3964 let Ok(progress_sender) = self
3965 .inner
3966 .progress_sender
3967 .lock()
3968 .map(|sender| sender.clone())
3969 else {
3970 return;
3971 };
3972 if let Some(sender) = progress_sender.as_ref() {
3973 sender(PushFrame::BashLongRunning(frame));
3974 }
3975 }
3976
3977 fn task(&self, task_id: &str) -> Option<Arc<BgTask>> {
3978 validate_task_id(task_id).ok()?;
3979 self.inner
3980 .tasks
3981 .lock()
3982 .ok()
3983 .and_then(|tasks| tasks.get(task_id).cloned())
3984 }
3985
3986 fn task_for_session(&self, task_id: &str, session_id: &str) -> Option<Arc<BgTask>> {
3987 self.task(task_id)
3988 .filter(|task| task.session_id == session_id)
3989 }
3990
3991 pub fn try_health_counts(&self) -> Option<BgTaskHealthCounts> {
3992 let running = self
3993 .inner
3994 .tasks
3995 .try_lock()
3996 .ok()
3997 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())?;
3998 let pending_completions = self.inner.completions.try_lock().ok().map(|q| q.len())?;
3999 Some(BgTaskHealthCounts {
4000 running,
4001 pending_completions,
4002 })
4003 }
4004
4005 pub fn estimated_memory(&self) -> crate::memory::MemoryEstimate {
4009 let tasks = match self.inner.tasks.try_lock() {
4010 Ok(tasks) => tasks.values().cloned().collect::<Vec<_>>(),
4011 Err(_) => return crate::memory::MemoryEstimate::busy(),
4012 };
4013 let mut bytes = 0u64;
4014 let mut terminal_output_caches = 0usize;
4015 let mut sessions = HashSet::new();
4016 for task in &tasks {
4017 sessions.insert(task.session_id.clone());
4018 let state = match task.state.try_lock() {
4019 Ok(state) => state,
4020 Err(_) => return crate::memory::MemoryEstimate::busy(),
4021 };
4022 if let Some(cache) = state.terminal_output_cache.as_ref() {
4023 terminal_output_caches = terminal_output_caches.saturating_add(1);
4024 bytes = bytes.saturating_add(terminal_output_cache_estimated_bytes(cache));
4025 }
4026 }
4027 let completion_count = match self.inner.completions.try_lock() {
4028 Ok(completions) => {
4029 for completion in completions.iter() {
4030 sessions.insert(completion.session_id.clone());
4031 bytes = bytes.saturating_add(completion_estimated_bytes(completion));
4032 }
4033 completions.len()
4034 }
4035 Err(_) => return crate::memory::MemoryEstimate::busy(),
4036 };
4037
4038 crate::memory::MemoryEstimate::estimated(bytes)
4039 .count("tasks", tasks.len())
4040 .count("sessions", sessions.len())
4041 .count("terminal_output_caches", terminal_output_caches)
4042 .count("completion_caches", completion_count)
4043 .count_u64("output_ring_bytes", 0)
4044 }
4045
4046 fn running_count(&self) -> usize {
4047 self.inner
4048 .tasks
4049 .lock()
4050 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())
4051 .unwrap_or(0)
4052 }
4053
4054 fn start_watchdog(&self) {
4055 if !self.inner.watchdog_started.swap(true, Ordering::SeqCst) {
4056 super::watchdog::start(self.clone());
4057 }
4058 }
4059
4060 fn running_metadata_is_stale(&self, metadata: &PersistedTask) -> bool {
4061 unix_millis().saturating_sub(metadata.started_at) > STALE_RUNNING_AFTER.as_millis() as u64
4062 }
4063
4064 #[cfg(test)]
4065 pub fn task_json_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
4066 self.task_for_session(task_id, session_id)
4067 .map(|task| task.paths.json.clone())
4068 }
4069
4070 #[cfg(test)]
4071 pub fn task_exit_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
4072 self.task_for_session(task_id, session_id)
4073 .map(|task| task.paths.exit.clone())
4074 }
4075}
4076
4077#[cfg(unix)]
4078fn should_capture_pipeline_status(
4079 spawn_plan: &SpawnPlan,
4080 has_pipeline: bool,
4081 shell: &Path,
4082) -> bool {
4083 if spawn_plan.is_native_launcher() {
4084 return false;
4087 }
4088 has_pipeline && super::process::pipeline_shell_kind(shell).is_some()
4089}
4090
4091fn canonical_artifact_root(paths: &TaskPaths) -> PathBuf {
4092 fs::canonicalize(&paths.io_dir).unwrap_or_else(|_| paths.io_dir.clone())
4093}
4094
4095fn append_pipeline_warning(
4100 cache: &mut TerminalOutputCache,
4101 metadata: &PersistedTask,
4102 paths: Option<&TaskPaths>,
4103) {
4104 if metadata.exit_code != Some(0) {
4105 return;
4106 }
4107 let Some(paths) = paths else {
4108 return;
4109 };
4110 if metadata.pipeline_segments.len() < 2 {
4111 return;
4112 }
4113 let Ok(mut status_file) = open_task_artifact(paths, TaskArtifact::PipelineStatus) else {
4114 return;
4115 };
4116 let Ok(status_bytes) = status_file.read_all() else {
4117 return;
4118 };
4119 let Some(statuses) = String::from_utf8_lossy(&status_bytes)
4120 .lines()
4121 .map(|line| line.trim().parse::<i32>().ok())
4122 .collect::<Option<Vec<_>>>()
4123 else {
4124 return;
4125 };
4126 if statuses.len() != metadata.pipeline_segments.len() {
4127 return;
4128 }
4129 let Some((failing_index, failing_code)) = statuses
4130 .iter()
4131 .enumerate()
4132 .take(statuses.len().saturating_sub(1))
4133 .find(|(_, code)| **code != 0)
4134 .map(|(index, code)| (index, *code))
4135 else {
4136 return;
4137 };
4138 let Some(final_segment) = metadata.pipeline_segments.last() else {
4139 return;
4140 };
4141 let failing_segment = &metadata.pipeline_segments[failing_index];
4142 let footer = format!(
4143 "note: `{}` (segment {} of {}) exited {}; the pipeline's exit code is `{}`'s.",
4144 failing_segment,
4145 failing_index + 1,
4146 metadata.pipeline_segments.len(),
4147 failing_code,
4148 final_segment,
4149 );
4150 if cache.output_preview.trim().is_empty() {
4151 cache.output_preview = footer;
4152 } else {
4153 cache.output_preview = format!("{}\n{}", cache.output_preview.trim_end(), footer,);
4154 }
4155}
4156
4157fn normalize_piped_display_output(text: &mut String) {
4160 if !text.contains('\r') {
4161 return;
4162 }
4163
4164 let mut rendered = String::with_capacity(text.len());
4165 let mut line = Vec::new();
4166 let mut column = 0;
4167 let mut chars = text.chars().peekable();
4168
4169 while let Some(character) = chars.next() {
4170 match character {
4171 '\r' if chars.peek() == Some(&'\n') => {
4172 chars.next();
4173 for character in &line {
4174 rendered.push(*character);
4175 }
4176 rendered.push('\n');
4177 line.clear();
4178 column = 0;
4179 }
4180 '\r' => column = 0,
4181 '\n' => {
4182 for character in &line {
4183 rendered.push(*character);
4184 }
4185 rendered.push('\n');
4186 line.clear();
4187 column = 0;
4188 }
4189 character => {
4190 if column < line.len() {
4191 line[column] = character;
4192 } else {
4193 line.resize(column, ' ');
4194 line.push(character);
4195 }
4196 column += 1;
4197 }
4198 }
4199 }
4200
4201 for character in &line {
4202 rendered.push(*character);
4203 }
4204 *text = rendered;
4205}
4206
4207fn render_compressed_with_recovery(
4208 buffer: &BgBuffer,
4209 mut compressed: CompressionResult,
4210 input_truncated: bool,
4211 disk_truncation: DiskTruncation,
4212 artifact_access: ArtifactRecoveryAccess,
4213) -> TerminalOutputCache {
4214 let had_trailing_newline = compressed.text.ends_with('\n');
4222 let mut text = strip_plain_truncation_marker_lines(&compressed.text)
4223 .trim_end()
4224 .to_string();
4225 if had_trailing_newline && !text.is_empty() {
4226 text.push('\n');
4227 }
4228 compressed.text = text;
4229
4230 let output_path = buffer.output_path().map(|path| path.display().to_string());
4231 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
4232 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
4233 let mut recovery = RecoveryContext {
4234 dropped_by_class: compressed.dropped_by_class,
4235 had_inner_drop: compressed.had_inner_drop,
4236 offset_hint_eligible: compressed.offset_hint_eligible,
4237 offset_start_line: compressed.offset_start_line,
4238 byte_truncated: input_truncated,
4239 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
4240 output_path: output_path.clone(),
4241 stderr_path: stderr_path.clone(),
4242 include_stderr_path,
4243 artifact_access: artifact_access.clone(),
4244 };
4245
4246 let (output_preview, output_truncated) =
4247 render_body_with_recovery_marker(&compressed.text, &mut recovery);
4248 TerminalOutputCache {
4249 output_preview,
4250 output_truncated,
4251 kind: TerminalOutputKind::Compressed,
4252 output_path,
4253 stderr_path,
4254 artifact_access,
4255 recovery: Some(recovery),
4256 }
4257}
4258
4259fn render_body_with_recovery_marker(body: &str, recovery: &mut RecoveryContext) -> (String, bool) {
4260 render_body_with_recovery_marker_at_cap(
4261 body,
4262 recovery,
4263 FINAL_OUTPUT_CAP_BYTES,
4264 cap_final_output,
4265 cap_final_output_with_marker,
4266 )
4267}
4268
4269fn render_raw_body_with_recovery_marker(
4270 body: &str,
4271 recovery: &mut RecoveryContext,
4272) -> (String, bool) {
4273 render_body_with_recovery_marker_at_cap(
4274 body,
4275 recovery,
4276 RAW_PASSTHROUGH_CAP_BYTES,
4277 |input| {
4278 super::output::cap_head_tail(
4279 input,
4280 RAW_PASSTHROUGH_CAP_BYTES,
4281 RAW_PASSTHROUGH_HEAD_BYTES,
4282 RAW_PASSTHROUGH_TAIL_BYTES,
4283 )
4284 },
4285 |input, marker| {
4286 super::output::cap_head_tail_with_marker(
4287 input,
4288 RAW_PASSTHROUGH_CAP_BYTES,
4289 RAW_PASSTHROUGH_HEAD_BYTES,
4290 RAW_PASSTHROUGH_TAIL_BYTES,
4291 marker,
4292 )
4293 },
4294 )
4295}
4296
4297fn render_body_with_recovery_marker_at_cap<F, G>(
4298 body: &str,
4299 recovery: &mut RecoveryContext,
4300 cap_bytes: usize,
4301 cap_plain: F,
4302 cap_with_marker: G,
4303) -> (String, bool)
4304where
4305 F: Fn(&str) -> super::output::CappedText,
4306 G: Fn(&str, &str) -> super::output::CappedText,
4307{
4308 let needs_marker = recovery.has_visible_drop();
4309 if body.len() > cap_bytes {
4310 recovery.byte_truncated = true;
4311 if let Some(marker) = recovery_marker(recovery) {
4312 let capped = cap_with_marker(body, &marker);
4313 return (capped.text, true);
4314 }
4315 let capped = cap_plain(body);
4316 return (capped.text, capped.truncated || needs_marker);
4317 }
4318
4319 if !needs_marker {
4320 return (body.to_string(), false);
4321 }
4322
4323 let Some(marker) = recovery_marker(recovery) else {
4324 return (body.to_string(), true);
4325 };
4326 let with_marker = append_recovery_marker(body, &marker);
4327 if with_marker.len() <= cap_bytes {
4328 return (with_marker, true);
4329 }
4330
4331 recovery.byte_truncated = true;
4332 let marker = recovery_marker(recovery).unwrap_or(marker);
4333 let capped = cap_with_marker(body, &marker);
4334 (capped.text, true)
4335}
4336
4337fn append_recovery_marker(body: &str, marker: &str) -> String {
4338 if body.is_empty() {
4339 return marker.to_string();
4340 }
4341 let mut output = body.trim_end().to_string();
4342 output.push('\n');
4343 output.push_str(marker);
4344 output
4345}
4346
4347fn recovery_marker(recovery: &RecoveryContext) -> Option<String> {
4348 let mut parts = Vec::new();
4349 for (class, count) in &recovery.dropped_by_class {
4350 let label = if *count == 1 {
4351 class.singular()
4352 } else {
4353 class.plural()
4354 };
4355 parts.push(format!("+{count} more {label}"));
4356 }
4357 if recovery.byte_truncated {
4358 parts.push("truncated output".to_string());
4359 }
4360 let disk_truncated_prefix_bytes = recovery.disk_truncated_prefix_bytes;
4361 if disk_truncated_prefix_bytes > 0 {
4362 parts.push(format!(
4363 "truncated {disk_truncated_prefix_bytes} bytes from saved output prefix"
4364 ));
4365 } else if recovery.had_inner_drop && parts.is_empty() {
4366 parts.push("omitted output".to_string());
4367 }
4368
4369 if parts.is_empty() {
4370 return None;
4371 }
4372
4373 let hint = recovery_hint(recovery);
4374 Some(format!("[{}; {hint}]", parts.join(", ")))
4375}
4376
4377fn bash_status_recovery_hint(access: &ArtifactRecoveryAccess) -> String {
4378 let task_id = serde_json::to_string(&access.task_id)
4379 .unwrap_or_else(|_| format!("\"{}\"", access.task_id));
4380 format!("use bash_status({{taskId: {task_id}}})")
4381}
4382
4383fn recovery_hint(recovery: &RecoveryContext) -> String {
4384 if !recovery.artifact_access.readable {
4385 return bash_status_recovery_hint(&recovery.artifact_access);
4386 }
4387
4388 if recovery.offset_hint_eligible
4392 && !recovery.byte_truncated
4393 && recovery.dropped_by_class.is_empty()
4394 && !recovery.include_stderr_path
4395 {
4396 if let (Some(path), Some(line)) =
4397 (recovery.output_path.as_deref(), recovery.offset_start_line)
4398 {
4399 return format!("see remaining: tail -n +{line} {}", quote_path(path));
4400 }
4401 }
4402
4403 let mut paths = Vec::new();
4404 if let Some(path) = recovery.output_path.as_deref() {
4405 paths.push(path);
4406 }
4407 if recovery.include_stderr_path {
4408 if let Some(path) = recovery.stderr_path.as_deref() {
4409 if !paths.contains(&path) {
4410 paths.push(path);
4411 }
4412 }
4413 }
4414
4415 if paths.is_empty() {
4416 return "full output unavailable".to_string();
4417 }
4418
4419 let reads = paths
4420 .into_iter()
4421 .map(|path| format!("read {}", quote_path(path)))
4422 .collect::<Vec<_>>()
4423 .join(" and ");
4424 if recovery.disk_truncated_prefix_bytes > 0 {
4425 format!("retained output: {reads}")
4426 } else {
4427 format!("full output: {reads}")
4428 }
4429}
4430
4431fn strip_plain_truncation_marker_lines(input: &str) -> String {
4432 input
4433 .lines()
4434 .filter(|line| !is_plain_truncation_marker(line.trim()))
4435 .collect::<Vec<_>>()
4436 .join("\n")
4437}
4438
4439fn strip_recovery_marker_lines(input: &str) -> String {
4440 input
4441 .lines()
4442 .filter(|line| !is_recovery_marker(line.trim()))
4443 .collect::<Vec<_>>()
4444 .join("\n")
4445}
4446
4447fn is_plain_truncation_marker(line: &str) -> bool {
4448 let Some(rest) = line.strip_prefix("...<truncated ") else {
4449 return false;
4450 };
4451 let Some(bytes) = rest.strip_suffix(" bytes>...") else {
4452 return false;
4453 };
4454 !bytes.is_empty() && bytes.chars().all(|ch| ch.is_ascii_digit())
4455}
4456
4457fn is_recovery_marker(line: &str) -> bool {
4458 line.starts_with('[')
4459 && line.ends_with(']')
4460 && (line.contains("full output: read ")
4461 || line.contains("retained output: read ")
4462 || line.contains("see remaining: tail -n +")
4463 || line.contains("use bash_status({taskId:")
4464 || line.contains("full output unavailable"))
4465}
4466
4467fn structured_output_pointer(
4468 total_bytes: u64,
4469 output_path: &str,
4470 truncated_prefix_bytes: u64,
4471 artifact_access: &ArtifactRecoveryAccess,
4472) -> String {
4473 if artifact_access.readable {
4474 return if truncated_prefix_bytes > 0 {
4475 retained_json_output_pointer(total_bytes, output_path, truncated_prefix_bytes)
4476 } else {
4477 json_output_pointer(total_bytes, output_path)
4478 };
4479 }
4480
4481 let kb = total_bytes.div_ceil(1024);
4482 let hint = bash_status_recovery_hint(artifact_access);
4483 if truncated_prefix_bytes > 0 {
4484 format!(
4485 "[JSON output {kb} KB; truncated {truncated_prefix_bytes} bytes from saved output prefix; retained output: {hint}]"
4486 )
4487 } else {
4488 format!("[JSON output {kb} KB; full output: {hint}]")
4489 }
4490}
4491
4492fn render_structured_output(
4493 command: &str,
4494 buffer: &BgBuffer,
4495 disk_truncation: DiskTruncation,
4496 artifact_access: ArtifactRecoveryAccess,
4497) -> Option<TerminalOutputCache> {
4498 if !is_gh_structured_command(command) {
4499 return None;
4500 }
4501
4502 let output_path = buffer
4503 .output_path()
4504 .map(|path| path.display().to_string())?;
4505 let stdout_bytes = buffer.stream_len(StreamKind::Stdout);
4506 if stdout_bytes == 0 {
4507 return None;
4508 }
4509
4510 if stdout_bytes > STRUCTURED_OUTPUT_CAP_BYTES as u64 {
4511 if !stream_starts_like_json(buffer, StreamKind::Stdout) {
4512 return None;
4513 }
4514 let output_preview = structured_output_pointer(
4515 stdout_bytes,
4516 &output_path,
4517 disk_truncation.total_prefix_bytes(),
4518 &artifact_access,
4519 );
4520 return Some(TerminalOutputCache {
4521 output_preview,
4522 output_truncated: true,
4523 kind: TerminalOutputKind::Structured,
4524 output_path: Some(output_path),
4525 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
4526 artifact_access,
4527 recovery: None,
4528 });
4529 }
4530
4531 let stdout = buffer.read_stream_bounded(StreamKind::Stdout, STRUCTURED_OUTPUT_CAP_BYTES);
4532 if stdout.truncated || !is_structured_body(&stdout.text) {
4533 return None;
4534 }
4535
4536 Some(TerminalOutputCache {
4537 output_preview: stdout.text,
4538 output_truncated: false,
4539 kind: TerminalOutputKind::Structured,
4540 output_path: Some(output_path),
4541 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
4542 artifact_access,
4543 recovery: None,
4544 })
4545}
4546
4547fn render_raw_passthrough(
4548 buffer: &BgBuffer,
4549 disk_truncation: DiskTruncation,
4550 artifact_access: ArtifactRecoveryAccess,
4551) -> TerminalOutputCache {
4552 let raw = buffer.read_combined_head_tail(
4553 RAW_PASSTHROUGH_CAP_BYTES,
4554 RAW_PASSTHROUGH_HEAD_BYTES,
4555 RAW_PASSTHROUGH_TAIL_BYTES,
4556 );
4557 let output_path = buffer.output_path().map(|path| path.display().to_string());
4558 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
4559 if !raw.truncated && disk_truncation.total_prefix_bytes() == 0 {
4560 return TerminalOutputCache {
4561 output_preview: raw.text,
4562 output_truncated: false,
4563 kind: TerminalOutputKind::Raw,
4564 output_path,
4565 stderr_path,
4566 artifact_access,
4567 recovery: None,
4568 };
4569 }
4570
4571 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
4572 let mut recovery = RecoveryContext {
4573 dropped_by_class: BTreeMap::new(),
4574 had_inner_drop: false,
4575 offset_hint_eligible: false,
4576 offset_start_line: None,
4577 byte_truncated: raw.truncated,
4578 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
4579 output_path: output_path.clone(),
4580 stderr_path: stderr_path.clone(),
4581 include_stderr_path,
4582 artifact_access: artifact_access.clone(),
4583 };
4584 let (output_preview, output_truncated) =
4585 render_raw_body_with_recovery_marker(&raw.text, &mut recovery);
4586 TerminalOutputCache {
4587 output_preview,
4588 output_truncated,
4589 kind: TerminalOutputKind::Raw,
4590 output_path,
4591 stderr_path,
4592 artifact_access,
4593 recovery: Some(recovery),
4594 }
4595}
4596
4597fn completion_preview_for_cache(
4598 cache: &TerminalOutputCache,
4599 exit_code: Option<i32>,
4600) -> (String, bool) {
4601 let exit_ok = exit_code == Some(0);
4604 let threshold = completion_preview_threshold(exit_ok);
4605 if cache.kind == TerminalOutputKind::Structured && cache.output_preview.len() > threshold {
4606 if let Some(path) = cache.output_path.as_deref() {
4607 return (
4608 structured_output_pointer(
4609 cache.output_preview.len() as u64,
4610 path,
4611 0,
4612 &cache.artifact_access,
4613 ),
4614 true,
4615 );
4616 }
4617 return (cache.output_preview.clone(), cache.output_truncated);
4618 }
4619
4620 if let Some(recovery) = cache.recovery.as_ref() {
4621 if cache.output_preview.len() <= threshold {
4622 return (cache.output_preview.clone(), cache.output_truncated);
4623 }
4624 let body = strip_recovery_marker_lines(&cache.output_preview);
4625 let mut completion_recovery = recovery.clone();
4626 completion_recovery.byte_truncated = true;
4627 if let Some(marker) = recovery_marker(&completion_recovery) {
4628 let capped = cap_completion_output_with_marker(&body, &marker, exit_ok);
4629 return (capped.text, true);
4630 }
4631 }
4632
4633 let capped = cap_completion_output(&cache.output_preview, exit_ok);
4634 (capped.text, cache.output_truncated || capped.truncated)
4635}
4636
4637fn is_gh_structured_command(command: &str) -> bool {
4638 let Some(normalized) = crate::compress::plain_command_for_structured_output(command) else {
4639 return false;
4640 };
4641 let tokens = shell_words_for_flags(&normalized);
4642 let Some(head) = tokens.first() else {
4643 return false;
4644 };
4645 let head_name = Path::new(head)
4646 .file_name()
4647 .and_then(|name| name.to_str())
4648 .unwrap_or(head);
4649 if !(head_name == "gh" || head_name.eq_ignore_ascii_case("gh.exe")) {
4650 return false;
4651 }
4652 tokens.iter().any(|token| {
4653 matches!(token.as_str(), "--json" | "--jq" | "--template")
4654 || token.starts_with("--json=")
4655 || token.starts_with("--jq=")
4656 || token.starts_with("--template=")
4657 })
4658}
4659
4660fn shell_words_for_flags(command: &str) -> Vec<String> {
4661 let mut words = Vec::new();
4662 let mut current = String::new();
4663 let mut in_single = false;
4664 let mut in_double = false;
4665 let mut escaped = false;
4666
4667 for ch in command.chars() {
4668 if escaped {
4669 current.push(ch);
4670 escaped = false;
4671 continue;
4672 }
4673 if ch == '\\' && !in_single {
4674 escaped = true;
4675 continue;
4676 }
4677 if ch == '\'' && !in_double {
4678 in_single = !in_single;
4679 continue;
4680 }
4681 if ch == '"' && !in_single {
4682 in_double = !in_double;
4683 continue;
4684 }
4685 if ch.is_whitespace() && !in_single && !in_double {
4686 if !current.is_empty() {
4687 words.push(std::mem::take(&mut current));
4688 }
4689 continue;
4690 }
4691 if matches!(ch, ';' | '&' | '|') && !in_single && !in_double {
4692 if !current.is_empty() {
4693 words.push(std::mem::take(&mut current));
4694 }
4695 continue;
4696 }
4697 current.push(ch);
4698 }
4699 if !current.is_empty() {
4700 words.push(current);
4701 }
4702 words
4703}
4704
4705fn is_structured_body(body: &str) -> bool {
4706 let trimmed = body.trim();
4707 if trimmed.is_empty() {
4708 return false;
4709 }
4710 if serde_json::from_str::<serde_json::Value>(trimmed).is_ok() {
4711 return true;
4712 }
4713
4714 let mut saw_line = false;
4715 for line in trimmed
4716 .lines()
4717 .map(str::trim)
4718 .filter(|line| !line.is_empty())
4719 {
4720 saw_line = true;
4721 if serde_json::from_str::<serde_json::Value>(line).is_err() {
4722 return false;
4723 }
4724 }
4725 saw_line
4726}
4727
4728fn stream_starts_like_json(buffer: &BgBuffer, stream: StreamKind) -> bool {
4729 buffer
4730 .read_stream_bounded(stream, 512)
4731 .text
4732 .chars()
4733 .find(|ch| !ch.is_whitespace())
4734 .is_some_and(|ch| matches!(ch, '{' | '[' | '"' | '-' | '0'..='9' | 't' | 'f' | 'n'))
4735}
4736
4737struct CompletionTokenCounts {
4738 original_tokens: Option<u32>,
4739 compressed_tokens: Option<u32>,
4740 original_bytes: Option<i64>,
4741 compressed_bytes: Option<i64>,
4742 tokens_skipped: bool,
4743}
4744
4745impl CompletionTokenCounts {
4746 fn skipped() -> Self {
4747 Self {
4748 original_tokens: None,
4749 compressed_tokens: None,
4750 original_bytes: None,
4751 compressed_bytes: None,
4752 tokens_skipped: true,
4753 }
4754 }
4755}
4756
4757fn completion_status_text(status: &BgTaskStatus, exit_code: Option<i32>) -> String {
4758 match status {
4759 BgTaskStatus::TimedOut => "timed out".to_string(),
4760 BgTaskStatus::Killed => "killed".to_string(),
4761 _ => exit_code
4762 .map(|code| format!("exit {code}"))
4763 .unwrap_or_else(|| format!("{status:?}").to_lowercase()),
4764 }
4765}
4766
4767fn token_count_u32(text: &str) -> u32 {
4768 aft_tokenizer::count_tokens(text)
4769 .try_into()
4770 .unwrap_or(u32::MAX)
4771}
4772
4773impl Default for BgTaskRegistry {
4774 fn default() -> Self {
4775 Self::new(Arc::new(Mutex::new(None)))
4776 }
4777}
4778
4779fn modified_within(path: &Path, grace: Duration) -> bool {
4780 fs::metadata(path)
4781 .and_then(|metadata| metadata.modified())
4782 .ok()
4783 .and_then(|modified| SystemTime::now().duration_since(modified).ok())
4784 .map(|age| age < grace)
4785 .unwrap_or(false)
4786}
4787
4788fn canonicalized_path(path: &Path) -> PathBuf {
4789 fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf())
4790}
4791
4792fn started_instant_from_unix_millis(started_at: u64) -> Instant {
4793 let now_ms = SystemTime::now()
4794 .duration_since(UNIX_EPOCH)
4795 .ok()
4796 .map(|duration| duration.as_millis() as u64)
4797 .unwrap_or(started_at);
4798 let elapsed_ms = now_ms.saturating_sub(started_at);
4799 Instant::now()
4800 .checked_sub(Duration::from_millis(elapsed_ms))
4801 .unwrap_or_else(Instant::now)
4802}
4803
4804fn gc_quarantine(storage_dir: &Path) {
4805 let quarantine_root = storage_dir.join("bash-tasks-quarantine");
4806 let Ok(session_dirs) = fs::read_dir(&quarantine_root) else {
4807 return;
4808 };
4809 for session_entry in session_dirs.flatten() {
4810 let session_quarantine_dir = session_entry.path();
4811 if !session_quarantine_dir.is_dir() {
4812 continue;
4813 }
4814 let entries = match fs::read_dir(&session_quarantine_dir) {
4815 Ok(entries) => entries,
4816 Err(error) => {
4817 crate::slog_warn!(
4818 "failed to read background task quarantine dir {}: {error}",
4819 session_quarantine_dir.display()
4820 );
4821 continue;
4822 }
4823 };
4824 for entry in entries.flatten() {
4825 let path = entry.path();
4826 if modified_within(&path, QUARANTINE_GC_GRACE) {
4827 continue;
4828 }
4829 let result = if path.is_dir() {
4830 fs::remove_dir_all(&path)
4831 } else {
4832 fs::remove_file(&path)
4833 };
4834 match result {
4835 Ok(()) => log::debug!(
4836 "deleted old background task quarantine entry {}",
4837 path.display()
4838 ),
4839 Err(error) => crate::slog_warn!(
4840 "failed to delete old background task quarantine entry {}: {error}",
4841 path.display()
4842 ),
4843 }
4844 }
4845 let _ = fs::remove_dir(&session_quarantine_dir);
4846 }
4847 let _ = fs::remove_dir(&quarantine_root);
4848}
4849
4850fn read_for_token_count_from_disk(
4851 metadata: &PersistedTask,
4852 paths: &TaskPaths,
4853 max_bytes_per_stream: usize,
4854) -> TokenCountInput {
4855 if metadata.mode == BgMode::Pty {
4856 return TokenCountInput::Skipped;
4857 }
4858 let stdout = read_file_tail_capped(paths, TaskArtifact::Stdout, max_bytes_per_stream);
4865 let stderr = read_file_tail_capped(paths, TaskArtifact::Stderr, max_bytes_per_stream);
4866 match (stdout, stderr) {
4867 (Ok(stdout), Ok(stderr)) => TokenCountInput::Text(combine_streams(
4868 String::from_utf8_lossy(&stdout).as_ref(),
4869 String::from_utf8_lossy(&stderr).as_ref(),
4870 )),
4871 (Ok(stdout), Err(_)) => TokenCountInput::Text(combine_streams(
4872 String::from_utf8_lossy(&stdout).as_ref(),
4873 "",
4874 )),
4875 (Err(_), Ok(stderr)) => TokenCountInput::Text(combine_streams(
4876 "",
4877 String::from_utf8_lossy(&stderr).as_ref(),
4878 )),
4879 (Err(_), Err(_)) => TokenCountInput::Skipped,
4880 }
4881}
4882
4883fn read_file_tail_capped(
4884 paths: &TaskPaths,
4885 artifact: TaskArtifact,
4886 max_bytes: usize,
4887) -> std::io::Result<Vec<u8>> {
4888 let mut file = open_task_artifact(paths, artifact)?;
4889 file.tail(max_bytes).map(|(bytes, _)| bytes)
4890}
4891
4892impl BgTask {
4893 fn snapshot(&self, preview_bytes: usize) -> BgTaskSnapshot {
4894 let state = self
4895 .state
4896 .lock()
4897 .unwrap_or_else(|poison| poison.into_inner());
4898 self.snapshot_locked(&state, preview_bytes)
4899 }
4900
4901 fn snapshot_locked(&self, state: &BgTaskState, preview_bytes: usize) -> BgTaskSnapshot {
4902 let metadata = &state.metadata;
4903 let duration_ms = metadata.duration_ms.or_else(|| {
4904 metadata
4905 .status
4906 .is_terminal()
4907 .then(|| self.started.elapsed().as_millis() as u64)
4908 });
4909 let (output_preview, output_truncated) = if metadata.mode == BgMode::Pty {
4910 (String::new(), false)
4911 } else if metadata.status.is_terminal() {
4912 state
4913 .terminal_output_cache
4914 .as_ref()
4915 .map(|cache| (cache.output_preview.clone(), cache.output_truncated))
4916 .unwrap_or_else(|| (String::new(), false))
4917 } else if preview_bytes == 0 {
4918 (String::new(), false)
4919 } else {
4920 state.buffer.read_tail(preview_bytes)
4921 };
4922 BgTaskSnapshot {
4923 info: BgTaskInfo {
4924 task_id: self.task_id.clone(),
4925 status: metadata.status.clone(),
4926 command: metadata.command.clone(),
4927 mode: metadata.mode.clone(),
4928 started_at: metadata.started_at,
4929 duration_ms,
4930 status_reason: metadata.status_reason.clone(),
4931 },
4932 exit_code: metadata.exit_code,
4933 child_pid: metadata.child_pid,
4934 workdir: metadata.workdir.display().to_string(),
4935 output_preview,
4936 output_truncated,
4937 output_path: state
4938 .buffer
4939 .output_path()
4940 .map(|path| path.display().to_string()),
4941 stderr_path: state
4942 .buffer
4943 .stderr_path()
4944 .map(|path| path.display().to_string()),
4945 pty_rows: (metadata.mode == BgMode::Pty).then_some(metadata.pty_rows.unwrap_or(24)),
4946 pty_cols: (metadata.mode == BgMode::Pty).then_some(metadata.pty_cols.unwrap_or(80)),
4947 pty_screen: None,
4948 scanner_report: metadata.scanner_report.clone(),
4949 sandbox_native: metadata.sandbox_native,
4950 sandbox_unavailable: metadata.sandbox_native
4951 && open_task_artifact(&self.paths, TaskArtifact::SandboxUnavailable)
4952 .and_then(|mut file| file.read_all())
4953 .is_ok_and(|bytes| bytes == b"sandbox_unavailable"),
4954 }
4955 }
4956
4957 pub(crate) fn is_running(&self) -> bool {
4958 self.state
4959 .lock()
4960 .map(|state| {
4961 state.metadata.status == BgTaskStatus::Running
4962 || (state.metadata.mode == BgMode::Pty
4963 && state.metadata.status == BgTaskStatus::Killing)
4964 })
4965 .unwrap_or(false)
4966 }
4967
4968 fn is_terminal(&self) -> bool {
4969 self.state
4970 .lock()
4971 .map(|state| state.metadata.status.is_terminal())
4972 .unwrap_or(false)
4973 }
4974
4975 fn mark_terminal_now(&self) {
4976 if let Ok(mut terminal_at) = self.terminal_at.lock() {
4977 if terminal_at.is_none() {
4978 *terminal_at = Some(Instant::now());
4979 }
4980 }
4981 }
4982
4983 fn set_completion_delivered(
4984 &self,
4985 delivered: bool,
4986 registry: &BgTaskRegistry,
4987 ) -> Result<(), String> {
4988 let mut state = self
4989 .state
4990 .lock()
4991 .map_err(|_| "background task lock poisoned".to_string())?;
4992 let updated = registry
4993 .update_task_metadata(&self.paths, |metadata| {
4994 metadata.completion_delivered = delivered;
4995 })
4996 .map_err(|e| format!("failed to update completion delivery: {e}"))?;
4997 state.metadata = updated;
4998 Ok(())
4999 }
5000}
5001
5002#[cfg(unix)]
5023fn reap_piped_child(child_slot: &mut Option<Child>) {
5024 if let Some(mut child) = child_slot.take() {
5025 if matches!(child.try_wait(), Ok(None)) {
5026 let _ = child.wait();
5027 }
5028 }
5029}
5030
5031#[cfg(windows)]
5036fn reap_piped_child(child_slot: &mut Option<Child>) {
5037 *child_slot = None;
5038}
5039
5040fn terminal_metadata_from_marker(
5041 mut metadata: PersistedTask,
5042 marker: ExitMarker,
5043 reason: Option<String>,
5044) -> PersistedTask {
5045 match marker {
5046 ExitMarker::Code(code) => {
5047 let status = if code == 0 {
5048 BgTaskStatus::Completed
5049 } else {
5050 BgTaskStatus::Failed
5051 };
5052 metadata.mark_terminal(status, Some(code), reason);
5053 }
5054 ExitMarker::Killed => metadata.mark_terminal(
5055 BgTaskStatus::Killed,
5056 terminal_exit_code_for_status(&BgTaskStatus::Killed),
5057 reason,
5058 ),
5059 }
5060 metadata
5061}
5062
5063fn terminal_exit_code_for_status(status: &BgTaskStatus) -> Option<i32> {
5064 match status {
5065 BgTaskStatus::TimedOut => Some(124),
5066 BgTaskStatus::Killed => Some(137),
5067 _ => None,
5068 }
5069}
5070
5071fn attach_sandbox_metadata(metadata: &mut PersistedTask, spawn_plan: &SpawnPlan) {
5072 metadata.sandbox_native = spawn_plan.is_native_launcher();
5073 metadata.sandbox_temp_dir = spawn_plan.temp_dir().map(Path::to_path_buf);
5074}
5075
5076#[cfg(unix)]
5077pub(crate) fn resolve_posix_shell() -> PathBuf {
5078 static POSIX_SHELL: OnceLock<PathBuf> = OnceLock::new();
5079 POSIX_SHELL
5080 .get_or_init(|| {
5081 std::env::var_os("BASH")
5082 .filter(|value| !value.is_empty())
5083 .map(PathBuf::from)
5084 .filter(|path| path.exists())
5085 .or_else(|| which::which("bash").ok())
5086 .or_else(|| which::which("zsh").ok())
5087 .unwrap_or_else(|| PathBuf::from("/bin/sh"))
5088 })
5089 .clone()
5090}
5091
5092#[cfg(windows)]
5093fn detached_shell_command_for(
5094 shell: crate::windows_shell::WindowsShell,
5095 command: &str,
5096 exit_path: &Path,
5097 paths: &TaskPaths,
5098 creation_flags: u32,
5099) -> Result<Command, String> {
5100 use crate::windows_shell::WindowsShell;
5101 let wrapper_body = shell.wrapper_script_bytes(command, exit_path);
5114 let wrapper_ext = match shell {
5115 WindowsShell::Pwsh | WindowsShell::Powershell => "ps1",
5116 WindowsShell::Cmd => "bat",
5117 WindowsShell::Posix(_) => "sh",
5121 };
5122 let wrapper_path = paths.dir.join(format!(
5123 "{}.{}",
5124 paths
5125 .json
5126 .file_stem()
5127 .and_then(|s| s.to_str())
5128 .unwrap_or("wrapper"),
5129 wrapper_ext
5130 ));
5131 fs::write(&wrapper_path, wrapper_body)
5132 .map_err(|e| format!("failed to write background bash wrapper script: {e}"))?;
5133
5134 let mut cmd = Command::new(shell.binary().as_ref());
5135 match shell {
5136 WindowsShell::Pwsh | WindowsShell::Powershell => {
5137 cmd.args([
5140 "-NoLogo",
5141 "-NoProfile",
5142 "-NonInteractive",
5143 "-ExecutionPolicy",
5144 "Bypass",
5145 "-File",
5146 ]);
5147 cmd.arg(&wrapper_path);
5148 }
5149 WindowsShell::Cmd => {
5150 cmd.args(["/D", "/C"]);
5157 cmd.arg(&wrapper_path);
5158 }
5159 WindowsShell::Posix(_) => {
5160 cmd.arg(&wrapper_path);
5165 }
5166 }
5167
5168 cmd.creation_flags(creation_flags);
5172 Ok(cmd)
5173}
5174
5175fn spawn_detached_child(
5191 spawn_plan: &SpawnPlan,
5192 command: &str,
5193 paths: &TaskPaths,
5194 workdir: &Path,
5195 env: &HashMap<String, String>,
5196 io_handles: &mut TaskIoHandles,
5197 capture_pipeline_status: bool,
5198) -> Result<std::process::Child, String> {
5199 #[cfg(windows)]
5200 let _ = capture_pipeline_status;
5201 #[cfg(not(windows))]
5202 let _ = command;
5203 #[cfg(not(windows))]
5204 {
5205 use std::os::fd::AsRawFd;
5206
5207 let stdout = io_handles
5208 .clone_file(TaskArtifact::Stdout)
5209 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
5210 let stderr = io_handles
5211 .clone_file(TaskArtifact::Stderr)
5212 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
5213 let prepared = spawn_plan
5214 .prepared_task()
5215 .ok_or_else(|| "background task payload was not prepared".to_string())?;
5216 let payload = prepared.invocation()?;
5217 let exit = io_handles
5218 .inheritable_file(TaskArtifact::Exit)
5219 .map_err(|e| format!("failed to inherit exit marker handle: {e}"))?;
5220 let failure = io_handles
5221 .inheritable_file(TaskArtifact::SandboxUnavailable)
5222 .map_err(|e| format!("failed to inherit sandbox failure marker handle: {e}"))?;
5223 let pipeline_status = capture_pipeline_status
5224 .then(|| io_handles.inheritable_file(TaskArtifact::PipelineStatus))
5225 .transpose()
5226 .map_err(|e| format!("failed to inherit pipeline status handle: {e}"))?;
5227 let shell = spawn_plan
5228 .host_shell_path()
5229 .map(Path::to_path_buf)
5230 .unwrap_or_else(resolve_posix_shell);
5231 let pipeline_shell = super::process::pipeline_shell_kind(&shell).unwrap_or("");
5232 let pipeline_status_fd = if capture_pipeline_status {
5233 crate::sandbox_spawn::CHILD_PIPE_STATUS_FD.to_string()
5234 } else {
5235 String::new()
5236 };
5237 let args = vec![
5238 OsString::from("-c"),
5239 payload.wrapper_text.clone(),
5240 OsString::from("aft-payload-wrapper"),
5241 shell.as_os_str().to_os_string(),
5242 payload.command_text.clone(),
5243 OsString::from(crate::sandbox_spawn::CHILD_EXIT_FD.to_string()),
5244 OsString::from(pipeline_status_fd),
5245 OsString::from(pipeline_shell),
5246 ];
5247 let (mut child_command, profile_handle) = crate::sandbox_spawn::detached_command_for_plan(
5248 spawn_plan,
5249 std::ffi::OsStr::new("/bin/sh"),
5250 &args,
5251 &paths.json,
5252 crate::sandbox_spawn::CHILD_EXIT_FD,
5253 crate::sandbox_spawn::CHILD_FAILURE_FD,
5254 )?;
5255 crate::sandbox_spawn::apply_marker_fd_allowlist(
5256 &mut child_command,
5257 exit.as_raw_fd(),
5258 failure.as_raw_fd(),
5259 pipeline_status.as_ref().map(|file| file.as_raw_fd()),
5260 )?;
5261 child_command
5262 .current_dir(workdir)
5263 .envs(env)
5264 .stdin(Stdio::null())
5265 .stdout(Stdio::from(stdout))
5266 .stderr(Stdio::from(stderr));
5267 crate::sandbox_spawn::apply_sandbox_environment(spawn_plan, &mut child_command, env);
5268 let child = child_command
5269 .spawn()
5270 .map_err(|e| format!("failed to spawn background bash command: {e}"));
5271 drop((payload, exit, failure, pipeline_status, profile_handle));
5272 child
5273 }
5274 #[cfg(windows)]
5275 {
5276 use crate::windows_shell::shell_candidates;
5277 match spawn_plan {
5278 SpawnPlan::Unsandboxed | SpawnPlan::Host { .. } => {}
5279 SpawnPlan::Refused { code, .. } => return Err((*code).to_string()),
5280 SpawnPlan::Launcher { .. } => return Err("sandbox_unavailable".to_string()),
5281 }
5282 let candidates: Vec<crate::windows_shell::WindowsShell> = shell_candidates();
5293 const FLAG_CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200;
5306 const FLAG_CREATE_BREAKAWAY_FROM_JOB: u32 = 0x0100_0000;
5307 const FLAG_CREATE_NO_WINDOW: u32 = 0x0800_0000;
5308 let with_breakaway =
5309 FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP | FLAG_CREATE_BREAKAWAY_FROM_JOB;
5310 let without_breakaway = FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP;
5311 let mut last_error: Option<String> = None;
5312 for (idx, shell) in candidates.iter().enumerate() {
5313 for &flags in &[with_breakaway, without_breakaway] {
5317 let stdout = io_handles
5320 .clone_file(TaskArtifact::Stdout)
5321 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
5322 let stderr = io_handles
5323 .clone_file(TaskArtifact::Stderr)
5324 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
5325 let mut cmd =
5326 detached_shell_command_for(shell.clone(), command, &paths.exit, paths, flags)?;
5327 cmd.current_dir(workdir)
5328 .envs(env)
5329 .stdin(Stdio::null())
5330 .stdout(Stdio::from(stdout))
5331 .stderr(Stdio::from(stderr));
5332 match cmd.spawn() {
5333 Ok(child) => {
5334 if idx > 0 {
5335 crate::slog_warn!("background bash spawn fell back to {} after {} earlier candidate(s) failed; \
5336 the cached PATH probe disagreed with runtime spawn — likely PATH \
5337 inheritance, antivirus / AppLocker / Defender ASR, or sandbox policy.",
5338 shell.binary(),
5339 idx);
5340 }
5341 if flags == without_breakaway {
5342 crate::slog_warn!(
5343 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected \
5344 (likely a restrictive Job Object — CI sandbox or MDM policy). \
5345 Spawned without breakaway; the bg task will be torn down if the \
5346 AFT process group is killed."
5347 );
5348 }
5349 return Ok(child);
5350 }
5351 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
5352 crate::slog_warn!("background bash spawn: {} returned NotFound at runtime — trying next candidate",
5353 shell.binary());
5354 last_error = Some(format!("{}: {e}", shell.binary()));
5355 break;
5358 }
5359 Err(e) if flags == with_breakaway && e.raw_os_error() == Some(5) => {
5360 crate::slog_warn!(
5362 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected with \
5363 Access Denied — retrying {} without breakaway",
5364 shell.binary()
5365 );
5366 last_error = Some(format!("{}: {e}", shell.binary()));
5367 continue;
5368 }
5369 Err(e) => {
5370 return Err(format!(
5371 "failed to spawn background bash command via {}: {e}",
5372 shell.binary()
5373 ));
5374 }
5375 }
5376 }
5377 }
5378 Err(format!(
5379 "failed to spawn background bash command: no Windows shell could be spawned. \
5380 Last error: {}. PATH-probed candidates: {:?}",
5381 last_error.unwrap_or_else(|| "no candidates were attempted".to_string()),
5382 candidates.iter().map(|s| s.binary()).collect::<Vec<_>>()
5383 ))
5384 }
5385}
5386
5387#[cfg(test)]
5388fn random_slug() -> String {
5389 let mut bytes = [0u8; 8];
5397 getrandom::fill(&mut bytes).unwrap_or_else(|_| {
5399 let t = SystemTime::now()
5401 .duration_since(UNIX_EPOCH)
5402 .map(|d| d.as_nanos() as u64)
5403 .unwrap_or(0);
5404 let p = u64::from(std::process::id());
5405 bytes.copy_from_slice(&(t ^ p.rotate_left(32)).to_le_bytes());
5406 });
5407 let hex: String = bytes.iter().map(|b| format!("{b:02x}")).collect();
5409 format!("bash-{hex}")
5410}
5411
5412#[cfg(test)]
5413mod tests {
5414 use std::collections::HashMap;
5415 use std::fs;
5416 use std::io::Write;
5417 #[cfg(unix)]
5418 use std::os::unix::fs::PermissionsExt;
5419 use std::sync::atomic::{AtomicBool, AtomicUsize};
5420 use std::sync::{Arc, Mutex};
5421 use std::time::{Duration, Instant, SystemTime};
5422
5423 use super::*;
5424 use crate::bash_background::persistence::{read_task, task_paths, write_task};
5425
5426 #[cfg(unix)]
5427 const QUICK_SUCCESS_COMMAND: &str = "true";
5428 #[cfg(windows)]
5429 const QUICK_SUCCESS_COMMAND: &str = "cmd /c exit 0";
5430
5431 #[cfg(unix)]
5432 const LONG_RUNNING_COMMAND: &str = "sleep 5";
5433
5434 #[cfg(unix)]
5435 #[test]
5436 fn launcher_plans_disable_pipeline_status_capture() {
5437 let launcher = SpawnPlan::launcher_for_test(
5438 crate::sandbox_profile::SandboxProfile {
5439 v: crate::sandbox_profile::SANDBOX_PROFILE_VERSION,
5440 writable_roots: Vec::new(),
5441 write_deny: Vec::new(),
5442 write_deny_nested: Vec::new(),
5443 read_allow: Vec::new(),
5444 read_deny: Vec::new(),
5445 socket_deny: Vec::new(),
5446 cache_roots: Vec::new(),
5447 temp_dir: PathBuf::from("/tmp/aft-test-sandbox"),
5448 },
5449 PathBuf::from("/bin/true"),
5450 );
5451 assert!(!should_capture_pipeline_status(
5452 &launcher,
5453 true,
5454 Path::new("/bin/bash")
5455 ));
5456 assert!(should_capture_pipeline_status(
5457 &SpawnPlan::Unsandboxed,
5458 true,
5459 Path::new("/bin/bash")
5460 ));
5461 }
5462
5463 #[cfg(windows)]
5464 const LONG_RUNNING_COMMAND: &str = "cmd /c timeout /t 5 /nobreak > nul";
5465
5466 #[test]
5467 fn bash_memory_estimate_is_zero_when_empty_and_nonzero_for_completion_cache() {
5468 let registry = BgTaskRegistry::default();
5469 assert_eq!(registry.estimated_memory().estimated_bytes, Some(0));
5470 registry
5471 .inner
5472 .completions
5473 .lock()
5474 .unwrap()
5475 .push_back(BgCompletion {
5476 task_id: "bash-memory".to_string(),
5477 session_id: "session-memory".to_string(),
5478 status: BgTaskStatus::Completed,
5479 exit_code: Some(0),
5480 command: "printf memory".to_string(),
5481 output_preview: "resident completion output".to_string(),
5482 output_truncated: false,
5483 original_tokens: None,
5484 compressed_tokens: None,
5485 tokens_skipped: false,
5486 status_reason: None,
5487 });
5488 let estimate = registry.estimated_memory();
5489 assert!(estimate.estimated_bytes.unwrap() > 0);
5490 assert_eq!(estimate.counts["completion_caches"], 1);
5491 assert_eq!(estimate.counts["sessions"], 1);
5492 }
5493
5494 #[test]
5495 fn gh_structured_detection_rejects_piped_commands() {
5496 assert!(is_gh_structured_command(
5497 "gh issue list --json number,title"
5498 ));
5499 assert!(is_gh_structured_command(
5500 "cd repo && gh issue list --json number,title"
5501 ));
5502
5503 assert!(!is_gh_structured_command(
5504 "gh issue list --json number,title | jq '.[]'"
5505 ));
5506 assert!(!is_gh_structured_command(
5507 "gh issue list --json number,title |"
5508 ));
5509 }
5510
5511 fn insert_terminal_piped_task(
5512 registry: &BgTaskRegistry,
5513 dir: &tempfile::TempDir,
5514 command: &str,
5515 stdout: &str,
5516 stderr: &str,
5517 compressed: bool,
5518 ) -> (String, Arc<BgTask>) {
5519 let task_id = random_slug();
5520 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5521 fs::create_dir_all(&paths.dir).unwrap();
5522 fs::write(&paths.stdout, stdout).unwrap();
5523 fs::write(&paths.stderr, stderr).unwrap();
5524 let mut metadata = PersistedTask::starting(
5525 task_id.clone(),
5526 "session".to_string(),
5527 command.to_string(),
5528 dir.path().to_path_buf(),
5529 Some(dir.path().to_path_buf()),
5530 Some(30_000),
5531 true,
5532 compressed,
5533 );
5534 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
5535 write_task(&paths.json, &metadata).unwrap();
5536 registry
5537 .insert_rehydrated_task(metadata, paths, true)
5538 .expect("insert terminal task");
5539 let task = registry.task_for_session(&task_id, "session").unwrap();
5540 (task_id, task)
5541 }
5542
5543 #[test]
5544 fn bash_zero_preview_running_status_skips_output_read_while_explicit_preview_reads() {
5545 let registry = BgTaskRegistry::default();
5546 let dir = tempfile::tempdir().unwrap();
5547 let task_id = random_slug();
5548 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5549 fs::create_dir_all(&paths.dir).unwrap();
5550 fs::write(&paths.stdout, "live output\n").unwrap();
5551 fs::write(&paths.stderr, "").unwrap();
5552 let stdout_path = paths.stdout.clone();
5553 let mut metadata = PersistedTask::starting(
5554 task_id.clone(),
5555 "session".to_string(),
5556 "sleep 60".to_string(),
5557 dir.path().to_path_buf(),
5558 Some(dir.path().to_path_buf()),
5559 Some(30_000),
5560 true,
5561 false,
5562 );
5563 metadata.status = BgTaskStatus::Running;
5564 write_task(&paths.json, &metadata).unwrap();
5565 registry
5566 .insert_rehydrated_task(metadata, paths, false)
5567 .expect("insert running task");
5568
5569 crate::bash_background::buffer::reset_tail_read_count(&stdout_path);
5570 for _ in 0..5 {
5571 let snapshot = registry
5572 .status(&task_id, "session", Some(dir.path()), Some(dir.path()), 0)
5573 .expect("running snapshot");
5574 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
5575 assert!(snapshot.output_preview.is_empty());
5576 }
5577 assert_eq!(
5578 crate::bash_background::buffer::tail_read_count(&stdout_path),
5579 0
5580 );
5581
5582 let snapshot = registry
5583 .status(
5584 &task_id,
5585 "session",
5586 Some(dir.path()),
5587 Some(dir.path()),
5588 RUNNING_OUTPUT_PREVIEW_BYTES,
5589 )
5590 .expect("explicit running snapshot");
5591 assert_eq!(snapshot.output_preview, "live output\n");
5592 assert_eq!(
5593 crate::bash_background::buffer::tail_read_count(&stdout_path),
5594 1
5595 );
5596 }
5597
5598 #[test]
5599 fn artifact_read_capability_requires_exact_canonical_path_and_session() {
5600 let registry = BgTaskRegistry::default();
5601 let dir = tempfile::tempdir().unwrap();
5602 let (_task_id, task) = insert_terminal_piped_task(
5603 ®istry,
5604 &dir,
5605 "printf output",
5606 "stdout\n",
5607 "stderr\n",
5608 true,
5609 );
5610 fs::write(&task.paths.exit, "0\n").unwrap();
5611
5612 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stdout));
5613 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stderr));
5614 assert!(registry.is_session_owned_artifact_path("session", &task.paths.exit));
5615 assert!(!registry.is_session_owned_artifact_path("different-session", &task.paths.stdout));
5616 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.json));
5617
5618 let unregistered = task.paths.dir.join("unregistered-output");
5619 fs::write(&unregistered, "not a task artifact\n").unwrap();
5620 assert!(!registry.is_session_owned_artifact_path("session", &unregistered));
5621 }
5622
5623 #[cfg(unix)]
5624 #[test]
5625 fn artifact_directory_symlink_does_not_create_a_prefix_exception() {
5626 let registry = BgTaskRegistry::default();
5627 let dir = tempfile::tempdir().unwrap();
5628 let project = dir.path().join("project");
5629 fs::create_dir_all(&project).unwrap();
5630 let (_task_id, task) =
5631 insert_terminal_piped_task(®istry, &dir, "printf output", "stdout\n", "", true);
5632 let link = project.join("task-artifacts");
5633 std::os::unix::fs::symlink(&task.paths.dir, &link).unwrap();
5634 let unregistered = task.paths.dir.join("unregistered-output");
5635 fs::write(&unregistered, "not registered\n").unwrap();
5636
5637 assert!(!registry.is_session_owned_artifact_path("session", &link));
5638 assert!(
5639 !registry.is_session_owned_artifact_path("session", &link.join("unregistered-output"))
5640 );
5641 assert!(registry.is_session_owned_artifact_path(
5642 "session",
5643 &link.join(task.paths.stdout.file_name().unwrap())
5644 ));
5645
5646 let outside = dir.path().join("outside-secret");
5647 fs::write(&outside, "must stay private\n").unwrap();
5648 fs::remove_file(&task.paths.stdout).unwrap();
5649 std::os::unix::fs::symlink(&outside, &task.paths.stdout).unwrap();
5650 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.stdout));
5651 }
5652
5653 #[test]
5654 fn recovery_footer_uses_bash_status_when_artifact_is_not_registered() {
5655 let registry = BgTaskRegistry::default();
5656 let dir = tempfile::tempdir().unwrap();
5657 let task_id = "bash-1111111111111111";
5658 let paths = task_paths(dir.path(), "session", task_id).unwrap();
5659 fs::create_dir_all(&paths.dir).unwrap();
5660 fs::write(
5661 &paths.stdout,
5662 format!("{}tail\n", "output-line\n".repeat(2_000)),
5663 )
5664 .unwrap();
5665 fs::write(&paths.stderr, "").unwrap();
5666 let mut metadata = PersistedTask::starting(
5667 task_id.to_string(),
5668 "session".to_string(),
5669 "printf output".to_string(),
5670 dir.path().to_path_buf(),
5671 Some(dir.path().to_path_buf()),
5672 Some(30_000),
5673 true,
5674 true,
5675 );
5676 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
5677 write_task(&paths.json, &metadata).unwrap();
5678
5679 let cache = registry
5680 .render_terminal_output_from_paths(&metadata, &paths)
5681 .expect("terminal render");
5682
5683 assert!(cache
5684 .output_preview
5685 .contains("use bash_status({taskId: \"bash-1111111111111111\"})"));
5686 assert!(!cache.output_preview.contains("full output: read "));
5687 }
5688
5689 fn insert_terminal_pty_task(
5690 registry: &BgTaskRegistry,
5691 dir: &tempfile::TempDir,
5692 pty_output: &str,
5693 ) -> (String, Arc<BgTask>) {
5694 let task_id = random_slug();
5695 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5696 fs::create_dir_all(&paths.dir).unwrap();
5697 fs::write(&paths.pty, pty_output).unwrap();
5698 let mut metadata = PersistedTask::starting(
5699 task_id.clone(),
5700 "session".to_string(),
5701 "python".to_string(),
5702 dir.path().to_path_buf(),
5703 Some(dir.path().to_path_buf()),
5704 Some(30_000),
5705 true,
5706 true,
5707 );
5708 metadata.mode = BgMode::Pty;
5709 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
5710 write_task(&paths.json, &metadata).unwrap();
5711 registry
5712 .insert_rehydrated_task(metadata, paths, true)
5713 .expect("insert terminal pty task");
5714 let task = registry.task_for_session(&task_id, "session").unwrap();
5715 (task_id, task)
5716 }
5717
5718 #[cfg(unix)]
5719 fn wait_for_terminal_snapshot(
5720 registry: &BgTaskRegistry,
5721 task_id: &str,
5722 session_id: &str,
5723 project: &Path,
5724 storage: &Path,
5725 ) -> BgTaskSnapshot {
5726 let started = Instant::now();
5727 loop {
5728 let snapshot = registry
5729 .status(task_id, session_id, Some(project), Some(storage), 4096)
5730 .expect("spawned task should be visible to status");
5731 if snapshot.info.status.is_terminal() {
5732 return snapshot;
5733 }
5734 assert!(
5735 started.elapsed() < Duration::from_secs(10),
5736 "timed out waiting for task {task_id} to finish; last status={:?}",
5737 snapshot.info.status
5738 );
5739 std::thread::sleep(Duration::from_millis(50));
5740 }
5741 }
5742
5743 fn write_running_project_task(storage: &Path, project: &Path, session: &str, task_id: &str) {
5744 let paths = task_paths(storage, session, task_id).unwrap();
5745 let mut metadata = PersistedTask::starting(
5746 task_id.to_string(),
5747 session.to_string(),
5748 "sleep 60".to_string(),
5749 project.to_path_buf(),
5750 Some(project.to_path_buf()),
5751 Some(30_000),
5752 true,
5753 true,
5754 );
5755 metadata.status = BgTaskStatus::Running;
5756 write_task(&paths.json, &metadata).unwrap();
5757 fs::write(&paths.stdout, "still running\n").unwrap();
5758 fs::write(&paths.stderr, "").unwrap();
5759 }
5760
5761 #[test]
5762 fn status_replay_filters_same_session_by_project_root() {
5763 let project_a = tempfile::tempdir().unwrap();
5764 let project_b = tempfile::tempdir().unwrap();
5765 let storage = tempfile::tempdir().unwrap();
5766 let session = "shared-session";
5767 let task_id = "bash-2222222222222222";
5768 write_running_project_task(storage.path(), project_a.path(), session, task_id);
5769
5770 let actor_b = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5771 assert!(actor_b
5772 .status(
5773 task_id,
5774 session,
5775 Some(project_b.path()),
5776 Some(storage.path()),
5777 1024,
5778 )
5779 .is_none());
5780 assert!(actor_b.task_for_session(task_id, session).is_none());
5781
5782 let actor_a = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5783 let snapshot = actor_a
5784 .status(
5785 task_id,
5786 session,
5787 Some(project_a.path()),
5788 Some(storage.path()),
5789 1024,
5790 )
5791 .expect("owning project should replay its task");
5792 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
5793 }
5794
5795 #[cfg(unix)]
5796 #[test]
5797 fn multiline_pipeline_stdout_persists_all_lines_after_terminal_status() {
5798 let cases = [
5799 (
5800 "long-first",
5801 "sleep 0.5; printf 'one\\n' | cat\nprintf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
5802 vec!["one", "1", "three"],
5803 ),
5804 (
5805 "short-first",
5806 "printf 'one\\n' | cat\nsleep 0.2; printf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
5807 vec!["one", "1", "three"],
5808 ),
5809 (
5810 "failing-middle",
5811 "sleep 0.2; printf 'one\\n' | cat\nfalse; printf 'after-false\\n' | cat\nprintf 'three\\n' | cat",
5812 vec!["one", "after-false", "three"],
5813 ),
5814 ];
5815
5816 for (name, command, expected_lines) in cases {
5817 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5818 let dir = tempfile::tempdir().unwrap();
5819 let session_id = format!("session-{name}");
5820 let task_id = registry
5821 .spawn(
5822 SpawnPlan::Unsandboxed,
5823 command,
5824 session_id.clone(),
5825 dir.path().to_path_buf(),
5826 HashMap::new(),
5827 Some(Duration::from_secs(30)),
5828 dir.path().to_path_buf(),
5829 10,
5830 true,
5831 true,
5832 Some(dir.path().to_path_buf()),
5833 )
5834 .unwrap();
5835
5836 let snapshot = wait_for_terminal_snapshot(
5837 ®istry,
5838 &task_id,
5839 &session_id,
5840 dir.path(),
5841 dir.path(),
5842 );
5843 assert_eq!(
5844 snapshot.info.status,
5845 BgTaskStatus::Completed,
5846 "{name}: task should complete; snapshot={snapshot:?}"
5847 );
5848 assert_eq!(
5849 snapshot.exit_code,
5850 Some(0),
5851 "{name}: script should use the final command's exit code"
5852 );
5853
5854 let stdout = String::from_utf8(
5855 registry
5856 .read_artifact(&task_id, &session_id, TaskArtifact::Stdout)
5857 .expect("read validated stdout artifact"),
5858 )
5859 .expect("stdout is UTF-8");
5860 let lines: Vec<&str> = stdout.lines().collect();
5861 assert_eq!(
5862 lines, expected_lines,
5863 "{name}: raw stdout artifact must include every newline-separated command's output"
5864 );
5865 }
5866 }
5867
5868 #[test]
5869 fn recognizes_all_recovery_marker_forms() {
5870 assert!(is_recovery_marker(
5871 "[truncated output; full output: read \"/tmp/out\"]"
5872 ));
5873 assert!(is_recovery_marker(
5874 "[omitted output; see remaining: tail -n +42 \"/tmp/out\"]"
5875 ));
5876 assert!(is_recovery_marker(
5877 "[truncated output; full output unavailable]"
5878 ));
5879 assert!(is_recovery_marker(
5880 r#"[truncated 123 bytes from saved output prefix; retained output: read "/tmp/out"]"#
5881 ));
5882 }
5883
5884 #[test]
5885 fn recovery_marker_reports_disk_prefix_truncation_as_retained_output() {
5886 let recovery = RecoveryContext {
5887 dropped_by_class: BTreeMap::new(),
5888 had_inner_drop: false,
5889 offset_hint_eligible: false,
5890 offset_start_line: None,
5891 byte_truncated: false,
5892 disk_truncated_prefix_bytes: 4096,
5893 output_path: Some("/tmp/stdout".to_string()),
5894 stderr_path: None,
5895 include_stderr_path: false,
5896 artifact_access: ArtifactRecoveryAccess {
5897 task_id: "bash-test".to_string(),
5898 readable: true,
5899 },
5900 };
5901
5902 let marker = recovery_marker(&recovery).expect("disk truncation must emit marker");
5903
5904 assert!(marker.contains("truncated 4096 bytes from saved output prefix"));
5905 assert!(marker.contains(r#"retained output: read "/tmp/stdout""#));
5906 assert!(!marker.contains("full output: read"));
5907 }
5908
5909 #[test]
5910 fn killed_exit_marker_sets_nonzero_sentinel_exit_code() {
5911 let metadata = PersistedTask::starting(
5912 "task".to_string(),
5913 "session".to_string(),
5914 "cargo test".to_string(),
5915 PathBuf::from("/tmp"),
5916 None,
5917 None,
5918 true,
5919 true,
5920 );
5921
5922 let terminal = terminal_metadata_from_marker(metadata, ExitMarker::Killed, None);
5923
5924 assert_eq!(terminal.status, BgTaskStatus::Killed);
5925 assert_eq!(terminal.exit_code, Some(137));
5926 }
5927
5928 #[test]
5929 fn terminal_status_polls_use_cached_render_once_and_off_lock() {
5930 let registry = BgTaskRegistry::default();
5931 let dir = tempfile::tempdir().unwrap();
5932 let (_task_id, task) = insert_terminal_piped_task(
5933 ®istry,
5934 &dir,
5935 "custom-tool --verbose",
5936 &"stdout line\n".repeat(200_000),
5937 "",
5938 true,
5939 );
5940 let calls = Arc::new(AtomicUsize::new(0));
5941 let saw_unlocked_state = Arc::new(AtomicBool::new(false));
5942 let task_holder = Arc::new(Mutex::new(Some(Arc::clone(&task))));
5943 let calls_for_closure = Arc::clone(&calls);
5944 let unlocked_for_closure = Arc::clone(&saw_unlocked_state);
5945 let task_for_closure = Arc::clone(&task_holder);
5946 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
5947 calls_for_closure.fetch_add(1, Ordering::SeqCst);
5948 if let Some(task) = task_for_closure.lock().unwrap().as_ref() {
5949 if task.state.try_lock().is_ok() {
5950 unlocked_for_closure.store(true, Ordering::SeqCst);
5951 }
5952 }
5953 CompressionResult::new(format!("compressed {} bytes", output.len()))
5954 });
5955
5956 let first = registry
5957 .status(
5958 &task.task_id,
5959 "session",
5960 None,
5961 Some(dir.path()),
5962 RUNNING_OUTPUT_PREVIEW_BYTES,
5963 )
5964 .unwrap();
5965 let second = registry
5966 .status(
5967 &task.task_id,
5968 "session",
5969 None,
5970 Some(dir.path()),
5971 RUNNING_OUTPUT_PREVIEW_BYTES,
5972 )
5973 .unwrap();
5974 let listed = registry.list(RUNNING_OUTPUT_PREVIEW_BYTES);
5975
5976 assert_eq!(
5977 calls.load(Ordering::SeqCst),
5978 1,
5979 "terminal render must be cached"
5980 );
5981 assert!(
5982 saw_unlocked_state.load(Ordering::SeqCst),
5983 "compressor must run after releasing the task state lock"
5984 );
5985 assert!(first.output_preview.starts_with("compressed "));
5986 assert_eq!(second.output_preview, first.output_preview);
5987 assert_eq!(listed[0].output_preview, first.output_preview);
5988 }
5989
5990 #[test]
5991 fn completion_preview_success_keeps_tail_only() {
5992 let registry = BgTaskRegistry::default();
5997 let dir = tempfile::tempdir().unwrap();
5998 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
5999 let (_task_id, task) =
6000 insert_terminal_piped_task(®istry, &dir, "cat big.log", &output, "", false);
6001
6002 registry.post_terminal_transition(&task, true).unwrap();
6003 let completions = registry.drain_completions_for_session(Some("session"));
6004 assert_eq!(completions.len(), 1);
6005 let preview = &completions[0].output_preview;
6006 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
6007 assert!(!preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
6008 assert!(completions[0].output_truncated);
6009 }
6010
6011 #[test]
6012 fn completion_preview_failure_keeps_head_and_tail() {
6013 let registry = BgTaskRegistry::default();
6016 let dir = tempfile::tempdir().unwrap();
6017 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
6018 let task_id = random_slug();
6019 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
6020 fs::create_dir_all(&paths.dir).unwrap();
6021 fs::write(&paths.stdout, &output).unwrap();
6022 fs::write(&paths.stderr, "").unwrap();
6023 let mut metadata = PersistedTask::starting(
6024 task_id.clone(),
6025 "session".to_string(),
6026 "cat big.log".to_string(),
6027 dir.path().to_path_buf(),
6028 Some(dir.path().to_path_buf()),
6029 Some(30_000),
6030 true,
6031 false,
6032 );
6033 metadata.mark_terminal(BgTaskStatus::Failed, Some(1), None);
6034 write_task(&paths.json, &metadata).unwrap();
6035 registry
6036 .insert_rehydrated_task(metadata, paths, true)
6037 .expect("insert terminal task");
6038 let task = registry.task_for_session(&task_id, "session").unwrap();
6039
6040 registry.post_terminal_transition(&task, true).unwrap();
6041 let completions = registry.drain_completions_for_session(Some("session"));
6042 assert_eq!(completions.len(), 1);
6043 let preview = &completions[0].output_preview;
6044 assert!(preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
6045 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
6046 }
6047
6048 #[test]
6049 fn has_completions_for_session_matches_pending_delivery() {
6050 let registry = BgTaskRegistry::default();
6051 assert!(!registry.has_completions_for_session(Some("session")));
6052 assert!(!registry.has_completions_for_session(None));
6053
6054 let dir = tempfile::tempdir().unwrap();
6055 let (_task_id, task) =
6056 insert_terminal_piped_task(®istry, &dir, QUICK_SUCCESS_COMMAND, "done\n", "", false);
6057 registry.post_terminal_transition(&task, true).unwrap();
6058
6059 assert!(registry.has_completions_for_session(Some("session")));
6060 assert!(registry.has_completions_for_session(None));
6061 assert!(!registry.has_completions_for_session(Some("other-session")));
6062
6063 let completions = registry.drain_completions_for_session(Some("session"));
6064 assert_eq!(completions.len(), 1);
6065 assert_eq!(completions[0].task_id, task.task_id);
6066 }
6067
6068 #[test]
6069 fn structured_gh_json_survives_intact_and_ignores_stderr() {
6070 let registry = BgTaskRegistry::default();
6071 let dir = tempfile::tempdir().unwrap();
6072 let calls = Arc::new(AtomicUsize::new(0));
6073 let calls_for_closure = Arc::clone(&calls);
6074 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
6075 calls_for_closure.fetch_add(1, Ordering::SeqCst);
6076 CompressionResult::new(output)
6077 });
6078 let (task_id, _task) = insert_terminal_piped_task(
6079 ®istry,
6080 &dir,
6081 "gh pr view 123 --json body",
6082 "{\"body\":\"hello\"}",
6083 "warning: stderr must not join json",
6084 true,
6085 );
6086
6087 let snapshot = registry
6088 .status(
6089 &task_id,
6090 "session",
6091 None,
6092 Some(dir.path()),
6093 RUNNING_OUTPUT_PREVIEW_BYTES,
6094 )
6095 .unwrap();
6096
6097 assert_eq!(snapshot.output_preview, "{\"body\":\"hello\"}");
6098 assert!(!snapshot.output_preview.contains("warning"));
6099 assert!(!snapshot.output_truncated);
6100 assert_eq!(
6101 calls.load(Ordering::SeqCst),
6102 0,
6103 "structured JSON bypasses compression"
6104 );
6105 }
6106
6107 #[test]
6108 fn registry_emits_single_recovery_marker_for_class_drops() {
6109 let registry = BgTaskRegistry::default();
6110 let dir = tempfile::tempdir().unwrap();
6111 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
6112 let mut dropped = BTreeMap::new();
6113 dropped.insert(DropClass::Error, 18);
6114 dropped.insert(DropClass::Warning, 6);
6115 CompressionResult::with_class_drops("kept diagnostic", dropped)
6116 });
6117 let (task_id, task) =
6118 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
6119
6120 let snapshot = registry
6121 .status(
6122 &task_id,
6123 "session",
6124 None,
6125 Some(dir.path()),
6126 RUNNING_OUTPUT_PREVIEW_BYTES,
6127 )
6128 .unwrap();
6129
6130 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
6131 assert!(snapshot.output_preview.contains("+18 more errors"));
6132 assert!(snapshot.output_preview.contains("+6 more warnings"));
6133 assert!(snapshot
6134 .output_preview
6135 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
6136 assert!(!snapshot.output_preview.contains("tail -n +"));
6137 assert!(snapshot.output_truncated);
6138 }
6139
6140 #[test]
6141 fn registry_marker_reports_semantic_and_byte_drops_once() {
6142 let registry = BgTaskRegistry::default();
6143 let dir = tempfile::tempdir().unwrap();
6144 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
6145 let mut dropped = BTreeMap::new();
6146 dropped.insert(DropClass::Error, 1);
6147 CompressionResult::with_class_drops(
6148 format!("HEAD-SIGNAL\n{}TAIL-SIGNAL", "middle\n".repeat(8_000)),
6149 dropped,
6150 )
6151 });
6152 let (task_id, _task) =
6153 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
6154
6155 let snapshot = registry
6156 .status(
6157 &task_id,
6158 "session",
6159 None,
6160 Some(dir.path()),
6161 RUNNING_OUTPUT_PREVIEW_BYTES,
6162 )
6163 .unwrap();
6164
6165 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
6166 assert!(snapshot.output_preview.contains("+1 more error"));
6167 assert!(snapshot.output_preview.contains("truncated output"));
6168 assert!(snapshot.output_preview.contains("HEAD-SIGNAL"));
6169 assert!(snapshot.output_preview.contains("TAIL-SIGNAL"));
6170 assert!(!snapshot.output_preview.contains("...<truncated"));
6171 assert!(snapshot.output_truncated);
6172 }
6173
6174 #[test]
6175 fn cargo_stderr_class_drops_name_both_capture_paths() {
6176 let registry = BgTaskRegistry::default();
6177 let dir = tempfile::tempdir().unwrap();
6178 let filter_registry = crate::compress::toml_filter::FilterRegistry::default();
6179 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
6180 crate::compress::compress_with_registry_exit_code(
6181 command,
6182 &output,
6183 exit_code,
6184 &filter_registry,
6185 )
6186 });
6187 let stderr = (0..22)
6188 .map(|index| {
6189 format!(
6190 "error: cargo failure {index}\n --> src/lib.rs:{}:1\n |\n{} | boom\n",
6191 index + 1,
6192 index + 1
6193 )
6194 })
6195 .collect::<Vec<_>>()
6196 .join("\n");
6197 let (task_id, task) = insert_terminal_piped_task(
6198 ®istry,
6199 &dir,
6200 "cargo check",
6201 "Finished dev [unoptimized] target(s) in 0.01s\n",
6202 &stderr,
6203 true,
6204 );
6205
6206 let snapshot = registry
6207 .status(
6208 &task_id,
6209 "session",
6210 None,
6211 Some(dir.path()),
6212 RUNNING_OUTPUT_PREVIEW_BYTES,
6213 )
6214 .unwrap();
6215
6216 assert!(snapshot.output_preview.contains("+2 more errors"));
6217 assert!(snapshot
6218 .output_preview
6219 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
6220 assert!(snapshot
6221 .output_preview
6222 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
6223 assert!(!snapshot.output_preview.contains("tail -n +"));
6224 }
6225
6226 #[test]
6227 fn over_ceiling_structured_json_uses_pointer_not_partial_json() {
6228 let registry = BgTaskRegistry::default();
6229 let dir = tempfile::tempdir().unwrap();
6230 let body = format!("{{\"body\":\"{}\"}}", "x".repeat(60 * 1024));
6231 let (task_id, task) = insert_terminal_piped_task(
6232 ®istry,
6233 &dir,
6234 "cd /repo && gh pr view 123 --json body",
6235 &body,
6236 "",
6237 true,
6238 );
6239
6240 let snapshot = registry
6241 .status(
6242 &task_id,
6243 "session",
6244 None,
6245 Some(dir.path()),
6246 RUNNING_OUTPUT_PREVIEW_BYTES,
6247 )
6248 .unwrap();
6249
6250 assert!(snapshot.output_preview.starts_with("[JSON output "));
6251 assert!(snapshot
6252 .output_preview
6253 .contains(&task.paths.stdout.display().to_string()));
6254 assert!(!snapshot.output_preview.contains(&"x".repeat(1024)));
6255 assert!(snapshot.output_truncated);
6256 }
6257
6258 #[test]
6259 fn toml_strip_tail_cap_uses_full_output_hint_not_offset_hint() {
6260 let registry = BgTaskRegistry::default();
6261 let dir = tempfile::tempdir().unwrap();
6262 let filter_registry = crate::compress::toml_filter::build_registry(
6263 crate::compress::builtin_filters::ALL,
6264 None,
6265 None,
6266 );
6267 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
6268 crate::compress::compress_with_registry_exit_code(
6269 command,
6270 &output,
6271 exit_code,
6272 &filter_registry,
6273 )
6274 });
6275 let stdout = format!(
6276 "make[1]: Entering directory `/tmp`\n{}",
6277 (0..100)
6278 .map(|index| format!("compile line {index}"))
6279 .collect::<Vec<_>>()
6280 .join("\n")
6281 );
6282 let (task_id, task) =
6283 insert_terminal_piped_task(®istry, &dir, "make all", &stdout, "", true);
6284
6285 let snapshot = registry
6286 .status(
6287 &task_id,
6288 "session",
6289 None,
6290 Some(dir.path()),
6291 RUNNING_OUTPUT_PREVIEW_BYTES,
6292 )
6293 .unwrap();
6294
6295 assert!(snapshot.output_preview.contains("compile line 99"));
6296 assert!(snapshot.output_preview.contains(&format!(
6297 "full output: read \"{}\"",
6298 task.paths.stdout.display()
6299 )));
6300 assert!(!snapshot
6301 .output_preview
6302 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
6303 assert!(!snapshot.output_preview.contains("tail -n +"));
6304 }
6305
6306 #[test]
6307 fn compressed_false_raw_passthrough_uses_wider_head_tail_cap() {
6308 let registry = BgTaskRegistry::default();
6309 let dir = tempfile::tempdir().unwrap();
6310 let output = format!("RAW-HEAD\n{}RAW-TAIL\n", "raw-middle\n".repeat(8_000));
6311 let (task_id, task) =
6312 insert_terminal_piped_task(®istry, &dir, "cat raw.log", &output, "RAW-ERR\n", false);
6313
6314 let snapshot = registry
6315 .status(
6316 &task_id,
6317 "session",
6318 None,
6319 Some(dir.path()),
6320 RUNNING_OUTPUT_PREVIEW_BYTES,
6321 )
6322 .unwrap();
6323
6324 assert!(snapshot.output_preview.contains("RAW-HEAD"));
6325 assert!(snapshot.output_preview.contains("RAW-TAIL"));
6326 assert!(snapshot.output_preview.contains("truncated output"));
6327 assert!(snapshot
6328 .output_preview
6329 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
6330 assert!(snapshot
6331 .output_preview
6332 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
6333 assert!(!snapshot.output_preview.contains("tail -n +"));
6334 assert!(snapshot.output_preview.len() > 16 * 1024);
6335 assert!(snapshot.output_truncated);
6336 }
6337
6338 #[test]
6339 fn pty_terminal_snapshot_bypasses_line_compression() {
6340 let registry = BgTaskRegistry::default();
6341 let dir = tempfile::tempdir().unwrap();
6342 let calls = Arc::new(AtomicUsize::new(0));
6343 let calls_for_closure = Arc::clone(&calls);
6344 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
6345 calls_for_closure.fetch_add(1, Ordering::SeqCst);
6346 CompressionResult::new(output)
6347 });
6348 let (task_id, _task) = insert_terminal_pty_task(®istry, &dir, "raw\u{1b}[31m pty bytes");
6349
6350 let snapshot = registry
6351 .status(
6352 &task_id,
6353 "session",
6354 None,
6355 Some(dir.path()),
6356 RUNNING_OUTPUT_PREVIEW_BYTES,
6357 )
6358 .unwrap();
6359
6360 assert_eq!(snapshot.info.mode, BgMode::Pty);
6361 assert_eq!(snapshot.output_preview, "");
6362 assert_eq!(calls.load(Ordering::SeqCst), 0);
6363 }
6364
6365 #[test]
6366 fn pty_dimensions_are_persisted_and_returned_in_snapshot() {
6367 let registry = BgTaskRegistry::default();
6368 let dir = tempfile::tempdir().unwrap();
6369 let task_id = registry
6370 .spawn_pty(
6371 SpawnPlan::Unsandboxed,
6372 QUICK_SUCCESS_COMMAND,
6373 "session".to_string(),
6374 dir.path().to_path_buf(),
6375 HashMap::new(),
6376 Some(Duration::from_secs(30)),
6377 dir.path().to_path_buf(),
6378 10,
6379 true,
6380 false,
6381 Some(dir.path().to_path_buf()),
6382 50,
6383 120,
6384 )
6385 .unwrap();
6386
6387 let resolved =
6388 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
6389 let metadata = read_task_at(&resolved).unwrap();
6390 assert_eq!(
6391 metadata.schema_version,
6392 crate::bash_background::persistence::SCHEMA_VERSION
6393 );
6394 assert_eq!(metadata.mode, BgMode::Pty);
6395 assert_eq!(metadata.pty_rows, Some(50));
6396 assert_eq!(metadata.pty_cols, Some(120));
6397
6398 let snapshot = registry
6399 .status(&task_id, "session", None, Some(dir.path()), 1024)
6400 .unwrap();
6401 assert_eq!(snapshot.pty_rows, Some(50));
6402 assert_eq!(snapshot.pty_cols, Some(120));
6403 }
6404
6405 fn spawn_dead_child() -> std::process::Child {
6410 #[cfg(unix)]
6411 let mut cmd = std::process::Command::new("true");
6412 #[cfg(windows)]
6413 let mut cmd = {
6414 let mut c = std::process::Command::new("cmd");
6415 c.args(["/c", "exit", "0"]);
6416 c
6417 };
6418 cmd.stdin(std::process::Stdio::null());
6419 cmd.stdout(std::process::Stdio::null());
6420 cmd.stderr(std::process::Stdio::null());
6421 let mut child = cmd.spawn().expect("spawn replacement child for reap test");
6422 let started = Instant::now();
6431 loop {
6432 match child.try_wait() {
6433 Ok(Some(_)) => break,
6434 Ok(None) => {
6435 if started.elapsed() > Duration::from_secs(5) {
6436 panic!("dead-child stand-in did not exit within 5s");
6437 }
6438 std::thread::sleep(Duration::from_millis(10));
6439 }
6440 Err(error) => panic!("dead-child try_wait failed: {error}"),
6441 }
6442 }
6443 child
6444 }
6445
6446 #[test]
6447 fn ack_marks_delivered_even_when_completion_was_already_consumed_locally() {
6448 let registry = BgTaskRegistry::default();
6449 let dir = tempfile::tempdir().unwrap();
6450 let task_id = registry
6451 .spawn(
6452 SpawnPlan::Unsandboxed,
6453 LONG_RUNNING_COMMAND,
6454 "session".to_string(),
6455 dir.path().to_path_buf(),
6456 HashMap::new(),
6457 Some(Duration::from_secs(30)),
6458 dir.path().to_path_buf(),
6459 10,
6460 true,
6461 false,
6462 Some(dir.path().to_path_buf()),
6463 )
6464 .unwrap();
6465 registry
6466 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6467 .unwrap();
6468 assert_eq!(
6469 registry
6470 .drain_completions_for_session(Some("session"))
6471 .len(),
6472 1
6473 );
6474
6475 registry.inner.completions.lock().unwrap().clear();
6478
6479 assert_eq!(
6480 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
6481 vec![task_id.clone()]
6482 );
6483 assert!(registry
6484 .drain_completions_for_session(Some("session"))
6485 .is_empty());
6486
6487 let resolved =
6488 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
6489 let metadata = read_task_at(&resolved).unwrap();
6490 assert!(metadata.completion_delivered);
6491
6492 let replayed = BgTaskRegistry::default();
6493 replayed
6494 .replay_session_inner(dir.path(), "session", None)
6495 .unwrap();
6496 assert!(replayed
6497 .drain_completions_for_session(Some("session"))
6498 .is_empty());
6499 }
6500
6501 #[test]
6502 fn reclaimed_root_kills_running_task_and_persists_reason() {
6503 let registry = BgTaskRegistry::default();
6504 let root = tempfile::tempdir().unwrap();
6505 let storage = tempfile::tempdir().unwrap();
6506 let task_id = registry
6507 .spawn(
6508 SpawnPlan::Unsandboxed,
6509 LONG_RUNNING_COMMAND,
6510 "session".to_string(),
6511 root.path().to_path_buf(),
6512 HashMap::new(),
6513 Some(Duration::from_secs(30)),
6514 storage.path().to_path_buf(),
6515 10,
6516 true,
6517 false,
6518 Some(root.path().to_path_buf()),
6519 )
6520 .unwrap();
6521 let pid = registry
6522 .status(
6523 &task_id,
6524 "session",
6525 Some(root.path()),
6526 Some(storage.path()),
6527 0,
6528 )
6529 .unwrap()
6530 .child_pid
6531 .unwrap();
6532 assert!(is_process_alive(pid));
6533
6534 assert_eq!(registry.kill_running_tasks_for_root(root.path()), 1);
6535 let deadline = Instant::now() + Duration::from_secs(5);
6536 while is_process_alive(pid) {
6537 assert!(
6538 Instant::now() < deadline,
6539 "reclaimed task process survived kill"
6540 );
6541 std::thread::sleep(Duration::from_millis(20));
6542 }
6543
6544 let snapshot = registry
6545 .status(
6546 &task_id,
6547 "session",
6548 Some(root.path()),
6549 Some(storage.path()),
6550 0,
6551 )
6552 .unwrap();
6553 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
6554 assert_eq!(
6555 snapshot.info.status_reason.as_deref(),
6556 Some(ROOT_RECLAIMED_REASON)
6557 );
6558 let persisted = read_task(
6559 ®istry
6560 .task_json_path(&task_id, "session")
6561 .expect("reclaimed task metadata path"),
6562 )
6563 .expect("persisted reclaimed task");
6564 assert_eq!(
6565 persisted.status_reason.as_deref(),
6566 Some(ROOT_RECLAIMED_REASON)
6567 );
6568 let completion = registry
6569 .drain_completions_for_session(Some("session"))
6570 .pop()
6571 .expect("reclaimed task completion");
6572 assert_eq!(
6573 completion.status_reason.as_deref(),
6574 Some(ROOT_RECLAIMED_REASON)
6575 );
6576 registry.detach();
6577 }
6578
6579 #[test]
6580 fn reclaimed_root_kills_pty_task_and_preserves_reason() {
6581 let registry = BgTaskRegistry::default();
6582 let root = tempfile::tempdir().unwrap();
6583 let storage = tempfile::tempdir().unwrap();
6584 let command = if cfg!(windows) {
6585 "Start-Sleep -Seconds 30"
6586 } else {
6587 "sleep 30"
6588 };
6589 let task_id = registry
6590 .spawn_pty(
6591 SpawnPlan::Unsandboxed,
6592 command,
6593 "session".to_string(),
6594 root.path().to_path_buf(),
6595 HashMap::new(),
6596 Some(Duration::from_secs(60)),
6597 storage.path().to_path_buf(),
6598 10,
6599 true,
6600 false,
6601 Some(root.path().to_path_buf()),
6602 24,
6603 80,
6604 )
6605 .unwrap();
6606 let pid = registry
6607 .status(
6608 &task_id,
6609 "session",
6610 Some(root.path()),
6611 Some(storage.path()),
6612 0,
6613 )
6614 .unwrap()
6615 .child_pid
6616 .unwrap();
6617 assert!(is_process_alive(pid));
6618
6619 assert_eq!(registry.kill_running_tasks_for_root(root.path()), 1);
6620 let deadline = Instant::now() + Duration::from_secs(10);
6621 loop {
6622 let snapshot = registry
6623 .status(
6624 &task_id,
6625 "session",
6626 Some(root.path()),
6627 Some(storage.path()),
6628 0,
6629 )
6630 .unwrap();
6631 if snapshot.info.status.is_terminal() {
6632 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
6633 assert_eq!(
6634 snapshot.info.status_reason.as_deref(),
6635 Some(ROOT_RECLAIMED_REASON)
6636 );
6637 break;
6638 }
6639 assert!(
6640 Instant::now() < deadline,
6641 "reclaimed PTY task did not terminate"
6642 );
6643 std::thread::sleep(Duration::from_millis(20));
6644 }
6645 assert!(!is_process_alive(pid));
6646 let completion = loop {
6651 if let Some(completion) = registry
6652 .drain_completions_for_session(Some("session"))
6653 .pop()
6654 {
6655 break completion;
6656 }
6657 assert!(
6658 Instant::now() < deadline,
6659 "reclaimed PTY completion never arrived"
6660 );
6661 std::thread::sleep(Duration::from_millis(20));
6662 };
6663 assert_eq!(
6664 completion.status_reason.as_deref(),
6665 Some(ROOT_RECLAIMED_REASON)
6666 );
6667 registry.detach();
6668 }
6669
6670 #[test]
6671 fn register_watch_rejects_unknown_task() {
6672 let registry = BgTaskRegistry::default();
6673
6674 let result = registry.register_watch(
6675 "missing-task".to_string(),
6676 WatchPattern::Substring("READY".into()),
6677 true,
6678 );
6679
6680 assert_eq!(result, Err("task_not_found"));
6681 }
6682
6683 #[test]
6684 fn register_watch_on_terminal_task_scans_existing_output() {
6685 let frames = Arc::new(Mutex::new(Vec::new()));
6686 let captured = Arc::clone(&frames);
6687 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
6688 captured.lock().unwrap().push(frame);
6689 })
6690 as Box<dyn Fn(PushFrame) + Send + Sync>);
6691 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
6692 let dir = tempfile::tempdir().unwrap();
6693 let task_id = registry
6694 .spawn(
6695 SpawnPlan::Unsandboxed,
6696 LONG_RUNNING_COMMAND,
6697 "session".to_string(),
6698 dir.path().to_path_buf(),
6699 HashMap::new(),
6700 Some(Duration::from_secs(30)),
6701 dir.path().to_path_buf(),
6702 10,
6703 true,
6704 false,
6705 Some(dir.path().to_path_buf()),
6706 )
6707 .unwrap();
6708 registry
6709 .inner
6710 .shutdown
6711 .store(true, std::sync::atomic::Ordering::SeqCst);
6712 let task = registry.task_for_session(&task_id, "session").unwrap();
6713 std::fs::write(&task.paths.stdout, "READY\n").unwrap();
6714 registry
6715 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6716 .unwrap();
6717 frames.lock().unwrap().clear();
6718 registry.inner.completions.lock().unwrap().clear();
6719
6720 registry
6721 .register_watch(
6722 task_id.clone(),
6723 WatchPattern::Substring("READY".into()),
6724 true,
6725 )
6726 .unwrap();
6727
6728 let frames = frames.lock().unwrap();
6729 let frame = frames
6730 .iter()
6731 .find_map(|frame| match frame {
6732 PushFrame::BashPatternMatch(frame) => Some(frame),
6733 _ => None,
6734 })
6735 .expect("terminal watch registration should emit pattern frame");
6736 assert_eq!(frame.reason, "pattern_match");
6737 assert_eq!(frame.task_id, task_id);
6738 assert_eq!(frame.session_id, "session");
6739 assert_eq!(frame.match_text, "READY");
6740 assert_eq!(frame.match_offset, 0);
6741 assert_eq!(registry.active_watch_count(&frame.task_id), 0);
6742 let metadata = read_task(&task.paths.json).unwrap();
6743 assert!(metadata.completion_delivered);
6744 }
6745
6746 #[test]
6747 fn cleanup_finished_removes_terminal_tasks_older_than_threshold() {
6748 let registry = BgTaskRegistry::default();
6749 let dir = tempfile::tempdir().unwrap();
6750 let task_id = registry
6751 .spawn(
6752 SpawnPlan::Unsandboxed,
6753 QUICK_SUCCESS_COMMAND,
6754 "session".to_string(),
6755 dir.path().to_path_buf(),
6756 HashMap::new(),
6757 Some(Duration::from_secs(30)),
6758 dir.path().to_path_buf(),
6759 10,
6760 true,
6761 false,
6762 Some(dir.path().to_path_buf()),
6763 )
6764 .unwrap();
6765 registry
6766 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6767 .unwrap();
6768 let completions = registry.drain_completions_for_session(Some("session"));
6769 assert_eq!(completions.len(), 1);
6770 assert_eq!(
6771 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
6772 vec![task_id.clone()]
6773 );
6774
6775 registry.cleanup_finished(Duration::ZERO);
6776
6777 assert!(registry.inner.tasks.lock().unwrap().is_empty());
6778 }
6779
6780 #[test]
6781 fn cleanup_finished_retains_undelivered_terminals() {
6782 let registry = BgTaskRegistry::default();
6783 let dir = tempfile::tempdir().unwrap();
6784 let task_id = registry
6785 .spawn(
6786 SpawnPlan::Unsandboxed,
6787 QUICK_SUCCESS_COMMAND,
6788 "session".to_string(),
6789 dir.path().to_path_buf(),
6790 HashMap::new(),
6791 Some(Duration::from_secs(30)),
6792 dir.path().to_path_buf(),
6793 10,
6794 true,
6795 false,
6796 Some(dir.path().to_path_buf()),
6797 )
6798 .unwrap();
6799 registry
6800 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6801 .unwrap();
6802
6803 registry.cleanup_finished(Duration::ZERO);
6804
6805 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
6806 }
6807
6808 #[test]
6816 fn reap_child_marks_failed_when_child_exits_without_exit_marker() {
6817 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6818 let dir = tempfile::tempdir().unwrap();
6819 let task_id = registry
6820 .spawn(
6821 SpawnPlan::Unsandboxed,
6822 QUICK_SUCCESS_COMMAND,
6823 "session".to_string(),
6824 dir.path().to_path_buf(),
6825 HashMap::new(),
6826 Some(Duration::from_secs(30)),
6827 dir.path().to_path_buf(),
6828 10,
6829 true,
6830 false,
6831 Some(dir.path().to_path_buf()),
6832 )
6833 .unwrap();
6834
6835 let task = registry.task_for_session(&task_id, "session").unwrap();
6836
6837 let started = Instant::now();
6842 loop {
6843 let exited = {
6844 let mut state = task.state.lock().unwrap();
6845 match &mut state.runtime {
6846 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
6847 _ => true,
6848 }
6849 };
6850 if exited {
6851 break;
6852 }
6853 assert!(
6854 started.elapsed() < Duration::from_secs(5),
6855 "child should exit quickly"
6856 );
6857 std::thread::sleep(Duration::from_millis(20));
6858 }
6859
6860 registry
6868 .inner
6869 .shutdown
6870 .store(true, std::sync::atomic::Ordering::SeqCst);
6871 std::thread::sleep(Duration::from_millis(550));
6875
6876 let _ = std::fs::remove_file(&task.paths.exit);
6879
6880 {
6895 let mut state = task.state.lock().unwrap();
6896 state.metadata.status = BgTaskStatus::Running;
6897 state.metadata.status_reason = None;
6898 state.metadata.exit_code = None;
6899 state.metadata.finished_at = None;
6900 state.metadata.duration_ms = None;
6901 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
6904 .expect("persist reset Running metadata for reap_child test");
6905 if matches!(state.runtime, TaskRuntime::Piped(None)) {
6909 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
6910 }
6911 }
6912 *task.terminal_at.lock().unwrap() = None;
6915
6916 assert!(
6919 task.is_running(),
6920 "precondition: metadata.status == Running"
6921 );
6922 assert!(
6923 !task.paths.exit.exists(),
6924 "precondition: exit marker absent"
6925 );
6926
6927 registry.reap_child(&task);
6932
6933 {
6934 let state = task.state.lock().unwrap();
6935 assert_eq!(
6936 state.metadata.status,
6937 BgTaskStatus::Running,
6938 "first reap must leave status Running while waiting one pass for marker"
6939 );
6940 assert_eq!(
6941 state.metadata.status_reason, None,
6942 "first reap must not record a failure reason"
6943 );
6944 assert!(
6945 matches!(state.runtime, TaskRuntime::Piped(None)),
6946 "child handle must be released after first reap"
6947 );
6948 assert!(
6949 state.detached,
6950 "task must be marked detached after first reap"
6951 );
6952 }
6953
6954 registry.reap_child(&task);
6958
6959 let state = task.state.lock().unwrap();
6960 assert!(
6961 state.metadata.status.is_terminal(),
6962 "second reap must transition to terminal when PID dead and no marker. Got status={:?}",
6963 state.metadata.status
6964 );
6965 assert_eq!(
6966 state.metadata.status,
6967 BgTaskStatus::Failed,
6968 "must specifically be Failed (not Killed): status={:?}",
6969 state.metadata.status
6970 );
6971 assert_eq!(
6972 state.metadata.status_reason.as_deref(),
6973 Some("process exited without exit marker"),
6974 "reason must match replay path's wording: {:?}",
6975 state.metadata.status_reason
6976 );
6977 assert!(
6978 matches!(state.runtime, TaskRuntime::Piped(None)),
6979 "child handle must stay released after second reap"
6980 );
6981 assert!(
6982 state.detached,
6983 "task must remain detached after second reap"
6984 );
6985 }
6986
6987 #[test]
6992 fn reap_child_preserves_running_when_exit_marker_exists() {
6993 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6994 let dir = tempfile::tempdir().unwrap();
6995 let task_id = registry
6996 .spawn(
6997 SpawnPlan::Unsandboxed,
6998 QUICK_SUCCESS_COMMAND,
6999 "session".to_string(),
7000 dir.path().to_path_buf(),
7001 HashMap::new(),
7002 Some(Duration::from_secs(30)),
7003 dir.path().to_path_buf(),
7004 10,
7005 true,
7006 false,
7007 Some(dir.path().to_path_buf()),
7008 )
7009 .unwrap();
7010
7011 let task = registry.task_for_session(&task_id, "session").unwrap();
7012
7013 let started = Instant::now();
7016 loop {
7017 let exited = {
7018 let mut state = task.state.lock().unwrap();
7019 match &mut state.runtime {
7020 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
7021 _ => true,
7022 }
7023 };
7024 if exited && task.paths.exit.exists() {
7025 break;
7026 }
7027 assert!(
7028 started.elapsed() < Duration::from_secs(5),
7029 "child should exit and write marker quickly"
7030 );
7031 std::thread::sleep(Duration::from_millis(20));
7032 }
7033
7034 registry
7040 .inner
7041 .shutdown
7042 .store(true, std::sync::atomic::Ordering::SeqCst);
7043 std::thread::sleep(Duration::from_millis(550));
7044
7045 {
7051 let mut state = task.state.lock().unwrap();
7052 state.metadata.status = BgTaskStatus::Running;
7053 state.metadata.status_reason = None;
7054 if matches!(state.runtime, TaskRuntime::Piped(None)) {
7055 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
7056 }
7057 }
7058 *task.terminal_at.lock().unwrap() = None;
7059 if !task.paths.exit.exists() {
7062 std::fs::write(&task.paths.exit, "0").expect("write replacement exit marker");
7063 }
7064
7065 registry.reap_child(&task);
7069
7070 let state = task.state.lock().unwrap();
7071 assert!(
7072 matches!(state.runtime, TaskRuntime::Piped(None)),
7073 "child handle still released even when marker exists"
7074 );
7075 assert!(
7076 state.detached,
7077 "task still marked detached even when marker exists"
7078 );
7079 assert_eq!(
7084 state.metadata.status,
7085 BgTaskStatus::Running,
7086 "reap_child must defer to poll_task when marker exists"
7087 );
7088 }
7089
7090 #[cfg(unix)]
7094 fn pid_stat(pid: u32) -> Option<String> {
7095 let output = std::process::Command::new("ps")
7096 .args(["-o", "stat=", "-p", &pid.to_string()])
7097 .output()
7098 .ok()?;
7099 if !output.status.success() {
7100 return None;
7101 }
7102 let stat = String::from_utf8_lossy(&output.stdout).trim().to_string();
7103 if stat.is_empty() {
7104 None
7105 } else {
7106 Some(stat)
7107 }
7108 }
7109
7110 #[cfg(unix)]
7112 fn is_zombie(pid: u32) -> bool {
7113 pid_stat(pid).is_some_and(|stat| stat.starts_with('Z'))
7114 }
7115
7116 #[cfg(unix)]
7122 fn spawn_unreaped_zombie() -> std::process::Child {
7123 let child = std::process::Command::new("true")
7124 .stdin(std::process::Stdio::null())
7125 .stdout(std::process::Stdio::null())
7126 .stderr(std::process::Stdio::null())
7127 .spawn()
7128 .expect("spawn zombie stand-in");
7129 let pid = child.id();
7130 let started = Instant::now();
7131 while !is_zombie(pid) {
7132 assert!(
7133 started.elapsed() < Duration::from_secs(5),
7134 "stand-in child should become a zombie within 5s"
7135 );
7136 std::thread::sleep(Duration::from_millis(10));
7137 }
7138 child
7140 }
7141
7142 #[cfg(unix)]
7152 #[test]
7153 fn finalize_from_marker_reaps_child_no_zombie() {
7154 use std::sync::atomic::Ordering;
7155
7156 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7157 let dir = tempfile::tempdir().unwrap();
7158 let task_id = registry
7159 .spawn(
7160 SpawnPlan::Unsandboxed,
7161 QUICK_SUCCESS_COMMAND,
7162 "session".to_string(),
7163 dir.path().to_path_buf(),
7164 HashMap::new(),
7165 Some(Duration::from_secs(30)),
7166 dir.path().to_path_buf(),
7167 10,
7168 true,
7169 false,
7170 Some(dir.path().to_path_buf()),
7171 )
7172 .unwrap();
7173
7174 registry.inner.shutdown.store(true, Ordering::SeqCst);
7178 std::thread::sleep(Duration::from_millis(550));
7179
7180 let task = registry.task_for_session(&task_id, "session").unwrap();
7181
7182 let started = Instant::now();
7186 while !task.paths.exit.exists() {
7187 assert!(
7188 started.elapsed() < Duration::from_secs(5),
7189 "exit marker should land quickly for `true`"
7190 );
7191 std::thread::sleep(Duration::from_millis(20));
7192 }
7193
7194 let zombie_pid;
7200 {
7201 let mut state = task.state.lock().unwrap();
7202 state.metadata.status = BgTaskStatus::Running;
7203 state.metadata.status_reason = None;
7204 state.metadata.exit_code = None;
7205 state.metadata.finished_at = None;
7206 state.metadata.duration_ms = None;
7207 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
7208 .expect("persist reset Running metadata");
7209 let zombie = spawn_unreaped_zombie();
7210 zombie_pid = zombie.id();
7211 state.runtime = TaskRuntime::Piped(Some(zombie));
7212 }
7213 *task.terminal_at.lock().unwrap() = None;
7214
7215 assert!(
7217 is_zombie(zombie_pid),
7218 "precondition: stand-in child {zombie_pid} must be a zombie before finalize"
7219 );
7220
7221 registry.poll_task(&task).unwrap();
7224
7225 {
7226 let state = task.state.lock().unwrap();
7227 assert!(
7228 matches!(state.runtime, TaskRuntime::Piped(None)),
7229 "child handle must be released after marker finalize"
7230 );
7231 assert!(
7232 state.metadata.status.is_terminal(),
7233 "task must be terminal after marker finalize: {:?}",
7234 state.metadata.status
7235 );
7236 }
7237
7238 assert!(
7241 !is_zombie(zombie_pid),
7242 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
7243 after the exit-marker terminal transition"
7244 );
7245 }
7246
7247 #[cfg(unix)]
7251 #[test]
7252 fn kill_with_existing_marker_reaps_child_no_zombie() {
7253 use std::sync::atomic::Ordering;
7254
7255 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7256 let dir = tempfile::tempdir().unwrap();
7257 let task_id = registry
7258 .spawn(
7259 SpawnPlan::Unsandboxed,
7260 QUICK_SUCCESS_COMMAND,
7261 "session".to_string(),
7262 dir.path().to_path_buf(),
7263 HashMap::new(),
7264 Some(Duration::from_secs(30)),
7265 dir.path().to_path_buf(),
7266 10,
7267 true,
7268 false,
7269 Some(dir.path().to_path_buf()),
7270 )
7271 .unwrap();
7272
7273 registry.inner.shutdown.store(true, Ordering::SeqCst);
7274 std::thread::sleep(Duration::from_millis(550));
7275
7276 let task = registry.task_for_session(&task_id, "session").unwrap();
7277
7278 let started = Instant::now();
7279 while !task.paths.exit.exists() {
7280 assert!(
7281 started.elapsed() < Duration::from_secs(5),
7282 "exit marker should land quickly for `true`"
7283 );
7284 std::thread::sleep(Duration::from_millis(20));
7285 }
7286
7287 let zombie_pid;
7288 {
7289 let mut state = task.state.lock().unwrap();
7290 state.metadata.status = BgTaskStatus::Running;
7291 state.metadata.status_reason = None;
7292 state.metadata.exit_code = None;
7293 state.metadata.finished_at = None;
7294 state.metadata.duration_ms = None;
7295 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
7296 .expect("persist reset Running metadata");
7297 let zombie = spawn_unreaped_zombie();
7298 zombie_pid = zombie.id();
7299 state.runtime = TaskRuntime::Piped(Some(zombie));
7300 }
7301 *task.terminal_at.lock().unwrap() = None;
7302
7303 assert!(
7304 is_zombie(zombie_pid),
7305 "precondition: stand-in child {zombie_pid} must be a zombie before kill"
7306 );
7307
7308 registry
7310 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
7311 .expect("kill should succeed");
7312
7313 {
7314 let state = task.state.lock().unwrap();
7315 assert!(
7316 matches!(state.runtime, TaskRuntime::Piped(None)),
7317 "child handle must be released after marker-aware kill"
7318 );
7319 assert!(state.metadata.status.is_terminal());
7320 }
7321
7322 assert!(
7323 !is_zombie(zombie_pid),
7324 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
7325 after a marker-aware kill"
7326 );
7327 }
7328
7329 #[test]
7330 fn cleanup_finished_keeps_running_tasks() {
7331 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7332 let dir = tempfile::tempdir().unwrap();
7333 let task_id = registry
7334 .spawn(
7335 SpawnPlan::Unsandboxed,
7336 LONG_RUNNING_COMMAND,
7337 "session".to_string(),
7338 dir.path().to_path_buf(),
7339 HashMap::new(),
7340 Some(Duration::from_secs(30)),
7341 dir.path().to_path_buf(),
7342 10,
7343 true,
7344 false,
7345 Some(dir.path().to_path_buf()),
7346 )
7347 .unwrap();
7348
7349 registry.cleanup_finished(Duration::ZERO);
7350
7351 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
7352 let _ = registry.kill(&task_id, "session");
7353 }
7354
7355 #[cfg(unix)]
7356 #[test]
7357 fn rehydrating_sandboxed_task_never_respawns_persisted_command() {
7358 let project = tempfile::tempdir().unwrap();
7359 let storage = tempfile::tempdir().unwrap();
7360 let sandbox_temp = storage.path().join("sandbox-temp");
7361 fs::create_dir(&sandbox_temp).unwrap();
7362 let launcher_script = project.path().join("sandbox-launch");
7363 let launcher = PathBuf::from("/bin/sh");
7364 fs::write(
7365 &launcher_script,
7366 "while [ \"$#\" -gt 0 ]; do\n if [ \"$1\" = -- ]; then\n shift\n exec \"$@\"\n fi\n shift\ndone\nexit 78\n",
7367 )
7368 .unwrap();
7369 let mut permissions = fs::metadata(&launcher_script).unwrap().permissions();
7370 permissions.set_mode(0o700);
7371 fs::set_permissions(&launcher_script, permissions).unwrap();
7372
7373 let profile = crate::sandbox_profile::SandboxProfile::build(
7374 vec![project.path().to_path_buf()],
7375 Vec::new(),
7376 Vec::new(),
7377 Vec::new(),
7378 Vec::new(),
7379 Vec::new(),
7380 Vec::new(),
7381 sandbox_temp,
7382 )
7383 .unwrap();
7384 let plan = SpawnPlan::launcher_for_test(profile, launcher);
7385 let spawn_marker = project.path().join("spawn-count");
7386 let stop_marker = project.path().join("stop-command");
7387 let quote =
7388 |path: &Path| format!("'{}'", path.display().to_string().replace('\'', "'\\''"));
7389 let command = format!(
7390 "printf 'spawn\\n' >> {}; while [ ! -e {} ]; do sleep 0.05; done",
7391 quote(&spawn_marker),
7392 quote(&stop_marker)
7393 );
7394
7395 let original = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7396 let task_id = original
7397 .spawn(
7398 plan,
7399 &command,
7400 "sandbox-rehydrate".to_string(),
7401 project.path().to_path_buf(),
7402 HashMap::new(),
7403 Some(Duration::from_secs(30)),
7404 storage.path().to_path_buf(),
7405 10,
7406 true,
7407 false,
7408 Some(project.path().to_path_buf()),
7409 )
7410 .unwrap();
7411 let started = Instant::now();
7412 while !spawn_marker.exists() {
7413 assert!(
7414 started.elapsed() < Duration::from_secs(20),
7415 "original sandboxed task did not start"
7416 );
7417 std::thread::sleep(Duration::from_millis(10));
7418 }
7419 original.detach();
7420
7421 let restarted = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7422 restarted
7423 .replay_session(storage.path(), "sandbox-rehydrate")
7424 .unwrap();
7425 let replayed = restarted
7426 .status(
7427 &task_id,
7428 "sandbox-rehydrate",
7429 Some(project.path()),
7430 Some(storage.path()),
7431 4096,
7432 )
7433 .expect("rehydrated sandbox task");
7434 assert_eq!(replayed.info.status, BgTaskStatus::Running);
7435 assert!(replayed.sandbox_native);
7436
7437 std::thread::sleep(Duration::from_millis(650));
7438 assert_eq!(
7439 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
7440 1,
7441 "registry replay must observe the persisted process without spawning its command"
7442 );
7443
7444 fs::write(&stop_marker, "stop").unwrap();
7445 let terminal = wait_for_terminal_snapshot(
7446 &restarted,
7447 &task_id,
7448 "sandbox-rehydrate",
7449 project.path(),
7450 storage.path(),
7451 );
7452 assert_eq!(terminal.info.status, BgTaskStatus::Completed);
7453 assert_eq!(
7454 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
7455 1
7456 );
7457 restarted.detach();
7458 }
7459
7460 #[cfg(windows)]
7461 fn wait_for_file(path: &Path) -> String {
7462 let started = Instant::now();
7469 loop {
7470 if let Ok(content) = fs::read_to_string(path) {
7471 if !content.trim().is_empty() {
7472 return content;
7473 }
7474 }
7475 assert!(
7476 started.elapsed() < Duration::from_secs(30),
7477 "timed out waiting for non-empty {}",
7478 path.display()
7479 );
7480 std::thread::sleep(Duration::from_millis(100));
7481 }
7482 }
7483
7484 #[cfg(windows)]
7485 fn spawn_windows_registry_command(
7486 command: &str,
7487 ) -> (BgTaskRegistry, tempfile::TempDir, String) {
7488 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7489 let dir = tempfile::tempdir().unwrap();
7490 let task_id = registry
7491 .spawn(
7492 SpawnPlan::Unsandboxed,
7493 command,
7494 "session".to_string(),
7495 dir.path().to_path_buf(),
7496 HashMap::new(),
7497 Some(Duration::from_secs(30)),
7498 dir.path().to_path_buf(),
7499 10,
7500 false,
7501 false,
7502 Some(dir.path().to_path_buf()),
7503 )
7504 .unwrap();
7505 (registry, dir, task_id)
7506 }
7507
7508 #[cfg(windows)]
7509 #[test]
7510 fn windows_spawn_writes_exit_marker_for_zero_exit() {
7511 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 0");
7512 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
7513
7514 let content = wait_for_file(&exit_path);
7515
7516 assert_eq!(content.trim(), "0");
7517 }
7518
7519 #[cfg(windows)]
7520 #[test]
7521 fn windows_spawn_writes_exit_marker_for_nonzero_exit() {
7522 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 42");
7523 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
7524
7525 let content = wait_for_file(&exit_path);
7526
7527 assert_eq!(content.trim(), "42");
7528 }
7529
7530 #[cfg(windows)]
7531 #[test]
7532 fn windows_spawn_captures_stdout_to_disk() {
7533 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c echo hello");
7534 let task = registry.task_for_session(&task_id, "session").unwrap();
7535 let stdout_path = task.paths.stdout.clone();
7536 let exit_path = task.paths.exit.clone();
7537
7538 let _ = wait_for_file(&exit_path);
7539 let stdout = fs::read_to_string(stdout_path).expect("read stdout");
7540
7541 assert!(stdout.contains("hello"), "stdout was {stdout:?}");
7542 }
7543
7544 #[cfg(windows)]
7545 #[test]
7546 fn windows_spawn_uses_pwsh_when_available() {
7547 let candidates = crate::windows_shell::shell_candidates_with(
7551 |binary| match binary {
7552 "pwsh.exe" => Some(std::path::PathBuf::from(r"C:\pwsh\pwsh.exe")),
7553 "powershell.exe" => Some(std::path::PathBuf::from(r"C:\ps\powershell.exe")),
7554 _ => None,
7555 },
7556 || None,
7557 );
7558 let shell = candidates.first().expect("at least one candidate").clone();
7559 assert_eq!(shell, crate::windows_shell::WindowsShell::Pwsh);
7560 assert_eq!(shell.binary().as_ref(), "pwsh.exe");
7561 }
7562
7563 #[cfg(windows)]
7566 #[test]
7567 fn windows_shell_cmd_wrapper_writes_marker_via_temp_rename() {
7568 let exit_path = Path::new(r"C:\Temp\bash-test.exit");
7569 let script =
7570 crate::windows_shell::WindowsShell::Cmd.wrapper_script("cmd /c exit 42", exit_path);
7571
7572 assert!(
7573 script.contains("set CODE=%ERRORLEVEL%"),
7574 "wrapper must capture the child exit code: {script}"
7575 );
7576 assert!(
7577 script.contains("exit /B %CODE%"),
7578 "wrapper must propagate the child exit code: {script}"
7579 );
7580 assert!(
7585 script.contains("bash-test.exit"),
7586 "wrapper must target the exit marker path: {script}"
7587 );
7588 assert!(
7589 script.contains("move /Y"),
7590 "wrapper must write the marker atomically via temp-file + rename: {script}"
7591 );
7592 }
7593
7594 #[cfg(windows)]
7600 #[test]
7601 fn windows_shell_cmd_bg_command_uses_minimal_cmd_flags() {
7602 use crate::windows_shell::WindowsShell;
7603 let cmd = WindowsShell::Cmd.bg_command("echo wrapped");
7604 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
7605 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
7606 assert_eq!(
7607 args_strs,
7608 vec!["/D", "/S", "/C", "echo wrapped"],
7609 "Cmd::bg_command must prepend /D /S /C"
7610 );
7611 }
7612
7613 #[cfg(windows)]
7616 #[test]
7617 fn windows_shell_pwsh_bg_command_uses_standard_args() {
7618 use crate::windows_shell::WindowsShell;
7619 let cmd = WindowsShell::Pwsh.bg_command("Get-Date");
7620 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
7621 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
7622 assert!(
7623 args_strs.contains(&"-Command"),
7624 "Pwsh::bg_command must use -Command: {args_strs:?}"
7625 );
7626 assert!(
7627 args_strs.contains(&"Get-Date"),
7628 "Pwsh::bg_command must include the user command body"
7629 );
7630 }
7631
7632 fn registry_with_db_and_frames(
7633 storage: &Path,
7634 ) -> (
7635 BgTaskRegistry,
7636 Arc<Mutex<Connection>>,
7637 Arc<Mutex<Vec<PushFrame>>>,
7638 ) {
7639 let frames = Arc::new(Mutex::new(Vec::new()));
7640 let captured = Arc::clone(&frames);
7641 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
7642 captured.lock().unwrap().push(frame);
7643 })
7644 as Box<dyn Fn(PushFrame) + Send + Sync>);
7645 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
7646 registry.set_harness(Harness::Opencode);
7647 let conn = crate::db::open(&storage.join("aft.db")).expect("open test DB");
7648 let shared = Arc::new(Mutex::new(conn));
7649 registry.set_db_pool(shared.clone());
7650 (registry, shared, frames)
7651 }
7652
7653 fn pattern_match_frames(frames: &Mutex<Vec<PushFrame>>) -> Vec<BashPatternMatchFrame> {
7654 frames
7655 .lock()
7656 .unwrap()
7657 .iter()
7658 .filter_map(|frame| match frame {
7659 PushFrame::BashPatternMatch(frame) => Some(frame.clone()),
7660 _ => None,
7661 })
7662 .collect()
7663 }
7664
7665 #[test]
7666 fn pattern_watch_survives_registry_teardown_and_rehydrate() {
7667 let dir = tempfile::tempdir().unwrap();
7668 let storage = dir.path();
7669 let (registry, _db, frames) = registry_with_db_and_frames(storage);
7670 let task_id = registry
7671 .spawn(
7672 SpawnPlan::Unsandboxed,
7673 LONG_RUNNING_COMMAND,
7674 "session".to_string(),
7675 storage.to_path_buf(),
7676 HashMap::new(),
7677 Some(Duration::from_secs(30)),
7678 storage.to_path_buf(),
7679 10,
7680 true,
7681 false,
7682 Some(storage.to_path_buf()),
7683 )
7684 .unwrap();
7685 registry
7686 .register_watch(
7687 task_id.clone(),
7688 WatchPattern::Substring("READY".into()),
7689 true,
7690 )
7691 .unwrap();
7692 let task = registry.task_for_session(&task_id, "session").unwrap();
7693 registry.clear_task_watch_state(&task_id);
7695 assert_eq!(registry.active_watch_count(&task_id), 0);
7696
7697 std::fs::OpenOptions::new()
7698 .append(true)
7699 .open(&task.paths.stdout)
7700 .unwrap()
7701 .write_all(b"READY\n")
7702 .unwrap();
7703 frames.lock().unwrap().clear();
7704
7705 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
7706 registry
7708 .inner
7709 .shutdown
7710 .store(true, std::sync::atomic::Ordering::SeqCst);
7711 replayed
7712 .replay_session_inner(storage, "session", None)
7713 .unwrap();
7714
7715 let matches = pattern_match_frames(&replay_frames);
7716 assert!(
7717 matches.iter().any(|frame| {
7718 frame.task_id == task_id
7719 && frame.reason == "pattern_match"
7720 && frame.match_text == "READY"
7721 }),
7722 "rehydrate should deliver gap match: {matches:?}"
7723 );
7724 }
7725
7726 #[test]
7727 fn pattern_watch_gap_match_between_teardown_and_rehydrate_delivers_once() {
7728 let dir = tempfile::tempdir().unwrap();
7729 let storage = dir.path();
7730 let (registry, _db, frames) = registry_with_db_and_frames(storage);
7731 let task_id = registry
7732 .spawn(
7733 SpawnPlan::Unsandboxed,
7734 LONG_RUNNING_COMMAND,
7735 "session".to_string(),
7736 storage.to_path_buf(),
7737 HashMap::new(),
7738 Some(Duration::from_secs(30)),
7739 storage.to_path_buf(),
7740 10,
7741 true,
7742 false,
7743 Some(storage.to_path_buf()),
7744 )
7745 .unwrap();
7746 registry
7747 .register_watch(
7748 task_id.clone(),
7749 WatchPattern::Substring("GAP-HIT".into()),
7750 true,
7751 )
7752 .unwrap();
7753 let task = registry.task_for_session(&task_id, "session").unwrap();
7754 let cursor_before = registry.watch_stream_cursors(&task_id).0;
7755 registry.clear_task_watch_state(&task_id);
7756
7757 std::fs::OpenOptions::new()
7759 .append(true)
7760 .open(&task.paths.stdout)
7761 .unwrap()
7762 .write_all(b"prefix GAP-HIT suffix\n")
7763 .unwrap();
7764 frames.lock().unwrap().clear();
7765
7766 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
7767 registry
7768 .inner
7769 .shutdown
7770 .store(true, std::sync::atomic::Ordering::SeqCst);
7771 replayed
7772 .replay_session_inner(storage, "session", None)
7773 .unwrap();
7774
7775 let matches: Vec<_> = pattern_match_frames(&replay_frames)
7776 .into_iter()
7777 .filter(|frame| frame.task_id == task_id && frame.match_text.contains("GAP-HIT"))
7778 .collect();
7779 assert_eq!(
7780 matches.len(),
7781 1,
7782 "gap match must deliver exactly once: {matches:?}"
7783 );
7784 assert!(
7785 matches[0].match_offset >= cursor_before,
7786 "match offset should be at/after the persisted cursor ({cursor_before}), got {}",
7787 matches[0].match_offset
7788 );
7789 }
7790
7791 #[test]
7792 fn pattern_watch_acked_match_does_not_redeliver_after_restart() {
7793 let dir = tempfile::tempdir().unwrap();
7794 let storage = dir.path();
7795 let (registry, db, frames) = registry_with_db_and_frames(storage);
7796 let task_id = registry
7797 .spawn(
7798 SpawnPlan::Unsandboxed,
7799 LONG_RUNNING_COMMAND,
7800 "session".to_string(),
7801 storage.to_path_buf(),
7802 HashMap::new(),
7803 Some(Duration::from_secs(30)),
7804 storage.to_path_buf(),
7805 10,
7806 true,
7807 false,
7808 Some(storage.to_path_buf()),
7809 )
7810 .unwrap();
7811 registry
7812 .register_watch(
7813 task_id.clone(),
7814 WatchPattern::Substring("READY".into()),
7815 true,
7816 )
7817 .unwrap();
7818 let task = registry.task_for_session(&task_id, "session").unwrap();
7819 std::fs::OpenOptions::new()
7820 .append(true)
7821 .open(&task.paths.stdout)
7822 .unwrap()
7823 .write_all(b"READY\n")
7824 .unwrap();
7825 registry.scan_task_watch_output(&task);
7826 let delivered = pattern_match_frames(&frames);
7827 assert!(
7828 delivered
7829 .iter()
7830 .any(|frame| frame.task_id == task_id && frame.match_text == "READY"),
7831 "live path should deliver match: {delivered:?}"
7832 );
7833 assert!(registry
7835 .ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id))
7836 .contains(&task_id));
7837 {
7838 let conn = db.lock().unwrap();
7839 let rows = crate::db::bash_watches::list_bash_pattern_watches_for_task(
7840 &conn, "opencode", "session", &task_id,
7841 )
7842 .unwrap();
7843 assert!(
7844 rows.is_empty(),
7845 "acked once-watch rows must be deleted: {rows:?}"
7846 );
7847 }
7848
7849 frames.lock().unwrap().clear();
7850 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
7851 registry
7852 .inner
7853 .shutdown
7854 .store(true, std::sync::atomic::Ordering::SeqCst);
7855 replayed
7856 .replay_session_inner(storage, "session", None)
7857 .unwrap();
7858 let matches = pattern_match_frames(&replay_frames)
7859 .into_iter()
7860 .filter(|frame| frame.task_id == task_id)
7861 .collect::<Vec<_>>();
7862 assert!(
7863 matches.is_empty(),
7864 "acked match must not re-deliver after restart: {matches:?}"
7865 );
7866 }
7867
7868 #[test]
7869 fn pattern_watch_rows_are_removed_when_task_is_gc_deleted() {
7870 let dir = tempfile::tempdir().unwrap();
7871 let storage = dir.path();
7872 let (registry, db, _frames) = registry_with_db_and_frames(storage);
7873 let task_id = "bash-aaaaaaaaaaaaaaaa";
7874 let paths = task_paths(storage, "session", task_id).unwrap();
7875 let mut metadata = PersistedTask::starting(
7876 task_id.to_string(),
7877 "session".to_string(),
7878 "true".to_string(),
7879 storage.to_path_buf(),
7880 Some(storage.to_path_buf()),
7881 None,
7882 true,
7883 true,
7884 );
7885 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
7886 metadata.completion_delivered = true;
7887 write_task(&paths.json, &metadata).unwrap();
7888 {
7889 let conn = db.lock().unwrap();
7890 crate::db::bash_tasks::upsert_bash_task(
7891 &conn,
7892 &metadata.to_bash_task_row("opencode", &paths).unwrap(),
7893 )
7894 .unwrap();
7895 crate::db::bash_watches::upsert_bash_pattern_watch(
7896 &conn,
7897 &BashPatternWatchRow {
7898 harness: "opencode".into(),
7899 session_id: "session".into(),
7900 task_id: task_id.into(),
7901 watch_id: "watch-00000001".into(),
7902 pattern_kind: "substring".into(),
7903 pattern: "x".into(),
7904 once: true,
7905 created_at: 1,
7906 stdout_offset: 0,
7907 stderr_offset: 0,
7908 pty_offset: 0,
7909 scanning: true,
7910 pending_match: false,
7911 match_text: None,
7912 match_offset: None,
7913 match_context: None,
7914 },
7915 )
7916 .unwrap();
7917 }
7918 let old = SystemTime::now()
7919 .checked_sub(Duration::from_secs(25 * 60 * 60))
7920 .unwrap();
7921 filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
7922
7923 let deleted = registry.maybe_gc_persisted(storage).unwrap();
7924 assert!(
7925 deleted >= 1,
7926 "expected GC to delete the terminal task bundle"
7927 );
7928 let conn = db.lock().unwrap();
7929 let watches = crate::db::bash_watches::list_bash_pattern_watches_for_task(
7930 &conn, "opencode", "session", task_id,
7931 )
7932 .unwrap();
7933 assert!(
7934 watches.is_empty(),
7935 "task GC must remove watch rows: {watches:?}"
7936 );
7937 }
7938}