1use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
2#[cfg(not(windows))]
3use std::ffi::OsString;
4use std::fs;
5use std::io::Write;
6use std::path::{Path, PathBuf};
7#[cfg(windows)]
8use std::process::Command;
9use std::process::{Child, Stdio};
10use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
11#[cfg(unix)]
12use std::sync::OnceLock;
13use std::sync::{Arc, Mutex, RwLock};
14use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
15
16use rusqlite::Connection;
17use serde::Serialize;
18
19use crate::bash_permissions::PermissionAsk;
20use crate::compress::caps::DropClass;
21#[cfg(unix)]
22use crate::compress::single_top_level_pipeline;
23use crate::compress::CompressionResult;
24use crate::context::SharedProgressSender;
25use crate::db::compression_events::CompressionAggregateCache;
26use crate::harness::Harness;
27use crate::protocol::{BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, PushFrame};
28use crate::sandbox_spawn::SpawnPlan;
29
30#[cfg(windows)]
31use std::os::windows::process::CommandExt;
32
33use super::buffer::{combine_streams, BgBuffer, DiskTruncation, StreamKind, TokenCountInput};
34use super::output::{
35 cap_completion_output, cap_completion_output_with_marker, cap_final_output,
36 cap_final_output_with_marker, completion_preview_threshold, json_output_pointer, quote_path,
37 retained_json_output_pointer, COMPRESS_INPUT_CAP_BYTES, COMPRESS_INPUT_HEAD_BYTES,
38 COMPRESS_INPUT_TAIL_BYTES, FINAL_OUTPUT_CAP_BYTES, RAW_PASSTHROUGH_CAP_BYTES,
39 RAW_PASSTHROUGH_HEAD_BYTES, RAW_PASSTHROUGH_TAIL_BYTES, RUNNING_OUTPUT_PREVIEW_BYTES,
40 STRUCTURED_OUTPUT_CAP_BYTES,
41};
42use super::persistence::{
43 allocate_task_layout, delete_resolved_task, delete_task_bundle, discover_task_ids,
44 open_task_artifact, quarantine_invalid_entry, quarantine_task_layout, read_exit_marker,
45 read_task_at, resolve_task_layout, session_tasks_dir, uninitialized_layout_is_recent,
46 unix_millis, update_task_at, validate_task_id, write_kill_marker_if_absent, write_task_at,
47 BgMode, ExitMarker, PersistedTask, TaskArtifact, TaskIoHandles, TaskPaths,
48};
49use super::process::is_process_alive;
50#[cfg(unix)]
51use super::process::terminate_pgid;
52#[cfg(windows)]
53use super::process::terminate_pid;
54use super::pty_process::spawn_pty_for_command;
55use super::pty_runtime::PtyRuntime;
56use super::watches::{PatternMatch, WatchPattern, WatchRegistry};
57use super::{BgTaskInfo, BgTaskStatus};
58use crate::db::bash_watches::BashPatternWatchRow;
59const DEFAULT_BG_TIMEOUT: Duration = Duration::from_secs(30 * 60);
62const STALE_RUNNING_AFTER: Duration = Duration::from_secs(24 * 60 * 60);
63const PERSISTED_GC_GRACE: Duration = Duration::from_secs(24 * 60 * 60);
64const QUARANTINE_GC_GRACE: Duration = Duration::from_secs(30 * 24 * 60 * 60);
65
66const TOKENIZE_CAP_BYTES_PER_STREAM: usize = 128 * 1024;
67pub const ROOT_RECLAIMED_REASON: &str = "root_reclaimed";
68
69#[derive(Debug, Clone, Serialize)]
70pub struct BgCompletion {
71 pub task_id: String,
72 #[serde(skip_serializing)]
75 pub session_id: String,
76 pub status: BgTaskStatus,
77 pub exit_code: Option<i32>,
78 pub command: String,
79 #[serde(default, skip_serializing_if = "String::is_empty")]
85 pub output_preview: String,
86 #[serde(default, skip_serializing_if = "is_false")]
91 pub output_truncated: bool,
92 #[serde(default, skip_serializing_if = "Option::is_none")]
95 pub original_tokens: Option<u32>,
96 #[serde(default, skip_serializing_if = "Option::is_none")]
99 pub compressed_tokens: Option<u32>,
100 #[serde(default, skip_serializing_if = "is_false")]
102 pub tokens_skipped: bool,
103 #[serde(default, skip_serializing_if = "Option::is_none")]
104 pub status_reason: Option<String>,
105}
106
107fn is_false(v: &bool) -> bool {
108 !*v
109}
110
111#[derive(Debug, Clone, Serialize)]
112pub struct BgTaskSnapshot {
113 #[serde(flatten)]
114 pub info: BgTaskInfo,
115 pub exit_code: Option<i32>,
116 pub child_pid: Option<u32>,
117 pub workdir: String,
118 pub output_preview: String,
119 pub output_truncated: bool,
120 pub output_path: Option<String>,
121 pub stderr_path: Option<String>,
122 #[serde(skip_serializing_if = "Option::is_none")]
123 pub pty_rows: Option<u16>,
124 #[serde(skip_serializing_if = "Option::is_none")]
125 pub pty_cols: Option<u16>,
126 #[serde(skip_serializing_if = "Option::is_none")]
127 pub pty_screen: Option<String>,
128 #[serde(default, skip_serializing_if = "Vec::is_empty")]
129 pub scanner_report: Vec<PermissionAsk>,
130 #[serde(default, skip_serializing_if = "is_false")]
131 pub sandbox_native: bool,
132 #[serde(default, skip_serializing_if = "is_false")]
133 pub sandbox_unavailable: bool,
134}
135
136#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
137pub struct BgTaskHealthCounts {
138 pub running: usize,
139 pub pending_completions: usize,
140}
141
142#[derive(Debug, Clone, Copy, PartialEq, Eq)]
143enum TerminalOutputKind {
144 Compressed,
145 Raw,
146 Structured,
147}
148
149#[derive(Debug, Clone, PartialEq, Eq)]
150struct TerminalOutputCache {
151 output_preview: String,
152 output_truncated: bool,
153 kind: TerminalOutputKind,
154 output_path: Option<String>,
155 stderr_path: Option<String>,
156 artifact_access: ArtifactRecoveryAccess,
157 recovery: Option<RecoveryContext>,
158}
159
160#[derive(Debug, Clone, PartialEq, Eq)]
161struct ArtifactRecoveryAccess {
162 task_id: String,
163 readable: bool,
164}
165
166#[derive(Debug, Clone, PartialEq, Eq)]
167struct RecoveryContext {
168 dropped_by_class: BTreeMap<DropClass, usize>,
169 had_inner_drop: bool,
170 offset_hint_eligible: bool,
171 offset_start_line: Option<usize>,
172 byte_truncated: bool,
173 disk_truncated_prefix_bytes: u64,
174 output_path: Option<String>,
175 stderr_path: Option<String>,
176 include_stderr_path: bool,
177 artifact_access: ArtifactRecoveryAccess,
178}
179
180fn optional_string_bytes(value: Option<&String>) -> u64 {
181 value
182 .map(|value| crate::memory::usize_to_u64(value.len()))
183 .unwrap_or(0)
184}
185
186fn terminal_output_cache_estimated_bytes(cache: &TerminalOutputCache) -> u64 {
187 let recovery_bytes = cache
188 .recovery
189 .as_ref()
190 .map(|recovery| {
191 crate::memory::usize_to_u64(recovery.dropped_by_class.len())
192 .saturating_mul(
193 (std::mem::size_of::<DropClass>() + std::mem::size_of::<usize>()) as u64,
194 )
195 .saturating_add(optional_string_bytes(recovery.output_path.as_ref()))
196 .saturating_add(optional_string_bytes(recovery.stderr_path.as_ref()))
197 .saturating_add(crate::memory::usize_to_u64(
198 recovery.artifact_access.task_id.len(),
199 ))
200 })
201 .unwrap_or(0);
202 (std::mem::size_of::<TerminalOutputCache>() as u64)
203 .saturating_add(crate::memory::usize_to_u64(cache.output_preview.len()))
204 .saturating_add(optional_string_bytes(cache.output_path.as_ref()))
205 .saturating_add(optional_string_bytes(cache.stderr_path.as_ref()))
206 .saturating_add(crate::memory::usize_to_u64(
207 cache.artifact_access.task_id.len(),
208 ))
209 .saturating_add(recovery_bytes)
210}
211
212fn completion_estimated_bytes(completion: &BgCompletion) -> u64 {
213 (std::mem::size_of::<BgCompletion>() as u64)
214 .saturating_add(crate::memory::usize_to_u64(completion.task_id.len()))
215 .saturating_add(crate::memory::usize_to_u64(completion.session_id.len()))
216 .saturating_add(crate::memory::usize_to_u64(completion.command.len()))
217 .saturating_add(crate::memory::usize_to_u64(completion.output_preview.len()))
218}
219
220impl RecoveryContext {
221 fn has_visible_drop(&self) -> bool {
222 self.byte_truncated
223 || self.disk_truncated_prefix_bytes > 0
224 || self.had_inner_drop
225 || !self.dropped_by_class.is_empty()
226 }
227}
228
229#[derive(Clone)]
230pub struct BgTaskRegistry {
231 pub(crate) inner: Arc<RegistryInner>,
232}
233
234pub(crate) struct RegistryInner {
235 pub(crate) tasks: Mutex<HashMap<String, Arc<BgTask>>>,
236 pub(crate) completions: Mutex<VecDeque<BgCompletion>>,
237 pub(crate) progress_sender: SharedProgressSender,
238 watchdog_started: AtomicBool,
239 pub(crate) shutdown: AtomicBool,
240 pub(crate) long_running_reminder_enabled: AtomicBool,
241 pub(crate) long_running_reminder_interval_ms: AtomicU64,
242 persisted_gc_started: AtomicBool,
243 #[cfg(test)]
244 persisted_gc_runs: AtomicU64,
245 pub(crate) compressor:
251 Mutex<Option<Box<dyn Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync>>>,
252 pub(crate) db_pool: RwLock<Option<Arc<Mutex<Connection>>>>,
253 pub(crate) db_harness: RwLock<Option<String>>,
254 pub(crate) compression_aggregates: Arc<CompressionAggregateCache>,
255 pub(crate) wake_tx: crossbeam_channel::Sender<()>,
256 pub(crate) wake_rx: crossbeam_channel::Receiver<()>,
257 pub(crate) watch_registry: Mutex<WatchRegistry>,
258 wait_detach_sessions: Mutex<HashSet<String>>,
259 active_wait_sessions: Mutex<HashMap<String, usize>>,
260 wait_registered_tasks: Mutex<HashMap<String, HashSet<String>>>,
261}
262
263pub(crate) struct BgTask {
264 pub(crate) task_id: String,
265 pub(crate) session_id: String,
266 pub(crate) paths: TaskPaths,
267 artifact_root: PathBuf,
268 pub(crate) started: Instant,
269 pub(crate) last_reminder_at: Mutex<Option<Instant>>,
270 pub(crate) terminal_at: Mutex<Option<Instant>>,
271 pub(crate) state: Mutex<BgTaskState>,
272}
273
274pub(crate) enum TaskRuntime {
275 Piped(Option<Child>),
276 Pty(Option<PtyRuntime>),
277}
278
279pub(crate) struct BgTaskState {
280 pub(crate) metadata: PersistedTask,
281 pub(crate) runtime: TaskRuntime,
282 pub(crate) io_handles: Option<TaskIoHandles>,
285 pub(crate) detached: bool,
286 pub(crate) child_exit_observed: bool,
297 pub(crate) buffer: BgBuffer,
298 terminal_output_cache: Option<TerminalOutputCache>,
299 pub(crate) pending_terminal_override: Option<BgTaskStatus>,
301}
302
303fn completion_matches_session(completion: &BgCompletion, session_id: Option<&str>) -> bool {
304 session_id
305 .map(|session_id| completion.session_id == session_id)
306 .unwrap_or(true)
307}
308
309impl BgTaskRegistry {
310 pub fn new(progress_sender: SharedProgressSender) -> Self {
311 let (wake_tx, wake_rx) = crossbeam_channel::bounded(1);
312 Self {
313 inner: Arc::new(RegistryInner {
314 tasks: Mutex::new(HashMap::new()),
315 completions: Mutex::new(VecDeque::new()),
316 progress_sender,
317 watchdog_started: AtomicBool::new(false),
318 shutdown: AtomicBool::new(false),
319 long_running_reminder_enabled: AtomicBool::new(true),
320 long_running_reminder_interval_ms: AtomicU64::new(600_000),
321 persisted_gc_started: AtomicBool::new(false),
322 #[cfg(test)]
323 persisted_gc_runs: AtomicU64::new(0),
324 compressor: Mutex::new(None),
325 db_pool: RwLock::new(None),
326 db_harness: RwLock::new(None),
327 compression_aggregates: Arc::new(CompressionAggregateCache::default()),
328 wake_tx,
329 wake_rx,
330 watch_registry: Mutex::new(WatchRegistry::default()),
331 wait_detach_sessions: Mutex::new(HashSet::new()),
332 active_wait_sessions: Mutex::new(HashMap::new()),
333 wait_registered_tasks: Mutex::new(HashMap::new()),
334 }),
335 }
336 }
337
338 pub fn is_session_owned_artifact_path(&self, session_id: &str, path: &Path) -> bool {
346 let Ok(requested) = fs::canonicalize(path) else {
347 return false;
348 };
349 let Ok(tasks) = self.inner.tasks.lock() else {
350 return false;
351 };
352
353 tasks.values().any(|task| {
354 task.session_id == session_id
355 && [
356 &task.paths.stdout,
357 &task.paths.stderr,
358 &task.paths.exit,
359 &task.paths.pty,
360 ]
361 .into_iter()
362 .filter_map(|known| known.file_name())
363 .any(|name| task.artifact_root.join(name) == requested)
364 })
365 }
366
367 pub fn read_artifact_path(
368 &self,
369 session_id: &str,
370 path: &Path,
371 ) -> Option<Result<Vec<u8>, String>> {
372 let requested = fs::canonicalize(path).ok()?;
373 let tasks = self.inner.tasks.lock().ok()?;
374 let (task, artifact) = tasks.values().find_map(|task| {
375 if task.session_id != session_id {
376 return None;
377 }
378 TaskArtifact::ALL.into_iter().find_map(|artifact| {
379 let expected = task
380 .paths
381 .artifact_path(artifact)
382 .file_name()
383 .map(|name| task.artifact_root.join(name));
384 (expected.as_deref() == Some(requested.as_path()))
385 .then(|| (Arc::clone(task), artifact))
386 })
387 })?;
388 drop(tasks);
389 Some(self.read_artifact(&task.task_id, session_id, artifact))
390 }
391
392 pub fn read_artifact_range(
393 &self,
394 task_id: &str,
395 session_id: &str,
396 artifact: TaskArtifact,
397 offset: u64,
398 ) -> Result<(Vec<u8>, u64), String> {
399 validate_task_id(task_id).map_err(|error| error.to_string())?;
400 let task = self
401 .task_for_session(task_id, session_id)
402 .ok_or_else(|| "task_not_found".to_string())?;
403 let mut file = open_task_artifact(&task.paths, artifact)
404 .map_err(|error| format!("artifact_refused: {error}"))?;
405 let len = file
406 .len()
407 .map_err(|error| format!("artifact_refused: {error}"))?;
408 let start = offset.min(len);
409 let bytes = file
410 .read_range(start, len.saturating_sub(start))
411 .map_err(|error| format!("artifact_refused: {error}"))?;
412 Ok((bytes, len))
413 }
414
415 pub fn read_artifact(
416 &self,
417 task_id: &str,
418 session_id: &str,
419 artifact: TaskArtifact,
420 ) -> Result<Vec<u8>, String> {
421 validate_task_id(task_id).map_err(|error| error.to_string())?;
422 let task = self
423 .task_for_session(task_id, session_id)
424 .ok_or_else(|| "task_not_found".to_string())?;
425 let mut file = open_task_artifact(&task.paths, artifact)
426 .map_err(|error| format!("artifact_refused: {error}"))?;
427 file.read_all()
428 .map_err(|error| format!("artifact_refused: {error}"))
429 }
430
431 pub fn set_harness(&self, harness: Harness) {
432 if let Ok(mut slot) = self.inner.db_harness.write() {
433 *slot = Some(harness.storage_segment());
434 }
435 }
436
437 pub fn set_db_pool(&self, conn: Arc<Mutex<Connection>>) {
438 if let Ok(mut slot) = self.inner.db_pool.write() {
439 *slot = Some(conn);
440 }
441 self.inner.compression_aggregates.clear();
442 }
443
444 pub fn clear_db_pool(&self) {
445 if let Ok(mut slot) = self.inner.db_pool.write() {
446 *slot = None;
447 }
448 self.inner.compression_aggregates.clear();
449 }
450
451 pub(crate) fn compression_aggregate_cache(&self) -> Arc<CompressionAggregateCache> {
452 Arc::clone(&self.inner.compression_aggregates)
453 }
454
455 pub fn register_foreground_task(&self, session_id: &str, task_id: &str) {
456 if let Ok(mut tasks) = self.inner.wait_registered_tasks.lock() {
457 tasks
458 .entry(session_id.to_string())
459 .or_default()
460 .insert(task_id.to_string());
461 }
462 }
463
464 pub fn begin_wait_mode_session(&self, session_id: &str, task_id: &str) {
465 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
466 *active.entry(session_id.to_string()).or_insert(0) += 1;
467 }
468 self.register_foreground_task(session_id, task_id);
469 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
470 detach.remove(session_id);
471 }
472 }
473
474 pub fn unregister_foreground_task(&self, session_id: &str, task_id: &str) {
475 if let Ok(mut tasks) = self.inner.wait_registered_tasks.lock() {
476 if let Some(session_tasks) = tasks.get_mut(session_id) {
477 session_tasks.remove(task_id);
478 if session_tasks.is_empty() {
479 tasks.remove(session_id);
480 }
481 }
482 }
483 }
484
485 pub fn end_wait_mode_session(&self, session_id: &str, task_id: &str) {
486 let no_active_wait = if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
487 match active.get_mut(session_id) {
488 Some(count) if *count > 1 => *count -= 1,
489 Some(_) => {
490 active.remove(session_id);
491 }
492 None => {}
493 }
494 !active.contains_key(session_id)
495 } else {
496 false
497 };
498 self.unregister_foreground_task(session_id, task_id);
499 if no_active_wait {
500 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
501 detach.remove(session_id);
502 }
503 }
504 }
505
506 pub fn abort_inflight(&self, session_id: &str) -> Result<usize, String> {
510 let task_ids = self
511 .inner
512 .wait_registered_tasks
513 .lock()
514 .map(|mut tasks| tasks.remove(session_id).unwrap_or_default())
515 .map_err(|_| "wait registration lock poisoned".to_string())?;
516 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
517 active.remove(session_id);
518 }
519 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
520 detach.remove(session_id);
521 }
522
523 let mut killed = 0;
524 for task_id in task_ids {
525 let Some(task) = self.task_for_session(&task_id, session_id) else {
526 continue;
527 };
528 let is_terminal = task
529 .state
530 .lock()
531 .map(|state| state.metadata.status.is_terminal())
532 .map_err(|_| "background task lock poisoned".to_string())?;
533 if is_terminal {
534 continue;
535 }
536 let snapshot = self.kill_with_status_reason(
537 &task_id,
538 session_id,
539 BgTaskStatus::Killed,
540 Some("call_aborted".to_string()),
541 )?;
542 if snapshot.info.status == BgTaskStatus::Killed
543 && snapshot.info.status_reason.as_deref() == Some("call_aborted")
544 {
545 killed += 1;
546 }
547 }
548 Ok(killed)
549 }
550
551 pub fn signal_wait_mode_detach(&self, session_id: &str) -> bool {
552 let is_waiting = self
553 .inner
554 .active_wait_sessions
555 .lock()
556 .map(|active| active.get(session_id).copied().unwrap_or(0) > 0)
557 .unwrap_or(false);
558 if !is_waiting {
559 return false;
560 }
561 self.inner
562 .wait_detach_sessions
563 .lock()
564 .map(|mut detach| detach.insert(session_id.to_string()))
565 .unwrap_or(false)
566 }
567
568 pub fn active_wait_session_count(&self) -> usize {
571 self.inner
572 .active_wait_sessions
573 .lock()
574 .map(|active| active.len())
575 .unwrap_or(0)
576 }
577
578 pub fn take_wait_mode_detach(&self, session_id: &str) -> bool {
579 self.inner
580 .wait_detach_sessions
581 .lock()
582 .map(|mut detach| detach.remove(session_id))
583 .unwrap_or(false)
584 }
585
586 pub fn set_compressor<F>(&self, compressor: F)
591 where
592 F: Fn(&str, String) -> CompressionResult + Send + Sync + 'static,
593 {
594 self.set_compressor_with_exit_code(move |command, output, _exit_code| {
595 compressor(command, output)
596 });
597 }
598
599 pub fn set_compressor_with_exit_code<F>(&self, compressor: F)
600 where
601 F: Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync + 'static,
602 {
603 if let Ok(mut slot) = self.inner.compressor.lock() {
604 *slot = Some(Box::new(compressor));
605 }
606 }
607
608 pub(crate) fn compress_output(
611 &self,
612 command: &str,
613 output: String,
614 exit_code: Option<i32>,
615 ) -> CompressionResult {
616 let Ok(slot) = self.inner.compressor.lock() else {
617 return CompressionResult::new(output);
618 };
619 match slot.as_ref() {
620 Some(compressor) => compressor(command, output, exit_code),
621 None => CompressionResult::new(output),
622 }
623 }
624
625 fn ensure_terminal_output_cache(&self, task: &Arc<BgTask>) -> Option<TerminalOutputCache> {
626 let (metadata, buffer) = {
627 let state = task.state.lock().ok()?;
628 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
629 return None;
630 }
631 if let Some(cache) = state.terminal_output_cache.clone() {
632 return Some(cache);
633 }
634 (state.metadata.clone(), state.buffer.clone())
635 };
636
637 let mut cap_buffer = buffer.clone();
638 let disk_truncation = cap_buffer.enforce_terminal_cap();
639 let cache =
640 self.render_terminal_output(&metadata, &cap_buffer, disk_truncation, Some(&task.paths));
641 let mut state = task.state.lock().ok()?;
642 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
643 return None;
644 }
645 if let Some(existing) = state.terminal_output_cache.clone() {
646 return Some(existing);
647 }
648 state.terminal_output_cache = Some(cache.clone());
649 Some(cache)
650 }
651
652 fn render_terminal_output(
653 &self,
654 metadata: &PersistedTask,
655 buffer: &BgBuffer,
656 disk_truncation: DiskTruncation,
657 paths: Option<&TaskPaths>,
658 ) -> TerminalOutputCache {
659 let output_readable = buffer
660 .output_path()
661 .is_some_and(|path| self.is_session_owned_artifact_path(&metadata.session_id, &path));
662 let stderr_readable = buffer
663 .stderr_path()
664 .map(|path| self.is_session_owned_artifact_path(&metadata.session_id, path))
665 .unwrap_or(true);
666 let artifact_access = ArtifactRecoveryAccess {
667 task_id: metadata.task_id.clone(),
668 readable: output_readable && stderr_readable,
669 };
670
671 if metadata.mode == BgMode::Pty {
672 return TerminalOutputCache {
673 output_preview: String::new(),
674 output_truncated: false,
675 kind: TerminalOutputKind::Raw,
676 output_path: buffer.output_path().map(|path| path.display().to_string()),
677 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
678 artifact_access,
679 recovery: None,
680 };
681 }
682
683 if let Some(mut structured) = render_structured_output(
684 &metadata.command,
685 buffer,
686 disk_truncation,
687 artifact_access.clone(),
688 ) {
689 append_pipeline_warning(&mut structured, metadata, paths);
690 return structured;
691 }
692
693 if !metadata.compressed {
694 let mut raw = render_raw_passthrough(buffer, disk_truncation, artifact_access);
695 append_pipeline_warning(&mut raw, metadata, paths);
696 return raw;
697 }
698
699 let raw = buffer.read_combined_head_tail(
700 COMPRESS_INPUT_CAP_BYTES,
701 COMPRESS_INPUT_HEAD_BYTES,
702 COMPRESS_INPUT_TAIL_BYTES,
703 );
704 let compressed = self.compress_output(&metadata.command, raw.text, metadata.exit_code);
705 let mut rendered = render_compressed_with_recovery(
706 buffer,
707 compressed,
708 raw.truncated,
709 disk_truncation,
710 artifact_access,
711 );
712 append_pipeline_warning(&mut rendered, metadata, paths);
713 rendered
714 }
715
716 fn snapshot_with_terminal_cache(
717 &self,
718 task: &Arc<BgTask>,
719 preview_bytes: usize,
720 ) -> BgTaskSnapshot {
721 let mut snapshot = task.snapshot(preview_bytes);
722 self.maybe_compress_snapshot(task, &mut snapshot);
723 snapshot
724 }
725
726 fn post_terminal_transition(&self, task: &Arc<BgTask>, emit_frame: bool) -> Result<(), String> {
727 let (metadata, buffer) = {
728 let state = task
729 .state
730 .lock()
731 .map_err(|_| "background task lock poisoned".to_string())?;
732 if !state.metadata.status.is_terminal() {
733 return Ok(());
734 }
735 (state.metadata.clone(), state.buffer.clone())
736 };
737
738 let cache = self.ensure_terminal_output_cache(task);
739 self.enqueue_completion_from_parts(
740 &metadata,
741 Some(&buffer),
742 None,
743 emit_frame,
744 cache.as_ref(),
745 );
746 Ok(())
747 }
748
749 fn persist_task(&self, paths: &TaskPaths, metadata: &PersistedTask) -> std::io::Result<()> {
750 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
751 write_task_at(&task, metadata)?;
752 self.dual_write_task(paths, metadata);
753 Ok(())
754 }
755
756 fn update_task_metadata<F>(
757 &self,
758 paths: &TaskPaths,
759 update: F,
760 ) -> std::io::Result<PersistedTask>
761 where
762 F: FnOnce(&mut PersistedTask),
763 {
764 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
765 let metadata = update_task_at(&task, update)?;
766 self.dual_write_task(paths, &metadata);
767 Ok(metadata)
768 }
769
770 fn dual_write_task(&self, paths: &TaskPaths, metadata: &PersistedTask) {
771 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
772 let Some(pool) = pool else {
773 return;
774 };
775 let harness = self
776 .inner
777 .db_harness
778 .read()
779 .ok()
780 .and_then(|slot| slot.clone());
781 let Some(harness) = harness else {
782 crate::slog_warn!(
783 "dual-write bash_task to DB skipped for {}: harness not configured",
784 metadata.task_id
785 );
786 return;
787 };
788 let row = match metadata.to_bash_task_row(&harness, paths) {
789 Ok(row) => row,
790 Err(error) => {
791 crate::slog_warn!(
792 "dual-write bash_task to DB failed for {}: {}",
793 metadata.task_id,
794 error
795 );
796 return;
797 }
798 };
799 let conn = match pool.lock() {
800 Ok(conn) => conn,
801 Err(_) => {
802 crate::slog_warn!(
803 "dual-write bash_task to DB failed for {}: db mutex poisoned",
804 metadata.task_id
805 );
806 return;
807 }
808 };
809 if let Err(error) = crate::db::bash_tasks::upsert_bash_task(&conn, &row) {
810 crate::slog_warn!(
811 "dual-write bash_task to DB failed for {}: {}",
812 metadata.task_id,
813 error
814 );
815 }
816 }
817
818 fn delete_gc_task_from_db(&self, metadata: &PersistedTask) {
819 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
820 let Some(pool) = pool else {
821 return;
822 };
823 let harness = self
824 .inner
825 .db_harness
826 .read()
827 .ok()
828 .and_then(|slot| slot.clone());
829 let Some(harness) = harness else {
830 crate::slog_warn!(
831 "GC bash_task DB delete skipped for {}: harness not configured",
832 metadata.task_id
833 );
834 return;
835 };
836 let conn = match pool.lock() {
837 Ok(conn) => conn,
838 Err(_) => {
839 crate::slog_warn!(
840 "GC bash_task DB delete failed for {}: db mutex poisoned",
841 metadata.task_id
842 );
843 return;
844 }
845 };
846 if let Err(error) = crate::db::bash_tasks::delete_delivered_terminal_bash_task(
847 &conn,
848 &harness,
849 &metadata.session_id,
850 &metadata.task_id,
851 ) {
852 crate::slog_warn!(
853 "GC bash_task DB delete failed for {}: {}",
854 metadata.task_id,
855 error
856 );
857 }
858 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
860 &conn,
861 &harness,
862 &metadata.session_id,
863 &metadata.task_id,
864 ) {
865 crate::slog_warn!(
866 "GC bash_pattern_watches DB delete failed for {}: {}",
867 metadata.task_id,
868 error
869 );
870 }
871 }
872
873 fn db_harness_and_pool(&self) -> Option<(String, Arc<Mutex<Connection>>)> {
874 let pool = self
875 .inner
876 .db_pool
877 .read()
878 .ok()
879 .and_then(|slot| slot.clone())?;
880 let harness = self
881 .inner
882 .db_harness
883 .read()
884 .ok()
885 .and_then(|slot| slot.clone())?;
886 Some((harness, pool))
887 }
888
889 fn persist_watch_registration(
890 &self,
891 session_id: &str,
892 task_id: &str,
893 watch_id: &str,
894 pattern: &WatchPattern,
895 once: bool,
896 stdout_offset: u64,
897 stderr_offset: u64,
898 pty_offset: u64,
899 ) {
900 let Some((harness, pool)) = self.db_harness_and_pool() else {
901 return;
902 };
903 let Ok(conn) = pool.lock() else {
904 return;
905 };
906 let row = BashPatternWatchRow {
907 harness,
908 session_id: session_id.to_string(),
909 task_id: task_id.to_string(),
910 watch_id: watch_id.to_string(),
911 pattern_kind: pattern.kind_name().to_string(),
912 pattern: pattern.pattern_text().to_string(),
913 once,
914 created_at: unix_millis() as i64,
915 stdout_offset: stdout_offset as i64,
916 stderr_offset: stderr_offset as i64,
917 pty_offset: pty_offset as i64,
918 scanning: true,
919 pending_match: false,
920 match_text: None,
921 match_offset: None,
922 match_context: None,
923 };
924 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
925 crate::slog_warn!(
926 "persist bash_pattern_watch failed for {task_id}/{watch_id}: {error}"
927 );
928 }
929 }
930
931 fn delete_persisted_watch(&self, session_id: &str, task_id: &str, watch_id: &str) {
932 let Some((harness, pool)) = self.db_harness_and_pool() else {
933 return;
934 };
935 let Ok(conn) = pool.lock() else {
936 return;
937 };
938 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watch(
939 &conn, &harness, session_id, task_id, watch_id,
940 ) {
941 crate::slog_warn!("delete bash_pattern_watch failed for {task_id}/{watch_id}: {error}");
942 }
943 }
944
945 fn delete_persisted_watches_for_task(&self, session_id: &str, task_id: &str) {
946 let Some((harness, pool)) = self.db_harness_and_pool() else {
947 return;
948 };
949 let Ok(conn) = pool.lock() else {
950 return;
951 };
952 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
953 &conn, &harness, session_id, task_id,
954 ) {
955 crate::slog_warn!("delete bash_pattern_watches for {task_id} failed: {error}");
956 }
957 }
958
959 fn persist_watch_match(
960 &self,
961 session_id: &str,
962 task_id: &str,
963 pattern_match: &PatternMatch,
964 stdout_offset: u64,
965 stderr_offset: u64,
966 pty_offset: u64,
967 ) {
968 let Some((harness, pool)) = self.db_harness_and_pool() else {
969 return;
970 };
971 let Ok(conn) = pool.lock() else {
972 return;
973 };
974 let Ok(Some(mut row)) = crate::db::bash_watches::get_bash_pattern_watch(
975 &conn,
976 &harness,
977 session_id,
978 task_id,
979 &pattern_match.watch_id,
980 ) else {
981 return;
982 };
983 row.stdout_offset = stdout_offset as i64;
984 row.stderr_offset = stderr_offset as i64;
985 row.pty_offset = pty_offset as i64;
986 row.pending_match = true;
987 row.match_text = Some(pattern_match.match_text.clone());
988 row.match_offset = Some(pattern_match.match_offset as i64);
989 row.match_context = Some(pattern_match.context.clone());
990 if pattern_match.once {
991 row.scanning = false;
994 }
995 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
996 crate::slog_warn!(
997 "persist bash_pattern_watch match failed for {}/{}: {error}",
998 task_id,
999 pattern_match.watch_id
1000 );
1001 }
1002 }
1003
1004 fn persist_task_watch_cursors(
1005 &self,
1006 session_id: &str,
1007 task_id: &str,
1008 stdout_offset: u64,
1009 stderr_offset: u64,
1010 pty_offset: u64,
1011 ) {
1012 let Some((harness, pool)) = self.db_harness_and_pool() else {
1013 return;
1014 };
1015 let Ok(conn) = pool.lock() else {
1016 return;
1017 };
1018 if let Err(error) = crate::db::bash_watches::update_watch_offsets_for_task(
1019 &conn,
1020 &harness,
1021 session_id,
1022 task_id,
1023 stdout_offset as i64,
1024 stderr_offset as i64,
1025 pty_offset as i64,
1026 ) {
1027 crate::slog_warn!("persist bash_pattern_watch cursors failed for {task_id}: {error}");
1028 }
1029 }
1030
1031 fn watch_stream_cursors(&self, task_id: &str) -> (u64, u64, u64) {
1032 let Ok(registry) = self.inner.watch_registry.lock() else {
1033 return (0, 0, 0);
1034 };
1035 let stdout = registry
1036 .file_cursor(&format!("{task_id}:stdout"))
1037 .unwrap_or(0);
1038 let stderr = registry
1039 .file_cursor(&format!("{task_id}:stderr"))
1040 .unwrap_or(0);
1041 let pty = registry.file_cursor(&format!("{task_id}:pty")).unwrap_or(0);
1042 (stdout, stderr, pty)
1043 }
1044
1045 fn ack_persisted_watches_for_task(&self, session_id: &str, task_id: &str, task_terminal: bool) {
1048 let Some((harness, pool)) = self.db_harness_and_pool() else {
1049 return;
1050 };
1051 let Ok(conn) = pool.lock() else {
1052 return;
1053 };
1054 if task_terminal {
1055 let _ = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
1056 &conn, &harness, session_id, task_id,
1057 );
1058 return;
1059 }
1060 let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
1061 &conn, &harness, session_id, task_id,
1062 ) else {
1063 return;
1064 };
1065 for mut row in rows {
1066 if row.once && (!row.scanning || row.pending_match) {
1067 let _ = crate::db::bash_watches::delete_bash_pattern_watch(
1068 &conn,
1069 &harness,
1070 session_id,
1071 task_id,
1072 &row.watch_id,
1073 );
1074 continue;
1075 }
1076 if row.pending_match {
1077 row.pending_match = false;
1078 row.match_text = None;
1079 row.match_offset = None;
1080 row.match_context = None;
1081 let _ = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row);
1082 }
1083 }
1084 }
1085
1086 pub fn record_scanner_report(
1087 &self,
1088 task_id: &str,
1089 session_id: &str,
1090 scanner_report: Vec<PermissionAsk>,
1091 ) -> Result<(), String> {
1092 if scanner_report.is_empty() {
1093 return Ok(());
1094 }
1095 let task = self.task_for_session(task_id, session_id).ok_or_else(|| {
1096 "background task not found while recording scanner report".to_string()
1097 })?;
1098 let metadata = {
1099 let mut state = task
1100 .state
1101 .lock()
1102 .map_err(|_| "background task lock poisoned".to_string())?;
1103 state.metadata.scanner_report = scanner_report;
1104 state.metadata.clone()
1105 };
1106 self.persist_task(&task.paths, &metadata)
1107 .map_err(|error| format!("failed to persist scanner report: {error}"))
1108 }
1109
1110 pub fn configure_long_running_reminders(&self, enabled: bool, interval_ms: u64) {
1111 self.inner
1112 .long_running_reminder_enabled
1113 .store(enabled, Ordering::SeqCst);
1114 self.inner
1115 .long_running_reminder_interval_ms
1116 .store(interval_ms, Ordering::SeqCst);
1117 }
1118
1119 #[cfg(unix)]
1120 #[allow(clippy::too_many_arguments)]
1121 pub fn spawn(
1122 &self,
1123 spawn_plan: SpawnPlan,
1124 command: &str,
1125 session_id: String,
1126 workdir: PathBuf,
1127 env: HashMap<String, String>,
1128 timeout: Option<Duration>,
1129 storage_dir: PathBuf,
1130 max_running: usize,
1131 notify_on_completion: bool,
1132 compressed: bool,
1133 project_root: Option<PathBuf>,
1134 ) -> Result<String, String> {
1135 self.start_watchdog();
1136
1137 let running = self.running_count();
1138 if running >= max_running {
1139 #[cfg(unix)]
1140 if let Some(prepared) = spawn_plan.prepared_task() {
1141 let _ = delete_resolved_task(&prepared.resolved_task());
1142 }
1143 return Err(format!(
1144 "background bash task limit exceeded: {running} running (max {max_running})"
1145 ));
1146 }
1147
1148 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1149 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1150 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1151 (spawn_plan.clone(), prepared.resolved_task())
1152 } else {
1153 let task = allocate_task_layout(&storage_dir, &session_id)
1154 .map_err(|error| format!("failed to create background task layout: {error}"))?;
1155 let shell = resolve_posix_shell();
1156 let root = project_root.as_deref().unwrap_or(&workdir);
1157 let environment =
1158 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1159 let prepared = match crate::sandbox_spawn::prepare_task_payload(
1160 &task,
1161 command.as_bytes(),
1162 root,
1163 &workdir,
1164 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1165 &shell,
1166 &environment,
1167 ) {
1168 Ok(prepared) => prepared,
1169 Err(error) => {
1170 let _ = delete_resolved_task(&task);
1171 return Err(error);
1172 }
1173 };
1174 let task = prepared.resolved_task();
1175 (spawn_plan.with_prepared_task(prepared), task)
1176 };
1177 let task_id = task_layout.paths.task_id.clone();
1178 let paths = task_layout.paths.clone();
1179
1180 if self.task(&task_id).is_some() {
1181 let _ = delete_resolved_task(&task_layout);
1182 return Err("background task id collided with a live task".to_string());
1183 }
1184
1185 let mut metadata = PersistedTask::starting(
1186 task_id.clone(),
1187 session_id.clone(),
1188 command.to_string(),
1189 workdir.clone(),
1190 project_root,
1191 timeout_ms,
1192 notify_on_completion,
1193 compressed,
1194 );
1195 #[cfg(unix)]
1199 let capture_pipeline_status = {
1200 let shell = spawn_plan
1201 .host_shell_path()
1202 .map(Path::to_path_buf)
1203 .unwrap_or_else(resolve_posix_shell);
1204 let pipeline = single_top_level_pipeline(command);
1205 let capture = should_capture_pipeline_status(&spawn_plan, pipeline.is_some(), &shell);
1206 if capture {
1207 metadata.pipeline_segments = pipeline
1208 .as_ref()
1209 .map(|pipeline| {
1210 pipeline
1211 .segments
1212 .iter()
1213 .map(|segment| segment.label.clone())
1214 .collect()
1215 })
1216 .unwrap_or_default();
1217 }
1218 capture
1219 };
1220 #[cfg(windows)]
1221 let capture_pipeline_status = false;
1222 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1223 if let Err(error) = write_task_at(&task_layout, &metadata) {
1224 let _ = delete_resolved_task(&task_layout);
1225 return Err(format!(
1226 "failed to persist background task metadata: {error}"
1227 ));
1228 }
1229 self.dual_write_task(&paths, &metadata);
1230
1231 let mut io_handles =
1232 TaskIoHandles::create(&task_layout, BgMode::Pipes, capture_pipeline_status)
1233 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
1234 let child = match spawn_detached_child(
1235 &spawn_plan,
1236 command,
1237 &paths,
1238 &workdir,
1239 &env,
1240 &mut io_handles,
1241 capture_pipeline_status,
1242 ) {
1243 Ok(child) => child,
1244 Err(error) => {
1245 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
1246 let _ = delete_task_bundle(&paths);
1247 return Err(error);
1248 }
1249 };
1250
1251 let child_pid = child.id();
1252 metadata.mark_running(child_pid, child_pid as i32);
1253 self.persist_task(&paths, &metadata)
1254 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1255
1256 let task = Arc::new(BgTask {
1257 task_id: task_id.clone(),
1258 session_id,
1259 paths: paths.clone(),
1260 artifact_root: canonical_artifact_root(&paths),
1261 started: Instant::now(),
1262 last_reminder_at: Mutex::new(None),
1263 terminal_at: Mutex::new(None),
1264 state: Mutex::new(BgTaskState {
1265 metadata,
1266 runtime: TaskRuntime::Piped(Some(child)),
1267 io_handles: Some(io_handles),
1268 detached: false,
1269 child_exit_observed: false,
1270 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
1271 terminal_output_cache: None,
1272 pending_terminal_override: None,
1273 }),
1274 });
1275
1276 self.inner
1277 .tasks
1278 .lock()
1279 .map_err(|_| "background task registry lock poisoned".to_string())?
1280 .insert(task_id.clone(), task);
1281
1282 Ok(task_id)
1283 }
1284
1285 #[allow(clippy::too_many_arguments)]
1286 pub fn spawn_pty(
1287 &self,
1288 spawn_plan: SpawnPlan,
1289 command: &str,
1290 session_id: String,
1291 workdir: PathBuf,
1292 env: HashMap<String, String>,
1293 timeout: Option<Duration>,
1294 storage_dir: PathBuf,
1295 max_running: usize,
1296 notify_on_completion: bool,
1297 compressed: bool,
1298 project_root: Option<PathBuf>,
1299 rows: u16,
1300 cols: u16,
1301 ) -> Result<String, String> {
1302 self.start_watchdog();
1303
1304 let running = self.running_count();
1305 if running >= max_running {
1306 #[cfg(unix)]
1307 if let Some(prepared) = spawn_plan.prepared_task() {
1308 let _ = delete_resolved_task(&prepared.resolved_task());
1309 }
1310 return Err(format!(
1311 "background bash task limit exceeded: {running} running (max {max_running})"
1312 ));
1313 }
1314
1315 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1316 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1317 #[cfg(unix)]
1318 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1319 (spawn_plan.clone(), prepared.resolved_task())
1320 } else {
1321 let task = allocate_task_layout(&storage_dir, &session_id)
1322 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1323 let shell = super::resolved_shell_path(true);
1324 let root = project_root.as_deref().unwrap_or(&workdir);
1325 let environment =
1326 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1327 let prepared = match crate::sandbox_spawn::prepare_task_payload(
1328 &task,
1329 command.as_bytes(),
1330 root,
1331 &workdir,
1332 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1333 &shell,
1334 &environment,
1335 ) {
1336 Ok(prepared) => prepared,
1337 Err(error) => {
1338 let _ = delete_resolved_task(&task);
1339 return Err(error);
1340 }
1341 };
1342 let task = prepared.resolved_task();
1343 (spawn_plan.with_prepared_task(prepared), task)
1344 };
1345 #[cfg(windows)]
1346 let task_layout = allocate_task_layout(&storage_dir, &session_id)
1347 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1348 let task_id = task_layout.paths.task_id.clone();
1349 let paths = task_layout.paths.clone();
1350
1351 let mut metadata = PersistedTask::starting(
1352 task_id.clone(),
1353 session_id.clone(),
1354 command.to_string(),
1355 workdir.clone(),
1356 project_root,
1357 timeout_ms,
1358 notify_on_completion,
1359 compressed,
1360 );
1361 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1362 metadata.mode = BgMode::Pty;
1363 metadata.pty_rows = Some(rows);
1364 metadata.pty_cols = Some(cols);
1365 if let Err(error) = write_task_at(&task_layout, &metadata) {
1366 let _ = delete_resolved_task(&task_layout);
1367 return Err(format!(
1368 "failed to persist background task metadata: {error}"
1369 ));
1370 }
1371 self.dual_write_task(&paths, &metadata);
1372 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pty, false)
1373 .map_err(|error| format!("failed to pre-open PTY output handles: {error}"))?;
1374
1375 let runtime = match spawn_pty_for_command(
1376 &spawn_plan,
1377 &task_id,
1378 &session_id,
1379 command,
1380 &paths,
1381 &workdir,
1382 &env,
1383 rows,
1384 cols,
1385 self.inner.wake_tx.clone(),
1386 &mut io_handles,
1387 ) {
1388 Ok(runtime) => runtime,
1389 Err(error) => {
1390 crate::slog_warn!(
1391 "failed to spawn PTY background bash task {task_id}; deleting partial bundle: {error}"
1392 );
1393 let _ = delete_task_bundle(&paths);
1394 return Err(error);
1395 }
1396 };
1397
1398 if let Some(child_pid) = runtime.child_pid {
1399 metadata.mark_running(child_pid, child_pid as i32);
1400 } else {
1401 metadata.status = BgTaskStatus::Running;
1402 metadata.pgid = None;
1403 }
1404 self.persist_task(&paths, &metadata)
1405 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1406
1407 let task = Arc::new(BgTask {
1408 task_id: task_id.clone(),
1409 session_id,
1410 paths: paths.clone(),
1411 artifact_root: canonical_artifact_root(&paths),
1412 started: Instant::now(),
1413 last_reminder_at: Mutex::new(None),
1414 terminal_at: Mutex::new(None),
1415 state: Mutex::new(BgTaskState {
1416 metadata,
1417 runtime: TaskRuntime::Pty(Some(runtime)),
1418 io_handles: Some(io_handles),
1419 detached: false,
1420 child_exit_observed: false,
1421 buffer: BgBuffer::registered(&paths, BgMode::Pty),
1422 terminal_output_cache: None,
1423 pending_terminal_override: None,
1424 }),
1425 });
1426
1427 self.inner
1428 .tasks
1429 .lock()
1430 .map_err(|_| "background task registry lock poisoned".to_string())?
1431 .insert(task_id.clone(), task);
1432
1433 Ok(task_id)
1434 }
1435
1436 #[cfg(windows)]
1437 #[allow(clippy::too_many_arguments)]
1438 pub fn spawn(
1439 &self,
1440 spawn_plan: SpawnPlan,
1441 command: &str,
1442 session_id: String,
1443 workdir: PathBuf,
1444 env: HashMap<String, String>,
1445 timeout: Option<Duration>,
1446 storage_dir: PathBuf,
1447 max_running: usize,
1448 notify_on_completion: bool,
1449 compressed: bool,
1450 project_root: Option<PathBuf>,
1451 ) -> Result<String, String> {
1452 self.start_watchdog();
1453
1454 let running = self.running_count();
1455 if running >= max_running {
1456 #[cfg(unix)]
1457 if let Some(prepared) = spawn_plan.prepared_task() {
1458 let _ = delete_resolved_task(&prepared.resolved_task());
1459 }
1460 return Err(format!(
1461 "background bash task limit exceeded: {running} running (max {max_running})"
1462 ));
1463 }
1464
1465 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1466 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1467 let task_layout = allocate_task_layout(&storage_dir, &session_id)
1468 .map_err(|error| format!("failed to create background task layout: {error}"))?;
1469 let task_id = task_layout.paths.task_id.clone();
1470 let paths = task_layout.paths.clone();
1471
1472 let mut metadata = PersistedTask::starting(
1473 task_id.clone(),
1474 session_id.clone(),
1475 command.to_string(),
1476 workdir.clone(),
1477 project_root,
1478 timeout_ms,
1479 notify_on_completion,
1480 compressed,
1481 );
1482 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1483 if let Err(error) = write_task_at(&task_layout, &metadata) {
1484 let _ = delete_resolved_task(&task_layout);
1485 return Err(format!(
1486 "failed to persist background task metadata: {error}"
1487 ));
1488 }
1489 self.dual_write_task(&paths, &metadata);
1490 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pipes, false)
1491 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
1492
1493 let child = match spawn_detached_child(
1494 &spawn_plan,
1495 command,
1496 &paths,
1497 &workdir,
1498 &env,
1499 &mut io_handles,
1500 false,
1501 ) {
1502 Ok(child) => child,
1503 Err(error) => {
1504 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
1505 let _ = delete_task_bundle(&paths);
1506 return Err(error);
1507 }
1508 };
1509
1510 let child_pid = child.id();
1511 metadata.status = BgTaskStatus::Running;
1512 metadata.child_pid = Some(child_pid);
1513 metadata.pgid = None;
1514 self.persist_task(&paths, &metadata)
1515 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1516
1517 let task = Arc::new(BgTask {
1518 task_id: task_id.clone(),
1519 session_id,
1520 paths: paths.clone(),
1521 artifact_root: canonical_artifact_root(&paths),
1522 started: Instant::now(),
1523 last_reminder_at: Mutex::new(None),
1524 terminal_at: Mutex::new(None),
1525 state: Mutex::new(BgTaskState {
1526 metadata,
1527 runtime: TaskRuntime::Piped(Some(child)),
1528 io_handles: Some(io_handles),
1529 detached: false,
1530 child_exit_observed: false,
1531 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
1532 terminal_output_cache: None,
1533 pending_terminal_override: None,
1534 }),
1535 });
1536
1537 self.inner
1538 .tasks
1539 .lock()
1540 .map_err(|_| "background task registry lock poisoned".to_string())?
1541 .insert(task_id.clone(), task);
1542
1543 Ok(task_id)
1544 }
1545
1546 pub fn write_pty(
1547 &self,
1548 task_id: &str,
1549 session_id: &str,
1550 input: &[u8],
1551 ) -> Result<usize, String> {
1552 let task = self
1553 .task_for_session(task_id, session_id)
1554 .ok_or_else(|| "task_not_found".to_string())?;
1555
1556 let writer = {
1557 let state = task
1558 .state
1559 .lock()
1560 .map_err(|_| "background task lock poisoned".to_string())?;
1561 if state.metadata.mode != BgMode::Pty {
1562 return Err("task_not_pty".to_string());
1563 }
1564 if state.metadata.status.is_terminal() {
1565 return Err("task_exited".to_string());
1566 }
1567 match &state.runtime {
1568 TaskRuntime::Pty(Some(runtime)) => Arc::clone(&runtime.writer),
1569 TaskRuntime::Pty(None) => return Err("task_exited".to_string()),
1570 TaskRuntime::Piped(_) => return Err("task_not_pty".to_string()),
1571 }
1572 };
1573
1574 let mut writer = writer
1575 .lock()
1576 .map_err(|_| "PTY writer lock poisoned".to_string())?;
1577 writer
1578 .write_all(input)
1579 .map_err(|error| format!("failed to write to PTY: {error}"))?;
1580 writer
1581 .flush()
1582 .map_err(|error| format!("failed to flush PTY writer: {error}"))?;
1583 Ok(input.len())
1584 }
1585
1586 pub fn replay_session(&self, storage_dir: &Path, session_id: &str) -> Result<(), String> {
1587 self.replay_session_inner(storage_dir, session_id, None)
1588 }
1589
1590 pub fn replay_session_for_project(
1591 &self,
1592 storage_dir: &Path,
1593 session_id: &str,
1594 project_root: &Path,
1595 ) -> Result<(), String> {
1596 self.replay_session_inner(storage_dir, session_id, Some(project_root))
1597 }
1598
1599 fn replay_session_inner(
1600 &self,
1601 storage_dir: &Path,
1602 session_id: &str,
1603 project_root: Option<&Path>,
1604 ) -> Result<(), String> {
1605 self.start_watchdog();
1606 if !self.inner.persisted_gc_started.swap(true, Ordering::SeqCst) {
1607 if let Err(error) = self.maybe_gc_persisted(storage_dir) {
1608 crate::slog_warn!("failed to GC persisted background bash tasks: {error}");
1609 }
1610 }
1611
1612 let canonical_project = project_root.map(canonicalized_path);
1613 let tasks = match self.replay_session_from_db(session_id) {
1625 Some(Ok(tasks)) if !tasks.is_empty() => tasks,
1626 Some(Ok(_)) => {
1627 let disk_tasks = self.replay_session_from_disk(storage_dir, session_id)?;
1628 if !disk_tasks.is_empty() {
1629 crate::slog_info!(
1630 "bash task replay: 0 in DB for session {}, {} from disk fallback",
1631 session_id,
1632 disk_tasks.len()
1633 );
1634 }
1635 disk_tasks
1636 }
1637 Some(Err(error)) => {
1638 crate::slog_warn!(
1639 "bash task replay DB lookup failed for session {}; falling back to disk: {}",
1640 session_id,
1641 error
1642 );
1643 self.replay_session_from_disk(storage_dir, session_id)?
1644 }
1645 None => {
1646 self.replay_session_from_disk(storage_dir, session_id)?
1648 }
1649 };
1650
1651 for mut metadata in tasks {
1652 if metadata.session_id != session_id {
1653 continue;
1654 }
1655 if let Some(canonical_project) = canonical_project.as_deref() {
1656 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
1657 if metadata_project.as_deref() != Some(canonical_project) {
1658 continue;
1659 }
1660 }
1661
1662 if validate_task_id(&metadata.task_id).is_err() {
1663 crate::slog_warn!(
1664 "ignoring persisted background task with invalid id {:?}",
1665 metadata.task_id
1666 );
1667 continue;
1668 }
1669 let session_dir = session_tasks_dir(storage_dir, session_id);
1670 let resolved = match resolve_task_layout(&session_dir, &metadata.task_id) {
1671 Ok(task) => task,
1672 Err(error) => {
1673 crate::slog_warn!(
1674 "quarantining unresolved background task {}: {error}",
1675 metadata.task_id
1676 );
1677 let _ = quarantine_task_layout(
1678 storage_dir,
1679 &session_dir,
1680 &metadata.task_id,
1681 "invalid",
1682 );
1683 continue;
1684 }
1685 };
1686 match read_task_at(&resolved) {
1687 Ok(disk) if disk.task_id == metadata.task_id && disk.session_id == session_id => {}
1688 Ok(_) | Err(_) => {
1689 let _ = quarantine_task_layout(
1690 storage_dir,
1691 &session_dir,
1692 &metadata.task_id,
1693 "mismatch",
1694 );
1695 continue;
1696 }
1697 }
1698 let paths = resolved.paths;
1699 match metadata.status {
1700 BgTaskStatus::Starting => {
1701 let completion_was_delivered = metadata.completion_delivered;
1702 metadata.mark_terminal(
1703 BgTaskStatus::Failed,
1704 None,
1705 Some("spawn aborted".to_string()),
1706 );
1707 metadata.completion_delivered |= completion_was_delivered;
1708 let _ = self.persist_task(&paths, &metadata);
1709 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1710 self.insert_rehydrated_task(metadata, paths, true)?;
1711 }
1712 BgTaskStatus::Running | BgTaskStatus::Killing => {
1713 if metadata.mode == BgMode::Pty {
1714 if let Ok(Some(marker)) = read_exit_marker(&paths) {
1715 let completion_was_delivered = metadata.completion_delivered;
1716 metadata = terminal_metadata_from_marker(metadata, marker, None);
1717 metadata.completion_delivered |= completion_was_delivered;
1718 let _ = self.persist_task(&paths, &metadata);
1719 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1720 self.insert_rehydrated_task(metadata, paths, true)?;
1721 } else if metadata.status.is_terminal() {
1722 self.insert_rehydrated_task(metadata, paths, true)?;
1723 } else {
1724 let completion_was_delivered = metadata.completion_delivered;
1725 metadata.mark_terminal(
1726 BgTaskStatus::Killed,
1727 None,
1728 Some("pty_lost_on_bridge_restart".to_string()),
1729 );
1730 metadata.completion_delivered |= completion_was_delivered;
1731 let _ = self.persist_task(&paths, &metadata);
1732 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1733 self.insert_rehydrated_task(metadata, paths, true)?;
1734 }
1735 } else if self.running_metadata_is_stale(&metadata) {
1736 let completion_was_delivered = metadata.completion_delivered;
1737 metadata.mark_terminal(
1738 BgTaskStatus::Killed,
1739 None,
1740 Some("orphaned (>24h)".to_string()),
1741 );
1742 metadata.completion_delivered |= completion_was_delivered;
1743 let _ = write_kill_marker_if_absent(&paths);
1744 let _ = self.persist_task(&paths, &metadata);
1745 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1746 self.insert_rehydrated_task(metadata, paths, true)?;
1747 } else if let Ok(Some(marker)) = read_exit_marker(&paths) {
1748 let reason = (metadata.status == BgTaskStatus::Killing).then(|| {
1749 "recovered from inconsistent killing state on replay".to_string()
1750 });
1751 if reason.is_some() {
1752 crate::slog_warn!("background task {} had killing state with exit marker; preferring marker",
1753 metadata.task_id);
1754 }
1755 let completion_was_delivered = metadata.completion_delivered;
1756 metadata = terminal_metadata_from_marker(metadata, marker, reason);
1757 metadata.completion_delivered |= completion_was_delivered;
1758 let _ = self.persist_task(&paths, &metadata);
1759 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1760 self.insert_rehydrated_task(metadata, paths, true)?;
1761 } else if metadata.status == BgTaskStatus::Killing {
1762 let _ = write_kill_marker_if_absent(&paths);
1763 let completion_was_delivered = metadata.completion_delivered;
1764 metadata.mark_terminal(
1765 BgTaskStatus::Killed,
1766 None,
1767 Some("recovered from inconsistent killing state on replay".to_string()),
1768 );
1769 metadata.completion_delivered |= completion_was_delivered;
1770 let _ = self.persist_task(&paths, &metadata);
1771 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1772 self.insert_rehydrated_task(metadata, paths, true)?;
1773 } else if metadata.child_pid.is_some_and(|pid| !is_process_alive(pid)) {
1774 let completion_was_delivered = metadata.completion_delivered;
1775 metadata.mark_terminal(
1776 BgTaskStatus::Failed,
1777 None,
1778 Some("process exited without exit marker".to_string()),
1779 );
1780 metadata.completion_delivered |= completion_was_delivered;
1781 let _ = self.persist_task(&paths, &metadata);
1782 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1783 self.insert_rehydrated_task(metadata, paths, true)?;
1784 } else {
1785 self.insert_rehydrated_task(metadata, paths, true)?;
1786 }
1787 }
1788 _ if metadata.status.is_terminal() => {
1789 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1795 self.insert_rehydrated_task(metadata, paths, true)?;
1796 }
1797 _ => {}
1798 }
1799 }
1800
1801 Ok(())
1802 }
1803
1804 fn replay_session_from_db(
1805 &self,
1806 session_id: &str,
1807 ) -> Option<Result<Vec<PersistedTask>, String>> {
1808 let pool = self
1809 .inner
1810 .db_pool
1811 .read()
1812 .ok()
1813 .and_then(|slot| slot.clone())?;
1814 let harness = self
1815 .inner
1816 .db_harness
1817 .read()
1818 .ok()
1819 .and_then(|slot| slot.clone())?;
1820 let conn = match pool.lock() {
1821 Ok(conn) => conn,
1822 Err(_) => return Some(Err("db mutex poisoned".to_string())),
1823 };
1824 Some(
1825 crate::db::bash_tasks::list_bash_tasks_for_session(&conn, &harness, session_id)
1826 .map(|rows| rows.into_iter().map(PersistedTask::from).collect())
1827 .map_err(|error| error.to_string()),
1828 )
1829 }
1830
1831 fn replay_session_from_disk(
1832 &self,
1833 storage_dir: &Path,
1834 session_id: &str,
1835 ) -> Result<Vec<PersistedTask>, String> {
1836 let dir = session_tasks_dir(storage_dir, session_id);
1837 if !dir.exists() {
1838 return Ok(Vec::new());
1839 }
1840
1841 let (task_ids, invalid_entries) = discover_task_ids(&dir)
1842 .map_err(|error| format!("failed to discover background task layouts: {error}"))?;
1843 for entry in invalid_entries {
1844 if let Err(error) = quarantine_invalid_entry(storage_dir, &dir, &entry) {
1845 crate::slog_warn!(
1846 "failed to quarantine invalid background task entry {:?}: {error}",
1847 entry
1848 );
1849 }
1850 }
1851
1852 let mut tasks = Vec::new();
1853 for task_id in task_ids {
1854 let task = match resolve_task_layout(&dir, &task_id) {
1855 Ok(task) => task,
1856 Err(error)
1857 if error.kind() == std::io::ErrorKind::NotFound
1858 && uninitialized_layout_is_recent(
1859 &dir,
1860 &task_id,
1861 Duration::from_secs(5 * 60),
1862 )
1863 .unwrap_or(false) =>
1864 {
1865 continue;
1866 }
1867 Err(error) => {
1868 crate::slog_warn!(
1869 "quarantining unresolved background task {task_id} during replay: {error}"
1870 );
1871 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
1872 continue;
1873 }
1874 };
1875 match read_task_at(&task) {
1876 Ok(metadata) if metadata.session_id == session_id => tasks.push(metadata),
1877 Ok(_) => {
1878 crate::slog_warn!(
1879 "quarantining background task {task_id} with mismatched session metadata"
1880 );
1881 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "mismatch");
1882 }
1883 Err(error) => {
1884 crate::slog_warn!(
1885 "quarantining invalid background task metadata {task_id} during replay: {error}"
1886 );
1887 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
1888 }
1889 }
1890 }
1891 Ok(tasks)
1892 }
1893
1894 pub fn register_watch(
1895 &self,
1896 task_id: String,
1897 pattern: WatchPattern,
1898 once: bool,
1899 ) -> Result<String, &'static str> {
1900 let task = self.task(&task_id).ok_or("task_not_found")?;
1901 validate_task_id(&task_id).map_err(|_| "invalid_task_id")?;
1902 let (mode, terminal_at_registration) = task
1903 .state
1904 .lock()
1905 .map(|state| {
1906 (
1907 state.metadata.mode.clone(),
1908 state.metadata.status.is_terminal(),
1909 )
1910 })
1911 .map_err(|_| "background_task_lock_poisoned")?;
1912 let mut stdout = (mode == BgMode::Pipes)
1913 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
1914 .transpose()
1915 .map_err(|_| "artifact_refused")?;
1916 let mut stderr = (mode == BgMode::Pipes)
1917 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
1918 .transpose()
1919 .map_err(|_| "artifact_refused")?;
1920 let mut pty = (mode == BgMode::Pty)
1921 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
1922 .transpose()
1923 .map_err(|_| "artifact_refused")?;
1924
1925 let mut terminal_matches = Vec::new();
1926 let scanned_terminal = terminal_at_registration;
1927 let watch_id = {
1928 let mut registry = self
1929 .inner
1930 .watch_registry
1931 .lock()
1932 .map_err(|_| "watch_registry_poisoned")?;
1933 let watch_id = registry.register(task_id.clone(), pattern.clone(), once)?;
1934 match &mode {
1935 BgMode::Pipes => {
1936 let stdout_key = format!("{task_id}:stdout");
1937 let stderr_key = format!("{task_id}:stderr");
1938 if terminal_at_registration {
1939 registry.set_file_cursor(&stdout_key, 0);
1940 registry.set_file_cursor(&stderr_key, 0);
1941 terminal_matches.extend(registry.scan_file_new_bytes(
1942 &stdout_key,
1943 &task_id,
1944 stdout.as_mut().expect("pipe stdout opened"),
1945 ));
1946 terminal_matches.extend(registry.scan_file_new_bytes(
1947 &stderr_key,
1948 &task_id,
1949 stderr.as_mut().expect("pipe stderr opened"),
1950 ));
1951 } else {
1952 registry.prime_file_cursor(
1953 &stdout_key,
1954 stdout.as_ref().expect("pipe stdout opened"),
1955 );
1956 registry.prime_file_cursor(
1957 &stderr_key,
1958 stderr.as_ref().expect("pipe stderr opened"),
1959 );
1960 }
1961 }
1962 BgMode::Pty => {
1963 let pty_key = format!("{task_id}:pty");
1964 if terminal_at_registration {
1965 registry.set_file_cursor(&pty_key, 0);
1966 terminal_matches.extend(registry.scan_file_new_bytes(
1967 &pty_key,
1968 &task_id,
1969 pty.as_mut().expect("PTY artifact opened"),
1970 ));
1971 } else {
1972 registry.prime_file_cursor(
1973 &pty_key,
1974 pty.as_ref().expect("PTY artifact opened"),
1975 );
1976 }
1977 }
1978 }
1979 watch_id
1980 };
1981
1982 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
1983 self.persist_watch_registration(
1984 &task.session_id,
1985 &task_id,
1986 &watch_id,
1987 &pattern,
1988 once,
1989 stdout_offset,
1990 stderr_offset,
1991 pty_offset,
1992 );
1993
1994 if task.is_terminal() {
1995 if !scanned_terminal {
1996 terminal_matches = {
1997 let mut registry = self
1998 .inner
1999 .watch_registry
2000 .lock()
2001 .map_err(|_| "watch_registry_poisoned")?;
2002 match &mode {
2003 BgMode::Pipes => {
2004 let stdout_key = format!("{task_id}:stdout");
2005 let stderr_key = format!("{task_id}:stderr");
2006 registry.set_file_cursor(&stdout_key, 0);
2007 registry.set_file_cursor(&stderr_key, 0);
2008 let mut matches = registry.scan_file_new_bytes(
2009 &stdout_key,
2010 &task_id,
2011 stdout.as_mut().expect("pipe stdout opened"),
2012 );
2013 matches.extend(registry.scan_file_new_bytes(
2014 &stderr_key,
2015 &task_id,
2016 stderr.as_mut().expect("pipe stderr opened"),
2017 ));
2018 matches
2019 }
2020 BgMode::Pty => {
2021 let pty_key = format!("{task_id}:pty");
2022 registry.set_file_cursor(&pty_key, 0);
2023 registry.scan_file_new_bytes(
2024 &pty_key,
2025 &task_id,
2026 pty.as_mut().expect("PTY artifact opened"),
2027 )
2028 }
2029 }
2030 };
2031 }
2032
2033 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
2034 let (watch_controlled, watch_matched) = self.task_watch_state(&task_id);
2035 if terminal_matches.is_empty() && (!watch_controlled || watch_matched) {
2036 if watch_matched {
2037 let _ = task.set_completion_delivered(true, self);
2038 self.clear_task_watch_state(&task_id);
2039 self.delete_persisted_watches_for_task(&task.session_id, &task_id);
2041 }
2042 return Ok(watch_id);
2043 }
2044
2045 let completion = self
2046 .remove_pending_completion(&task_id)
2047 .or_else(|| self.completion_snapshot_for_task(&task));
2048 if terminal_matches.is_empty() {
2049 if let Some(completion) = completion.as_ref() {
2050 self.emit_bash_watch_exit(completion);
2051 }
2052 } else {
2053 for pattern_match in &terminal_matches {
2054 self.persist_watch_match(
2055 &task.session_id,
2056 &task_id,
2057 pattern_match,
2058 stdout_offset,
2059 stderr_offset,
2060 pty_offset,
2061 );
2062 self.emit_bash_pattern_match(&task.session_id, pattern_match.clone());
2063 }
2064 }
2065 let _ = task.set_completion_delivered(true, self);
2066 self.clear_task_watch_state(&task_id);
2067 self.delete_persisted_watches_for_task(&task.session_id, &task_id);
2069 }
2070
2071 Ok(watch_id)
2072 }
2073
2074 pub fn unregister_watch(&self, task_id: &str, watch_id: &str) {
2075 let session_id = self.task(task_id).map(|task| task.session_id.clone());
2076 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2077 registry.unregister(task_id, watch_id);
2078 }
2079 if let Some(session_id) = session_id {
2080 self.delete_persisted_watch(&session_id, task_id, watch_id);
2081 }
2082 }
2083
2084 pub fn active_watch_count(&self, task_id: &str) -> usize {
2085 self.inner
2086 .watch_registry
2087 .lock()
2088 .map(|registry| registry.active_count(task_id))
2089 .unwrap_or(0)
2090 }
2091
2092 fn task_watch_state(&self, task_id: &str) -> (bool, bool) {
2093 self.inner
2094 .watch_registry
2095 .lock()
2096 .map(|registry| {
2097 (
2098 registry.has_controlled_task(task_id),
2099 registry.has_matched_task(task_id),
2100 )
2101 })
2102 .unwrap_or((false, false))
2103 }
2104
2105 fn task_has_watch_control(&self, task_id: &str) -> bool {
2106 self.inner
2107 .watch_registry
2108 .lock()
2109 .map(|registry| registry.has_controlled_task(task_id))
2110 .unwrap_or(false)
2111 }
2112
2113 fn clear_task_watch_state(&self, task_id: &str) {
2114 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2115 registry.clear_task(task_id);
2116 }
2117 }
2118
2119 pub(crate) fn scan_task_watch_output(&self, task: &Arc<BgTask>) {
2120 let mode = match task.state.lock() {
2121 Ok(state) => state.metadata.mode.clone(),
2122 Err(_) => return,
2123 };
2124 let mut stdout = (mode == BgMode::Pipes)
2125 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
2126 .transpose()
2127 .ok()
2128 .flatten();
2129 let mut stderr = (mode == BgMode::Pipes)
2130 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
2131 .transpose()
2132 .ok()
2133 .flatten();
2134 let mut pty = (mode == BgMode::Pty)
2135 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
2136 .transpose()
2137 .ok()
2138 .flatten();
2139 let mut matches = Vec::new();
2140 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2141 match mode {
2142 BgMode::Pipes => {
2143 let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) else {
2144 return;
2145 };
2146 let stdout_key = format!("{}:stdout", task.task_id);
2147 let stderr_key = format!("{}:stderr", task.task_id);
2148 matches.extend(registry.scan_file_new_bytes(
2149 &stdout_key,
2150 &task.task_id,
2151 stdout,
2152 ));
2153 matches.extend(registry.scan_file_new_bytes(
2154 &stderr_key,
2155 &task.task_id,
2156 stderr,
2157 ));
2158 }
2159 BgMode::Pty => {
2160 let Some(pty) = pty.as_mut() else {
2161 return;
2162 };
2163 let pty_key = format!("{}:pty", task.task_id);
2164 matches.extend(registry.scan_file_new_bytes(&pty_key, &task.task_id, pty));
2165 }
2166 }
2167 }
2168 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
2169 if matches.is_empty() {
2170 if self.task_has_watch_control(&task.task_id) {
2173 self.persist_task_watch_cursors(
2174 &task.session_id,
2175 &task.task_id,
2176 stdout_offset,
2177 stderr_offset,
2178 pty_offset,
2179 );
2180 }
2181 return;
2182 }
2183 for pattern_match in matches {
2184 self.persist_watch_match(
2185 &task.session_id,
2186 &task.task_id,
2187 &pattern_match,
2188 stdout_offset,
2189 stderr_offset,
2190 pty_offset,
2191 );
2192 self.emit_bash_pattern_match(&task.session_id, pattern_match);
2193 }
2194 self.persist_task_watch_cursors(
2195 &task.session_id,
2196 &task.task_id,
2197 stdout_offset,
2198 stderr_offset,
2199 pty_offset,
2200 );
2201 }
2202
2203 pub fn status(
2204 &self,
2205 task_id: &str,
2206 session_id: &str,
2207 project_root: Option<&Path>,
2208 storage_dir: Option<&Path>,
2209 preview_bytes: usize,
2210 ) -> Option<BgTaskSnapshot> {
2211 validate_task_id(task_id).ok()?;
2212 let mut task = self.task_for_session(task_id, session_id);
2213 if task.is_none() {
2214 if let Some(storage_dir) = storage_dir {
2215 let _ = if let Some(project_root) = project_root {
2216 self.replay_session_for_project(storage_dir, session_id, project_root)
2217 } else {
2218 self.replay_session(storage_dir, session_id)
2219 };
2220 task = self.task_for_session(task_id, session_id);
2221 }
2222 }
2223 let Some(task) = task else {
2224 return self.status_relaxed(
2225 task_id,
2226 session_id,
2227 project_root?,
2228 storage_dir?,
2229 preview_bytes,
2230 );
2231 };
2232 let _ = self.poll_task(&task);
2233 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
2234 }
2235
2236 fn status_relaxed_task(
2237 &self,
2238 task_id: &str,
2239 project_root: &Path,
2240 storage_dir: &Path,
2241 ) -> Option<Arc<BgTask>> {
2242 validate_task_id(task_id).ok()?;
2243 let canonical_project = canonicalized_path(project_root);
2244 match self.lookup_relaxed_task_from_db(task_id, project_root) {
2245 Some(Ok(Some(metadata))) => {
2246 if let Some(task) = self.task(task_id) {
2247 let matches_project = task
2248 .state
2249 .lock()
2250 .map(|state| {
2251 state
2252 .metadata
2253 .project_root
2254 .as_deref()
2255 .map(canonicalized_path)
2256 .as_deref()
2257 == Some(canonical_project.as_path())
2258 })
2259 .unwrap_or(false);
2260 return matches_project.then_some(task);
2261 }
2262 let resolved = resolve_task_layout(
2263 &session_tasks_dir(storage_dir, &metadata.session_id),
2264 &metadata.task_id,
2265 )
2266 .ok()?;
2267 let disk = read_task_at(&resolved).ok()?;
2268 if disk.task_id != metadata.task_id || disk.session_id != metadata.session_id {
2269 return None;
2270 }
2271 if self
2272 .insert_rehydrated_task(metadata, resolved.paths, true)
2273 .is_err()
2274 {
2275 return None;
2276 }
2277 return self.task(task_id);
2278 }
2279 Some(Ok(None)) => {
2280 crate::slog_info!(
2281 "bash task relaxed DB miss for {}; falling back to disk",
2282 task_id
2283 );
2284 }
2285 Some(Err(error)) => {
2286 crate::slog_warn!(
2287 "bash task relaxed DB lookup failed for {}; falling back to disk: {}",
2288 task_id,
2289 error
2290 );
2291 }
2292 None => {
2293 crate::slog_info!(
2294 "bash task relaxed DB unavailable for {}; falling back to disk",
2295 task_id
2296 );
2297 }
2298 }
2299 let root = storage_dir.join("bash-tasks");
2300 let entries = fs::read_dir(&root).ok()?;
2301 for entry in entries.flatten() {
2302 let dir = entry.path();
2303 if !dir.is_dir() {
2304 continue;
2305 }
2306 let resolved = match resolve_task_layout(&dir, task_id) {
2307 Ok(task) => task,
2308 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
2309 Err(error) => {
2310 crate::slog_warn!(
2311 "quarantining unresolved background task {task_id} during relaxed lookup: {error}"
2312 );
2313 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
2314 continue;
2315 }
2316 };
2317 let metadata = match read_task_at(&resolved) {
2318 Ok(metadata) => metadata,
2319 Err(error) => {
2320 crate::slog_warn!(
2321 "quarantining invalid background task metadata {task_id} during relaxed lookup: {error}"
2322 );
2323 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
2324 continue;
2325 }
2326 };
2327 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
2328 if metadata_project.as_deref() != Some(canonical_project.as_path()) {
2329 continue;
2330 }
2331 if let Some(task) = self.task(task_id) {
2332 let matches_project = task
2333 .state
2334 .lock()
2335 .map(|state| {
2336 state
2337 .metadata
2338 .project_root
2339 .as_deref()
2340 .map(canonicalized_path)
2341 .as_deref()
2342 == Some(canonical_project.as_path())
2343 })
2344 .unwrap_or(false);
2345 return matches_project.then_some(task);
2346 }
2347 if self
2348 .insert_rehydrated_task(metadata, resolved.paths, true)
2349 .is_err()
2350 {
2351 return None;
2352 }
2353 return self.task(task_id);
2354 }
2355 None
2356 }
2357
2358 fn lookup_relaxed_task_from_db(
2359 &self,
2360 task_id: &str,
2361 project_root: &Path,
2362 ) -> Option<Result<Option<PersistedTask>, String>> {
2363 let pool = self
2364 .inner
2365 .db_pool
2366 .read()
2367 .ok()
2368 .and_then(|slot| slot.clone())?;
2369 let harness = self
2370 .inner
2371 .db_harness
2372 .read()
2373 .ok()
2374 .and_then(|slot| slot.clone())?;
2375 let conn = match pool.lock() {
2376 Ok(conn) => conn,
2377 Err(_) => return Some(Err("db mutex poisoned".to_string())),
2378 };
2379 let project_key = crate::path_identity::project_scope_key(project_root);
2380 Some(
2381 crate::db::bash_tasks::find_bash_task_for_project(
2382 &conn,
2383 &harness,
2384 &project_key,
2385 task_id,
2386 )
2387 .map(|row| row.map(PersistedTask::from))
2388 .map_err(|error| error.to_string()),
2389 )
2390 }
2391
2392 pub(super) fn status_relaxed(
2393 &self,
2394 task_id: &str,
2395 _session_id: &str,
2396 project_root: &Path,
2397 storage_dir: &Path,
2398 preview_bytes: usize,
2399 ) -> Option<BgTaskSnapshot> {
2400 let task = self.status_relaxed_task(task_id, project_root, storage_dir)?;
2401 let _ = self.poll_task(&task);
2402 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
2403 }
2404
2405 pub fn kill_relaxed(
2406 &self,
2407 task_id: &str,
2408 project_root: &Path,
2409 storage_dir: &Path,
2410 ) -> Result<BgTaskSnapshot, String> {
2411 let task = self
2412 .status_relaxed_task(task_id, project_root, storage_dir)
2413 .ok_or_else(|| format!("background task not found: {task_id}"))?;
2414 self.kill_with_status(task_id, &task.session_id, BgTaskStatus::Killed)
2415 }
2416
2417 pub fn maybe_gc_persisted(&self, storage_dir: &Path) -> Result<usize, String> {
2418 #[cfg(test)]
2419 self.inner.persisted_gc_runs.fetch_add(1, Ordering::SeqCst);
2420
2421 let mut deleted = 0usize;
2422
2423 let root = storage_dir.join("bash-tasks");
2424 if root.exists() {
2425 let session_dirs = fs::read_dir(&root).map_err(|e| {
2426 format!(
2427 "failed to read background task root {}: {e}",
2428 root.display()
2429 )
2430 })?;
2431 for session_entry in session_dirs.flatten() {
2432 let session_dir = session_entry.path();
2433 if !session_dir.is_dir() {
2434 continue;
2435 }
2436 let (task_ids, invalid_entries) = match discover_task_ids(&session_dir) {
2437 Ok(discovery) => discovery,
2438 Err(error) => {
2439 crate::slog_warn!(
2440 "failed to discover background task session {}: {error}",
2441 session_dir.display()
2442 );
2443 continue;
2444 }
2445 };
2446 for entry in invalid_entries {
2447 let _ = quarantine_invalid_entry(storage_dir, &session_dir, &entry);
2448 }
2449 for task_id in task_ids {
2450 let resolved = match resolve_task_layout(&session_dir, &task_id) {
2451 Ok(task) => task,
2452 Err(error)
2453 if error.kind() == std::io::ErrorKind::NotFound
2454 && uninitialized_layout_is_recent(
2455 &session_dir,
2456 &task_id,
2457 Duration::from_secs(5 * 60),
2458 )
2459 .unwrap_or(false) =>
2460 {
2461 continue;
2462 }
2463 Err(error) => {
2464 crate::slog_warn!(
2465 "quarantining unresolved background task {task_id}: {error}"
2466 );
2467 quarantine_task_layout(storage_dir, &session_dir, &task_id, "invalid")
2468 .map_err(|error| error.to_string())?;
2469 continue;
2470 }
2471 };
2472 if modified_within(&resolved.paths.json, PERSISTED_GC_GRACE) {
2473 continue;
2474 }
2475 let metadata = match read_task_at(&resolved) {
2476 Ok(metadata) => metadata,
2477 Err(error) => {
2478 crate::slog_warn!(
2479 "quarantining corrupt background task metadata {task_id}: {error}"
2480 );
2481 quarantine_task_layout(storage_dir, &session_dir, &task_id, "corrupt")
2482 .map_err(|error| error.to_string())?;
2483 continue;
2484 }
2485 };
2486 if !(metadata.status.is_terminal() && metadata.completion_delivered) {
2487 continue;
2488 }
2489 match delete_task_bundle(&resolved.paths) {
2490 Ok(()) => {
2491 self.delete_gc_task_from_db(&metadata);
2492 deleted += 1;
2493 log::debug!(
2494 "deleted persisted background task bundle {}",
2495 metadata.task_id
2496 );
2497 }
2498 Err(error) => {
2499 crate::slog_warn!(
2500 "failed to delete background task bundle {}: {error}",
2501 metadata.task_id
2502 );
2503 }
2504 }
2505 }
2506 }
2507 }
2508 gc_quarantine(storage_dir);
2509 Ok(deleted)
2510 }
2511
2512 pub fn list(&self, preview_bytes: usize) -> Vec<BgTaskSnapshot> {
2513 let tasks = self
2514 .inner
2515 .tasks
2516 .lock()
2517 .map(|tasks| tasks.values().cloned().collect::<Vec<_>>())
2518 .unwrap_or_default();
2519 tasks
2520 .into_iter()
2521 .map(|task| {
2522 let _ = self.poll_task(&task);
2523 self.snapshot_with_terminal_cache(&task, preview_bytes)
2524 })
2525 .collect()
2526 }
2527
2528 fn maybe_compress_snapshot(&self, task: &Arc<BgTask>, snapshot: &mut BgTaskSnapshot) {
2534 if !snapshot.info.status.is_terminal() || snapshot.info.mode == BgMode::Pty {
2535 return;
2536 }
2537 if let Some(cache) = self.ensure_terminal_output_cache(task) {
2538 snapshot.output_preview = cache.output_preview;
2539 snapshot.output_truncated = cache.output_truncated;
2540 }
2541 }
2542
2543 pub fn kill(&self, task_id: &str, session_id: &str) -> Result<BgTaskSnapshot, String> {
2544 self.kill_with_status(task_id, session_id, BgTaskStatus::Killed)
2545 }
2546
2547 pub fn kill_running_tasks_for_root(&self, project_root: &Path) -> usize {
2557 let canonical_root = canonicalized_path(project_root);
2558 let targets = self
2559 .inner
2560 .tasks
2561 .lock()
2562 .map(|tasks| {
2563 tasks
2564 .values()
2565 .filter_map(|task| {
2566 let state = task.state.lock().ok()?;
2567 let status = &state.metadata.status;
2568 let running = matches!(status, BgTaskStatus::Running)
2569 || (state.metadata.mode == BgMode::Pty
2570 && matches!(status, BgTaskStatus::Killing));
2571 if !running {
2572 return None;
2573 }
2574 let task_root = state
2575 .metadata
2576 .project_root
2577 .as_deref()
2578 .unwrap_or(&state.metadata.workdir);
2579 (canonicalized_path(task_root) == canonical_root)
2580 .then(|| (task.task_id.clone(), task.session_id.clone()))
2581 })
2582 .collect::<Vec<_>>()
2583 })
2584 .unwrap_or_default();
2585
2586 let mut killed = 0;
2587 for (task_id, session_id) in targets {
2588 match self.kill_with_status_reason(
2589 &task_id,
2590 &session_id,
2591 BgTaskStatus::Killed,
2592 Some(ROOT_RECLAIMED_REASON.to_string()),
2593 ) {
2594 Ok(_) => killed += 1,
2595 Err(error) => crate::slog_warn!(
2596 "failed to terminate background task {task_id} for reclaimed root {}: {error}",
2597 project_root.display()
2598 ),
2599 }
2600 }
2601 killed
2602 }
2603
2604 pub fn promote(&self, task_id: &str, session_id: &str) -> Result<bool, String> {
2605 let task = self
2606 .task_for_session(task_id, session_id)
2607 .ok_or_else(|| format!("background task not found: {task_id}"))?;
2608 let terminal_after_promote = {
2609 let mut state = task
2610 .state
2611 .lock()
2612 .map_err(|_| "background task lock poisoned".to_string())?;
2613 let updated = self
2614 .update_task_metadata(&task.paths, |metadata| {
2615 metadata.notify_on_completion = true;
2616 metadata.completion_delivered = false;
2617 })
2618 .map_err(|e| format!("failed to promote background task: {e}"))?;
2619 state.metadata = updated;
2620 state.metadata.status.is_terminal()
2621 };
2622 if terminal_after_promote {
2623 self.post_terminal_transition(&task, true)?;
2624 }
2625 Ok(true)
2626 }
2627
2628 pub(crate) fn kill_for_timeout(&self, task_id: &str, session_id: &str) -> Result<(), String> {
2629 self.kill_with_status(task_id, session_id, BgTaskStatus::TimedOut)
2630 .map(|_| ())
2631 }
2632
2633 pub fn cleanup_finished(&self, older_than: Duration) {
2634 let cutoff = Instant::now().checked_sub(older_than);
2635 let removable_paths: Vec<(String, TaskPaths)> =
2636 if let Ok(mut tasks) = self.inner.tasks.lock() {
2637 let removable = tasks
2638 .iter()
2639 .filter_map(|(task_id, task)| {
2640 let delivered_terminal = task
2641 .state
2642 .lock()
2643 .map(|state| {
2644 state.metadata.status.is_terminal()
2645 && state.metadata.completion_delivered
2646 })
2647 .unwrap_or(false);
2648 if !delivered_terminal {
2649 return None;
2650 }
2651
2652 let terminal_at = task.terminal_at.lock().ok().and_then(|at| *at);
2653 let expired = match (terminal_at, cutoff) {
2654 (Some(terminal_at), Some(cutoff)) => terminal_at <= cutoff,
2655 (Some(_), None) => true,
2656 (None, _) => false,
2657 };
2658 expired.then(|| task_id.clone())
2659 })
2660 .collect::<Vec<_>>();
2661
2662 removable
2663 .into_iter()
2664 .filter_map(|task_id| {
2665 tasks
2666 .remove(&task_id)
2667 .map(|task| (task_id, task.paths.clone()))
2668 })
2669 .collect()
2670 } else {
2671 Vec::new()
2672 };
2673
2674 for (task_id, paths) in removable_paths {
2675 match delete_task_bundle(&paths) {
2676 Ok(()) => log::debug!("deleted persisted background task bundle {task_id}"),
2677 Err(error) => crate::slog_warn!(
2678 "failed to delete persisted background task bundle {task_id}: {error}"
2679 ),
2680 }
2681 }
2682 }
2683
2684 pub fn drain_completions(&self) -> Vec<BgCompletion> {
2685 self.drain_completions_for_session(None)
2686 }
2687
2688 pub fn drain_completions_for_session(&self, session_id: Option<&str>) -> Vec<BgCompletion> {
2689 let completions = match self.inner.completions.lock() {
2690 Ok(completions) => completions,
2691 Err(_) => return Vec::new(),
2692 };
2693
2694 completions
2695 .iter()
2696 .filter(|completion| completion_matches_session(completion, session_id))
2697 .cloned()
2698 .collect()
2699 }
2700
2701 pub fn has_completions_for_session(&self, session_id: Option<&str>) -> bool {
2702 match self.inner.completions.lock() {
2703 Ok(completions) => completions
2704 .iter()
2705 .any(|completion| completion_matches_session(completion, session_id)),
2706 Err(_) => true,
2710 }
2711 }
2712
2713 pub fn ack_completions_for_session(
2714 &self,
2715 session_id: Option<&str>,
2716 task_ids: &[String],
2717 ) -> Vec<String> {
2718 if task_ids.is_empty() {
2719 return Vec::new();
2720 }
2721 let requested_task_ids = task_ids.iter().map(String::as_str).collect::<HashSet<_>>();
2722 let mut completion_sessions = HashMap::new();
2723 if let Ok(mut completions) = self.inner.completions.lock() {
2724 completions.retain(|completion| {
2725 let session_matches = session_id
2726 .map(|session_id| completion.session_id == session_id)
2727 .unwrap_or(true);
2728 if session_matches && requested_task_ids.contains(completion.task_id.as_str()) {
2729 completion_sessions
2730 .insert(completion.task_id.clone(), completion.session_id.clone());
2731 false
2732 } else {
2733 true
2734 }
2735 });
2736 }
2737
2738 let mut delivered = Vec::new();
2739 for task_id in task_ids {
2740 let task = if let Some(session_id) = session_id {
2741 self.task_for_session(task_id, session_id)
2742 } else if let Some(completion_session_id) = completion_sessions.get(task_id) {
2743 self.task_for_session(task_id, completion_session_id)
2744 } else {
2745 self.task(task_id)
2746 };
2747 if let Some(task) = task {
2748 let terminal = task
2749 .state
2750 .lock()
2751 .map(|state| state.metadata.status.is_terminal())
2752 .unwrap_or(false);
2753 self.ack_persisted_watches_for_task(&task.session_id, task_id, terminal);
2757 if terminal {
2758 self.clear_task_watch_state(task_id);
2759 if task.set_completion_delivered(true, self).is_ok() {
2760 delivered.push(task_id.clone());
2761 }
2762 } else {
2763 self.sync_memory_watches_after_ack(task_id);
2766 delivered.push(task_id.clone());
2767 }
2768 } else if let Some(session_id) = session_id {
2769 self.ack_persisted_watches_for_task(session_id, task_id, true);
2771 delivered.push(task_id.clone());
2772 }
2773 }
2774
2775 delivered
2776 }
2777
2778 fn sync_memory_watches_after_ack(&self, task_id: &str) {
2779 let Some((harness, pool)) = self.db_harness_and_pool() else {
2780 return;
2781 };
2782 let session_id = match self.task(task_id) {
2783 Some(task) => task.session_id.clone(),
2784 None => return,
2785 };
2786 let Ok(conn) = pool.lock() else {
2787 return;
2788 };
2789 let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
2790 &conn,
2791 &harness,
2792 &session_id,
2793 task_id,
2794 ) else {
2795 return;
2796 };
2797 let remaining: HashSet<String> = rows.into_iter().map(|row| row.watch_id).collect();
2798 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2799 registry.retain_watch_ids(task_id, &remaining);
2800 }
2801 }
2802
2803 pub fn pending_completions_for_session(&self, session_id: &str) -> Vec<BgCompletion> {
2804 self.inner
2805 .completions
2806 .lock()
2807 .map(|completions| {
2808 completions
2809 .iter()
2810 .filter(|completion| completion.session_id == session_id)
2811 .cloned()
2812 .collect()
2813 })
2814 .unwrap_or_default()
2815 }
2816
2817 fn remove_pending_completion(&self, task_id: &str) -> Option<BgCompletion> {
2818 let mut completions = self.inner.completions.lock().ok()?;
2819 let idx = completions
2820 .iter()
2821 .position(|completion| completion.task_id == task_id)?;
2822 completions.remove(idx)
2823 }
2824
2825 fn completion_snapshot_for_task(&self, task: &Arc<BgTask>) -> Option<BgCompletion> {
2826 let snapshot = self.snapshot_with_terminal_cache(task, RUNNING_OUTPUT_PREVIEW_BYTES);
2827 if !snapshot.info.status.is_terminal() {
2828 return None;
2829 }
2830 let (output_preview, output_truncated) = if snapshot.info.mode == BgMode::Pty {
2831 (String::new(), false)
2832 } else {
2833 self.ensure_terminal_output_cache(task)
2834 .map(|cache| completion_preview_for_cache(&cache, snapshot.exit_code))
2835 .unwrap_or_else(|| (String::new(), false))
2836 };
2837 Some(BgCompletion {
2838 task_id: snapshot.info.task_id,
2839 session_id: task.session_id.clone(),
2840 status: snapshot.info.status,
2841 exit_code: snapshot.exit_code,
2842 command: snapshot.info.command,
2843 output_preview,
2844 output_truncated,
2845 original_tokens: None,
2846 compressed_tokens: None,
2847 tokens_skipped: false,
2848 status_reason: snapshot.info.status_reason,
2849 })
2850 }
2851
2852 pub fn detach(&self) {
2853 self.inner.shutdown.store(true, Ordering::SeqCst);
2854 if let Ok(mut tasks) = self.inner.tasks.lock() {
2855 for task in tasks.values() {
2856 if let Ok(mut state) = task.state.lock() {
2857 match &mut state.runtime {
2858 TaskRuntime::Piped(child) => *child = None,
2859 TaskRuntime::Pty(runtime) => *runtime = None,
2860 }
2861 state.detached = true;
2862 }
2863 }
2864 tasks.clear();
2865 }
2866 }
2867
2868 pub fn shutdown(&self) {
2869 let tasks = self
2870 .inner
2871 .tasks
2872 .lock()
2873 .map(|tasks| {
2874 tasks
2875 .values()
2876 .map(|task| (task.task_id.clone(), task.session_id.clone()))
2877 .collect::<Vec<_>>()
2878 })
2879 .unwrap_or_default();
2880 for (task_id, session_id) in tasks {
2881 let _ = self.kill(&task_id, &session_id);
2882 }
2883 }
2884
2885 pub(crate) fn poll_task(&self, task: &Arc<BgTask>) -> Result<(), String> {
2886 if let Ok(state) = task.state.lock() {
2887 if let TaskRuntime::Pty(Some(pty)) = &state.runtime {
2888 if !pty.exit_observed.load(Ordering::SeqCst) {
2896 return Ok(());
2897 }
2898 }
2899 }
2900 let marker = match read_exit_marker(&task.paths) {
2901 Ok(Some(marker)) => marker,
2902 Ok(None) => return Ok(()),
2903 Err(error) => return Err(format!("failed to read exit marker: {error}")),
2904 };
2905 self.finalize_from_marker(task, marker, None)
2906 }
2907
2908 pub(crate) fn reap_child(&self, task: &Arc<BgTask>) {
2909 let mut needs_completion = false;
2910 {
2911 let Ok(mut state) = task.state.lock() else {
2912 return;
2913 };
2914 match &mut state.runtime {
2915 TaskRuntime::Piped(child_slot) => {
2916 if let Some(child) = child_slot.as_mut() {
2917 if let Ok(Some(status)) = child.try_wait() {
2918 *child_slot = None;
2919 state.detached = true;
2920 state.child_exit_observed = true;
2921 if let Some(handles) = state.io_handles.as_mut() {
2922 if handles.artifact_len(TaskArtifact::Exit).unwrap_or(1) == 0 {
2923 let marker = status
2924 .code()
2925 .map(|code| code.to_string())
2926 .unwrap_or_else(|| "1".to_string());
2927 let _ = handles.write(TaskArtifact::Exit, marker.as_bytes());
2928 }
2929 }
2930 }
2931 } else if state.detached {
2932 let child_known_dead = state.child_exit_observed
2933 || state
2934 .metadata
2935 .child_pid
2936 .is_some_and(|pid| !is_process_alive(pid));
2937 if child_known_dead {
2938 needs_completion =
2939 self.fail_without_exit_marker_if_needed(task, &mut state);
2940 }
2941 }
2942 }
2943 TaskRuntime::Pty(Some(pty)) => {
2944 if pty.exit_observed.load(Ordering::SeqCst) {
2945 drop(state);
2946 let _ = self.poll_task(task);
2947 return;
2948 }
2949 }
2950 TaskRuntime::Pty(None) => {}
2951 }
2952 }
2953 if needs_completion {
2954 let _ = self.post_terminal_transition(task, true);
2955 }
2956 }
2957
2958 fn fail_without_exit_marker_if_needed(
2959 &self,
2960 task: &Arc<BgTask>,
2961 state: &mut BgTaskState,
2962 ) -> bool {
2963 if state.metadata.status.is_terminal() {
2964 return false;
2965 }
2966 if matches!(read_exit_marker(&task.paths), Ok(Some(_))) {
2967 return false;
2968 }
2969 let watch_controlled = self.task_has_watch_control(&task.task_id);
2970 let updated = self.update_task_metadata(&task.paths, |metadata| {
2971 metadata.mark_terminal(
2972 BgTaskStatus::Failed,
2973 None,
2974 Some("process exited without exit marker".to_string()),
2975 );
2976 if watch_controlled {
2977 metadata.completion_delivered = true;
2978 }
2979 });
2980 if let Ok(metadata) = updated {
2981 state.pending_terminal_override = None;
2982 state.metadata = metadata;
2983 task.mark_terminal_now();
2984 return true;
2985 }
2986 false
2987 }
2988
2989 pub(crate) fn running_tasks(&self) -> Vec<Arc<BgTask>> {
2990 self.inner
2991 .tasks
2992 .lock()
2993 .map(|tasks| {
2994 tasks
2995 .values()
2996 .filter(|task| task.is_running())
2997 .cloned()
2998 .collect()
2999 })
3000 .unwrap_or_default()
3001 }
3002
3003 fn insert_rehydrated_task(
3004 &self,
3005 metadata: PersistedTask,
3006 paths: TaskPaths,
3007 detached: bool,
3008 ) -> Result<(), String> {
3009 let task_id = metadata.task_id.clone();
3010 let session_id = metadata.session_id.clone();
3011 let started = started_instant_from_unix_millis(metadata.started_at);
3012 let suppress_replayed_running_reminder = metadata.status == BgTaskStatus::Running;
3013 let mode = metadata.mode.clone();
3014 let task = Arc::new(BgTask {
3015 task_id: task_id.clone(),
3016 session_id,
3017 paths: paths.clone(),
3018 artifact_root: canonical_artifact_root(&paths),
3019 started,
3020 last_reminder_at: Mutex::new(suppress_replayed_running_reminder.then(Instant::now)),
3021 terminal_at: Mutex::new(metadata.status.is_terminal().then(Instant::now)),
3022 state: Mutex::new(BgTaskState {
3023 metadata,
3024 runtime: if mode == BgMode::Pty {
3025 TaskRuntime::Pty(None)
3026 } else {
3027 TaskRuntime::Piped(None)
3028 },
3029 io_handles: None,
3030 detached,
3031 child_exit_observed: false,
3038 buffer: BgBuffer::registered(&paths, mode.clone()),
3039 terminal_output_cache: None,
3040 pending_terminal_override: None,
3041 }),
3042 });
3043 self.inner
3044 .tasks
3045 .lock()
3046 .map_err(|_| "background task registry lock poisoned".to_string())?
3047 .insert(task_id.clone(), Arc::clone(&task));
3048 self.rearm_persisted_watches(&task);
3052 Ok(())
3053 }
3054
3055 fn rearm_persisted_watches(&self, task: &Arc<BgTask>) {
3056 let Some((harness, pool)) = self.db_harness_and_pool() else {
3057 return;
3058 };
3059 let rows = {
3060 let Ok(conn) = pool.lock() else {
3061 return;
3062 };
3063 match crate::db::bash_watches::list_bash_pattern_watches_for_task(
3064 &conn,
3065 &harness,
3066 &task.session_id,
3067 &task.task_id,
3068 ) {
3069 Ok(rows) if !rows.is_empty() => rows,
3070 _ => return,
3071 }
3072 };
3073
3074 let mode = match task.state.lock() {
3075 Ok(state) => state.metadata.mode.clone(),
3076 Err(_) => return,
3077 };
3078 let terminal = task
3079 .state
3080 .lock()
3081 .map(|state| state.metadata.status.is_terminal())
3082 .unwrap_or(false);
3083 let completion_delivered = task
3084 .state
3085 .lock()
3086 .map(|state| state.metadata.completion_delivered)
3087 .unwrap_or(true);
3088
3089 let mut stdout = (mode == BgMode::Pipes)
3090 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
3091 .transpose()
3092 .ok()
3093 .flatten();
3094 let mut stderr = (mode == BgMode::Pipes)
3095 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
3096 .transpose()
3097 .ok()
3098 .flatten();
3099 let mut pty = (mode == BgMode::Pty)
3100 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
3101 .transpose()
3102 .ok()
3103 .flatten();
3104
3105 let mut pending_to_emit = Vec::new();
3106 let mut gap_matches = Vec::new();
3107 {
3108 let Ok(mut registry) = self.inner.watch_registry.lock() else {
3109 return;
3110 };
3111 let stdout_key = format!("{}:stdout", task.task_id);
3112 let stderr_key = format!("{}:stderr", task.task_id);
3113 let pty_key = format!("{}:pty", task.task_id);
3114
3115 let first = &rows[0];
3117 match mode {
3118 BgMode::Pipes => {
3119 registry.set_file_cursor(&stdout_key, first.stdout_offset.max(0) as u64);
3120 registry.set_file_cursor(&stderr_key, first.stderr_offset.max(0) as u64);
3121 }
3122 BgMode::Pty => {
3123 registry.set_file_cursor(&pty_key, first.pty_offset.max(0) as u64);
3124 }
3125 }
3126
3127 for row in &rows {
3128 let Ok(pattern) = WatchPattern::from_persisted(&row.pattern_kind, &row.pattern)
3129 else {
3130 crate::slog_warn!(
3131 "skipping unreadable persisted watch {}/{}",
3132 row.task_id,
3133 row.watch_id
3134 );
3135 continue;
3136 };
3137 if let Err(error) = registry.restore(
3138 row.watch_id.clone(),
3139 row.task_id.clone(),
3140 pattern,
3141 row.once,
3142 row.scanning,
3143 ) {
3144 crate::slog_warn!(
3145 "failed to restore watch {}/{}: {error}",
3146 row.task_id,
3147 row.watch_id
3148 );
3149 continue;
3150 }
3151 if row.pending_match {
3152 if let (Some(match_text), Some(match_offset), Some(context)) = (
3153 row.match_text.clone(),
3154 row.match_offset,
3155 row.match_context.clone(),
3156 ) {
3157 pending_to_emit.push(PatternMatch {
3158 watch_id: row.watch_id.clone(),
3159 task_id: row.task_id.clone(),
3160 match_text,
3161 match_offset: match_offset.max(0) as u64,
3162 context,
3163 once: row.once,
3164 });
3165 }
3166 }
3167 }
3168
3169 let should_gap_scan =
3173 rows.iter().any(|row| row.scanning) && !pending_to_emit.iter().any(|m| m.once);
3174 if should_gap_scan {
3175 match mode {
3176 BgMode::Pipes => {
3177 if let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) {
3178 gap_matches.extend(registry.scan_file_new_bytes(
3179 &stdout_key,
3180 &task.task_id,
3181 stdout,
3182 ));
3183 gap_matches.extend(registry.scan_file_new_bytes(
3184 &stderr_key,
3185 &task.task_id,
3186 stderr,
3187 ));
3188 }
3189 }
3190 BgMode::Pty => {
3191 if let Some(pty) = pty.as_mut() {
3192 gap_matches.extend(registry.scan_file_new_bytes(
3193 &pty_key,
3194 &task.task_id,
3195 pty,
3196 ));
3197 }
3198 }
3199 }
3200 }
3201 }
3202
3203 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
3204 for pattern_match in &gap_matches {
3205 self.persist_watch_match(
3206 &task.session_id,
3207 &task.task_id,
3208 pattern_match,
3209 stdout_offset,
3210 stderr_offset,
3211 pty_offset,
3212 );
3213 }
3214 if !gap_matches.is_empty() || rows.iter().any(|row| row.scanning) {
3215 self.persist_task_watch_cursors(
3216 &task.session_id,
3217 &task.task_id,
3218 stdout_offset,
3219 stderr_offset,
3220 pty_offset,
3221 );
3222 }
3223
3224 let emitted_pending = !pending_to_emit.is_empty();
3226 let to_emit = if emitted_pending {
3227 pending_to_emit
3228 } else {
3229 gap_matches
3230 };
3231 let emitted_once_ids: Vec<String> = to_emit
3232 .iter()
3233 .filter(|m| m.once)
3234 .map(|m| m.watch_id.clone())
3235 .collect();
3236 for pattern_match in to_emit {
3237 self.emit_bash_pattern_match(&task.session_id, pattern_match);
3238 }
3239 if emitted_pending {
3243 for watch_id in emitted_once_ids {
3244 self.delete_persisted_watch(&task.session_id, &task.task_id, &watch_id);
3245 }
3246 }
3247
3248 if !terminal {
3249 return;
3250 }
3251
3252 let _ = self.remove_pending_completion(&task.task_id);
3255 let (watch_controlled, watch_matched) = self.task_watch_state(&task.task_id);
3256 if !watch_controlled {
3257 return;
3258 }
3259 if watch_matched {
3260 return;
3262 }
3263 if completion_delivered {
3264 self.clear_task_watch_state(&task.task_id);
3266 self.delete_persisted_watches_for_task(&task.session_id, &task.task_id);
3267 return;
3268 }
3269 if let Some(completion) = self.completion_snapshot_for_task(task) {
3270 self.emit_bash_watch_exit(&completion);
3271 }
3272 self.clear_task_watch_state(&task.task_id);
3274 }
3275
3276 fn kill_with_status(
3277 &self,
3278 task_id: &str,
3279 session_id: &str,
3280 terminal_status: BgTaskStatus,
3281 ) -> Result<BgTaskSnapshot, String> {
3282 self.kill_with_status_reason(task_id, session_id, terminal_status, None)
3283 }
3284
3285 fn kill_with_status_reason(
3286 &self,
3287 task_id: &str,
3288 session_id: &str,
3289 terminal_status: BgTaskStatus,
3290 reason: Option<String>,
3291 ) -> Result<BgTaskSnapshot, String> {
3292 let task = self
3293 .task_for_session(task_id, session_id)
3294 .ok_or_else(|| format!("background task not found: {task_id}"))?;
3295 let mut terminalized = false;
3296
3297 {
3298 let mut state = task
3299 .state
3300 .lock()
3301 .map_err(|_| "background task lock poisoned".to_string())?;
3302 if state.metadata.status.is_terminal() {
3303 state.pending_terminal_override = None;
3304 } else if let Ok(Some(marker)) = read_exit_marker(&task.paths) {
3305 state.metadata =
3306 terminal_metadata_from_marker(state.metadata.clone(), marker, reason.clone());
3307 if self.task_has_watch_control(&task.task_id) {
3308 state.metadata.completion_delivered = true;
3309 }
3310 state.pending_terminal_override = None;
3311 task.mark_terminal_now();
3312 match &mut state.runtime {
3313 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
3320 TaskRuntime::Pty(runtime) => *runtime = None,
3321 }
3322 state.detached = true;
3323 self.persist_task(&task.paths, &state.metadata)
3324 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
3325 terminalized = true;
3326 } else {
3327 let was_already_killing = state.metadata.status == BgTaskStatus::Killing;
3328 if !was_already_killing {
3329 state.metadata.status = BgTaskStatus::Killing;
3330 }
3331 if reason.is_some() {
3332 state.metadata.status_reason = reason.clone();
3333 }
3334 if !was_already_killing || reason.is_some() {
3335 self.persist_task(&task.paths, &state.metadata)
3336 .map_err(|e| format!("failed to persist killing state: {e}"))?;
3337 }
3338
3339 #[cfg(unix)]
3340 let pgid = state.metadata.pgid;
3341 #[cfg(windows)]
3342 let child_pid = state.metadata.child_pid;
3343 if !was_already_killing
3344 && state.metadata.mode == BgMode::Pty
3345 && terminal_status == BgTaskStatus::TimedOut
3346 {
3347 state.pending_terminal_override = Some(BgTaskStatus::TimedOut);
3348 }
3349
3350 #[cfg(windows)]
3351 let mut pty_forced_terminal_status: Option<BgTaskStatus> = None;
3352
3353 match &mut state.runtime {
3354 TaskRuntime::Piped(child_slot) => {
3355 #[cfg(unix)]
3356 if let Some(pgid) = pgid {
3357 terminate_pgid(pgid, child_slot.as_mut());
3358 }
3359 #[cfg(windows)]
3360 if let Some(child) = child_slot.as_mut() {
3361 super::process::terminate_process(child);
3362 } else if let Some(pid) = child_pid {
3363 terminate_pid(pid);
3364 }
3365 if let Some(child) = child_slot.as_mut() {
3366 let _ = child.wait();
3367 }
3368 *child_slot = None;
3369 state.detached = true;
3370
3371 if let Some(handles) = state.io_handles.as_mut() {
3372 handles.write(TaskArtifact::Exit, b"killed").map_err(|e| {
3373 format!("failed to write retained kill marker: {e}")
3374 })?;
3375 } else {
3376 write_kill_marker_if_absent(&task.paths)
3377 .map_err(|e| format!("failed to write kill marker: {e}"))?;
3378 }
3379
3380 let exit_code = terminal_exit_code_for_status(&terminal_status);
3381 state
3382 .metadata
3383 .mark_terminal(terminal_status, exit_code, reason.clone());
3384 if self.task_has_watch_control(&task.task_id) {
3385 state.metadata.completion_delivered = true;
3386 }
3387 state.pending_terminal_override = None;
3388 task.mark_terminal_now();
3389 self.persist_task(&task.paths, &state.metadata)
3390 .map_err(|e| format!("failed to persist killed state: {e}"))?;
3391 terminalized = true;
3392 }
3393 TaskRuntime::Pty(Some(pty)) => {
3394 pty.was_killed.store(true, Ordering::SeqCst);
3395 if let Err(error) = pty.killer.kill() {
3396 crate::slog_warn!(
3397 "[pty-kill] {task_id} ChildKiller::kill failed: {error}"
3398 );
3399 }
3400 if let Some(pid) = pty.child_pid {
3401 #[cfg(unix)]
3402 terminate_pgid(pid as i32, None);
3403 #[cfg(windows)]
3404 terminate_pid(pid);
3405 }
3406 drop(pty.master.take());
3407
3408 #[cfg(windows)]
3409 {
3410 let default_status = if terminal_status == BgTaskStatus::TimedOut {
3411 BgTaskStatus::TimedOut
3412 } else {
3413 BgTaskStatus::Killed
3414 };
3415 pty_forced_terminal_status = Some(
3416 state
3417 .pending_terminal_override
3418 .take()
3419 .unwrap_or(default_status),
3420 );
3421 }
3422 }
3423 TaskRuntime::Pty(None) => {}
3424 }
3425
3426 #[cfg(windows)]
3427 if let Some(target_status) = pty_forced_terminal_status {
3428 if !task.paths.exit.exists() {
3429 write_kill_marker_if_absent(&task.paths)
3430 .map_err(|e| format!("failed to write kill marker: {e}"))?;
3431 }
3432
3433 let exit_code = terminal_exit_code_for_status(&target_status);
3434 state
3435 .metadata
3436 .mark_terminal(target_status, exit_code, reason.clone());
3437 if self.task_has_watch_control(&task.task_id) {
3438 state.metadata.completion_delivered = true;
3439 }
3440 state.pending_terminal_override = None;
3441 task.mark_terminal_now();
3442 if let TaskRuntime::Pty(runtime) = &mut state.runtime {
3443 *runtime = None;
3444 }
3445 state.detached = true;
3446 self.persist_task(&task.paths, &state.metadata)
3447 .map_err(|e| format!("failed to persist killed PTY state: {e}"))?;
3448 terminalized = true;
3449 }
3450 }
3451 }
3452
3453 if terminalized {
3454 self.post_terminal_transition(&task, true)?;
3455 }
3456 Ok(self.snapshot_with_terminal_cache(&task, RUNNING_OUTPUT_PREVIEW_BYTES))
3457 }
3458
3459 fn finalize_from_marker(
3460 &self,
3461 task: &Arc<BgTask>,
3462 marker: ExitMarker,
3463 reason: Option<String>,
3464 ) -> Result<(), String> {
3465 let watch_controlled = self.task_has_watch_control(&task.task_id);
3466 let mut pty_reader_done = None;
3467 {
3468 let mut state = task
3469 .state
3470 .lock()
3471 .map_err(|_| "background task lock poisoned".to_string())?;
3472 if state.metadata.status.is_terminal() {
3473 state.pending_terminal_override = None;
3474 return Ok(());
3475 }
3476
3477 let pending_override = state.pending_terminal_override.take();
3478 let is_pty = state.metadata.mode == BgMode::Pty;
3479 let reason = reason.or_else(|| state.metadata.status_reason.clone());
3480 let updated = self
3481 .update_task_metadata(&task.paths, |metadata| {
3482 let mut new_metadata = if is_pty && marker == ExitMarker::Killed {
3483 let mut metadata = metadata.clone();
3484 let target_status = pending_override.unwrap_or(BgTaskStatus::Killed);
3485 let exit_code = terminal_exit_code_for_status(&target_status);
3486 metadata.mark_terminal(target_status, exit_code, reason.clone());
3487 metadata
3488 } else {
3489 terminal_metadata_from_marker(metadata.clone(), marker, reason.clone())
3490 };
3491 if watch_controlled {
3492 new_metadata.completion_delivered = true;
3493 }
3494 *metadata = new_metadata;
3495 })
3496 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
3497 state.metadata = updated;
3498 task.mark_terminal_now();
3499 match &mut state.runtime {
3500 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
3505 TaskRuntime::Pty(runtime) => {
3506 pty_reader_done = runtime
3507 .as_ref()
3508 .map(|runtime| Arc::clone(&runtime.reader_done));
3509 *runtime = None;
3510 }
3511 }
3512 state.detached = true;
3513 }
3514
3515 if let Some(reader_done) = pty_reader_done {
3516 let deadline = Instant::now() + Duration::from_millis(200);
3517 while !reader_done.load(Ordering::SeqCst) && Instant::now() < deadline {
3518 std::thread::sleep(Duration::from_millis(10));
3519 }
3520 }
3521
3522 self.scan_task_watch_output(task);
3525
3526 self.post_terminal_transition(task, true)
3527 }
3528
3529 fn enqueue_completion_if_needed(
3530 &self,
3531 metadata: &PersistedTask,
3532 paths: Option<&TaskPaths>,
3533 emit_frame: bool,
3534 ) {
3535 if metadata.status.is_terminal() && !metadata.completion_delivered {
3536 let cache =
3537 paths.and_then(|paths| self.render_terminal_output_from_paths(metadata, paths));
3538 self.enqueue_completion_from_parts(metadata, None, paths, emit_frame, cache.as_ref());
3539 }
3540 }
3541
3542 fn render_terminal_output_from_paths(
3543 &self,
3544 metadata: &PersistedTask,
3545 paths: &TaskPaths,
3546 ) -> Option<TerminalOutputCache> {
3547 if metadata.mode == BgMode::Pty {
3548 return None;
3549 }
3550 let mut buffer = BgBuffer::registered(paths, BgMode::Pipes);
3551 let disk_truncation = buffer.enforce_terminal_cap();
3552 Some(self.render_terminal_output(metadata, &buffer, disk_truncation, Some(paths)))
3553 }
3554
3555 fn enqueue_completion_from_parts(
3556 &self,
3557 metadata: &PersistedTask,
3558 buffer: Option<&BgBuffer>,
3559 paths: Option<&TaskPaths>,
3560 emit_frame: bool,
3561 terminal_render: Option<&TerminalOutputCache>,
3562 ) {
3563 if !metadata.status.is_terminal() {
3574 return;
3575 }
3576
3577 let owned_buffer = if buffer.is_none() && metadata.mode != BgMode::Pty {
3578 paths.map(|paths| BgBuffer::registered(paths, BgMode::Pipes))
3579 } else {
3580 None
3581 };
3582 let render_buffer = buffer.or(owned_buffer.as_ref());
3583 let owned_render = if terminal_render.is_none() {
3584 render_buffer.map(|buffer| {
3585 let mut capped_buffer = buffer.clone();
3586 let disk_truncation = capped_buffer.enforce_terminal_cap();
3587 self.render_terminal_output(metadata, &capped_buffer, disk_truncation, paths)
3588 })
3589 } else {
3590 None
3591 };
3592 let render = terminal_render.or(owned_render.as_ref());
3593
3594 let (output_preview, output_truncated) = render
3598 .map(|cache| completion_preview_for_cache(cache, metadata.exit_code))
3599 .unwrap_or_else(|| (String::new(), false));
3600
3601 let token_counts = self.completion_token_counts(
3602 metadata,
3603 buffer,
3604 paths,
3605 render.map(|render| render.output_preview.as_str()),
3606 );
3607 let completion = BgCompletion {
3608 task_id: metadata.task_id.clone(),
3609 session_id: metadata.session_id.clone(),
3610 status: metadata.status.clone(),
3611 exit_code: metadata.exit_code,
3612 command: metadata.command.clone(),
3613 output_preview,
3614 output_truncated,
3615 original_tokens: token_counts.original_tokens,
3616 compressed_tokens: token_counts.compressed_tokens,
3617 tokens_skipped: token_counts.tokens_skipped,
3618 status_reason: metadata.status_reason.clone(),
3619 };
3620
3621 self.record_compression_event_if_applicable(metadata, &token_counts);
3632
3633 let (watch_controlled, watch_matched) = self.task_watch_state(&metadata.task_id);
3634 if watch_controlled {
3635 if emit_frame && !watch_matched {
3636 self.emit_bash_watch_exit(&completion);
3637 } else if watch_matched {
3638 if let Some(task) = self.task(&metadata.task_id) {
3643 let _ = task.set_completion_delivered(true, self);
3644 }
3645 }
3646 self.clear_task_watch_state(&metadata.task_id);
3648 return;
3649 }
3650
3651 if metadata.completion_delivered {
3661 return;
3662 }
3663
3664 let pushed = if let Ok(mut completions) = self.inner.completions.lock() {
3667 if completions
3668 .iter()
3669 .any(|existing| existing.task_id == metadata.task_id)
3670 {
3671 false
3672 } else {
3673 completions.push_back(completion.clone());
3674 true
3675 }
3676 } else {
3677 false
3678 };
3679
3680 if pushed && emit_frame {
3681 self.emit_bash_completed(completion);
3682 }
3683 }
3684
3685 fn record_compression_event_if_applicable(
3686 &self,
3687 metadata: &PersistedTask,
3688 token_counts: &CompletionTokenCounts,
3689 ) {
3690 if metadata.mode == BgMode::Pty {
3691 return;
3692 }
3693
3694 let (original_tokens, compressed_tokens, original_bytes, compressed_bytes) = match (
3695 token_counts.original_tokens,
3696 token_counts.compressed_tokens,
3697 token_counts.original_bytes,
3698 token_counts.compressed_bytes,
3699 ) {
3700 (
3701 Some(original_tokens),
3702 Some(compressed_tokens),
3703 Some(original_bytes),
3704 Some(compressed_bytes),
3705 ) => (
3706 original_tokens,
3707 compressed_tokens,
3708 original_bytes,
3709 compressed_bytes,
3710 ),
3711 _ => {
3712 crate::slog_warn!(
3713 "compression event skipped for {}: token counts unavailable (likely spill file missing or unreadable)",
3714 metadata.task_id
3715 );
3716 return;
3717 }
3718 };
3719
3720 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
3721 let Some(pool) = pool else {
3722 crate::slog_warn!(
3723 "compression event skipped for {}: db_pool not initialized — was configure run?",
3724 metadata.task_id
3725 );
3726 return;
3727 };
3728 let harness = self
3729 .inner
3730 .db_harness
3731 .read()
3732 .ok()
3733 .and_then(|slot| slot.clone());
3734 let Some(harness) = harness else {
3735 crate::slog_warn!(
3736 "compression event insert skipped for {}: harness not configured",
3737 metadata.task_id
3738 );
3739 return;
3740 };
3741
3742 let project_root = metadata
3743 .project_root
3744 .as_deref()
3745 .unwrap_or(&metadata.workdir);
3746 let project_key = crate::path_identity::project_scope_key(project_root);
3747 let row = crate::db::compression_events::CompressionEventRow {
3748 harness: &harness,
3749 session_id: Some(&metadata.session_id),
3750 project_key: &project_key,
3751 tool: "bash",
3752 task_id: Some(&metadata.task_id),
3753 command: Some(&metadata.command),
3754 compressor: if metadata.compressed {
3755 "registry"
3756 } else {
3757 "none"
3758 },
3759 original_bytes,
3760 compressed_bytes,
3761 original_tokens,
3762 compressed_tokens,
3763 created_at: unix_millis() as i64,
3764 };
3765
3766 let conn = match pool.lock() {
3767 Ok(conn) => conn,
3768 Err(_) => {
3769 crate::slog_warn!(
3770 "compression event insert failed for {}: db mutex poisoned",
3771 metadata.task_id
3772 );
3773 return;
3774 }
3775 };
3776 match crate::db::compression_events::insert_compression_event(&conn, &row) {
3777 Ok(Some(row_id)) => {
3778 self.inner
3782 .compression_aggregates
3783 .record_successful_insert(&conn, &row, row_id);
3784 crate::slog_debug!(
3788 "compression event recorded for {} (project={}, session={}, {} → {} tokens)",
3789 metadata.task_id,
3790 project_key,
3791 metadata.session_id,
3792 original_tokens,
3793 compressed_tokens
3794 );
3795 }
3796 Ok(None) => {
3797 crate::slog_debug!(
3798 "duplicate compression event ignored for {} (project={}, session={})",
3799 metadata.task_id,
3800 project_key,
3801 metadata.session_id
3802 );
3803 }
3804 Err(error) => {
3805 crate::slog_warn!(
3806 "compression event insert failed for {}: {}",
3807 metadata.task_id,
3808 error
3809 );
3810 }
3811 }
3812 }
3813
3814 fn emit_bash_pattern_match(&self, session_id: &str, pattern_match: PatternMatch) {
3815 let Ok(progress_sender) = self
3816 .inner
3817 .progress_sender
3818 .lock()
3819 .map(|sender| sender.clone())
3820 else {
3821 return;
3822 };
3823 if let Some(sender) = progress_sender.as_ref() {
3824 sender(PushFrame::BashPatternMatch(BashPatternMatchFrame::new(
3825 pattern_match.task_id,
3826 session_id.to_string(),
3827 pattern_match.watch_id,
3828 pattern_match.match_text,
3829 pattern_match.match_offset,
3830 pattern_match.context,
3831 pattern_match.once,
3832 )));
3833 }
3834 }
3835
3836 fn emit_bash_watch_exit(&self, completion: &BgCompletion) {
3837 let Ok(progress_sender) = self
3838 .inner
3839 .progress_sender
3840 .lock()
3841 .map(|sender| sender.clone())
3842 else {
3843 return;
3844 };
3845 let Some(sender) = progress_sender.as_ref() else {
3846 return;
3847 };
3848 let status = completion_status_text(&completion.status, completion.exit_code);
3849 let preview = completion.output_preview.trim_end();
3850 let context = if preview.is_empty() {
3851 format!("task {} exited ({status})", completion.task_id)
3852 } else {
3853 format!(
3854 "task {} exited ({status})
3855{preview}",
3856 completion.task_id
3857 )
3858 };
3859 sender(PushFrame::BashPatternMatch(
3860 BashPatternMatchFrame::task_exit(
3861 completion.task_id.clone(),
3862 completion.session_id.clone(),
3863 format!("exited ({status})"),
3864 context,
3865 ),
3866 ));
3867 }
3868
3869 fn emit_bash_completed(&self, completion: BgCompletion) {
3870 let Ok(progress_sender) = self
3871 .inner
3872 .progress_sender
3873 .lock()
3874 .map(|sender| sender.clone())
3875 else {
3876 return;
3877 };
3878 let Some(sender) = progress_sender.as_ref() else {
3879 return;
3880 };
3881 let mut frame = BashCompletedFrame::new(
3889 completion.task_id,
3890 completion.session_id,
3891 completion.status,
3892 completion.exit_code,
3893 completion.command,
3894 completion.output_preview,
3895 completion.output_truncated,
3896 completion.original_tokens,
3897 completion.compressed_tokens,
3898 completion.tokens_skipped,
3899 );
3900 frame.status_reason = completion.status_reason;
3901 sender(PushFrame::BashCompleted(frame));
3902 }
3903
3904 fn completion_token_counts(
3905 &self,
3906 metadata: &PersistedTask,
3907 buffer: Option<&BgBuffer>,
3908 paths: Option<&TaskPaths>,
3909 rendered_output: Option<&str>,
3910 ) -> CompletionTokenCounts {
3911 if metadata.mode == BgMode::Pty {
3912 return CompletionTokenCounts::skipped();
3913 }
3914
3915 let raw = match buffer {
3916 Some(buffer) => buffer.read_for_token_count(TOKENIZE_CAP_BYTES_PER_STREAM),
3917 None => paths
3918 .map(|paths| {
3919 read_for_token_count_from_disk(metadata, paths, TOKENIZE_CAP_BYTES_PER_STREAM)
3920 })
3921 .unwrap_or(TokenCountInput::Skipped),
3922 };
3923
3924 let TokenCountInput::Text(raw_output) = raw else {
3925 return CompletionTokenCounts::skipped();
3926 };
3927
3928 let original_tokens = token_count_u32(&raw_output);
3929 let original_bytes = raw_output.len() as i64;
3930 let compressed_output = rendered_output.unwrap_or(&raw_output);
3931 let compressed_tokens = token_count_u32(compressed_output);
3932 let compressed_bytes = compressed_output.len() as i64;
3933 CompletionTokenCounts {
3934 original_tokens: Some(original_tokens),
3935 compressed_tokens: Some(compressed_tokens),
3936 original_bytes: Some(original_bytes),
3937 compressed_bytes: Some(compressed_bytes),
3938 tokens_skipped: false,
3939 }
3940 }
3941
3942 pub(crate) fn maybe_emit_long_running_reminder(&self, task: &Arc<BgTask>) {
3943 if !self
3944 .inner
3945 .long_running_reminder_enabled
3946 .load(Ordering::SeqCst)
3947 {
3948 return;
3949 }
3950 let interval_ms = self
3951 .inner
3952 .long_running_reminder_interval_ms
3953 .load(Ordering::SeqCst);
3954 if interval_ms == 0 {
3955 return;
3956 }
3957 let interval = Duration::from_millis(interval_ms);
3958 let now = Instant::now();
3959 let Ok(mut last_reminder_at) = task.last_reminder_at.lock() else {
3960 return;
3961 };
3962 let since = last_reminder_at.unwrap_or(task.started);
3963 if now.duration_since(since) < interval {
3964 return;
3965 }
3966 let command = task
3967 .state
3968 .lock()
3969 .map(|state| state.metadata.command.clone())
3970 .unwrap_or_default();
3971 *last_reminder_at = Some(now);
3972 self.emit_bash_long_running(BashLongRunningFrame::new(
3973 task.task_id.clone(),
3974 task.session_id.clone(),
3975 command,
3976 task.started.elapsed().as_millis() as u64,
3977 ));
3978 }
3979
3980 fn emit_bash_long_running(&self, frame: BashLongRunningFrame) {
3981 let Ok(progress_sender) = self
3982 .inner
3983 .progress_sender
3984 .lock()
3985 .map(|sender| sender.clone())
3986 else {
3987 return;
3988 };
3989 if let Some(sender) = progress_sender.as_ref() {
3990 sender(PushFrame::BashLongRunning(frame));
3991 }
3992 }
3993
3994 fn task(&self, task_id: &str) -> Option<Arc<BgTask>> {
3995 validate_task_id(task_id).ok()?;
3996 self.inner
3997 .tasks
3998 .lock()
3999 .ok()
4000 .and_then(|tasks| tasks.get(task_id).cloned())
4001 }
4002
4003 fn task_for_session(&self, task_id: &str, session_id: &str) -> Option<Arc<BgTask>> {
4004 self.task(task_id)
4005 .filter(|task| task.session_id == session_id)
4006 }
4007
4008 pub fn try_health_counts(&self) -> Option<BgTaskHealthCounts> {
4009 let running = self
4010 .inner
4011 .tasks
4012 .try_lock()
4013 .ok()
4014 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())?;
4015 let pending_completions = self.inner.completions.try_lock().ok().map(|q| q.len())?;
4016 Some(BgTaskHealthCounts {
4017 running,
4018 pending_completions,
4019 })
4020 }
4021
4022 pub fn estimated_memory(&self) -> crate::memory::MemoryEstimate {
4026 let tasks = match self.inner.tasks.try_lock() {
4027 Ok(tasks) => tasks.values().cloned().collect::<Vec<_>>(),
4028 Err(_) => return crate::memory::MemoryEstimate::busy(),
4029 };
4030 let mut bytes = 0u64;
4031 let mut terminal_output_caches = 0usize;
4032 let mut sessions = HashSet::new();
4033 for task in &tasks {
4034 sessions.insert(task.session_id.clone());
4035 let state = match task.state.try_lock() {
4036 Ok(state) => state,
4037 Err(_) => return crate::memory::MemoryEstimate::busy(),
4038 };
4039 if let Some(cache) = state.terminal_output_cache.as_ref() {
4040 terminal_output_caches = terminal_output_caches.saturating_add(1);
4041 bytes = bytes.saturating_add(terminal_output_cache_estimated_bytes(cache));
4042 }
4043 }
4044 let completion_count = match self.inner.completions.try_lock() {
4045 Ok(completions) => {
4046 for completion in completions.iter() {
4047 sessions.insert(completion.session_id.clone());
4048 bytes = bytes.saturating_add(completion_estimated_bytes(completion));
4049 }
4050 completions.len()
4051 }
4052 Err(_) => return crate::memory::MemoryEstimate::busy(),
4053 };
4054
4055 crate::memory::MemoryEstimate::estimated(bytes)
4056 .count("tasks", tasks.len())
4057 .count("sessions", sessions.len())
4058 .count("terminal_output_caches", terminal_output_caches)
4059 .count("completion_caches", completion_count)
4060 .count_u64("output_ring_bytes", 0)
4061 }
4062
4063 fn running_count(&self) -> usize {
4064 self.inner
4065 .tasks
4066 .lock()
4067 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())
4068 .unwrap_or(0)
4069 }
4070
4071 fn start_watchdog(&self) {
4072 if !self.inner.watchdog_started.swap(true, Ordering::SeqCst) {
4073 super::watchdog::start(self.clone());
4074 }
4075 }
4076
4077 fn running_metadata_is_stale(&self, metadata: &PersistedTask) -> bool {
4078 unix_millis().saturating_sub(metadata.started_at) > STALE_RUNNING_AFTER.as_millis() as u64
4079 }
4080
4081 #[cfg(test)]
4082 pub fn task_json_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
4083 self.task_for_session(task_id, session_id)
4084 .map(|task| task.paths.json.clone())
4085 }
4086
4087 #[cfg(test)]
4088 pub fn task_exit_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
4089 self.task_for_session(task_id, session_id)
4090 .map(|task| task.paths.exit.clone())
4091 }
4092}
4093
4094#[cfg(unix)]
4095fn should_capture_pipeline_status(
4096 spawn_plan: &SpawnPlan,
4097 has_pipeline: bool,
4098 shell: &Path,
4099) -> bool {
4100 if spawn_plan.is_native_launcher() {
4101 return false;
4104 }
4105 has_pipeline && super::process::pipeline_shell_kind(shell).is_some()
4106}
4107
4108fn canonical_artifact_root(paths: &TaskPaths) -> PathBuf {
4109 fs::canonicalize(&paths.io_dir).unwrap_or_else(|_| paths.io_dir.clone())
4110}
4111
4112fn append_pipeline_warning(
4117 cache: &mut TerminalOutputCache,
4118 metadata: &PersistedTask,
4119 paths: Option<&TaskPaths>,
4120) {
4121 if metadata.exit_code != Some(0) {
4122 return;
4123 }
4124 let Some(paths) = paths else {
4125 return;
4126 };
4127 if metadata.pipeline_segments.len() < 2 {
4128 return;
4129 }
4130 let Ok(mut status_file) = open_task_artifact(paths, TaskArtifact::PipelineStatus) else {
4131 return;
4132 };
4133 let Ok(status_bytes) = status_file.read_all() else {
4134 return;
4135 };
4136 let Some(statuses) = String::from_utf8_lossy(&status_bytes)
4137 .lines()
4138 .map(|line| line.trim().parse::<i32>().ok())
4139 .collect::<Option<Vec<_>>>()
4140 else {
4141 return;
4142 };
4143 if statuses.len() != metadata.pipeline_segments.len() {
4144 return;
4145 }
4146 let Some((failing_index, failing_code)) = statuses
4147 .iter()
4148 .enumerate()
4149 .take(statuses.len().saturating_sub(1))
4150 .find(|(_, code)| **code != 0)
4151 .map(|(index, code)| (index, *code))
4152 else {
4153 return;
4154 };
4155 let Some(final_segment) = metadata.pipeline_segments.last() else {
4156 return;
4157 };
4158 let failing_segment = &metadata.pipeline_segments[failing_index];
4159 let footer = format!(
4160 "note: `{}` (segment {} of {}) exited {}; the pipeline's exit code is `{}`'s.",
4161 failing_segment,
4162 failing_index + 1,
4163 metadata.pipeline_segments.len(),
4164 failing_code,
4165 final_segment,
4166 );
4167 if cache.output_preview.trim().is_empty() {
4168 cache.output_preview = footer;
4169 } else {
4170 cache.output_preview = format!("{}\n{}", cache.output_preview.trim_end(), footer,);
4171 }
4172}
4173
4174fn render_compressed_with_recovery(
4175 buffer: &BgBuffer,
4176 mut compressed: CompressionResult,
4177 input_truncated: bool,
4178 disk_truncation: DiskTruncation,
4179 artifact_access: ArtifactRecoveryAccess,
4180) -> TerminalOutputCache {
4181 let had_trailing_newline = compressed.text.ends_with('\n');
4189 let mut text = strip_plain_truncation_marker_lines(&compressed.text)
4190 .trim_end()
4191 .to_string();
4192 if had_trailing_newline && !text.is_empty() {
4193 text.push('\n');
4194 }
4195 compressed.text = text;
4196
4197 let output_path = buffer.output_path().map(|path| path.display().to_string());
4198 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
4199 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
4200 let mut recovery = RecoveryContext {
4201 dropped_by_class: compressed.dropped_by_class,
4202 had_inner_drop: compressed.had_inner_drop,
4203 offset_hint_eligible: compressed.offset_hint_eligible,
4204 offset_start_line: compressed.offset_start_line,
4205 byte_truncated: input_truncated,
4206 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
4207 output_path: output_path.clone(),
4208 stderr_path: stderr_path.clone(),
4209 include_stderr_path,
4210 artifact_access: artifact_access.clone(),
4211 };
4212
4213 let (output_preview, output_truncated) =
4214 render_body_with_recovery_marker(&compressed.text, &mut recovery);
4215 TerminalOutputCache {
4216 output_preview,
4217 output_truncated,
4218 kind: TerminalOutputKind::Compressed,
4219 output_path,
4220 stderr_path,
4221 artifact_access,
4222 recovery: Some(recovery),
4223 }
4224}
4225
4226fn render_body_with_recovery_marker(body: &str, recovery: &mut RecoveryContext) -> (String, bool) {
4227 render_body_with_recovery_marker_at_cap(
4228 body,
4229 recovery,
4230 FINAL_OUTPUT_CAP_BYTES,
4231 cap_final_output,
4232 cap_final_output_with_marker,
4233 )
4234}
4235
4236fn render_raw_body_with_recovery_marker(
4237 body: &str,
4238 recovery: &mut RecoveryContext,
4239) -> (String, bool) {
4240 render_body_with_recovery_marker_at_cap(
4241 body,
4242 recovery,
4243 RAW_PASSTHROUGH_CAP_BYTES,
4244 |input| {
4245 super::output::cap_head_tail(
4246 input,
4247 RAW_PASSTHROUGH_CAP_BYTES,
4248 RAW_PASSTHROUGH_HEAD_BYTES,
4249 RAW_PASSTHROUGH_TAIL_BYTES,
4250 )
4251 },
4252 |input, marker| {
4253 super::output::cap_head_tail_with_marker(
4254 input,
4255 RAW_PASSTHROUGH_CAP_BYTES,
4256 RAW_PASSTHROUGH_HEAD_BYTES,
4257 RAW_PASSTHROUGH_TAIL_BYTES,
4258 marker,
4259 )
4260 },
4261 )
4262}
4263
4264fn render_body_with_recovery_marker_at_cap<F, G>(
4265 body: &str,
4266 recovery: &mut RecoveryContext,
4267 cap_bytes: usize,
4268 cap_plain: F,
4269 cap_with_marker: G,
4270) -> (String, bool)
4271where
4272 F: Fn(&str) -> super::output::CappedText,
4273 G: Fn(&str, &str) -> super::output::CappedText,
4274{
4275 let needs_marker = recovery.has_visible_drop();
4276 if body.len() > cap_bytes {
4277 recovery.byte_truncated = true;
4278 if let Some(marker) = recovery_marker(recovery) {
4279 let capped = cap_with_marker(body, &marker);
4280 return (capped.text, true);
4281 }
4282 let capped = cap_plain(body);
4283 return (capped.text, capped.truncated || needs_marker);
4284 }
4285
4286 if !needs_marker {
4287 return (body.to_string(), false);
4288 }
4289
4290 let Some(marker) = recovery_marker(recovery) else {
4291 return (body.to_string(), true);
4292 };
4293 let with_marker = append_recovery_marker(body, &marker);
4294 if with_marker.len() <= cap_bytes {
4295 return (with_marker, true);
4296 }
4297
4298 recovery.byte_truncated = true;
4299 let marker = recovery_marker(recovery).unwrap_or(marker);
4300 let capped = cap_with_marker(body, &marker);
4301 (capped.text, true)
4302}
4303
4304fn append_recovery_marker(body: &str, marker: &str) -> String {
4305 if body.is_empty() {
4306 return marker.to_string();
4307 }
4308 let mut output = body.trim_end().to_string();
4309 output.push('\n');
4310 output.push_str(marker);
4311 output
4312}
4313
4314fn recovery_marker(recovery: &RecoveryContext) -> Option<String> {
4315 let mut parts = Vec::new();
4316 for (class, count) in &recovery.dropped_by_class {
4317 let label = if *count == 1 {
4318 class.singular()
4319 } else {
4320 class.plural()
4321 };
4322 parts.push(format!("+{count} more {label}"));
4323 }
4324 if recovery.byte_truncated {
4325 parts.push("truncated output".to_string());
4326 }
4327 let disk_truncated_prefix_bytes = recovery.disk_truncated_prefix_bytes;
4328 if disk_truncated_prefix_bytes > 0 {
4329 parts.push(format!(
4330 "truncated {disk_truncated_prefix_bytes} bytes from saved output prefix"
4331 ));
4332 } else if recovery.had_inner_drop && parts.is_empty() {
4333 parts.push("omitted output".to_string());
4334 }
4335
4336 if parts.is_empty() {
4337 return None;
4338 }
4339
4340 let hint = recovery_hint(recovery);
4341 Some(format!("[{}; {hint}]", parts.join(", ")))
4342}
4343
4344fn bash_status_recovery_hint(access: &ArtifactRecoveryAccess) -> String {
4345 let task_id = serde_json::to_string(&access.task_id)
4346 .unwrap_or_else(|_| format!("\"{}\"", access.task_id));
4347 format!("use bash_status({{taskId: {task_id}}})")
4348}
4349
4350fn recovery_hint(recovery: &RecoveryContext) -> String {
4351 if !recovery.artifact_access.readable {
4352 return bash_status_recovery_hint(&recovery.artifact_access);
4353 }
4354
4355 if recovery.offset_hint_eligible
4359 && !recovery.byte_truncated
4360 && recovery.dropped_by_class.is_empty()
4361 && !recovery.include_stderr_path
4362 {
4363 if let (Some(path), Some(line)) =
4364 (recovery.output_path.as_deref(), recovery.offset_start_line)
4365 {
4366 return format!("see remaining: tail -n +{line} {}", quote_path(path));
4367 }
4368 }
4369
4370 let mut paths = Vec::new();
4371 if let Some(path) = recovery.output_path.as_deref() {
4372 paths.push(path);
4373 }
4374 if recovery.include_stderr_path {
4375 if let Some(path) = recovery.stderr_path.as_deref() {
4376 if !paths.contains(&path) {
4377 paths.push(path);
4378 }
4379 }
4380 }
4381
4382 if paths.is_empty() {
4383 return "full output unavailable".to_string();
4384 }
4385
4386 let reads = paths
4387 .into_iter()
4388 .map(|path| format!("read {}", quote_path(path)))
4389 .collect::<Vec<_>>()
4390 .join(" and ");
4391 if recovery.disk_truncated_prefix_bytes > 0 {
4392 format!("retained output: {reads}")
4393 } else {
4394 format!("full output: {reads}")
4395 }
4396}
4397
4398fn strip_plain_truncation_marker_lines(input: &str) -> String {
4399 input
4400 .lines()
4401 .filter(|line| !is_plain_truncation_marker(line.trim()))
4402 .collect::<Vec<_>>()
4403 .join("\n")
4404}
4405
4406fn strip_recovery_marker_lines(input: &str) -> String {
4407 input
4408 .lines()
4409 .filter(|line| !is_recovery_marker(line.trim()))
4410 .collect::<Vec<_>>()
4411 .join("\n")
4412}
4413
4414fn is_plain_truncation_marker(line: &str) -> bool {
4415 let Some(rest) = line.strip_prefix("...<truncated ") else {
4416 return false;
4417 };
4418 let Some(bytes) = rest.strip_suffix(" bytes>...") else {
4419 return false;
4420 };
4421 !bytes.is_empty() && bytes.chars().all(|ch| ch.is_ascii_digit())
4422}
4423
4424fn is_recovery_marker(line: &str) -> bool {
4425 line.starts_with('[')
4426 && line.ends_with(']')
4427 && (line.contains("full output: read ")
4428 || line.contains("retained output: read ")
4429 || line.contains("see remaining: tail -n +")
4430 || line.contains("use bash_status({taskId:")
4431 || line.contains("full output unavailable"))
4432}
4433
4434fn structured_output_pointer(
4435 total_bytes: u64,
4436 output_path: &str,
4437 truncated_prefix_bytes: u64,
4438 artifact_access: &ArtifactRecoveryAccess,
4439) -> String {
4440 if artifact_access.readable {
4441 return if truncated_prefix_bytes > 0 {
4442 retained_json_output_pointer(total_bytes, output_path, truncated_prefix_bytes)
4443 } else {
4444 json_output_pointer(total_bytes, output_path)
4445 };
4446 }
4447
4448 let kb = total_bytes.div_ceil(1024);
4449 let hint = bash_status_recovery_hint(artifact_access);
4450 if truncated_prefix_bytes > 0 {
4451 format!(
4452 "[JSON output {kb} KB; truncated {truncated_prefix_bytes} bytes from saved output prefix; retained output: {hint}]"
4453 )
4454 } else {
4455 format!("[JSON output {kb} KB; full output: {hint}]")
4456 }
4457}
4458
4459fn render_structured_output(
4460 command: &str,
4461 buffer: &BgBuffer,
4462 disk_truncation: DiskTruncation,
4463 artifact_access: ArtifactRecoveryAccess,
4464) -> Option<TerminalOutputCache> {
4465 if !is_gh_structured_command(command) {
4466 return None;
4467 }
4468
4469 let output_path = buffer
4470 .output_path()
4471 .map(|path| path.display().to_string())?;
4472 let stdout_bytes = buffer.stream_len(StreamKind::Stdout);
4473 if stdout_bytes == 0 {
4474 return None;
4475 }
4476
4477 if stdout_bytes > STRUCTURED_OUTPUT_CAP_BYTES as u64 {
4478 if !stream_starts_like_json(buffer, StreamKind::Stdout) {
4479 return None;
4480 }
4481 let output_preview = structured_output_pointer(
4482 stdout_bytes,
4483 &output_path,
4484 disk_truncation.total_prefix_bytes(),
4485 &artifact_access,
4486 );
4487 return Some(TerminalOutputCache {
4488 output_preview,
4489 output_truncated: true,
4490 kind: TerminalOutputKind::Structured,
4491 output_path: Some(output_path),
4492 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
4493 artifact_access,
4494 recovery: None,
4495 });
4496 }
4497
4498 let stdout = buffer.read_stream_bounded(StreamKind::Stdout, STRUCTURED_OUTPUT_CAP_BYTES);
4499 if stdout.truncated || !is_structured_body(&stdout.text) {
4500 return None;
4501 }
4502
4503 Some(TerminalOutputCache {
4504 output_preview: stdout.text,
4505 output_truncated: false,
4506 kind: TerminalOutputKind::Structured,
4507 output_path: Some(output_path),
4508 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
4509 artifact_access,
4510 recovery: None,
4511 })
4512}
4513
4514fn render_raw_passthrough(
4515 buffer: &BgBuffer,
4516 disk_truncation: DiskTruncation,
4517 artifact_access: ArtifactRecoveryAccess,
4518) -> TerminalOutputCache {
4519 let raw = buffer.read_combined_head_tail(
4520 RAW_PASSTHROUGH_CAP_BYTES,
4521 RAW_PASSTHROUGH_HEAD_BYTES,
4522 RAW_PASSTHROUGH_TAIL_BYTES,
4523 );
4524 let output_path = buffer.output_path().map(|path| path.display().to_string());
4525 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
4526 if !raw.truncated && disk_truncation.total_prefix_bytes() == 0 {
4527 return TerminalOutputCache {
4528 output_preview: raw.text,
4529 output_truncated: false,
4530 kind: TerminalOutputKind::Raw,
4531 output_path,
4532 stderr_path,
4533 artifact_access,
4534 recovery: None,
4535 };
4536 }
4537
4538 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
4539 let mut recovery = RecoveryContext {
4540 dropped_by_class: BTreeMap::new(),
4541 had_inner_drop: false,
4542 offset_hint_eligible: false,
4543 offset_start_line: None,
4544 byte_truncated: raw.truncated,
4545 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
4546 output_path: output_path.clone(),
4547 stderr_path: stderr_path.clone(),
4548 include_stderr_path,
4549 artifact_access: artifact_access.clone(),
4550 };
4551 let (output_preview, output_truncated) =
4552 render_raw_body_with_recovery_marker(&raw.text, &mut recovery);
4553 TerminalOutputCache {
4554 output_preview,
4555 output_truncated,
4556 kind: TerminalOutputKind::Raw,
4557 output_path,
4558 stderr_path,
4559 artifact_access,
4560 recovery: Some(recovery),
4561 }
4562}
4563
4564fn completion_preview_for_cache(
4565 cache: &TerminalOutputCache,
4566 exit_code: Option<i32>,
4567) -> (String, bool) {
4568 let exit_ok = exit_code == Some(0);
4571 let threshold = completion_preview_threshold(exit_ok);
4572 if cache.kind == TerminalOutputKind::Structured && cache.output_preview.len() > threshold {
4573 if let Some(path) = cache.output_path.as_deref() {
4574 return (
4575 structured_output_pointer(
4576 cache.output_preview.len() as u64,
4577 path,
4578 0,
4579 &cache.artifact_access,
4580 ),
4581 true,
4582 );
4583 }
4584 return (cache.output_preview.clone(), cache.output_truncated);
4585 }
4586
4587 if let Some(recovery) = cache.recovery.as_ref() {
4588 if cache.output_preview.len() <= threshold {
4589 return (cache.output_preview.clone(), cache.output_truncated);
4590 }
4591 let body = strip_recovery_marker_lines(&cache.output_preview);
4592 let mut completion_recovery = recovery.clone();
4593 completion_recovery.byte_truncated = true;
4594 if let Some(marker) = recovery_marker(&completion_recovery) {
4595 let capped = cap_completion_output_with_marker(&body, &marker, exit_ok);
4596 return (capped.text, true);
4597 }
4598 }
4599
4600 let capped = cap_completion_output(&cache.output_preview, exit_ok);
4601 (capped.text, cache.output_truncated || capped.truncated)
4602}
4603
4604fn is_gh_structured_command(command: &str) -> bool {
4605 let Some(normalized) = crate::compress::plain_command_for_structured_output(command) else {
4606 return false;
4607 };
4608 let tokens = shell_words_for_flags(&normalized);
4609 let Some(head) = tokens.first() else {
4610 return false;
4611 };
4612 let head_name = Path::new(head)
4613 .file_name()
4614 .and_then(|name| name.to_str())
4615 .unwrap_or(head);
4616 if !(head_name == "gh" || head_name.eq_ignore_ascii_case("gh.exe")) {
4617 return false;
4618 }
4619 tokens.iter().any(|token| {
4620 matches!(token.as_str(), "--json" | "--jq" | "--template")
4621 || token.starts_with("--json=")
4622 || token.starts_with("--jq=")
4623 || token.starts_with("--template=")
4624 })
4625}
4626
4627fn shell_words_for_flags(command: &str) -> Vec<String> {
4628 let mut words = Vec::new();
4629 let mut current = String::new();
4630 let mut in_single = false;
4631 let mut in_double = false;
4632 let mut escaped = false;
4633
4634 for ch in command.chars() {
4635 if escaped {
4636 current.push(ch);
4637 escaped = false;
4638 continue;
4639 }
4640 if ch == '\\' && !in_single {
4641 escaped = true;
4642 continue;
4643 }
4644 if ch == '\'' && !in_double {
4645 in_single = !in_single;
4646 continue;
4647 }
4648 if ch == '"' && !in_single {
4649 in_double = !in_double;
4650 continue;
4651 }
4652 if ch.is_whitespace() && !in_single && !in_double {
4653 if !current.is_empty() {
4654 words.push(std::mem::take(&mut current));
4655 }
4656 continue;
4657 }
4658 if matches!(ch, ';' | '&' | '|') && !in_single && !in_double {
4659 if !current.is_empty() {
4660 words.push(std::mem::take(&mut current));
4661 }
4662 continue;
4663 }
4664 current.push(ch);
4665 }
4666 if !current.is_empty() {
4667 words.push(current);
4668 }
4669 words
4670}
4671
4672fn is_structured_body(body: &str) -> bool {
4673 let trimmed = body.trim();
4674 if trimmed.is_empty() {
4675 return false;
4676 }
4677 if serde_json::from_str::<serde_json::Value>(trimmed).is_ok() {
4678 return true;
4679 }
4680
4681 let mut saw_line = false;
4682 for line in trimmed
4683 .lines()
4684 .map(str::trim)
4685 .filter(|line| !line.is_empty())
4686 {
4687 saw_line = true;
4688 if serde_json::from_str::<serde_json::Value>(line).is_err() {
4689 return false;
4690 }
4691 }
4692 saw_line
4693}
4694
4695fn stream_starts_like_json(buffer: &BgBuffer, stream: StreamKind) -> bool {
4696 buffer
4697 .read_stream_bounded(stream, 512)
4698 .text
4699 .chars()
4700 .find(|ch| !ch.is_whitespace())
4701 .is_some_and(|ch| matches!(ch, '{' | '[' | '"' | '-' | '0'..='9' | 't' | 'f' | 'n'))
4702}
4703
4704struct CompletionTokenCounts {
4705 original_tokens: Option<u32>,
4706 compressed_tokens: Option<u32>,
4707 original_bytes: Option<i64>,
4708 compressed_bytes: Option<i64>,
4709 tokens_skipped: bool,
4710}
4711
4712impl CompletionTokenCounts {
4713 fn skipped() -> Self {
4714 Self {
4715 original_tokens: None,
4716 compressed_tokens: None,
4717 original_bytes: None,
4718 compressed_bytes: None,
4719 tokens_skipped: true,
4720 }
4721 }
4722}
4723
4724fn completion_status_text(status: &BgTaskStatus, exit_code: Option<i32>) -> String {
4725 match status {
4726 BgTaskStatus::TimedOut => "timed out".to_string(),
4727 BgTaskStatus::Killed => "killed".to_string(),
4728 _ => exit_code
4729 .map(|code| format!("exit {code}"))
4730 .unwrap_or_else(|| format!("{status:?}").to_lowercase()),
4731 }
4732}
4733
4734fn token_count_u32(text: &str) -> u32 {
4735 aft_tokenizer::count_tokens(text)
4736 .try_into()
4737 .unwrap_or(u32::MAX)
4738}
4739
4740impl Default for BgTaskRegistry {
4741 fn default() -> Self {
4742 Self::new(Arc::new(Mutex::new(None)))
4743 }
4744}
4745
4746fn modified_within(path: &Path, grace: Duration) -> bool {
4747 fs::metadata(path)
4748 .and_then(|metadata| metadata.modified())
4749 .ok()
4750 .and_then(|modified| SystemTime::now().duration_since(modified).ok())
4751 .map(|age| age < grace)
4752 .unwrap_or(false)
4753}
4754
4755fn canonicalized_path(path: &Path) -> PathBuf {
4756 fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf())
4757}
4758
4759fn started_instant_from_unix_millis(started_at: u64) -> Instant {
4760 let now_ms = SystemTime::now()
4761 .duration_since(UNIX_EPOCH)
4762 .ok()
4763 .map(|duration| duration.as_millis() as u64)
4764 .unwrap_or(started_at);
4765 let elapsed_ms = now_ms.saturating_sub(started_at);
4766 Instant::now()
4767 .checked_sub(Duration::from_millis(elapsed_ms))
4768 .unwrap_or_else(Instant::now)
4769}
4770
4771fn gc_quarantine(storage_dir: &Path) {
4772 let quarantine_root = storage_dir.join("bash-tasks-quarantine");
4773 let Ok(session_dirs) = fs::read_dir(&quarantine_root) else {
4774 return;
4775 };
4776 for session_entry in session_dirs.flatten() {
4777 let session_quarantine_dir = session_entry.path();
4778 if !session_quarantine_dir.is_dir() {
4779 continue;
4780 }
4781 let entries = match fs::read_dir(&session_quarantine_dir) {
4782 Ok(entries) => entries,
4783 Err(error) => {
4784 crate::slog_warn!(
4785 "failed to read background task quarantine dir {}: {error}",
4786 session_quarantine_dir.display()
4787 );
4788 continue;
4789 }
4790 };
4791 for entry in entries.flatten() {
4792 let path = entry.path();
4793 if modified_within(&path, QUARANTINE_GC_GRACE) {
4794 continue;
4795 }
4796 let result = if path.is_dir() {
4797 fs::remove_dir_all(&path)
4798 } else {
4799 fs::remove_file(&path)
4800 };
4801 match result {
4802 Ok(()) => log::debug!(
4803 "deleted old background task quarantine entry {}",
4804 path.display()
4805 ),
4806 Err(error) => crate::slog_warn!(
4807 "failed to delete old background task quarantine entry {}: {error}",
4808 path.display()
4809 ),
4810 }
4811 }
4812 let _ = fs::remove_dir(&session_quarantine_dir);
4813 }
4814 let _ = fs::remove_dir(&quarantine_root);
4815}
4816
4817fn read_for_token_count_from_disk(
4818 metadata: &PersistedTask,
4819 paths: &TaskPaths,
4820 max_bytes_per_stream: usize,
4821) -> TokenCountInput {
4822 if metadata.mode == BgMode::Pty {
4823 return TokenCountInput::Skipped;
4824 }
4825 let stdout = read_file_tail_capped(paths, TaskArtifact::Stdout, max_bytes_per_stream);
4832 let stderr = read_file_tail_capped(paths, TaskArtifact::Stderr, max_bytes_per_stream);
4833 match (stdout, stderr) {
4834 (Ok(stdout), Ok(stderr)) => TokenCountInput::Text(combine_streams(
4835 String::from_utf8_lossy(&stdout).as_ref(),
4836 String::from_utf8_lossy(&stderr).as_ref(),
4837 )),
4838 (Ok(stdout), Err(_)) => TokenCountInput::Text(combine_streams(
4839 String::from_utf8_lossy(&stdout).as_ref(),
4840 "",
4841 )),
4842 (Err(_), Ok(stderr)) => TokenCountInput::Text(combine_streams(
4843 "",
4844 String::from_utf8_lossy(&stderr).as_ref(),
4845 )),
4846 (Err(_), Err(_)) => TokenCountInput::Skipped,
4847 }
4848}
4849
4850fn read_file_tail_capped(
4851 paths: &TaskPaths,
4852 artifact: TaskArtifact,
4853 max_bytes: usize,
4854) -> std::io::Result<Vec<u8>> {
4855 let mut file = open_task_artifact(paths, artifact)?;
4856 file.tail(max_bytes).map(|(bytes, _)| bytes)
4857}
4858
4859impl BgTask {
4860 fn snapshot(&self, preview_bytes: usize) -> BgTaskSnapshot {
4861 let state = self
4862 .state
4863 .lock()
4864 .unwrap_or_else(|poison| poison.into_inner());
4865 self.snapshot_locked(&state, preview_bytes)
4866 }
4867
4868 fn snapshot_locked(&self, state: &BgTaskState, preview_bytes: usize) -> BgTaskSnapshot {
4869 let metadata = &state.metadata;
4870 let duration_ms = metadata.duration_ms.or_else(|| {
4871 metadata
4872 .status
4873 .is_terminal()
4874 .then(|| self.started.elapsed().as_millis() as u64)
4875 });
4876 let (output_preview, output_truncated) = if metadata.mode == BgMode::Pty {
4877 (String::new(), false)
4878 } else if metadata.status.is_terminal() {
4879 state
4880 .terminal_output_cache
4881 .as_ref()
4882 .map(|cache| (cache.output_preview.clone(), cache.output_truncated))
4883 .unwrap_or_else(|| (String::new(), false))
4884 } else if preview_bytes == 0 {
4885 (String::new(), false)
4886 } else {
4887 state.buffer.read_tail(preview_bytes)
4888 };
4889 BgTaskSnapshot {
4890 info: BgTaskInfo {
4891 task_id: self.task_id.clone(),
4892 status: metadata.status.clone(),
4893 command: metadata.command.clone(),
4894 mode: metadata.mode.clone(),
4895 started_at: metadata.started_at,
4896 duration_ms,
4897 status_reason: metadata.status_reason.clone(),
4898 },
4899 exit_code: metadata.exit_code,
4900 child_pid: metadata.child_pid,
4901 workdir: metadata.workdir.display().to_string(),
4902 output_preview,
4903 output_truncated,
4904 output_path: state
4905 .buffer
4906 .output_path()
4907 .map(|path| path.display().to_string()),
4908 stderr_path: state
4909 .buffer
4910 .stderr_path()
4911 .map(|path| path.display().to_string()),
4912 pty_rows: (metadata.mode == BgMode::Pty).then_some(metadata.pty_rows.unwrap_or(24)),
4913 pty_cols: (metadata.mode == BgMode::Pty).then_some(metadata.pty_cols.unwrap_or(80)),
4914 pty_screen: None,
4915 scanner_report: metadata.scanner_report.clone(),
4916 sandbox_native: metadata.sandbox_native,
4917 sandbox_unavailable: metadata.sandbox_native
4918 && open_task_artifact(&self.paths, TaskArtifact::SandboxUnavailable)
4919 .and_then(|mut file| file.read_all())
4920 .is_ok_and(|bytes| bytes == b"sandbox_unavailable"),
4921 }
4922 }
4923
4924 pub(crate) fn is_running(&self) -> bool {
4925 self.state
4926 .lock()
4927 .map(|state| {
4928 state.metadata.status == BgTaskStatus::Running
4929 || (state.metadata.mode == BgMode::Pty
4930 && state.metadata.status == BgTaskStatus::Killing)
4931 })
4932 .unwrap_or(false)
4933 }
4934
4935 fn is_terminal(&self) -> bool {
4936 self.state
4937 .lock()
4938 .map(|state| state.metadata.status.is_terminal())
4939 .unwrap_or(false)
4940 }
4941
4942 fn mark_terminal_now(&self) {
4943 if let Ok(mut terminal_at) = self.terminal_at.lock() {
4944 if terminal_at.is_none() {
4945 *terminal_at = Some(Instant::now());
4946 }
4947 }
4948 }
4949
4950 fn set_completion_delivered(
4951 &self,
4952 delivered: bool,
4953 registry: &BgTaskRegistry,
4954 ) -> Result<(), String> {
4955 let mut state = self
4956 .state
4957 .lock()
4958 .map_err(|_| "background task lock poisoned".to_string())?;
4959 let updated = registry
4960 .update_task_metadata(&self.paths, |metadata| {
4961 metadata.completion_delivered = delivered;
4962 })
4963 .map_err(|e| format!("failed to update completion delivery: {e}"))?;
4964 state.metadata = updated;
4965 Ok(())
4966 }
4967}
4968
4969#[cfg(unix)]
4990fn reap_piped_child(child_slot: &mut Option<Child>) {
4991 if let Some(mut child) = child_slot.take() {
4992 if matches!(child.try_wait(), Ok(None)) {
4993 let _ = child.wait();
4994 }
4995 }
4996}
4997
4998#[cfg(windows)]
5003fn reap_piped_child(child_slot: &mut Option<Child>) {
5004 *child_slot = None;
5005}
5006
5007fn terminal_metadata_from_marker(
5008 mut metadata: PersistedTask,
5009 marker: ExitMarker,
5010 reason: Option<String>,
5011) -> PersistedTask {
5012 match marker {
5013 ExitMarker::Code(code) => {
5014 let status = if code == 0 {
5015 BgTaskStatus::Completed
5016 } else {
5017 BgTaskStatus::Failed
5018 };
5019 metadata.mark_terminal(status, Some(code), reason);
5020 }
5021 ExitMarker::Killed => metadata.mark_terminal(
5022 BgTaskStatus::Killed,
5023 terminal_exit_code_for_status(&BgTaskStatus::Killed),
5024 reason,
5025 ),
5026 }
5027 metadata
5028}
5029
5030fn terminal_exit_code_for_status(status: &BgTaskStatus) -> Option<i32> {
5031 match status {
5032 BgTaskStatus::TimedOut => Some(124),
5033 BgTaskStatus::Killed => Some(137),
5034 _ => None,
5035 }
5036}
5037
5038fn attach_sandbox_metadata(metadata: &mut PersistedTask, spawn_plan: &SpawnPlan) {
5039 metadata.sandbox_native = spawn_plan.is_native_launcher();
5040 metadata.sandbox_temp_dir = spawn_plan.temp_dir().map(Path::to_path_buf);
5041}
5042
5043#[cfg(unix)]
5044pub(crate) fn resolve_posix_shell() -> PathBuf {
5045 static POSIX_SHELL: OnceLock<PathBuf> = OnceLock::new();
5046 POSIX_SHELL
5047 .get_or_init(|| {
5048 std::env::var_os("BASH")
5049 .filter(|value| !value.is_empty())
5050 .map(PathBuf::from)
5051 .filter(|path| path.exists())
5052 .or_else(|| which::which("bash").ok())
5053 .or_else(|| which::which("zsh").ok())
5054 .unwrap_or_else(|| PathBuf::from("/bin/sh"))
5055 })
5056 .clone()
5057}
5058
5059#[cfg(windows)]
5060fn detached_shell_command_for(
5061 shell: crate::windows_shell::WindowsShell,
5062 command: &str,
5063 exit_path: &Path,
5064 paths: &TaskPaths,
5065 creation_flags: u32,
5066) -> Result<Command, String> {
5067 use crate::windows_shell::WindowsShell;
5068 let wrapper_body = shell.wrapper_script_bytes(command, exit_path);
5081 let wrapper_ext = match shell {
5082 WindowsShell::Pwsh | WindowsShell::Powershell => "ps1",
5083 WindowsShell::Cmd => "bat",
5084 WindowsShell::Posix(_) => "sh",
5088 };
5089 let wrapper_path = paths.dir.join(format!(
5090 "{}.{}",
5091 paths
5092 .json
5093 .file_stem()
5094 .and_then(|s| s.to_str())
5095 .unwrap_or("wrapper"),
5096 wrapper_ext
5097 ));
5098 fs::write(&wrapper_path, wrapper_body)
5099 .map_err(|e| format!("failed to write background bash wrapper script: {e}"))?;
5100
5101 let mut cmd = Command::new(shell.binary().as_ref());
5102 match shell {
5103 WindowsShell::Pwsh | WindowsShell::Powershell => {
5104 cmd.args([
5107 "-NoLogo",
5108 "-NoProfile",
5109 "-NonInteractive",
5110 "-ExecutionPolicy",
5111 "Bypass",
5112 "-File",
5113 ]);
5114 cmd.arg(&wrapper_path);
5115 }
5116 WindowsShell::Cmd => {
5117 cmd.args(["/D", "/C"]);
5124 cmd.arg(&wrapper_path);
5125 }
5126 WindowsShell::Posix(_) => {
5127 cmd.arg(&wrapper_path);
5132 }
5133 }
5134
5135 cmd.creation_flags(creation_flags);
5139 Ok(cmd)
5140}
5141
5142fn spawn_detached_child(
5158 spawn_plan: &SpawnPlan,
5159 command: &str,
5160 paths: &TaskPaths,
5161 workdir: &Path,
5162 env: &HashMap<String, String>,
5163 io_handles: &mut TaskIoHandles,
5164 capture_pipeline_status: bool,
5165) -> Result<std::process::Child, String> {
5166 #[cfg(windows)]
5167 let _ = capture_pipeline_status;
5168 #[cfg(not(windows))]
5169 let _ = command;
5170 #[cfg(not(windows))]
5171 {
5172 use std::os::fd::AsRawFd;
5173
5174 let stdout = io_handles
5175 .clone_file(TaskArtifact::Stdout)
5176 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
5177 let stderr = io_handles
5178 .clone_file(TaskArtifact::Stderr)
5179 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
5180 let prepared = spawn_plan
5181 .prepared_task()
5182 .ok_or_else(|| "background task payload was not prepared".to_string())?;
5183 let payload = prepared.invocation()?;
5184 let exit = io_handles
5185 .inheritable_file(TaskArtifact::Exit)
5186 .map_err(|e| format!("failed to inherit exit marker handle: {e}"))?;
5187 let failure = io_handles
5188 .inheritable_file(TaskArtifact::SandboxUnavailable)
5189 .map_err(|e| format!("failed to inherit sandbox failure marker handle: {e}"))?;
5190 let pipeline_status = capture_pipeline_status
5191 .then(|| io_handles.inheritable_file(TaskArtifact::PipelineStatus))
5192 .transpose()
5193 .map_err(|e| format!("failed to inherit pipeline status handle: {e}"))?;
5194 let shell = spawn_plan
5195 .host_shell_path()
5196 .map(Path::to_path_buf)
5197 .unwrap_or_else(resolve_posix_shell);
5198 let pipeline_shell = super::process::pipeline_shell_kind(&shell).unwrap_or("");
5199 let pipeline_status_fd = if capture_pipeline_status {
5200 crate::sandbox_spawn::CHILD_PIPE_STATUS_FD.to_string()
5201 } else {
5202 String::new()
5203 };
5204 let args = vec![
5205 OsString::from("-c"),
5206 payload.wrapper_text.clone(),
5207 OsString::from("aft-payload-wrapper"),
5208 shell.as_os_str().to_os_string(),
5209 payload.command_text.clone(),
5210 OsString::from(crate::sandbox_spawn::CHILD_EXIT_FD.to_string()),
5211 OsString::from(pipeline_status_fd),
5212 OsString::from(pipeline_shell),
5213 ];
5214 let (mut child_command, profile_handle) = crate::sandbox_spawn::detached_command_for_plan(
5215 spawn_plan,
5216 std::ffi::OsStr::new("/bin/sh"),
5217 &args,
5218 &paths.json,
5219 crate::sandbox_spawn::CHILD_EXIT_FD,
5220 crate::sandbox_spawn::CHILD_FAILURE_FD,
5221 )?;
5222 crate::sandbox_spawn::apply_marker_fd_allowlist(
5223 &mut child_command,
5224 exit.as_raw_fd(),
5225 failure.as_raw_fd(),
5226 pipeline_status.as_ref().map(|file| file.as_raw_fd()),
5227 )?;
5228 child_command
5229 .current_dir(workdir)
5230 .envs(env)
5231 .stdin(Stdio::null())
5232 .stdout(Stdio::from(stdout))
5233 .stderr(Stdio::from(stderr));
5234 crate::sandbox_spawn::apply_sandbox_environment(spawn_plan, &mut child_command, env);
5235 let child = child_command
5236 .spawn()
5237 .map_err(|e| format!("failed to spawn background bash command: {e}"));
5238 drop((payload, exit, failure, pipeline_status, profile_handle));
5239 child
5240 }
5241 #[cfg(windows)]
5242 {
5243 use crate::windows_shell::shell_candidates;
5244 match spawn_plan {
5245 SpawnPlan::Unsandboxed | SpawnPlan::Host { .. } => {}
5246 SpawnPlan::Refused { code, .. } => return Err((*code).to_string()),
5247 SpawnPlan::Launcher { .. } => return Err("sandbox_unavailable".to_string()),
5248 }
5249 let candidates: Vec<crate::windows_shell::WindowsShell> = shell_candidates();
5260 const FLAG_CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200;
5273 const FLAG_CREATE_BREAKAWAY_FROM_JOB: u32 = 0x0100_0000;
5274 const FLAG_CREATE_NO_WINDOW: u32 = 0x0800_0000;
5275 let with_breakaway =
5276 FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP | FLAG_CREATE_BREAKAWAY_FROM_JOB;
5277 let without_breakaway = FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP;
5278 let mut last_error: Option<String> = None;
5279 for (idx, shell) in candidates.iter().enumerate() {
5280 for &flags in &[with_breakaway, without_breakaway] {
5284 let stdout = io_handles
5287 .clone_file(TaskArtifact::Stdout)
5288 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
5289 let stderr = io_handles
5290 .clone_file(TaskArtifact::Stderr)
5291 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
5292 let mut cmd =
5293 detached_shell_command_for(shell.clone(), command, &paths.exit, paths, flags)?;
5294 cmd.current_dir(workdir)
5295 .envs(env)
5296 .stdin(Stdio::null())
5297 .stdout(Stdio::from(stdout))
5298 .stderr(Stdio::from(stderr));
5299 match cmd.spawn() {
5300 Ok(child) => {
5301 if idx > 0 {
5302 crate::slog_warn!("background bash spawn fell back to {} after {} earlier candidate(s) failed; \
5303 the cached PATH probe disagreed with runtime spawn — likely PATH \
5304 inheritance, antivirus / AppLocker / Defender ASR, or sandbox policy.",
5305 shell.binary(),
5306 idx);
5307 }
5308 if flags == without_breakaway {
5309 crate::slog_warn!(
5310 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected \
5311 (likely a restrictive Job Object — CI sandbox or MDM policy). \
5312 Spawned without breakaway; the bg task will be torn down if the \
5313 AFT process group is killed."
5314 );
5315 }
5316 return Ok(child);
5317 }
5318 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
5319 crate::slog_warn!("background bash spawn: {} returned NotFound at runtime — trying next candidate",
5320 shell.binary());
5321 last_error = Some(format!("{}: {e}", shell.binary()));
5322 break;
5325 }
5326 Err(e) if flags == with_breakaway && e.raw_os_error() == Some(5) => {
5327 crate::slog_warn!(
5329 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected with \
5330 Access Denied — retrying {} without breakaway",
5331 shell.binary()
5332 );
5333 last_error = Some(format!("{}: {e}", shell.binary()));
5334 continue;
5335 }
5336 Err(e) => {
5337 return Err(format!(
5338 "failed to spawn background bash command via {}: {e}",
5339 shell.binary()
5340 ));
5341 }
5342 }
5343 }
5344 }
5345 Err(format!(
5346 "failed to spawn background bash command: no Windows shell could be spawned. \
5347 Last error: {}. PATH-probed candidates: {:?}",
5348 last_error.unwrap_or_else(|| "no candidates were attempted".to_string()),
5349 candidates.iter().map(|s| s.binary()).collect::<Vec<_>>()
5350 ))
5351 }
5352}
5353
5354#[cfg(test)]
5355fn random_slug() -> String {
5356 let mut bytes = [0u8; 8];
5364 getrandom::fill(&mut bytes).unwrap_or_else(|_| {
5366 let t = SystemTime::now()
5368 .duration_since(UNIX_EPOCH)
5369 .map(|d| d.as_nanos() as u64)
5370 .unwrap_or(0);
5371 let p = u64::from(std::process::id());
5372 bytes.copy_from_slice(&(t ^ p.rotate_left(32)).to_le_bytes());
5373 });
5374 let hex: String = bytes.iter().map(|b| format!("{b:02x}")).collect();
5376 format!("bash-{hex}")
5377}
5378
5379#[cfg(test)]
5380mod tests {
5381 use std::collections::HashMap;
5382 use std::fs;
5383 use std::io::Write;
5384 #[cfg(unix)]
5385 use std::os::unix::fs::PermissionsExt;
5386 use std::sync::atomic::{AtomicBool, AtomicUsize};
5387 use std::sync::{Arc, Mutex};
5388 use std::time::{Duration, Instant, SystemTime};
5389
5390 use super::*;
5391 use crate::bash_background::persistence::{read_task, task_paths, write_task};
5392
5393 #[cfg(unix)]
5394 const QUICK_SUCCESS_COMMAND: &str = "true";
5395 #[cfg(windows)]
5396 const QUICK_SUCCESS_COMMAND: &str = "cmd /c exit 0";
5397
5398 #[cfg(unix)]
5399 const LONG_RUNNING_COMMAND: &str = "sleep 5";
5400
5401 #[cfg(unix)]
5402 #[test]
5403 fn launcher_plans_disable_pipeline_status_capture() {
5404 let launcher = SpawnPlan::launcher_for_test(
5405 crate::sandbox_profile::SandboxProfile {
5406 v: crate::sandbox_profile::SANDBOX_PROFILE_VERSION,
5407 writable_roots: Vec::new(),
5408 write_deny: Vec::new(),
5409 write_deny_nested: Vec::new(),
5410 read_allow: Vec::new(),
5411 read_deny: Vec::new(),
5412 socket_deny: Vec::new(),
5413 cache_roots: Vec::new(),
5414 temp_dir: PathBuf::from("/tmp/aft-test-sandbox"),
5415 },
5416 PathBuf::from("/bin/true"),
5417 );
5418 assert!(!should_capture_pipeline_status(
5419 &launcher,
5420 true,
5421 Path::new("/bin/bash")
5422 ));
5423 assert!(should_capture_pipeline_status(
5424 &SpawnPlan::Unsandboxed,
5425 true,
5426 Path::new("/bin/bash")
5427 ));
5428 }
5429
5430 #[cfg(windows)]
5431 const LONG_RUNNING_COMMAND: &str = "cmd /c timeout /t 5 /nobreak > nul";
5432
5433 #[test]
5434 fn bash_memory_estimate_is_zero_when_empty_and_nonzero_for_completion_cache() {
5435 let registry = BgTaskRegistry::default();
5436 assert_eq!(registry.estimated_memory().estimated_bytes, Some(0));
5437 registry
5438 .inner
5439 .completions
5440 .lock()
5441 .unwrap()
5442 .push_back(BgCompletion {
5443 task_id: "bash-memory".to_string(),
5444 session_id: "session-memory".to_string(),
5445 status: BgTaskStatus::Completed,
5446 exit_code: Some(0),
5447 command: "printf memory".to_string(),
5448 output_preview: "resident completion output".to_string(),
5449 output_truncated: false,
5450 original_tokens: None,
5451 compressed_tokens: None,
5452 tokens_skipped: false,
5453 status_reason: None,
5454 });
5455 let estimate = registry.estimated_memory();
5456 assert!(estimate.estimated_bytes.unwrap() > 0);
5457 assert_eq!(estimate.counts["completion_caches"], 1);
5458 assert_eq!(estimate.counts["sessions"], 1);
5459 }
5460
5461 #[test]
5462 fn gh_structured_detection_rejects_piped_commands() {
5463 assert!(is_gh_structured_command(
5464 "gh issue list --json number,title"
5465 ));
5466 assert!(is_gh_structured_command(
5467 "cd repo && gh issue list --json number,title"
5468 ));
5469
5470 assert!(!is_gh_structured_command(
5471 "gh issue list --json number,title | jq '.[]'"
5472 ));
5473 assert!(!is_gh_structured_command(
5474 "gh issue list --json number,title |"
5475 ));
5476 }
5477
5478 fn insert_terminal_piped_task(
5479 registry: &BgTaskRegistry,
5480 dir: &tempfile::TempDir,
5481 command: &str,
5482 stdout: &str,
5483 stderr: &str,
5484 compressed: bool,
5485 ) -> (String, Arc<BgTask>) {
5486 let task_id = random_slug();
5487 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5488 fs::create_dir_all(&paths.dir).unwrap();
5489 fs::write(&paths.stdout, stdout).unwrap();
5490 fs::write(&paths.stderr, stderr).unwrap();
5491 let mut metadata = PersistedTask::starting(
5492 task_id.clone(),
5493 "session".to_string(),
5494 command.to_string(),
5495 dir.path().to_path_buf(),
5496 Some(dir.path().to_path_buf()),
5497 Some(30_000),
5498 true,
5499 compressed,
5500 );
5501 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
5502 write_task(&paths.json, &metadata).unwrap();
5503 registry
5504 .insert_rehydrated_task(metadata, paths, true)
5505 .expect("insert terminal task");
5506 let task = registry.task_for_session(&task_id, "session").unwrap();
5507 (task_id, task)
5508 }
5509
5510 #[test]
5511 fn bash_zero_preview_running_status_skips_output_read_while_explicit_preview_reads() {
5512 let registry = BgTaskRegistry::default();
5513 let dir = tempfile::tempdir().unwrap();
5514 let task_id = random_slug();
5515 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5516 fs::create_dir_all(&paths.dir).unwrap();
5517 fs::write(&paths.stdout, "live output\n").unwrap();
5518 fs::write(&paths.stderr, "").unwrap();
5519 let stdout_path = paths.stdout.clone();
5520 let mut metadata = PersistedTask::starting(
5521 task_id.clone(),
5522 "session".to_string(),
5523 "sleep 60".to_string(),
5524 dir.path().to_path_buf(),
5525 Some(dir.path().to_path_buf()),
5526 Some(30_000),
5527 true,
5528 false,
5529 );
5530 metadata.status = BgTaskStatus::Running;
5531 write_task(&paths.json, &metadata).unwrap();
5532 registry
5533 .insert_rehydrated_task(metadata, paths, false)
5534 .expect("insert running task");
5535
5536 crate::bash_background::buffer::reset_tail_read_count(&stdout_path);
5537 for _ in 0..5 {
5538 let snapshot = registry
5539 .status(&task_id, "session", Some(dir.path()), Some(dir.path()), 0)
5540 .expect("running snapshot");
5541 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
5542 assert!(snapshot.output_preview.is_empty());
5543 }
5544 assert_eq!(
5545 crate::bash_background::buffer::tail_read_count(&stdout_path),
5546 0
5547 );
5548
5549 let snapshot = registry
5550 .status(
5551 &task_id,
5552 "session",
5553 Some(dir.path()),
5554 Some(dir.path()),
5555 RUNNING_OUTPUT_PREVIEW_BYTES,
5556 )
5557 .expect("explicit running snapshot");
5558 assert_eq!(snapshot.output_preview, "live output\n");
5559 assert_eq!(
5560 crate::bash_background::buffer::tail_read_count(&stdout_path),
5561 1
5562 );
5563 }
5564
5565 #[test]
5566 fn artifact_read_capability_requires_exact_canonical_path_and_session() {
5567 let registry = BgTaskRegistry::default();
5568 let dir = tempfile::tempdir().unwrap();
5569 let (_task_id, task) = insert_terminal_piped_task(
5570 ®istry,
5571 &dir,
5572 "printf output",
5573 "stdout\n",
5574 "stderr\n",
5575 true,
5576 );
5577 fs::write(&task.paths.exit, "0\n").unwrap();
5578
5579 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stdout));
5580 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stderr));
5581 assert!(registry.is_session_owned_artifact_path("session", &task.paths.exit));
5582 assert!(!registry.is_session_owned_artifact_path("different-session", &task.paths.stdout));
5583 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.json));
5584
5585 let unregistered = task.paths.dir.join("unregistered-output");
5586 fs::write(&unregistered, "not a task artifact\n").unwrap();
5587 assert!(!registry.is_session_owned_artifact_path("session", &unregistered));
5588 }
5589
5590 #[cfg(unix)]
5591 #[test]
5592 fn artifact_directory_symlink_does_not_create_a_prefix_exception() {
5593 let registry = BgTaskRegistry::default();
5594 let dir = tempfile::tempdir().unwrap();
5595 let project = dir.path().join("project");
5596 fs::create_dir_all(&project).unwrap();
5597 let (_task_id, task) =
5598 insert_terminal_piped_task(®istry, &dir, "printf output", "stdout\n", "", true);
5599 let link = project.join("task-artifacts");
5600 std::os::unix::fs::symlink(&task.paths.dir, &link).unwrap();
5601 let unregistered = task.paths.dir.join("unregistered-output");
5602 fs::write(&unregistered, "not registered\n").unwrap();
5603
5604 assert!(!registry.is_session_owned_artifact_path("session", &link));
5605 assert!(
5606 !registry.is_session_owned_artifact_path("session", &link.join("unregistered-output"))
5607 );
5608 assert!(registry.is_session_owned_artifact_path(
5609 "session",
5610 &link.join(task.paths.stdout.file_name().unwrap())
5611 ));
5612
5613 let outside = dir.path().join("outside-secret");
5614 fs::write(&outside, "must stay private\n").unwrap();
5615 fs::remove_file(&task.paths.stdout).unwrap();
5616 std::os::unix::fs::symlink(&outside, &task.paths.stdout).unwrap();
5617 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.stdout));
5618 }
5619
5620 #[test]
5621 fn recovery_footer_uses_bash_status_when_artifact_is_not_registered() {
5622 let registry = BgTaskRegistry::default();
5623 let dir = tempfile::tempdir().unwrap();
5624 let task_id = "bash-1111111111111111";
5625 let paths = task_paths(dir.path(), "session", task_id).unwrap();
5626 fs::create_dir_all(&paths.dir).unwrap();
5627 fs::write(
5628 &paths.stdout,
5629 format!("{}tail\n", "output-line\n".repeat(2_000)),
5630 )
5631 .unwrap();
5632 fs::write(&paths.stderr, "").unwrap();
5633 let mut metadata = PersistedTask::starting(
5634 task_id.to_string(),
5635 "session".to_string(),
5636 "printf output".to_string(),
5637 dir.path().to_path_buf(),
5638 Some(dir.path().to_path_buf()),
5639 Some(30_000),
5640 true,
5641 true,
5642 );
5643 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
5644 write_task(&paths.json, &metadata).unwrap();
5645
5646 let cache = registry
5647 .render_terminal_output_from_paths(&metadata, &paths)
5648 .expect("terminal render");
5649
5650 assert!(cache
5651 .output_preview
5652 .contains("use bash_status({taskId: \"bash-1111111111111111\"})"));
5653 assert!(!cache.output_preview.contains("full output: read "));
5654 }
5655
5656 fn insert_terminal_pty_task(
5657 registry: &BgTaskRegistry,
5658 dir: &tempfile::TempDir,
5659 pty_output: &str,
5660 ) -> (String, Arc<BgTask>) {
5661 let task_id = random_slug();
5662 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5663 fs::create_dir_all(&paths.dir).unwrap();
5664 fs::write(&paths.pty, pty_output).unwrap();
5665 let mut metadata = PersistedTask::starting(
5666 task_id.clone(),
5667 "session".to_string(),
5668 "python".to_string(),
5669 dir.path().to_path_buf(),
5670 Some(dir.path().to_path_buf()),
5671 Some(30_000),
5672 true,
5673 true,
5674 );
5675 metadata.mode = BgMode::Pty;
5676 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
5677 write_task(&paths.json, &metadata).unwrap();
5678 registry
5679 .insert_rehydrated_task(metadata, paths, true)
5680 .expect("insert terminal pty task");
5681 let task = registry.task_for_session(&task_id, "session").unwrap();
5682 (task_id, task)
5683 }
5684
5685 #[cfg(unix)]
5686 fn wait_for_terminal_snapshot(
5687 registry: &BgTaskRegistry,
5688 task_id: &str,
5689 session_id: &str,
5690 project: &Path,
5691 storage: &Path,
5692 ) -> BgTaskSnapshot {
5693 let started = Instant::now();
5694 loop {
5695 let snapshot = registry
5696 .status(task_id, session_id, Some(project), Some(storage), 4096)
5697 .expect("spawned task should be visible to status");
5698 if snapshot.info.status.is_terminal() {
5699 return snapshot;
5700 }
5701 assert!(
5702 started.elapsed() < Duration::from_secs(10),
5703 "timed out waiting for task {task_id} to finish; last status={:?}",
5704 snapshot.info.status
5705 );
5706 std::thread::sleep(Duration::from_millis(50));
5707 }
5708 }
5709
5710 fn write_running_project_task(storage: &Path, project: &Path, session: &str, task_id: &str) {
5711 let paths = task_paths(storage, session, task_id).unwrap();
5712 let mut metadata = PersistedTask::starting(
5713 task_id.to_string(),
5714 session.to_string(),
5715 "sleep 60".to_string(),
5716 project.to_path_buf(),
5717 Some(project.to_path_buf()),
5718 Some(30_000),
5719 true,
5720 true,
5721 );
5722 metadata.status = BgTaskStatus::Running;
5723 write_task(&paths.json, &metadata).unwrap();
5724 fs::write(&paths.stdout, "still running\n").unwrap();
5725 fs::write(&paths.stderr, "").unwrap();
5726 }
5727
5728 #[test]
5729 fn status_replay_filters_same_session_by_project_root() {
5730 let project_a = tempfile::tempdir().unwrap();
5731 let project_b = tempfile::tempdir().unwrap();
5732 let storage = tempfile::tempdir().unwrap();
5733 let session = "shared-session";
5734 let task_id = "bash-2222222222222222";
5735 write_running_project_task(storage.path(), project_a.path(), session, task_id);
5736
5737 let actor_b = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5738 assert!(actor_b
5739 .status(
5740 task_id,
5741 session,
5742 Some(project_b.path()),
5743 Some(storage.path()),
5744 1024,
5745 )
5746 .is_none());
5747 assert!(actor_b.task_for_session(task_id, session).is_none());
5748
5749 let actor_a = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5750 let snapshot = actor_a
5751 .status(
5752 task_id,
5753 session,
5754 Some(project_a.path()),
5755 Some(storage.path()),
5756 1024,
5757 )
5758 .expect("owning project should replay its task");
5759 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
5760 }
5761
5762 #[cfg(unix)]
5763 #[test]
5764 fn multiline_pipeline_stdout_persists_all_lines_after_terminal_status() {
5765 let cases = [
5766 (
5767 "long-first",
5768 "sleep 0.5; printf 'one\\n' | cat\nprintf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
5769 vec!["one", "1", "three"],
5770 ),
5771 (
5772 "short-first",
5773 "printf 'one\\n' | cat\nsleep 0.2; printf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
5774 vec!["one", "1", "three"],
5775 ),
5776 (
5777 "failing-middle",
5778 "sleep 0.2; printf 'one\\n' | cat\nfalse; printf 'after-false\\n' | cat\nprintf 'three\\n' | cat",
5779 vec!["one", "after-false", "three"],
5780 ),
5781 ];
5782
5783 for (name, command, expected_lines) in cases {
5784 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5785 let dir = tempfile::tempdir().unwrap();
5786 let session_id = format!("session-{name}");
5787 let task_id = registry
5788 .spawn(
5789 SpawnPlan::Unsandboxed,
5790 command,
5791 session_id.clone(),
5792 dir.path().to_path_buf(),
5793 HashMap::new(),
5794 Some(Duration::from_secs(30)),
5795 dir.path().to_path_buf(),
5796 10,
5797 true,
5798 true,
5799 Some(dir.path().to_path_buf()),
5800 )
5801 .unwrap();
5802
5803 let snapshot = wait_for_terminal_snapshot(
5804 ®istry,
5805 &task_id,
5806 &session_id,
5807 dir.path(),
5808 dir.path(),
5809 );
5810 assert_eq!(
5811 snapshot.info.status,
5812 BgTaskStatus::Completed,
5813 "{name}: task should complete; snapshot={snapshot:?}"
5814 );
5815 assert_eq!(
5816 snapshot.exit_code,
5817 Some(0),
5818 "{name}: script should use the final command's exit code"
5819 );
5820
5821 let stdout = String::from_utf8(
5822 registry
5823 .read_artifact(&task_id, &session_id, TaskArtifact::Stdout)
5824 .expect("read validated stdout artifact"),
5825 )
5826 .expect("stdout is UTF-8");
5827 let lines: Vec<&str> = stdout.lines().collect();
5828 assert_eq!(
5829 lines, expected_lines,
5830 "{name}: raw stdout artifact must include every newline-separated command's output"
5831 );
5832 }
5833 }
5834
5835 #[test]
5836 fn recognizes_all_recovery_marker_forms() {
5837 assert!(is_recovery_marker(
5838 "[truncated output; full output: read \"/tmp/out\"]"
5839 ));
5840 assert!(is_recovery_marker(
5841 "[omitted output; see remaining: tail -n +42 \"/tmp/out\"]"
5842 ));
5843 assert!(is_recovery_marker(
5844 "[truncated output; full output unavailable]"
5845 ));
5846 assert!(is_recovery_marker(
5847 r#"[truncated 123 bytes from saved output prefix; retained output: read "/tmp/out"]"#
5848 ));
5849 }
5850
5851 #[test]
5852 fn recovery_marker_reports_disk_prefix_truncation_as_retained_output() {
5853 let recovery = RecoveryContext {
5854 dropped_by_class: BTreeMap::new(),
5855 had_inner_drop: false,
5856 offset_hint_eligible: false,
5857 offset_start_line: None,
5858 byte_truncated: false,
5859 disk_truncated_prefix_bytes: 4096,
5860 output_path: Some("/tmp/stdout".to_string()),
5861 stderr_path: None,
5862 include_stderr_path: false,
5863 artifact_access: ArtifactRecoveryAccess {
5864 task_id: "bash-test".to_string(),
5865 readable: true,
5866 },
5867 };
5868
5869 let marker = recovery_marker(&recovery).expect("disk truncation must emit marker");
5870
5871 assert!(marker.contains("truncated 4096 bytes from saved output prefix"));
5872 assert!(marker.contains(r#"retained output: read "/tmp/stdout""#));
5873 assert!(!marker.contains("full output: read"));
5874 }
5875
5876 #[test]
5877 fn killed_exit_marker_sets_nonzero_sentinel_exit_code() {
5878 let metadata = PersistedTask::starting(
5879 "task".to_string(),
5880 "session".to_string(),
5881 "cargo test".to_string(),
5882 PathBuf::from("/tmp"),
5883 None,
5884 None,
5885 true,
5886 true,
5887 );
5888
5889 let terminal = terminal_metadata_from_marker(metadata, ExitMarker::Killed, None);
5890
5891 assert_eq!(terminal.status, BgTaskStatus::Killed);
5892 assert_eq!(terminal.exit_code, Some(137));
5893 }
5894
5895 #[test]
5896 fn terminal_status_polls_use_cached_render_once_and_off_lock() {
5897 let registry = BgTaskRegistry::default();
5898 let dir = tempfile::tempdir().unwrap();
5899 let (_task_id, task) = insert_terminal_piped_task(
5900 ®istry,
5901 &dir,
5902 "custom-tool --verbose",
5903 &"stdout line\n".repeat(200_000),
5904 "",
5905 true,
5906 );
5907 let calls = Arc::new(AtomicUsize::new(0));
5908 let saw_unlocked_state = Arc::new(AtomicBool::new(false));
5909 let task_holder = Arc::new(Mutex::new(Some(Arc::clone(&task))));
5910 let calls_for_closure = Arc::clone(&calls);
5911 let unlocked_for_closure = Arc::clone(&saw_unlocked_state);
5912 let task_for_closure = Arc::clone(&task_holder);
5913 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
5914 calls_for_closure.fetch_add(1, Ordering::SeqCst);
5915 if let Some(task) = task_for_closure.lock().unwrap().as_ref() {
5916 if task.state.try_lock().is_ok() {
5917 unlocked_for_closure.store(true, Ordering::SeqCst);
5918 }
5919 }
5920 CompressionResult::new(format!("compressed {} bytes", output.len()))
5921 });
5922
5923 let first = registry
5924 .status(
5925 &task.task_id,
5926 "session",
5927 None,
5928 Some(dir.path()),
5929 RUNNING_OUTPUT_PREVIEW_BYTES,
5930 )
5931 .unwrap();
5932 let second = registry
5933 .status(
5934 &task.task_id,
5935 "session",
5936 None,
5937 Some(dir.path()),
5938 RUNNING_OUTPUT_PREVIEW_BYTES,
5939 )
5940 .unwrap();
5941 let listed = registry.list(RUNNING_OUTPUT_PREVIEW_BYTES);
5942
5943 assert_eq!(
5944 calls.load(Ordering::SeqCst),
5945 1,
5946 "terminal render must be cached"
5947 );
5948 assert!(
5949 saw_unlocked_state.load(Ordering::SeqCst),
5950 "compressor must run after releasing the task state lock"
5951 );
5952 assert!(first.output_preview.starts_with("compressed "));
5953 assert_eq!(second.output_preview, first.output_preview);
5954 assert_eq!(listed[0].output_preview, first.output_preview);
5955 }
5956
5957 #[test]
5958 fn completion_preview_success_keeps_tail_only() {
5959 let registry = BgTaskRegistry::default();
5964 let dir = tempfile::tempdir().unwrap();
5965 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
5966 let (_task_id, task) =
5967 insert_terminal_piped_task(®istry, &dir, "cat big.log", &output, "", false);
5968
5969 registry.post_terminal_transition(&task, true).unwrap();
5970 let completions = registry.drain_completions_for_session(Some("session"));
5971 assert_eq!(completions.len(), 1);
5972 let preview = &completions[0].output_preview;
5973 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
5974 assert!(!preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
5975 assert!(completions[0].output_truncated);
5976 }
5977
5978 #[test]
5979 fn completion_preview_failure_keeps_head_and_tail() {
5980 let registry = BgTaskRegistry::default();
5983 let dir = tempfile::tempdir().unwrap();
5984 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
5985 let task_id = random_slug();
5986 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5987 fs::create_dir_all(&paths.dir).unwrap();
5988 fs::write(&paths.stdout, &output).unwrap();
5989 fs::write(&paths.stderr, "").unwrap();
5990 let mut metadata = PersistedTask::starting(
5991 task_id.clone(),
5992 "session".to_string(),
5993 "cat big.log".to_string(),
5994 dir.path().to_path_buf(),
5995 Some(dir.path().to_path_buf()),
5996 Some(30_000),
5997 true,
5998 false,
5999 );
6000 metadata.mark_terminal(BgTaskStatus::Failed, Some(1), None);
6001 write_task(&paths.json, &metadata).unwrap();
6002 registry
6003 .insert_rehydrated_task(metadata, paths, true)
6004 .expect("insert terminal task");
6005 let task = registry.task_for_session(&task_id, "session").unwrap();
6006
6007 registry.post_terminal_transition(&task, true).unwrap();
6008 let completions = registry.drain_completions_for_session(Some("session"));
6009 assert_eq!(completions.len(), 1);
6010 let preview = &completions[0].output_preview;
6011 assert!(preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
6012 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
6013 }
6014
6015 #[test]
6016 fn has_completions_for_session_matches_pending_delivery() {
6017 let registry = BgTaskRegistry::default();
6018 assert!(!registry.has_completions_for_session(Some("session")));
6019 assert!(!registry.has_completions_for_session(None));
6020
6021 let dir = tempfile::tempdir().unwrap();
6022 let (_task_id, task) =
6023 insert_terminal_piped_task(®istry, &dir, QUICK_SUCCESS_COMMAND, "done\n", "", false);
6024 registry.post_terminal_transition(&task, true).unwrap();
6025
6026 assert!(registry.has_completions_for_session(Some("session")));
6027 assert!(registry.has_completions_for_session(None));
6028 assert!(!registry.has_completions_for_session(Some("other-session")));
6029
6030 let completions = registry.drain_completions_for_session(Some("session"));
6031 assert_eq!(completions.len(), 1);
6032 assert_eq!(completions[0].task_id, task.task_id);
6033 }
6034
6035 #[test]
6036 fn structured_gh_json_survives_intact_and_ignores_stderr() {
6037 let registry = BgTaskRegistry::default();
6038 let dir = tempfile::tempdir().unwrap();
6039 let calls = Arc::new(AtomicUsize::new(0));
6040 let calls_for_closure = Arc::clone(&calls);
6041 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
6042 calls_for_closure.fetch_add(1, Ordering::SeqCst);
6043 CompressionResult::new(output)
6044 });
6045 let (task_id, _task) = insert_terminal_piped_task(
6046 ®istry,
6047 &dir,
6048 "gh pr view 123 --json body",
6049 "{\"body\":\"hello\"}",
6050 "warning: stderr must not join json",
6051 true,
6052 );
6053
6054 let snapshot = registry
6055 .status(
6056 &task_id,
6057 "session",
6058 None,
6059 Some(dir.path()),
6060 RUNNING_OUTPUT_PREVIEW_BYTES,
6061 )
6062 .unwrap();
6063
6064 assert_eq!(snapshot.output_preview, "{\"body\":\"hello\"}");
6065 assert!(!snapshot.output_preview.contains("warning"));
6066 assert!(!snapshot.output_truncated);
6067 assert_eq!(
6068 calls.load(Ordering::SeqCst),
6069 0,
6070 "structured JSON bypasses compression"
6071 );
6072 }
6073
6074 #[test]
6075 fn registry_emits_single_recovery_marker_for_class_drops() {
6076 let registry = BgTaskRegistry::default();
6077 let dir = tempfile::tempdir().unwrap();
6078 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
6079 let mut dropped = BTreeMap::new();
6080 dropped.insert(DropClass::Error, 18);
6081 dropped.insert(DropClass::Warning, 6);
6082 CompressionResult::with_class_drops("kept diagnostic", dropped)
6083 });
6084 let (task_id, task) =
6085 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
6086
6087 let snapshot = registry
6088 .status(
6089 &task_id,
6090 "session",
6091 None,
6092 Some(dir.path()),
6093 RUNNING_OUTPUT_PREVIEW_BYTES,
6094 )
6095 .unwrap();
6096
6097 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
6098 assert!(snapshot.output_preview.contains("+18 more errors"));
6099 assert!(snapshot.output_preview.contains("+6 more warnings"));
6100 assert!(snapshot
6101 .output_preview
6102 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
6103 assert!(!snapshot.output_preview.contains("tail -n +"));
6104 assert!(snapshot.output_truncated);
6105 }
6106
6107 #[test]
6108 fn registry_marker_reports_semantic_and_byte_drops_once() {
6109 let registry = BgTaskRegistry::default();
6110 let dir = tempfile::tempdir().unwrap();
6111 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
6112 let mut dropped = BTreeMap::new();
6113 dropped.insert(DropClass::Error, 1);
6114 CompressionResult::with_class_drops(
6115 format!("HEAD-SIGNAL\n{}TAIL-SIGNAL", "middle\n".repeat(8_000)),
6116 dropped,
6117 )
6118 });
6119 let (task_id, _task) =
6120 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
6121
6122 let snapshot = registry
6123 .status(
6124 &task_id,
6125 "session",
6126 None,
6127 Some(dir.path()),
6128 RUNNING_OUTPUT_PREVIEW_BYTES,
6129 )
6130 .unwrap();
6131
6132 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
6133 assert!(snapshot.output_preview.contains("+1 more error"));
6134 assert!(snapshot.output_preview.contains("truncated output"));
6135 assert!(snapshot.output_preview.contains("HEAD-SIGNAL"));
6136 assert!(snapshot.output_preview.contains("TAIL-SIGNAL"));
6137 assert!(!snapshot.output_preview.contains("...<truncated"));
6138 assert!(snapshot.output_truncated);
6139 }
6140
6141 #[test]
6142 fn cargo_stderr_class_drops_name_both_capture_paths() {
6143 let registry = BgTaskRegistry::default();
6144 let dir = tempfile::tempdir().unwrap();
6145 let filter_registry = crate::compress::toml_filter::FilterRegistry::default();
6146 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
6147 crate::compress::compress_with_registry_exit_code(
6148 command,
6149 &output,
6150 exit_code,
6151 &filter_registry,
6152 )
6153 });
6154 let stderr = (0..22)
6155 .map(|index| {
6156 format!(
6157 "error: cargo failure {index}\n --> src/lib.rs:{}:1\n |\n{} | boom\n",
6158 index + 1,
6159 index + 1
6160 )
6161 })
6162 .collect::<Vec<_>>()
6163 .join("\n");
6164 let (task_id, task) = insert_terminal_piped_task(
6165 ®istry,
6166 &dir,
6167 "cargo check",
6168 "Finished dev [unoptimized] target(s) in 0.01s\n",
6169 &stderr,
6170 true,
6171 );
6172
6173 let snapshot = registry
6174 .status(
6175 &task_id,
6176 "session",
6177 None,
6178 Some(dir.path()),
6179 RUNNING_OUTPUT_PREVIEW_BYTES,
6180 )
6181 .unwrap();
6182
6183 assert!(snapshot.output_preview.contains("+2 more errors"));
6184 assert!(snapshot
6185 .output_preview
6186 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
6187 assert!(snapshot
6188 .output_preview
6189 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
6190 assert!(!snapshot.output_preview.contains("tail -n +"));
6191 }
6192
6193 #[test]
6194 fn over_ceiling_structured_json_uses_pointer_not_partial_json() {
6195 let registry = BgTaskRegistry::default();
6196 let dir = tempfile::tempdir().unwrap();
6197 let body = format!("{{\"body\":\"{}\"}}", "x".repeat(60 * 1024));
6198 let (task_id, task) = insert_terminal_piped_task(
6199 ®istry,
6200 &dir,
6201 "cd /repo && gh pr view 123 --json body",
6202 &body,
6203 "",
6204 true,
6205 );
6206
6207 let snapshot = registry
6208 .status(
6209 &task_id,
6210 "session",
6211 None,
6212 Some(dir.path()),
6213 RUNNING_OUTPUT_PREVIEW_BYTES,
6214 )
6215 .unwrap();
6216
6217 assert!(snapshot.output_preview.starts_with("[JSON output "));
6218 assert!(snapshot
6219 .output_preview
6220 .contains(&task.paths.stdout.display().to_string()));
6221 assert!(!snapshot.output_preview.contains(&"x".repeat(1024)));
6222 assert!(snapshot.output_truncated);
6223 }
6224
6225 #[test]
6226 fn toml_strip_tail_cap_uses_full_output_hint_not_offset_hint() {
6227 let registry = BgTaskRegistry::default();
6228 let dir = tempfile::tempdir().unwrap();
6229 let filter_registry = crate::compress::toml_filter::build_registry(
6230 crate::compress::builtin_filters::ALL,
6231 None,
6232 None,
6233 );
6234 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
6235 crate::compress::compress_with_registry_exit_code(
6236 command,
6237 &output,
6238 exit_code,
6239 &filter_registry,
6240 )
6241 });
6242 let stdout = format!(
6243 "make[1]: Entering directory `/tmp`\n{}",
6244 (0..100)
6245 .map(|index| format!("compile line {index}"))
6246 .collect::<Vec<_>>()
6247 .join("\n")
6248 );
6249 let (task_id, task) =
6250 insert_terminal_piped_task(®istry, &dir, "make all", &stdout, "", true);
6251
6252 let snapshot = registry
6253 .status(
6254 &task_id,
6255 "session",
6256 None,
6257 Some(dir.path()),
6258 RUNNING_OUTPUT_PREVIEW_BYTES,
6259 )
6260 .unwrap();
6261
6262 assert!(snapshot.output_preview.contains("compile line 99"));
6263 assert!(snapshot.output_preview.contains(&format!(
6264 "full output: read \"{}\"",
6265 task.paths.stdout.display()
6266 )));
6267 assert!(!snapshot
6268 .output_preview
6269 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
6270 assert!(!snapshot.output_preview.contains("tail -n +"));
6271 }
6272
6273 #[test]
6274 fn compressed_false_raw_passthrough_uses_wider_head_tail_cap() {
6275 let registry = BgTaskRegistry::default();
6276 let dir = tempfile::tempdir().unwrap();
6277 let output = format!("RAW-HEAD\n{}RAW-TAIL\n", "raw-middle\n".repeat(8_000));
6278 let (task_id, task) =
6279 insert_terminal_piped_task(®istry, &dir, "cat raw.log", &output, "RAW-ERR\n", false);
6280
6281 let snapshot = registry
6282 .status(
6283 &task_id,
6284 "session",
6285 None,
6286 Some(dir.path()),
6287 RUNNING_OUTPUT_PREVIEW_BYTES,
6288 )
6289 .unwrap();
6290
6291 assert!(snapshot.output_preview.contains("RAW-HEAD"));
6292 assert!(snapshot.output_preview.contains("RAW-TAIL"));
6293 assert!(snapshot.output_preview.contains("truncated output"));
6294 assert!(snapshot
6295 .output_preview
6296 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
6297 assert!(snapshot
6298 .output_preview
6299 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
6300 assert!(!snapshot.output_preview.contains("tail -n +"));
6301 assert!(snapshot.output_preview.len() > 16 * 1024);
6302 assert!(snapshot.output_truncated);
6303 }
6304
6305 #[test]
6306 fn pty_terminal_snapshot_bypasses_line_compression() {
6307 let registry = BgTaskRegistry::default();
6308 let dir = tempfile::tempdir().unwrap();
6309 let calls = Arc::new(AtomicUsize::new(0));
6310 let calls_for_closure = Arc::clone(&calls);
6311 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
6312 calls_for_closure.fetch_add(1, Ordering::SeqCst);
6313 CompressionResult::new(output)
6314 });
6315 let (task_id, _task) = insert_terminal_pty_task(®istry, &dir, "raw\u{1b}[31m pty bytes");
6316
6317 let snapshot = registry
6318 .status(
6319 &task_id,
6320 "session",
6321 None,
6322 Some(dir.path()),
6323 RUNNING_OUTPUT_PREVIEW_BYTES,
6324 )
6325 .unwrap();
6326
6327 assert_eq!(snapshot.info.mode, BgMode::Pty);
6328 assert_eq!(snapshot.output_preview, "");
6329 assert_eq!(calls.load(Ordering::SeqCst), 0);
6330 }
6331
6332 #[test]
6333 fn pty_dimensions_are_persisted_and_returned_in_snapshot() {
6334 let registry = BgTaskRegistry::default();
6335 let dir = tempfile::tempdir().unwrap();
6336 let task_id = registry
6337 .spawn_pty(
6338 SpawnPlan::Unsandboxed,
6339 QUICK_SUCCESS_COMMAND,
6340 "session".to_string(),
6341 dir.path().to_path_buf(),
6342 HashMap::new(),
6343 Some(Duration::from_secs(30)),
6344 dir.path().to_path_buf(),
6345 10,
6346 true,
6347 false,
6348 Some(dir.path().to_path_buf()),
6349 50,
6350 120,
6351 )
6352 .unwrap();
6353
6354 let resolved =
6355 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
6356 let metadata = read_task_at(&resolved).unwrap();
6357 assert_eq!(
6358 metadata.schema_version,
6359 crate::bash_background::persistence::SCHEMA_VERSION
6360 );
6361 assert_eq!(metadata.mode, BgMode::Pty);
6362 assert_eq!(metadata.pty_rows, Some(50));
6363 assert_eq!(metadata.pty_cols, Some(120));
6364
6365 let snapshot = registry
6366 .status(&task_id, "session", None, Some(dir.path()), 1024)
6367 .unwrap();
6368 assert_eq!(snapshot.pty_rows, Some(50));
6369 assert_eq!(snapshot.pty_cols, Some(120));
6370 }
6371
6372 fn spawn_dead_child() -> std::process::Child {
6377 #[cfg(unix)]
6378 let mut cmd = std::process::Command::new("true");
6379 #[cfg(windows)]
6380 let mut cmd = {
6381 let mut c = std::process::Command::new("cmd");
6382 c.args(["/c", "exit", "0"]);
6383 c
6384 };
6385 cmd.stdin(std::process::Stdio::null());
6386 cmd.stdout(std::process::Stdio::null());
6387 cmd.stderr(std::process::Stdio::null());
6388 let mut child = cmd.spawn().expect("spawn replacement child for reap test");
6389 let started = Instant::now();
6398 loop {
6399 match child.try_wait() {
6400 Ok(Some(_)) => break,
6401 Ok(None) => {
6402 if started.elapsed() > Duration::from_secs(5) {
6403 panic!("dead-child stand-in did not exit within 5s");
6404 }
6405 std::thread::sleep(Duration::from_millis(10));
6406 }
6407 Err(error) => panic!("dead-child try_wait failed: {error}"),
6408 }
6409 }
6410 child
6411 }
6412
6413 #[test]
6414 fn ack_marks_delivered_even_when_completion_was_already_consumed_locally() {
6415 let registry = BgTaskRegistry::default();
6416 let dir = tempfile::tempdir().unwrap();
6417 let task_id = registry
6418 .spawn(
6419 SpawnPlan::Unsandboxed,
6420 LONG_RUNNING_COMMAND,
6421 "session".to_string(),
6422 dir.path().to_path_buf(),
6423 HashMap::new(),
6424 Some(Duration::from_secs(30)),
6425 dir.path().to_path_buf(),
6426 10,
6427 true,
6428 false,
6429 Some(dir.path().to_path_buf()),
6430 )
6431 .unwrap();
6432 registry
6433 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6434 .unwrap();
6435 assert_eq!(
6436 registry
6437 .drain_completions_for_session(Some("session"))
6438 .len(),
6439 1
6440 );
6441
6442 registry.inner.completions.lock().unwrap().clear();
6445
6446 assert_eq!(
6447 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
6448 vec![task_id.clone()]
6449 );
6450 assert!(registry
6451 .drain_completions_for_session(Some("session"))
6452 .is_empty());
6453
6454 let resolved =
6455 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
6456 let metadata = read_task_at(&resolved).unwrap();
6457 assert!(metadata.completion_delivered);
6458
6459 let replayed = BgTaskRegistry::default();
6460 replayed
6461 .replay_session_inner(dir.path(), "session", None)
6462 .unwrap();
6463 assert!(replayed
6464 .drain_completions_for_session(Some("session"))
6465 .is_empty());
6466 }
6467
6468 #[test]
6469 fn reclaimed_root_kills_running_task_and_persists_reason() {
6470 let registry = BgTaskRegistry::default();
6471 let root = tempfile::tempdir().unwrap();
6472 let storage = tempfile::tempdir().unwrap();
6473 let task_id = registry
6474 .spawn(
6475 SpawnPlan::Unsandboxed,
6476 LONG_RUNNING_COMMAND,
6477 "session".to_string(),
6478 root.path().to_path_buf(),
6479 HashMap::new(),
6480 Some(Duration::from_secs(30)),
6481 storage.path().to_path_buf(),
6482 10,
6483 true,
6484 false,
6485 Some(root.path().to_path_buf()),
6486 )
6487 .unwrap();
6488 let pid = registry
6489 .status(
6490 &task_id,
6491 "session",
6492 Some(root.path()),
6493 Some(storage.path()),
6494 0,
6495 )
6496 .unwrap()
6497 .child_pid
6498 .unwrap();
6499 assert!(is_process_alive(pid));
6500
6501 assert_eq!(registry.kill_running_tasks_for_root(root.path()), 1);
6502 let deadline = Instant::now() + Duration::from_secs(5);
6503 while is_process_alive(pid) {
6504 assert!(
6505 Instant::now() < deadline,
6506 "reclaimed task process survived kill"
6507 );
6508 std::thread::sleep(Duration::from_millis(20));
6509 }
6510
6511 let snapshot = registry
6512 .status(
6513 &task_id,
6514 "session",
6515 Some(root.path()),
6516 Some(storage.path()),
6517 0,
6518 )
6519 .unwrap();
6520 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
6521 assert_eq!(
6522 snapshot.info.status_reason.as_deref(),
6523 Some(ROOT_RECLAIMED_REASON)
6524 );
6525 let persisted = read_task(
6526 ®istry
6527 .task_json_path(&task_id, "session")
6528 .expect("reclaimed task metadata path"),
6529 )
6530 .expect("persisted reclaimed task");
6531 assert_eq!(
6532 persisted.status_reason.as_deref(),
6533 Some(ROOT_RECLAIMED_REASON)
6534 );
6535 let completion = registry
6536 .drain_completions_for_session(Some("session"))
6537 .pop()
6538 .expect("reclaimed task completion");
6539 assert_eq!(
6540 completion.status_reason.as_deref(),
6541 Some(ROOT_RECLAIMED_REASON)
6542 );
6543 registry.detach();
6544 }
6545
6546 #[test]
6547 fn reclaimed_root_kills_pty_task_and_preserves_reason() {
6548 let registry = BgTaskRegistry::default();
6549 let root = tempfile::tempdir().unwrap();
6550 let storage = tempfile::tempdir().unwrap();
6551 let command = if cfg!(windows) {
6552 "Start-Sleep -Seconds 30"
6553 } else {
6554 "sleep 30"
6555 };
6556 let task_id = registry
6557 .spawn_pty(
6558 SpawnPlan::Unsandboxed,
6559 command,
6560 "session".to_string(),
6561 root.path().to_path_buf(),
6562 HashMap::new(),
6563 Some(Duration::from_secs(60)),
6564 storage.path().to_path_buf(),
6565 10,
6566 true,
6567 false,
6568 Some(root.path().to_path_buf()),
6569 24,
6570 80,
6571 )
6572 .unwrap();
6573 let pid = registry
6574 .status(
6575 &task_id,
6576 "session",
6577 Some(root.path()),
6578 Some(storage.path()),
6579 0,
6580 )
6581 .unwrap()
6582 .child_pid
6583 .unwrap();
6584 assert!(is_process_alive(pid));
6585
6586 assert_eq!(registry.kill_running_tasks_for_root(root.path()), 1);
6587 let deadline = Instant::now() + Duration::from_secs(10);
6588 loop {
6589 let snapshot = registry
6590 .status(
6591 &task_id,
6592 "session",
6593 Some(root.path()),
6594 Some(storage.path()),
6595 0,
6596 )
6597 .unwrap();
6598 if snapshot.info.status.is_terminal() {
6599 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
6600 assert_eq!(
6601 snapshot.info.status_reason.as_deref(),
6602 Some(ROOT_RECLAIMED_REASON)
6603 );
6604 break;
6605 }
6606 assert!(
6607 Instant::now() < deadline,
6608 "reclaimed PTY task did not terminate"
6609 );
6610 std::thread::sleep(Duration::from_millis(20));
6611 }
6612 assert!(!is_process_alive(pid));
6613 let completion = loop {
6618 if let Some(completion) = registry
6619 .drain_completions_for_session(Some("session"))
6620 .pop()
6621 {
6622 break completion;
6623 }
6624 assert!(
6625 Instant::now() < deadline,
6626 "reclaimed PTY completion never arrived"
6627 );
6628 std::thread::sleep(Duration::from_millis(20));
6629 };
6630 assert_eq!(
6631 completion.status_reason.as_deref(),
6632 Some(ROOT_RECLAIMED_REASON)
6633 );
6634 registry.detach();
6635 }
6636
6637 #[test]
6638 fn register_watch_rejects_unknown_task() {
6639 let registry = BgTaskRegistry::default();
6640
6641 let result = registry.register_watch(
6642 "missing-task".to_string(),
6643 WatchPattern::Substring("READY".into()),
6644 true,
6645 );
6646
6647 assert_eq!(result, Err("task_not_found"));
6648 }
6649
6650 #[test]
6651 fn register_watch_on_terminal_task_scans_existing_output() {
6652 let frames = Arc::new(Mutex::new(Vec::new()));
6653 let captured = Arc::clone(&frames);
6654 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
6655 captured.lock().unwrap().push(frame);
6656 })
6657 as Box<dyn Fn(PushFrame) + Send + Sync>);
6658 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
6659 let dir = tempfile::tempdir().unwrap();
6660 let task_id = registry
6661 .spawn(
6662 SpawnPlan::Unsandboxed,
6663 LONG_RUNNING_COMMAND,
6664 "session".to_string(),
6665 dir.path().to_path_buf(),
6666 HashMap::new(),
6667 Some(Duration::from_secs(30)),
6668 dir.path().to_path_buf(),
6669 10,
6670 true,
6671 false,
6672 Some(dir.path().to_path_buf()),
6673 )
6674 .unwrap();
6675 registry
6676 .inner
6677 .shutdown
6678 .store(true, std::sync::atomic::Ordering::SeqCst);
6679 let task = registry.task_for_session(&task_id, "session").unwrap();
6680 std::fs::write(&task.paths.stdout, "READY\n").unwrap();
6681 registry
6682 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6683 .unwrap();
6684 frames.lock().unwrap().clear();
6685 registry.inner.completions.lock().unwrap().clear();
6686
6687 registry
6688 .register_watch(
6689 task_id.clone(),
6690 WatchPattern::Substring("READY".into()),
6691 true,
6692 )
6693 .unwrap();
6694
6695 let frames = frames.lock().unwrap();
6696 let frame = frames
6697 .iter()
6698 .find_map(|frame| match frame {
6699 PushFrame::BashPatternMatch(frame) => Some(frame),
6700 _ => None,
6701 })
6702 .expect("terminal watch registration should emit pattern frame");
6703 assert_eq!(frame.reason, "pattern_match");
6704 assert_eq!(frame.task_id, task_id);
6705 assert_eq!(frame.session_id, "session");
6706 assert_eq!(frame.match_text, "READY");
6707 assert_eq!(frame.match_offset, 0);
6708 assert_eq!(registry.active_watch_count(&frame.task_id), 0);
6709 let metadata = read_task(&task.paths.json).unwrap();
6710 assert!(metadata.completion_delivered);
6711 }
6712
6713 #[test]
6714 fn cleanup_finished_removes_terminal_tasks_older_than_threshold() {
6715 let registry = BgTaskRegistry::default();
6716 let dir = tempfile::tempdir().unwrap();
6717 let task_id = registry
6718 .spawn(
6719 SpawnPlan::Unsandboxed,
6720 QUICK_SUCCESS_COMMAND,
6721 "session".to_string(),
6722 dir.path().to_path_buf(),
6723 HashMap::new(),
6724 Some(Duration::from_secs(30)),
6725 dir.path().to_path_buf(),
6726 10,
6727 true,
6728 false,
6729 Some(dir.path().to_path_buf()),
6730 )
6731 .unwrap();
6732 registry
6733 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6734 .unwrap();
6735 let completions = registry.drain_completions_for_session(Some("session"));
6736 assert_eq!(completions.len(), 1);
6737 assert_eq!(
6738 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
6739 vec![task_id.clone()]
6740 );
6741
6742 registry.cleanup_finished(Duration::ZERO);
6743
6744 assert!(registry.inner.tasks.lock().unwrap().is_empty());
6745 }
6746
6747 #[test]
6748 fn cleanup_finished_retains_undelivered_terminals() {
6749 let registry = BgTaskRegistry::default();
6750 let dir = tempfile::tempdir().unwrap();
6751 let task_id = registry
6752 .spawn(
6753 SpawnPlan::Unsandboxed,
6754 QUICK_SUCCESS_COMMAND,
6755 "session".to_string(),
6756 dir.path().to_path_buf(),
6757 HashMap::new(),
6758 Some(Duration::from_secs(30)),
6759 dir.path().to_path_buf(),
6760 10,
6761 true,
6762 false,
6763 Some(dir.path().to_path_buf()),
6764 )
6765 .unwrap();
6766 registry
6767 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6768 .unwrap();
6769
6770 registry.cleanup_finished(Duration::ZERO);
6771
6772 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
6773 }
6774
6775 #[test]
6783 fn reap_child_marks_failed_when_child_exits_without_exit_marker() {
6784 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6785 let dir = tempfile::tempdir().unwrap();
6786 let task_id = registry
6787 .spawn(
6788 SpawnPlan::Unsandboxed,
6789 QUICK_SUCCESS_COMMAND,
6790 "session".to_string(),
6791 dir.path().to_path_buf(),
6792 HashMap::new(),
6793 Some(Duration::from_secs(30)),
6794 dir.path().to_path_buf(),
6795 10,
6796 true,
6797 false,
6798 Some(dir.path().to_path_buf()),
6799 )
6800 .unwrap();
6801
6802 let task = registry.task_for_session(&task_id, "session").unwrap();
6803
6804 let started = Instant::now();
6809 loop {
6810 let exited = {
6811 let mut state = task.state.lock().unwrap();
6812 match &mut state.runtime {
6813 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
6814 _ => true,
6815 }
6816 };
6817 if exited {
6818 break;
6819 }
6820 assert!(
6821 started.elapsed() < Duration::from_secs(5),
6822 "child should exit quickly"
6823 );
6824 std::thread::sleep(Duration::from_millis(20));
6825 }
6826
6827 registry
6835 .inner
6836 .shutdown
6837 .store(true, std::sync::atomic::Ordering::SeqCst);
6838 std::thread::sleep(Duration::from_millis(550));
6842
6843 let _ = std::fs::remove_file(&task.paths.exit);
6846
6847 {
6862 let mut state = task.state.lock().unwrap();
6863 state.metadata.status = BgTaskStatus::Running;
6864 state.metadata.status_reason = None;
6865 state.metadata.exit_code = None;
6866 state.metadata.finished_at = None;
6867 state.metadata.duration_ms = None;
6868 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
6871 .expect("persist reset Running metadata for reap_child test");
6872 if matches!(state.runtime, TaskRuntime::Piped(None)) {
6876 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
6877 }
6878 }
6879 *task.terminal_at.lock().unwrap() = None;
6882
6883 assert!(
6886 task.is_running(),
6887 "precondition: metadata.status == Running"
6888 );
6889 assert!(
6890 !task.paths.exit.exists(),
6891 "precondition: exit marker absent"
6892 );
6893
6894 registry.reap_child(&task);
6899
6900 {
6901 let state = task.state.lock().unwrap();
6902 assert_eq!(
6903 state.metadata.status,
6904 BgTaskStatus::Running,
6905 "first reap must leave status Running while waiting one pass for marker"
6906 );
6907 assert_eq!(
6908 state.metadata.status_reason, None,
6909 "first reap must not record a failure reason"
6910 );
6911 assert!(
6912 matches!(state.runtime, TaskRuntime::Piped(None)),
6913 "child handle must be released after first reap"
6914 );
6915 assert!(
6916 state.detached,
6917 "task must be marked detached after first reap"
6918 );
6919 }
6920
6921 registry.reap_child(&task);
6925
6926 let state = task.state.lock().unwrap();
6927 assert!(
6928 state.metadata.status.is_terminal(),
6929 "second reap must transition to terminal when PID dead and no marker. Got status={:?}",
6930 state.metadata.status
6931 );
6932 assert_eq!(
6933 state.metadata.status,
6934 BgTaskStatus::Failed,
6935 "must specifically be Failed (not Killed): status={:?}",
6936 state.metadata.status
6937 );
6938 assert_eq!(
6939 state.metadata.status_reason.as_deref(),
6940 Some("process exited without exit marker"),
6941 "reason must match replay path's wording: {:?}",
6942 state.metadata.status_reason
6943 );
6944 assert!(
6945 matches!(state.runtime, TaskRuntime::Piped(None)),
6946 "child handle must stay released after second reap"
6947 );
6948 assert!(
6949 state.detached,
6950 "task must remain detached after second reap"
6951 );
6952 }
6953
6954 #[test]
6959 fn reap_child_preserves_running_when_exit_marker_exists() {
6960 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6961 let dir = tempfile::tempdir().unwrap();
6962 let task_id = registry
6963 .spawn(
6964 SpawnPlan::Unsandboxed,
6965 QUICK_SUCCESS_COMMAND,
6966 "session".to_string(),
6967 dir.path().to_path_buf(),
6968 HashMap::new(),
6969 Some(Duration::from_secs(30)),
6970 dir.path().to_path_buf(),
6971 10,
6972 true,
6973 false,
6974 Some(dir.path().to_path_buf()),
6975 )
6976 .unwrap();
6977
6978 let task = registry.task_for_session(&task_id, "session").unwrap();
6979
6980 let started = Instant::now();
6983 loop {
6984 let exited = {
6985 let mut state = task.state.lock().unwrap();
6986 match &mut state.runtime {
6987 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
6988 _ => true,
6989 }
6990 };
6991 if exited && task.paths.exit.exists() {
6992 break;
6993 }
6994 assert!(
6995 started.elapsed() < Duration::from_secs(5),
6996 "child should exit and write marker quickly"
6997 );
6998 std::thread::sleep(Duration::from_millis(20));
6999 }
7000
7001 registry
7007 .inner
7008 .shutdown
7009 .store(true, std::sync::atomic::Ordering::SeqCst);
7010 std::thread::sleep(Duration::from_millis(550));
7011
7012 {
7018 let mut state = task.state.lock().unwrap();
7019 state.metadata.status = BgTaskStatus::Running;
7020 state.metadata.status_reason = None;
7021 if matches!(state.runtime, TaskRuntime::Piped(None)) {
7022 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
7023 }
7024 }
7025 *task.terminal_at.lock().unwrap() = None;
7026 if !task.paths.exit.exists() {
7029 std::fs::write(&task.paths.exit, "0").expect("write replacement exit marker");
7030 }
7031
7032 registry.reap_child(&task);
7036
7037 let state = task.state.lock().unwrap();
7038 assert!(
7039 matches!(state.runtime, TaskRuntime::Piped(None)),
7040 "child handle still released even when marker exists"
7041 );
7042 assert!(
7043 state.detached,
7044 "task still marked detached even when marker exists"
7045 );
7046 assert_eq!(
7051 state.metadata.status,
7052 BgTaskStatus::Running,
7053 "reap_child must defer to poll_task when marker exists"
7054 );
7055 }
7056
7057 #[cfg(unix)]
7061 fn pid_stat(pid: u32) -> Option<String> {
7062 let output = std::process::Command::new("ps")
7063 .args(["-o", "stat=", "-p", &pid.to_string()])
7064 .output()
7065 .ok()?;
7066 if !output.status.success() {
7067 return None;
7068 }
7069 let stat = String::from_utf8_lossy(&output.stdout).trim().to_string();
7070 if stat.is_empty() {
7071 None
7072 } else {
7073 Some(stat)
7074 }
7075 }
7076
7077 #[cfg(unix)]
7079 fn is_zombie(pid: u32) -> bool {
7080 pid_stat(pid).is_some_and(|stat| stat.starts_with('Z'))
7081 }
7082
7083 #[cfg(unix)]
7089 fn spawn_unreaped_zombie() -> std::process::Child {
7090 let child = std::process::Command::new("true")
7091 .stdin(std::process::Stdio::null())
7092 .stdout(std::process::Stdio::null())
7093 .stderr(std::process::Stdio::null())
7094 .spawn()
7095 .expect("spawn zombie stand-in");
7096 let pid = child.id();
7097 let started = Instant::now();
7098 while !is_zombie(pid) {
7099 assert!(
7100 started.elapsed() < Duration::from_secs(5),
7101 "stand-in child should become a zombie within 5s"
7102 );
7103 std::thread::sleep(Duration::from_millis(10));
7104 }
7105 child
7107 }
7108
7109 #[cfg(unix)]
7119 #[test]
7120 fn finalize_from_marker_reaps_child_no_zombie() {
7121 use std::sync::atomic::Ordering;
7122
7123 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7124 let dir = tempfile::tempdir().unwrap();
7125 let task_id = registry
7126 .spawn(
7127 SpawnPlan::Unsandboxed,
7128 QUICK_SUCCESS_COMMAND,
7129 "session".to_string(),
7130 dir.path().to_path_buf(),
7131 HashMap::new(),
7132 Some(Duration::from_secs(30)),
7133 dir.path().to_path_buf(),
7134 10,
7135 true,
7136 false,
7137 Some(dir.path().to_path_buf()),
7138 )
7139 .unwrap();
7140
7141 registry.inner.shutdown.store(true, Ordering::SeqCst);
7145 std::thread::sleep(Duration::from_millis(550));
7146
7147 let task = registry.task_for_session(&task_id, "session").unwrap();
7148
7149 let started = Instant::now();
7153 while !task.paths.exit.exists() {
7154 assert!(
7155 started.elapsed() < Duration::from_secs(5),
7156 "exit marker should land quickly for `true`"
7157 );
7158 std::thread::sleep(Duration::from_millis(20));
7159 }
7160
7161 let zombie_pid;
7167 {
7168 let mut state = task.state.lock().unwrap();
7169 state.metadata.status = BgTaskStatus::Running;
7170 state.metadata.status_reason = None;
7171 state.metadata.exit_code = None;
7172 state.metadata.finished_at = None;
7173 state.metadata.duration_ms = None;
7174 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
7175 .expect("persist reset Running metadata");
7176 let zombie = spawn_unreaped_zombie();
7177 zombie_pid = zombie.id();
7178 state.runtime = TaskRuntime::Piped(Some(zombie));
7179 }
7180 *task.terminal_at.lock().unwrap() = None;
7181
7182 assert!(
7184 is_zombie(zombie_pid),
7185 "precondition: stand-in child {zombie_pid} must be a zombie before finalize"
7186 );
7187
7188 registry.poll_task(&task).unwrap();
7191
7192 {
7193 let state = task.state.lock().unwrap();
7194 assert!(
7195 matches!(state.runtime, TaskRuntime::Piped(None)),
7196 "child handle must be released after marker finalize"
7197 );
7198 assert!(
7199 state.metadata.status.is_terminal(),
7200 "task must be terminal after marker finalize: {:?}",
7201 state.metadata.status
7202 );
7203 }
7204
7205 assert!(
7208 !is_zombie(zombie_pid),
7209 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
7210 after the exit-marker terminal transition"
7211 );
7212 }
7213
7214 #[cfg(unix)]
7218 #[test]
7219 fn kill_with_existing_marker_reaps_child_no_zombie() {
7220 use std::sync::atomic::Ordering;
7221
7222 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7223 let dir = tempfile::tempdir().unwrap();
7224 let task_id = registry
7225 .spawn(
7226 SpawnPlan::Unsandboxed,
7227 QUICK_SUCCESS_COMMAND,
7228 "session".to_string(),
7229 dir.path().to_path_buf(),
7230 HashMap::new(),
7231 Some(Duration::from_secs(30)),
7232 dir.path().to_path_buf(),
7233 10,
7234 true,
7235 false,
7236 Some(dir.path().to_path_buf()),
7237 )
7238 .unwrap();
7239
7240 registry.inner.shutdown.store(true, Ordering::SeqCst);
7241 std::thread::sleep(Duration::from_millis(550));
7242
7243 let task = registry.task_for_session(&task_id, "session").unwrap();
7244
7245 let started = Instant::now();
7246 while !task.paths.exit.exists() {
7247 assert!(
7248 started.elapsed() < Duration::from_secs(5),
7249 "exit marker should land quickly for `true`"
7250 );
7251 std::thread::sleep(Duration::from_millis(20));
7252 }
7253
7254 let zombie_pid;
7255 {
7256 let mut state = task.state.lock().unwrap();
7257 state.metadata.status = BgTaskStatus::Running;
7258 state.metadata.status_reason = None;
7259 state.metadata.exit_code = None;
7260 state.metadata.finished_at = None;
7261 state.metadata.duration_ms = None;
7262 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
7263 .expect("persist reset Running metadata");
7264 let zombie = spawn_unreaped_zombie();
7265 zombie_pid = zombie.id();
7266 state.runtime = TaskRuntime::Piped(Some(zombie));
7267 }
7268 *task.terminal_at.lock().unwrap() = None;
7269
7270 assert!(
7271 is_zombie(zombie_pid),
7272 "precondition: stand-in child {zombie_pid} must be a zombie before kill"
7273 );
7274
7275 registry
7277 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
7278 .expect("kill should succeed");
7279
7280 {
7281 let state = task.state.lock().unwrap();
7282 assert!(
7283 matches!(state.runtime, TaskRuntime::Piped(None)),
7284 "child handle must be released after marker-aware kill"
7285 );
7286 assert!(state.metadata.status.is_terminal());
7287 }
7288
7289 assert!(
7290 !is_zombie(zombie_pid),
7291 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
7292 after a marker-aware kill"
7293 );
7294 }
7295
7296 #[test]
7297 fn cleanup_finished_keeps_running_tasks() {
7298 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7299 let dir = tempfile::tempdir().unwrap();
7300 let task_id = registry
7301 .spawn(
7302 SpawnPlan::Unsandboxed,
7303 LONG_RUNNING_COMMAND,
7304 "session".to_string(),
7305 dir.path().to_path_buf(),
7306 HashMap::new(),
7307 Some(Duration::from_secs(30)),
7308 dir.path().to_path_buf(),
7309 10,
7310 true,
7311 false,
7312 Some(dir.path().to_path_buf()),
7313 )
7314 .unwrap();
7315
7316 registry.cleanup_finished(Duration::ZERO);
7317
7318 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
7319 let _ = registry.kill(&task_id, "session");
7320 }
7321
7322 #[cfg(unix)]
7323 #[test]
7324 fn rehydrating_sandboxed_task_never_respawns_persisted_command() {
7325 let project = tempfile::tempdir().unwrap();
7326 let storage = tempfile::tempdir().unwrap();
7327 let sandbox_temp = storage.path().join("sandbox-temp");
7328 fs::create_dir(&sandbox_temp).unwrap();
7329 let launcher_script = project.path().join("sandbox-launch");
7330 let launcher = PathBuf::from("/bin/sh");
7331 fs::write(
7332 &launcher_script,
7333 "while [ \"$#\" -gt 0 ]; do\n if [ \"$1\" = -- ]; then\n shift\n exec \"$@\"\n fi\n shift\ndone\nexit 78\n",
7334 )
7335 .unwrap();
7336 let mut permissions = fs::metadata(&launcher_script).unwrap().permissions();
7337 permissions.set_mode(0o700);
7338 fs::set_permissions(&launcher_script, permissions).unwrap();
7339
7340 let profile = crate::sandbox_profile::SandboxProfile::build(
7341 vec![project.path().to_path_buf()],
7342 Vec::new(),
7343 Vec::new(),
7344 Vec::new(),
7345 Vec::new(),
7346 Vec::new(),
7347 Vec::new(),
7348 sandbox_temp,
7349 )
7350 .unwrap();
7351 let plan = SpawnPlan::launcher_for_test(profile, launcher);
7352 let spawn_marker = project.path().join("spawn-count");
7353 let stop_marker = project.path().join("stop-command");
7354 let quote =
7355 |path: &Path| format!("'{}'", path.display().to_string().replace('\'', "'\\''"));
7356 let command = format!(
7357 "printf 'spawn\\n' >> {}; while [ ! -e {} ]; do sleep 0.05; done",
7358 quote(&spawn_marker),
7359 quote(&stop_marker)
7360 );
7361
7362 let original = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7363 let task_id = original
7364 .spawn(
7365 plan,
7366 &command,
7367 "sandbox-rehydrate".to_string(),
7368 project.path().to_path_buf(),
7369 HashMap::new(),
7370 Some(Duration::from_secs(30)),
7371 storage.path().to_path_buf(),
7372 10,
7373 true,
7374 false,
7375 Some(project.path().to_path_buf()),
7376 )
7377 .unwrap();
7378 let started = Instant::now();
7379 while !spawn_marker.exists() {
7380 assert!(
7381 started.elapsed() < Duration::from_secs(20),
7382 "original sandboxed task did not start"
7383 );
7384 std::thread::sleep(Duration::from_millis(10));
7385 }
7386 original.detach();
7387
7388 let restarted = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7389 restarted
7390 .replay_session(storage.path(), "sandbox-rehydrate")
7391 .unwrap();
7392 let replayed = restarted
7393 .status(
7394 &task_id,
7395 "sandbox-rehydrate",
7396 Some(project.path()),
7397 Some(storage.path()),
7398 4096,
7399 )
7400 .expect("rehydrated sandbox task");
7401 assert_eq!(replayed.info.status, BgTaskStatus::Running);
7402 assert!(replayed.sandbox_native);
7403
7404 std::thread::sleep(Duration::from_millis(650));
7405 assert_eq!(
7406 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
7407 1,
7408 "registry replay must observe the persisted process without spawning its command"
7409 );
7410
7411 fs::write(&stop_marker, "stop").unwrap();
7412 let terminal = wait_for_terminal_snapshot(
7413 &restarted,
7414 &task_id,
7415 "sandbox-rehydrate",
7416 project.path(),
7417 storage.path(),
7418 );
7419 assert_eq!(terminal.info.status, BgTaskStatus::Completed);
7420 assert_eq!(
7421 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
7422 1
7423 );
7424 restarted.detach();
7425 }
7426
7427 #[cfg(windows)]
7428 fn wait_for_file(path: &Path) -> String {
7429 let started = Instant::now();
7436 loop {
7437 if let Ok(content) = fs::read_to_string(path) {
7438 if !content.trim().is_empty() {
7439 return content;
7440 }
7441 }
7442 assert!(
7443 started.elapsed() < Duration::from_secs(30),
7444 "timed out waiting for non-empty {}",
7445 path.display()
7446 );
7447 std::thread::sleep(Duration::from_millis(100));
7448 }
7449 }
7450
7451 #[cfg(windows)]
7452 fn spawn_windows_registry_command(
7453 command: &str,
7454 ) -> (BgTaskRegistry, tempfile::TempDir, String) {
7455 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7456 let dir = tempfile::tempdir().unwrap();
7457 let task_id = registry
7458 .spawn(
7459 SpawnPlan::Unsandboxed,
7460 command,
7461 "session".to_string(),
7462 dir.path().to_path_buf(),
7463 HashMap::new(),
7464 Some(Duration::from_secs(30)),
7465 dir.path().to_path_buf(),
7466 10,
7467 false,
7468 false,
7469 Some(dir.path().to_path_buf()),
7470 )
7471 .unwrap();
7472 (registry, dir, task_id)
7473 }
7474
7475 #[cfg(windows)]
7476 #[test]
7477 fn windows_spawn_writes_exit_marker_for_zero_exit() {
7478 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 0");
7479 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
7480
7481 let content = wait_for_file(&exit_path);
7482
7483 assert_eq!(content.trim(), "0");
7484 }
7485
7486 #[cfg(windows)]
7487 #[test]
7488 fn windows_spawn_writes_exit_marker_for_nonzero_exit() {
7489 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 42");
7490 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
7491
7492 let content = wait_for_file(&exit_path);
7493
7494 assert_eq!(content.trim(), "42");
7495 }
7496
7497 #[cfg(windows)]
7498 #[test]
7499 fn windows_spawn_captures_stdout_to_disk() {
7500 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c echo hello");
7501 let task = registry.task_for_session(&task_id, "session").unwrap();
7502 let stdout_path = task.paths.stdout.clone();
7503 let exit_path = task.paths.exit.clone();
7504
7505 let _ = wait_for_file(&exit_path);
7506 let stdout = fs::read_to_string(stdout_path).expect("read stdout");
7507
7508 assert!(stdout.contains("hello"), "stdout was {stdout:?}");
7509 }
7510
7511 #[cfg(windows)]
7512 #[test]
7513 fn windows_spawn_uses_pwsh_when_available() {
7514 let candidates = crate::windows_shell::shell_candidates_with(
7518 |binary| match binary {
7519 "pwsh.exe" => Some(std::path::PathBuf::from(r"C:\pwsh\pwsh.exe")),
7520 "powershell.exe" => Some(std::path::PathBuf::from(r"C:\ps\powershell.exe")),
7521 _ => None,
7522 },
7523 || None,
7524 );
7525 let shell = candidates.first().expect("at least one candidate").clone();
7526 assert_eq!(shell, crate::windows_shell::WindowsShell::Pwsh);
7527 assert_eq!(shell.binary().as_ref(), "pwsh.exe");
7528 }
7529
7530 #[cfg(windows)]
7533 #[test]
7534 fn windows_shell_cmd_wrapper_writes_marker_via_temp_rename() {
7535 let exit_path = Path::new(r"C:\Temp\bash-test.exit");
7536 let script =
7537 crate::windows_shell::WindowsShell::Cmd.wrapper_script("cmd /c exit 42", exit_path);
7538
7539 assert!(
7540 script.contains("set CODE=%ERRORLEVEL%"),
7541 "wrapper must capture the child exit code: {script}"
7542 );
7543 assert!(
7544 script.contains("exit /B %CODE%"),
7545 "wrapper must propagate the child exit code: {script}"
7546 );
7547 assert!(
7552 script.contains("bash-test.exit"),
7553 "wrapper must target the exit marker path: {script}"
7554 );
7555 assert!(
7556 script.contains("move /Y"),
7557 "wrapper must write the marker atomically via temp-file + rename: {script}"
7558 );
7559 }
7560
7561 #[cfg(windows)]
7567 #[test]
7568 fn windows_shell_cmd_bg_command_uses_minimal_cmd_flags() {
7569 use crate::windows_shell::WindowsShell;
7570 let cmd = WindowsShell::Cmd.bg_command("echo wrapped");
7571 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
7572 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
7573 assert_eq!(
7574 args_strs,
7575 vec!["/D", "/S", "/C", "echo wrapped"],
7576 "Cmd::bg_command must prepend /D /S /C"
7577 );
7578 }
7579
7580 #[cfg(windows)]
7583 #[test]
7584 fn windows_shell_pwsh_bg_command_uses_standard_args() {
7585 use crate::windows_shell::WindowsShell;
7586 let cmd = WindowsShell::Pwsh.bg_command("Get-Date");
7587 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
7588 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
7589 assert!(
7590 args_strs.contains(&"-Command"),
7591 "Pwsh::bg_command must use -Command: {args_strs:?}"
7592 );
7593 assert!(
7594 args_strs.contains(&"Get-Date"),
7595 "Pwsh::bg_command must include the user command body"
7596 );
7597 }
7598
7599 fn registry_with_db_and_frames(
7600 storage: &Path,
7601 ) -> (
7602 BgTaskRegistry,
7603 Arc<Mutex<Connection>>,
7604 Arc<Mutex<Vec<PushFrame>>>,
7605 ) {
7606 let frames = Arc::new(Mutex::new(Vec::new()));
7607 let captured = Arc::clone(&frames);
7608 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
7609 captured.lock().unwrap().push(frame);
7610 })
7611 as Box<dyn Fn(PushFrame) + Send + Sync>);
7612 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
7613 registry.set_harness(Harness::Opencode);
7614 let conn = crate::db::open(&storage.join("aft.db")).expect("open test DB");
7615 let shared = Arc::new(Mutex::new(conn));
7616 registry.set_db_pool(shared.clone());
7617 (registry, shared, frames)
7618 }
7619
7620 fn pattern_match_frames(frames: &Mutex<Vec<PushFrame>>) -> Vec<BashPatternMatchFrame> {
7621 frames
7622 .lock()
7623 .unwrap()
7624 .iter()
7625 .filter_map(|frame| match frame {
7626 PushFrame::BashPatternMatch(frame) => Some(frame.clone()),
7627 _ => None,
7628 })
7629 .collect()
7630 }
7631
7632 #[test]
7633 fn pattern_watch_survives_registry_teardown_and_rehydrate() {
7634 let dir = tempfile::tempdir().unwrap();
7635 let storage = dir.path();
7636 let (registry, _db, frames) = registry_with_db_and_frames(storage);
7637 let task_id = registry
7638 .spawn(
7639 SpawnPlan::Unsandboxed,
7640 LONG_RUNNING_COMMAND,
7641 "session".to_string(),
7642 storage.to_path_buf(),
7643 HashMap::new(),
7644 Some(Duration::from_secs(30)),
7645 storage.to_path_buf(),
7646 10,
7647 true,
7648 false,
7649 Some(storage.to_path_buf()),
7650 )
7651 .unwrap();
7652 registry
7653 .register_watch(
7654 task_id.clone(),
7655 WatchPattern::Substring("READY".into()),
7656 true,
7657 )
7658 .unwrap();
7659 let task = registry.task_for_session(&task_id, "session").unwrap();
7660 registry.clear_task_watch_state(&task_id);
7662 assert_eq!(registry.active_watch_count(&task_id), 0);
7663
7664 std::fs::OpenOptions::new()
7665 .append(true)
7666 .open(&task.paths.stdout)
7667 .unwrap()
7668 .write_all(b"READY\n")
7669 .unwrap();
7670 frames.lock().unwrap().clear();
7671
7672 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
7673 registry
7675 .inner
7676 .shutdown
7677 .store(true, std::sync::atomic::Ordering::SeqCst);
7678 replayed
7679 .replay_session_inner(storage, "session", None)
7680 .unwrap();
7681
7682 let matches = pattern_match_frames(&replay_frames);
7683 assert!(
7684 matches.iter().any(|frame| {
7685 frame.task_id == task_id
7686 && frame.reason == "pattern_match"
7687 && frame.match_text == "READY"
7688 }),
7689 "rehydrate should deliver gap match: {matches:?}"
7690 );
7691 }
7692
7693 #[test]
7694 fn pattern_watch_gap_match_between_teardown_and_rehydrate_delivers_once() {
7695 let dir = tempfile::tempdir().unwrap();
7696 let storage = dir.path();
7697 let (registry, _db, frames) = registry_with_db_and_frames(storage);
7698 let task_id = registry
7699 .spawn(
7700 SpawnPlan::Unsandboxed,
7701 LONG_RUNNING_COMMAND,
7702 "session".to_string(),
7703 storage.to_path_buf(),
7704 HashMap::new(),
7705 Some(Duration::from_secs(30)),
7706 storage.to_path_buf(),
7707 10,
7708 true,
7709 false,
7710 Some(storage.to_path_buf()),
7711 )
7712 .unwrap();
7713 registry
7714 .register_watch(
7715 task_id.clone(),
7716 WatchPattern::Substring("GAP-HIT".into()),
7717 true,
7718 )
7719 .unwrap();
7720 let task = registry.task_for_session(&task_id, "session").unwrap();
7721 let cursor_before = registry.watch_stream_cursors(&task_id).0;
7722 registry.clear_task_watch_state(&task_id);
7723
7724 std::fs::OpenOptions::new()
7726 .append(true)
7727 .open(&task.paths.stdout)
7728 .unwrap()
7729 .write_all(b"prefix GAP-HIT suffix\n")
7730 .unwrap();
7731 frames.lock().unwrap().clear();
7732
7733 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
7734 registry
7735 .inner
7736 .shutdown
7737 .store(true, std::sync::atomic::Ordering::SeqCst);
7738 replayed
7739 .replay_session_inner(storage, "session", None)
7740 .unwrap();
7741
7742 let matches: Vec<_> = pattern_match_frames(&replay_frames)
7743 .into_iter()
7744 .filter(|frame| frame.task_id == task_id && frame.match_text.contains("GAP-HIT"))
7745 .collect();
7746 assert_eq!(
7747 matches.len(),
7748 1,
7749 "gap match must deliver exactly once: {matches:?}"
7750 );
7751 assert!(
7752 matches[0].match_offset >= cursor_before,
7753 "match offset should be at/after the persisted cursor ({cursor_before}), got {}",
7754 matches[0].match_offset
7755 );
7756 }
7757
7758 #[test]
7759 fn pattern_watch_acked_match_does_not_redeliver_after_restart() {
7760 let dir = tempfile::tempdir().unwrap();
7761 let storage = dir.path();
7762 let (registry, db, frames) = registry_with_db_and_frames(storage);
7763 let task_id = registry
7764 .spawn(
7765 SpawnPlan::Unsandboxed,
7766 LONG_RUNNING_COMMAND,
7767 "session".to_string(),
7768 storage.to_path_buf(),
7769 HashMap::new(),
7770 Some(Duration::from_secs(30)),
7771 storage.to_path_buf(),
7772 10,
7773 true,
7774 false,
7775 Some(storage.to_path_buf()),
7776 )
7777 .unwrap();
7778 registry
7779 .register_watch(
7780 task_id.clone(),
7781 WatchPattern::Substring("READY".into()),
7782 true,
7783 )
7784 .unwrap();
7785 let task = registry.task_for_session(&task_id, "session").unwrap();
7786 std::fs::OpenOptions::new()
7787 .append(true)
7788 .open(&task.paths.stdout)
7789 .unwrap()
7790 .write_all(b"READY\n")
7791 .unwrap();
7792 registry.scan_task_watch_output(&task);
7793 let delivered = pattern_match_frames(&frames);
7794 assert!(
7795 delivered
7796 .iter()
7797 .any(|frame| frame.task_id == task_id && frame.match_text == "READY"),
7798 "live path should deliver match: {delivered:?}"
7799 );
7800 assert!(registry
7802 .ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id))
7803 .contains(&task_id));
7804 {
7805 let conn = db.lock().unwrap();
7806 let rows = crate::db::bash_watches::list_bash_pattern_watches_for_task(
7807 &conn, "opencode", "session", &task_id,
7808 )
7809 .unwrap();
7810 assert!(
7811 rows.is_empty(),
7812 "acked once-watch rows must be deleted: {rows:?}"
7813 );
7814 }
7815
7816 frames.lock().unwrap().clear();
7817 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
7818 registry
7819 .inner
7820 .shutdown
7821 .store(true, std::sync::atomic::Ordering::SeqCst);
7822 replayed
7823 .replay_session_inner(storage, "session", None)
7824 .unwrap();
7825 let matches = pattern_match_frames(&replay_frames)
7826 .into_iter()
7827 .filter(|frame| frame.task_id == task_id)
7828 .collect::<Vec<_>>();
7829 assert!(
7830 matches.is_empty(),
7831 "acked match must not re-deliver after restart: {matches:?}"
7832 );
7833 }
7834
7835 #[test]
7836 fn pattern_watch_rows_are_removed_when_task_is_gc_deleted() {
7837 let dir = tempfile::tempdir().unwrap();
7838 let storage = dir.path();
7839 let (registry, db, _frames) = registry_with_db_and_frames(storage);
7840 let task_id = "bash-aaaaaaaaaaaaaaaa";
7841 let paths = task_paths(storage, "session", task_id).unwrap();
7842 let mut metadata = PersistedTask::starting(
7843 task_id.to_string(),
7844 "session".to_string(),
7845 "true".to_string(),
7846 storage.to_path_buf(),
7847 Some(storage.to_path_buf()),
7848 None,
7849 true,
7850 true,
7851 );
7852 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
7853 metadata.completion_delivered = true;
7854 write_task(&paths.json, &metadata).unwrap();
7855 {
7856 let conn = db.lock().unwrap();
7857 crate::db::bash_tasks::upsert_bash_task(
7858 &conn,
7859 &metadata.to_bash_task_row("opencode", &paths).unwrap(),
7860 )
7861 .unwrap();
7862 crate::db::bash_watches::upsert_bash_pattern_watch(
7863 &conn,
7864 &BashPatternWatchRow {
7865 harness: "opencode".into(),
7866 session_id: "session".into(),
7867 task_id: task_id.into(),
7868 watch_id: "watch-00000001".into(),
7869 pattern_kind: "substring".into(),
7870 pattern: "x".into(),
7871 once: true,
7872 created_at: 1,
7873 stdout_offset: 0,
7874 stderr_offset: 0,
7875 pty_offset: 0,
7876 scanning: true,
7877 pending_match: false,
7878 match_text: None,
7879 match_offset: None,
7880 match_context: None,
7881 },
7882 )
7883 .unwrap();
7884 }
7885 let old = SystemTime::now()
7886 .checked_sub(Duration::from_secs(25 * 60 * 60))
7887 .unwrap();
7888 filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
7889
7890 let deleted = registry.maybe_gc_persisted(storage).unwrap();
7891 assert!(
7892 deleted >= 1,
7893 "expected GC to delete the terminal task bundle"
7894 );
7895 let conn = db.lock().unwrap();
7896 let watches = crate::db::bash_watches::list_bash_pattern_watches_for_task(
7897 &conn, "opencode", "session", task_id,
7898 )
7899 .unwrap();
7900 assert!(
7901 watches.is_empty(),
7902 "task GC must remove watch rows: {watches:?}"
7903 );
7904 }
7905}