1use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
2#[cfg(not(windows))]
3use std::ffi::OsString;
4use std::fs;
5use std::io::Write;
6use std::path::{Path, PathBuf};
7#[cfg(windows)]
8use std::process::Command;
9use std::process::{Child, Stdio};
10use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
11#[cfg(unix)]
12use std::sync::OnceLock;
13use std::sync::{Arc, Mutex, RwLock};
14use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
15
16use rusqlite::Connection;
17use serde::Serialize;
18
19use crate::bash_permissions::PermissionAsk;
20use crate::compress::caps::DropClass;
21#[cfg(unix)]
22use crate::compress::single_top_level_pipeline;
23use crate::compress::CompressionResult;
24use crate::context::SharedProgressSender;
25use crate::db::compression_events::CompressionAggregateCache;
26use crate::harness::Harness;
27use crate::protocol::{BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, PushFrame};
28use crate::sandbox_spawn::SpawnPlan;
29
30#[cfg(windows)]
31use std::os::windows::process::CommandExt;
32
33use super::buffer::{combine_streams, BgBuffer, DiskTruncation, StreamKind, TokenCountInput};
34use super::output::{
35 cap_completion_output, cap_completion_output_with_marker, cap_final_output,
36 cap_final_output_with_marker, completion_preview_threshold, json_output_pointer, quote_path,
37 retained_json_output_pointer, COMPRESS_INPUT_CAP_BYTES, COMPRESS_INPUT_HEAD_BYTES,
38 COMPRESS_INPUT_TAIL_BYTES, FINAL_OUTPUT_CAP_BYTES, RAW_PASSTHROUGH_CAP_BYTES,
39 RAW_PASSTHROUGH_HEAD_BYTES, RAW_PASSTHROUGH_TAIL_BYTES, RUNNING_OUTPUT_PREVIEW_BYTES,
40 STRUCTURED_OUTPUT_CAP_BYTES,
41};
42use super::persistence::{
43 allocate_task_layout, delete_resolved_task, delete_task_bundle, discover_task_ids,
44 open_task_artifact, quarantine_invalid_entry, quarantine_task_layout, read_exit_marker,
45 read_task_at, resolve_task_layout, session_tasks_dir, uninitialized_layout_is_recent,
46 unix_millis, update_task_at, validate_task_id, write_kill_marker_if_absent, write_task_at,
47 BgMode, ExitMarker, PersistedTask, TaskArtifact, TaskIoHandles, TaskPaths,
48};
49use super::process::is_process_alive;
50#[cfg(unix)]
51use super::process::terminate_pgid;
52#[cfg(windows)]
53use super::process::terminate_pid;
54use super::pty_process::spawn_pty_for_command;
55use super::pty_runtime::PtyRuntime;
56use super::watches::{PatternMatch, WatchPattern, WatchRegistry};
57use super::{BgTaskInfo, BgTaskStatus};
58use crate::db::bash_watches::BashPatternWatchRow;
59const DEFAULT_BG_TIMEOUT: Duration = Duration::from_secs(30 * 60);
62const STALE_RUNNING_AFTER: Duration = Duration::from_secs(24 * 60 * 60);
63const PERSISTED_GC_GRACE: Duration = Duration::from_secs(24 * 60 * 60);
64const QUARANTINE_GC_GRACE: Duration = Duration::from_secs(30 * 24 * 60 * 60);
65
66const TOKENIZE_CAP_BYTES_PER_STREAM: usize = 128 * 1024;
67pub const ROOT_RECLAIMED_REASON: &str = "root_reclaimed";
68
69#[derive(Debug, Clone, Serialize)]
70pub struct BgCompletion {
71 pub task_id: String,
72 #[serde(skip_serializing)]
75 pub session_id: String,
76 pub status: BgTaskStatus,
77 pub exit_code: Option<i32>,
78 pub command: String,
79 #[serde(default, skip_serializing_if = "String::is_empty")]
85 pub output_preview: String,
86 #[serde(default, skip_serializing_if = "is_false")]
91 pub output_truncated: bool,
92 #[serde(default, skip_serializing_if = "Option::is_none")]
95 pub original_tokens: Option<u32>,
96 #[serde(default, skip_serializing_if = "Option::is_none")]
99 pub compressed_tokens: Option<u32>,
100 #[serde(default, skip_serializing_if = "is_false")]
102 pub tokens_skipped: bool,
103 #[serde(default, skip_serializing_if = "Option::is_none")]
104 pub status_reason: Option<String>,
105}
106
107fn is_false(v: &bool) -> bool {
108 !*v
109}
110
111#[derive(Debug, Clone, Serialize)]
112pub struct BgTaskSnapshot {
113 #[serde(flatten)]
114 pub info: BgTaskInfo,
115 pub exit_code: Option<i32>,
116 pub child_pid: Option<u32>,
117 pub workdir: String,
118 pub output_preview: String,
119 pub output_truncated: bool,
120 pub output_path: Option<String>,
121 pub stderr_path: Option<String>,
122 #[serde(skip_serializing_if = "Option::is_none")]
123 pub pty_rows: Option<u16>,
124 #[serde(skip_serializing_if = "Option::is_none")]
125 pub pty_cols: Option<u16>,
126 #[serde(skip_serializing_if = "Option::is_none")]
127 pub pty_screen: Option<String>,
128 #[serde(default, skip_serializing_if = "Vec::is_empty")]
129 pub scanner_report: Vec<PermissionAsk>,
130 #[serde(default, skip_serializing_if = "is_false")]
131 pub sandbox_native: bool,
132 #[serde(default, skip_serializing_if = "is_false")]
133 pub sandbox_unavailable: bool,
134}
135
136#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
137pub struct BgTaskHealthCounts {
138 pub running: usize,
139 pub pending_completions: usize,
140}
141
142#[derive(Debug, Clone, Copy, PartialEq, Eq)]
143enum TerminalOutputKind {
144 Compressed,
145 Raw,
146 Structured,
147}
148
149#[derive(Debug, Clone, PartialEq, Eq)]
150struct TerminalOutputCache {
151 output_preview: String,
152 output_truncated: bool,
153 kind: TerminalOutputKind,
154 output_path: Option<String>,
155 stderr_path: Option<String>,
156 artifact_access: ArtifactRecoveryAccess,
157 recovery: Option<RecoveryContext>,
158}
159
160#[derive(Debug, Clone, PartialEq, Eq)]
161struct ArtifactRecoveryAccess {
162 task_id: String,
163 readable: bool,
164}
165
166#[derive(Debug, Clone, PartialEq, Eq)]
167struct RecoveryContext {
168 dropped_by_class: BTreeMap<DropClass, usize>,
169 had_inner_drop: bool,
170 offset_hint_eligible: bool,
171 offset_start_line: Option<usize>,
172 byte_truncated: bool,
173 disk_truncated_prefix_bytes: u64,
174 output_path: Option<String>,
175 stderr_path: Option<String>,
176 include_stderr_path: bool,
177 artifact_access: ArtifactRecoveryAccess,
178}
179
180fn optional_string_bytes(value: Option<&String>) -> u64 {
181 value
182 .map(|value| crate::memory::usize_to_u64(value.len()))
183 .unwrap_or(0)
184}
185
186fn terminal_output_cache_estimated_bytes(cache: &TerminalOutputCache) -> u64 {
187 let recovery_bytes = cache
188 .recovery
189 .as_ref()
190 .map(|recovery| {
191 crate::memory::usize_to_u64(recovery.dropped_by_class.len())
192 .saturating_mul(
193 (std::mem::size_of::<DropClass>() + std::mem::size_of::<usize>()) as u64,
194 )
195 .saturating_add(optional_string_bytes(recovery.output_path.as_ref()))
196 .saturating_add(optional_string_bytes(recovery.stderr_path.as_ref()))
197 .saturating_add(crate::memory::usize_to_u64(
198 recovery.artifact_access.task_id.len(),
199 ))
200 })
201 .unwrap_or(0);
202 (std::mem::size_of::<TerminalOutputCache>() as u64)
203 .saturating_add(crate::memory::usize_to_u64(cache.output_preview.len()))
204 .saturating_add(optional_string_bytes(cache.output_path.as_ref()))
205 .saturating_add(optional_string_bytes(cache.stderr_path.as_ref()))
206 .saturating_add(crate::memory::usize_to_u64(
207 cache.artifact_access.task_id.len(),
208 ))
209 .saturating_add(recovery_bytes)
210}
211
212fn completion_estimated_bytes(completion: &BgCompletion) -> u64 {
213 (std::mem::size_of::<BgCompletion>() as u64)
214 .saturating_add(crate::memory::usize_to_u64(completion.task_id.len()))
215 .saturating_add(crate::memory::usize_to_u64(completion.session_id.len()))
216 .saturating_add(crate::memory::usize_to_u64(completion.command.len()))
217 .saturating_add(crate::memory::usize_to_u64(completion.output_preview.len()))
218}
219
220impl RecoveryContext {
221 fn has_visible_drop(&self) -> bool {
222 self.byte_truncated
223 || self.disk_truncated_prefix_bytes > 0
224 || self.had_inner_drop
225 || !self.dropped_by_class.is_empty()
226 }
227}
228
229#[derive(Clone)]
230pub struct BgTaskRegistry {
231 pub(crate) inner: Arc<RegistryInner>,
232}
233
234pub(crate) struct RegistryInner {
235 pub(crate) tasks: Mutex<HashMap<String, Arc<BgTask>>>,
236 pub(crate) completions: Mutex<VecDeque<BgCompletion>>,
237 pub(crate) progress_sender: SharedProgressSender,
238 watchdog_started: AtomicBool,
239 pub(crate) shutdown: AtomicBool,
240 pub(crate) long_running_reminder_enabled: AtomicBool,
241 pub(crate) long_running_reminder_interval_ms: AtomicU64,
242 persisted_gc_started: AtomicBool,
243 #[cfg(test)]
244 persisted_gc_runs: AtomicU64,
245 pub(crate) compressor:
251 Mutex<Option<Box<dyn Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync>>>,
252 pub(crate) db_pool: RwLock<Option<Arc<Mutex<Connection>>>>,
253 pub(crate) db_harness: RwLock<Option<String>>,
254 pub(crate) compression_aggregates: Arc<CompressionAggregateCache>,
255 pub(crate) wake_tx: crossbeam_channel::Sender<()>,
256 pub(crate) wake_rx: crossbeam_channel::Receiver<()>,
257 pub(crate) watch_registry: Mutex<WatchRegistry>,
258 wait_detach_sessions: Mutex<HashSet<String>>,
259 active_wait_sessions: Mutex<HashMap<String, usize>>,
260 wait_registered_tasks: Mutex<HashMap<String, HashSet<String>>>,
261}
262
263pub(crate) struct BgTask {
264 pub(crate) task_id: String,
265 pub(crate) session_id: String,
266 pub(crate) paths: TaskPaths,
267 artifact_root: PathBuf,
268 pub(crate) started: Instant,
269 pub(crate) last_reminder_at: Mutex<Option<Instant>>,
270 pub(crate) terminal_at: Mutex<Option<Instant>>,
271 pub(crate) state: Mutex<BgTaskState>,
272}
273
274pub(crate) enum TaskRuntime {
275 Piped(Option<Child>),
276 Pty(Option<PtyRuntime>),
277}
278
279pub(crate) struct BgTaskState {
280 pub(crate) metadata: PersistedTask,
281 pub(crate) runtime: TaskRuntime,
282 pub(crate) io_handles: Option<TaskIoHandles>,
285 pub(crate) detached: bool,
286 pub(crate) child_exit_observed: bool,
297 pub(crate) buffer: BgBuffer,
298 terminal_output_cache: Option<TerminalOutputCache>,
299 pub(crate) pending_terminal_override: Option<BgTaskStatus>,
301}
302
303fn completion_matches_session(completion: &BgCompletion, session_id: Option<&str>) -> bool {
304 session_id
305 .map(|session_id| completion.session_id == session_id)
306 .unwrap_or(true)
307}
308
309impl BgTaskRegistry {
310 pub fn new(progress_sender: SharedProgressSender) -> Self {
311 let (wake_tx, wake_rx) = crossbeam_channel::bounded(1);
312 Self {
313 inner: Arc::new(RegistryInner {
314 tasks: Mutex::new(HashMap::new()),
315 completions: Mutex::new(VecDeque::new()),
316 progress_sender,
317 watchdog_started: AtomicBool::new(false),
318 shutdown: AtomicBool::new(false),
319 long_running_reminder_enabled: AtomicBool::new(true),
320 long_running_reminder_interval_ms: AtomicU64::new(600_000),
321 persisted_gc_started: AtomicBool::new(false),
322 #[cfg(test)]
323 persisted_gc_runs: AtomicU64::new(0),
324 compressor: Mutex::new(None),
325 db_pool: RwLock::new(None),
326 db_harness: RwLock::new(None),
327 compression_aggregates: Arc::new(CompressionAggregateCache::default()),
328 wake_tx,
329 wake_rx,
330 watch_registry: Mutex::new(WatchRegistry::default()),
331 wait_detach_sessions: Mutex::new(HashSet::new()),
332 active_wait_sessions: Mutex::new(HashMap::new()),
333 wait_registered_tasks: Mutex::new(HashMap::new()),
334 }),
335 }
336 }
337
338 pub fn is_session_owned_artifact_path(&self, session_id: &str, path: &Path) -> bool {
346 let Ok(requested) = fs::canonicalize(path) else {
347 return false;
348 };
349 let Ok(tasks) = self.inner.tasks.lock() else {
350 return false;
351 };
352
353 tasks.values().any(|task| {
354 task.session_id == session_id
355 && [
356 &task.paths.stdout,
357 &task.paths.stderr,
358 &task.paths.exit,
359 &task.paths.pty,
360 ]
361 .into_iter()
362 .filter_map(|known| known.file_name())
363 .any(|name| task.artifact_root.join(name) == requested)
364 })
365 }
366
367 pub fn read_artifact_path(
368 &self,
369 session_id: &str,
370 path: &Path,
371 ) -> Option<Result<Vec<u8>, String>> {
372 let requested = fs::canonicalize(path).ok()?;
373 let tasks = self.inner.tasks.lock().ok()?;
374 let (task, artifact) = tasks.values().find_map(|task| {
375 if task.session_id != session_id {
376 return None;
377 }
378 TaskArtifact::ALL.into_iter().find_map(|artifact| {
379 let expected = task
380 .paths
381 .artifact_path(artifact)
382 .file_name()
383 .map(|name| task.artifact_root.join(name));
384 (expected.as_deref() == Some(requested.as_path()))
385 .then(|| (Arc::clone(task), artifact))
386 })
387 })?;
388 drop(tasks);
389 Some(self.read_artifact(&task.task_id, session_id, artifact))
390 }
391
392 pub fn read_artifact_range(
393 &self,
394 task_id: &str,
395 session_id: &str,
396 artifact: TaskArtifact,
397 offset: u64,
398 ) -> Result<(Vec<u8>, u64), String> {
399 validate_task_id(task_id).map_err(|error| error.to_string())?;
400 let task = self
401 .task_for_session(task_id, session_id)
402 .ok_or_else(|| "task_not_found".to_string())?;
403 let mut file = open_task_artifact(&task.paths, artifact)
404 .map_err(|error| format!("artifact_refused: {error}"))?;
405 let len = file
406 .len()
407 .map_err(|error| format!("artifact_refused: {error}"))?;
408 let start = offset.min(len);
409 let bytes = file
410 .read_range(start, len.saturating_sub(start))
411 .map_err(|error| format!("artifact_refused: {error}"))?;
412 Ok((bytes, len))
413 }
414
415 pub fn read_artifact(
416 &self,
417 task_id: &str,
418 session_id: &str,
419 artifact: TaskArtifact,
420 ) -> Result<Vec<u8>, String> {
421 validate_task_id(task_id).map_err(|error| error.to_string())?;
422 let task = self
423 .task_for_session(task_id, session_id)
424 .ok_or_else(|| "task_not_found".to_string())?;
425 let mut file = open_task_artifact(&task.paths, artifact)
426 .map_err(|error| format!("artifact_refused: {error}"))?;
427 file.read_all()
428 .map_err(|error| format!("artifact_refused: {error}"))
429 }
430
431 pub fn set_harness(&self, harness: Harness) {
432 if let Ok(mut slot) = self.inner.db_harness.write() {
433 *slot = Some(harness.storage_segment());
434 }
435 }
436
437 pub fn set_db_pool(&self, conn: Arc<Mutex<Connection>>) {
438 if let Ok(mut slot) = self.inner.db_pool.write() {
439 *slot = Some(conn);
440 }
441 self.inner.compression_aggregates.clear();
442 }
443
444 pub fn clear_db_pool(&self) {
445 if let Ok(mut slot) = self.inner.db_pool.write() {
446 *slot = None;
447 }
448 self.inner.compression_aggregates.clear();
449 }
450
451 pub(crate) fn compression_aggregate_cache(&self) -> Arc<CompressionAggregateCache> {
452 Arc::clone(&self.inner.compression_aggregates)
453 }
454
455 pub fn register_foreground_task(&self, session_id: &str, task_id: &str) {
456 if let Ok(mut tasks) = self.inner.wait_registered_tasks.lock() {
457 tasks
458 .entry(session_id.to_string())
459 .or_default()
460 .insert(task_id.to_string());
461 }
462 }
463
464 pub fn begin_wait_mode_session(&self, session_id: &str, task_id: &str) {
465 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
466 *active.entry(session_id.to_string()).or_insert(0) += 1;
467 }
468 self.register_foreground_task(session_id, task_id);
469 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
470 detach.remove(session_id);
471 }
472 }
473
474 pub fn unregister_foreground_task(&self, session_id: &str, task_id: &str) {
475 if let Ok(mut tasks) = self.inner.wait_registered_tasks.lock() {
476 if let Some(session_tasks) = tasks.get_mut(session_id) {
477 session_tasks.remove(task_id);
478 if session_tasks.is_empty() {
479 tasks.remove(session_id);
480 }
481 }
482 }
483 }
484
485 pub fn end_wait_mode_session(&self, session_id: &str, task_id: &str) {
486 let no_active_wait = if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
487 match active.get_mut(session_id) {
488 Some(count) if *count > 1 => *count -= 1,
489 Some(_) => {
490 active.remove(session_id);
491 }
492 None => {}
493 }
494 !active.contains_key(session_id)
495 } else {
496 false
497 };
498 self.unregister_foreground_task(session_id, task_id);
499 if no_active_wait {
500 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
501 detach.remove(session_id);
502 }
503 }
504 }
505
506 pub fn abort_inflight(&self, session_id: &str) -> Result<usize, String> {
510 let task_ids = self
511 .inner
512 .wait_registered_tasks
513 .lock()
514 .map(|mut tasks| tasks.remove(session_id).unwrap_or_default())
515 .map_err(|_| "wait registration lock poisoned".to_string())?;
516 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
517 active.remove(session_id);
518 }
519 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
520 detach.remove(session_id);
521 }
522
523 let mut killed = 0;
524 for task_id in task_ids {
525 let Some(task) = self.task_for_session(&task_id, session_id) else {
526 continue;
527 };
528 let is_terminal = task
529 .state
530 .lock()
531 .map(|state| state.metadata.status.is_terminal())
532 .map_err(|_| "background task lock poisoned".to_string())?;
533 if is_terminal {
534 continue;
535 }
536 let snapshot = self.kill_with_status_reason(
537 &task_id,
538 session_id,
539 BgTaskStatus::Killed,
540 Some("call_aborted".to_string()),
541 )?;
542 if snapshot.info.status == BgTaskStatus::Killed
543 && snapshot.info.status_reason.as_deref() == Some("call_aborted")
544 {
545 killed += 1;
546 }
547 }
548 Ok(killed)
549 }
550
551 pub fn signal_wait_mode_detach(&self, session_id: &str) -> bool {
552 let is_waiting = self
553 .inner
554 .active_wait_sessions
555 .lock()
556 .map(|active| active.get(session_id).copied().unwrap_or(0) > 0)
557 .unwrap_or(false);
558 if !is_waiting {
559 return false;
560 }
561 self.inner
562 .wait_detach_sessions
563 .lock()
564 .map(|mut detach| detach.insert(session_id.to_string()))
565 .unwrap_or(false)
566 }
567
568 pub fn active_wait_session_count(&self) -> usize {
571 self.inner
572 .active_wait_sessions
573 .lock()
574 .map(|active| active.len())
575 .unwrap_or(0)
576 }
577
578 pub fn take_wait_mode_detach(&self, session_id: &str) -> bool {
579 self.inner
580 .wait_detach_sessions
581 .lock()
582 .map(|mut detach| detach.remove(session_id))
583 .unwrap_or(false)
584 }
585
586 pub fn set_compressor<F>(&self, compressor: F)
591 where
592 F: Fn(&str, String) -> CompressionResult + Send + Sync + 'static,
593 {
594 self.set_compressor_with_exit_code(move |command, output, _exit_code| {
595 compressor(command, output)
596 });
597 }
598
599 pub fn set_compressor_with_exit_code<F>(&self, compressor: F)
600 where
601 F: Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync + 'static,
602 {
603 if let Ok(mut slot) = self.inner.compressor.lock() {
604 *slot = Some(Box::new(compressor));
605 }
606 }
607
608 pub(crate) fn compress_output(
611 &self,
612 command: &str,
613 output: String,
614 exit_code: Option<i32>,
615 ) -> CompressionResult {
616 let Ok(slot) = self.inner.compressor.lock() else {
617 return CompressionResult::new(output);
618 };
619 match slot.as_ref() {
620 Some(compressor) => compressor(command, output, exit_code),
621 None => CompressionResult::new(output),
622 }
623 }
624
625 fn ensure_terminal_output_cache(&self, task: &Arc<BgTask>) -> Option<TerminalOutputCache> {
626 let (metadata, buffer) = {
627 let state = task.state.lock().ok()?;
628 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
629 return None;
630 }
631 if let Some(cache) = state.terminal_output_cache.clone() {
632 return Some(cache);
633 }
634 (state.metadata.clone(), state.buffer.clone())
635 };
636
637 let mut cap_buffer = buffer.clone();
638 let disk_truncation = cap_buffer.enforce_terminal_cap();
639 let cache =
640 self.render_terminal_output(&metadata, &cap_buffer, disk_truncation, Some(&task.paths));
641 let mut state = task.state.lock().ok()?;
642 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
643 return None;
644 }
645 if let Some(existing) = state.terminal_output_cache.clone() {
646 return Some(existing);
647 }
648 state.terminal_output_cache = Some(cache.clone());
649 Some(cache)
650 }
651
652 fn render_terminal_output(
653 &self,
654 metadata: &PersistedTask,
655 buffer: &BgBuffer,
656 disk_truncation: DiskTruncation,
657 paths: Option<&TaskPaths>,
658 ) -> TerminalOutputCache {
659 let output_readable = buffer
660 .output_path()
661 .is_some_and(|path| self.is_session_owned_artifact_path(&metadata.session_id, &path));
662 let stderr_readable = buffer
663 .stderr_path()
664 .map(|path| self.is_session_owned_artifact_path(&metadata.session_id, path))
665 .unwrap_or(true);
666 let artifact_access = ArtifactRecoveryAccess {
667 task_id: metadata.task_id.clone(),
668 readable: output_readable && stderr_readable,
669 };
670
671 if metadata.mode == BgMode::Pty {
672 return TerminalOutputCache {
673 output_preview: String::new(),
674 output_truncated: false,
675 kind: TerminalOutputKind::Raw,
676 output_path: buffer.output_path().map(|path| path.display().to_string()),
677 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
678 artifact_access,
679 recovery: None,
680 };
681 }
682
683 if let Some(mut structured) = render_structured_output(
684 &metadata.command,
685 buffer,
686 disk_truncation,
687 artifact_access.clone(),
688 ) {
689 append_pipeline_warning(&mut structured, metadata, paths);
690 return structured;
691 }
692
693 if !metadata.compressed {
694 let mut raw = render_raw_passthrough(buffer, disk_truncation, artifact_access);
695 append_pipeline_warning(&mut raw, metadata, paths);
696 return raw;
697 }
698
699 let raw = buffer.read_combined_head_tail(
700 COMPRESS_INPUT_CAP_BYTES,
701 COMPRESS_INPUT_HEAD_BYTES,
702 COMPRESS_INPUT_TAIL_BYTES,
703 );
704 let compressed = self.compress_output(&metadata.command, raw.text, metadata.exit_code);
705 let mut rendered = render_compressed_with_recovery(
706 buffer,
707 compressed,
708 raw.truncated,
709 disk_truncation,
710 artifact_access,
711 );
712 append_pipeline_warning(&mut rendered, metadata, paths);
713 rendered
714 }
715
716 fn snapshot_with_terminal_cache(
717 &self,
718 task: &Arc<BgTask>,
719 preview_bytes: usize,
720 ) -> BgTaskSnapshot {
721 let mut snapshot = task.snapshot(preview_bytes);
722 self.maybe_compress_snapshot(task, &mut snapshot);
723 snapshot
724 }
725
726 fn post_terminal_transition(&self, task: &Arc<BgTask>, emit_frame: bool) -> Result<(), String> {
727 let (metadata, buffer) = {
728 let state = task
729 .state
730 .lock()
731 .map_err(|_| "background task lock poisoned".to_string())?;
732 if !state.metadata.status.is_terminal() {
733 return Ok(());
734 }
735 (state.metadata.clone(), state.buffer.clone())
736 };
737
738 let cache = self.ensure_terminal_output_cache(task);
739 self.enqueue_completion_from_parts(
740 &metadata,
741 Some(&buffer),
742 None,
743 emit_frame,
744 cache.as_ref(),
745 );
746 Ok(())
747 }
748
749 fn persist_task(&self, paths: &TaskPaths, metadata: &PersistedTask) -> std::io::Result<()> {
750 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
751 write_task_at(&task, metadata)?;
752 self.dual_write_task(paths, metadata);
753 Ok(())
754 }
755
756 fn update_task_metadata<F>(
757 &self,
758 paths: &TaskPaths,
759 update: F,
760 ) -> std::io::Result<PersistedTask>
761 where
762 F: FnOnce(&mut PersistedTask),
763 {
764 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
765 let metadata = update_task_at(&task, update)?;
766 self.dual_write_task(paths, &metadata);
767 Ok(metadata)
768 }
769
770 fn dual_write_task(&self, paths: &TaskPaths, metadata: &PersistedTask) {
771 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
772 let Some(pool) = pool else {
773 return;
774 };
775 let harness = self
776 .inner
777 .db_harness
778 .read()
779 .ok()
780 .and_then(|slot| slot.clone());
781 let Some(harness) = harness else {
782 crate::slog_warn!(
783 "dual-write bash_task to DB skipped for {}: harness not configured",
784 metadata.task_id
785 );
786 return;
787 };
788 let row = match metadata.to_bash_task_row(&harness, paths) {
789 Ok(row) => row,
790 Err(error) => {
791 crate::slog_warn!(
792 "dual-write bash_task to DB failed for {}: {}",
793 metadata.task_id,
794 error
795 );
796 return;
797 }
798 };
799 let conn = match pool.lock() {
800 Ok(conn) => conn,
801 Err(_) => {
802 crate::slog_warn!(
803 "dual-write bash_task to DB failed for {}: db mutex poisoned",
804 metadata.task_id
805 );
806 return;
807 }
808 };
809 if let Err(error) = crate::db::bash_tasks::upsert_bash_task(&conn, &row) {
810 crate::slog_warn!(
811 "dual-write bash_task to DB failed for {}: {}",
812 metadata.task_id,
813 error
814 );
815 }
816 }
817
818 fn delete_gc_task_from_db(&self, metadata: &PersistedTask) {
819 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
820 let Some(pool) = pool else {
821 return;
822 };
823 let harness = self
824 .inner
825 .db_harness
826 .read()
827 .ok()
828 .and_then(|slot| slot.clone());
829 let Some(harness) = harness else {
830 crate::slog_warn!(
831 "GC bash_task DB delete skipped for {}: harness not configured",
832 metadata.task_id
833 );
834 return;
835 };
836 let conn = match pool.lock() {
837 Ok(conn) => conn,
838 Err(_) => {
839 crate::slog_warn!(
840 "GC bash_task DB delete failed for {}: db mutex poisoned",
841 metadata.task_id
842 );
843 return;
844 }
845 };
846 if let Err(error) = crate::db::bash_tasks::delete_delivered_terminal_bash_task(
847 &conn,
848 &harness,
849 &metadata.session_id,
850 &metadata.task_id,
851 ) {
852 crate::slog_warn!(
853 "GC bash_task DB delete failed for {}: {}",
854 metadata.task_id,
855 error
856 );
857 }
858 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
860 &conn,
861 &harness,
862 &metadata.session_id,
863 &metadata.task_id,
864 ) {
865 crate::slog_warn!(
866 "GC bash_pattern_watches DB delete failed for {}: {}",
867 metadata.task_id,
868 error
869 );
870 }
871 }
872
873 fn db_harness_and_pool(&self) -> Option<(String, Arc<Mutex<Connection>>)> {
874 let pool = self
875 .inner
876 .db_pool
877 .read()
878 .ok()
879 .and_then(|slot| slot.clone())?;
880 let harness = self
881 .inner
882 .db_harness
883 .read()
884 .ok()
885 .and_then(|slot| slot.clone())?;
886 Some((harness, pool))
887 }
888
889 fn persist_watch_registration(
890 &self,
891 session_id: &str,
892 task_id: &str,
893 watch_id: &str,
894 pattern: &WatchPattern,
895 once: bool,
896 stdout_offset: u64,
897 stderr_offset: u64,
898 pty_offset: u64,
899 ) {
900 let Some((harness, pool)) = self.db_harness_and_pool() else {
901 return;
902 };
903 let Ok(conn) = pool.lock() else {
904 return;
905 };
906 let row = BashPatternWatchRow {
907 harness,
908 session_id: session_id.to_string(),
909 task_id: task_id.to_string(),
910 watch_id: watch_id.to_string(),
911 pattern_kind: pattern.kind_name().to_string(),
912 pattern: pattern.pattern_text().to_string(),
913 once,
914 created_at: unix_millis() as i64,
915 stdout_offset: stdout_offset as i64,
916 stderr_offset: stderr_offset as i64,
917 pty_offset: pty_offset as i64,
918 scanning: true,
919 pending_match: false,
920 match_text: None,
921 match_offset: None,
922 match_context: None,
923 };
924 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
925 crate::slog_warn!(
926 "persist bash_pattern_watch failed for {task_id}/{watch_id}: {error}"
927 );
928 }
929 }
930
931 fn delete_persisted_watch(&self, session_id: &str, task_id: &str, watch_id: &str) {
932 let Some((harness, pool)) = self.db_harness_and_pool() else {
933 return;
934 };
935 let Ok(conn) = pool.lock() else {
936 return;
937 };
938 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watch(
939 &conn, &harness, session_id, task_id, watch_id,
940 ) {
941 crate::slog_warn!("delete bash_pattern_watch failed for {task_id}/{watch_id}: {error}");
942 }
943 }
944
945 fn delete_persisted_watches_for_task(&self, session_id: &str, task_id: &str) {
946 let Some((harness, pool)) = self.db_harness_and_pool() else {
947 return;
948 };
949 let Ok(conn) = pool.lock() else {
950 return;
951 };
952 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
953 &conn, &harness, session_id, task_id,
954 ) {
955 crate::slog_warn!("delete bash_pattern_watches for {task_id} failed: {error}");
956 }
957 }
958
959 fn persist_watch_match(
960 &self,
961 session_id: &str,
962 task_id: &str,
963 pattern_match: &PatternMatch,
964 stdout_offset: u64,
965 stderr_offset: u64,
966 pty_offset: u64,
967 ) {
968 let Some((harness, pool)) = self.db_harness_and_pool() else {
969 return;
970 };
971 let Ok(conn) = pool.lock() else {
972 return;
973 };
974 let Ok(Some(mut row)) = crate::db::bash_watches::get_bash_pattern_watch(
975 &conn,
976 &harness,
977 session_id,
978 task_id,
979 &pattern_match.watch_id,
980 ) else {
981 return;
982 };
983 row.stdout_offset = stdout_offset as i64;
984 row.stderr_offset = stderr_offset as i64;
985 row.pty_offset = pty_offset as i64;
986 row.pending_match = true;
987 row.match_text = Some(pattern_match.match_text.clone());
988 row.match_offset = Some(pattern_match.match_offset as i64);
989 row.match_context = Some(pattern_match.context.clone());
990 if pattern_match.once {
991 row.scanning = false;
994 }
995 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
996 crate::slog_warn!(
997 "persist bash_pattern_watch match failed for {}/{}: {error}",
998 task_id,
999 pattern_match.watch_id
1000 );
1001 }
1002 }
1003
1004 fn persist_task_watch_cursors(
1005 &self,
1006 session_id: &str,
1007 task_id: &str,
1008 stdout_offset: u64,
1009 stderr_offset: u64,
1010 pty_offset: u64,
1011 ) {
1012 let Some((harness, pool)) = self.db_harness_and_pool() else {
1013 return;
1014 };
1015 let Ok(conn) = pool.lock() else {
1016 return;
1017 };
1018 if let Err(error) = crate::db::bash_watches::update_watch_offsets_for_task(
1019 &conn,
1020 &harness,
1021 session_id,
1022 task_id,
1023 stdout_offset as i64,
1024 stderr_offset as i64,
1025 pty_offset as i64,
1026 ) {
1027 crate::slog_warn!("persist bash_pattern_watch cursors failed for {task_id}: {error}");
1028 }
1029 }
1030
1031 fn watch_stream_cursors(&self, task_id: &str) -> (u64, u64, u64) {
1032 let Ok(registry) = self.inner.watch_registry.lock() else {
1033 return (0, 0, 0);
1034 };
1035 let stdout = registry
1036 .file_cursor(&format!("{task_id}:stdout"))
1037 .unwrap_or(0);
1038 let stderr = registry
1039 .file_cursor(&format!("{task_id}:stderr"))
1040 .unwrap_or(0);
1041 let pty = registry.file_cursor(&format!("{task_id}:pty")).unwrap_or(0);
1042 (stdout, stderr, pty)
1043 }
1044
1045 fn ack_persisted_watches_for_task(&self, session_id: &str, task_id: &str, task_terminal: bool) {
1048 let Some((harness, pool)) = self.db_harness_and_pool() else {
1049 return;
1050 };
1051 let Ok(conn) = pool.lock() else {
1052 return;
1053 };
1054 if task_terminal {
1055 let _ = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
1056 &conn, &harness, session_id, task_id,
1057 );
1058 return;
1059 }
1060 let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
1061 &conn, &harness, session_id, task_id,
1062 ) else {
1063 return;
1064 };
1065 for mut row in rows {
1066 if row.once && (!row.scanning || row.pending_match) {
1067 let _ = crate::db::bash_watches::delete_bash_pattern_watch(
1068 &conn,
1069 &harness,
1070 session_id,
1071 task_id,
1072 &row.watch_id,
1073 );
1074 continue;
1075 }
1076 if row.pending_match {
1077 row.pending_match = false;
1078 row.match_text = None;
1079 row.match_offset = None;
1080 row.match_context = None;
1081 let _ = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row);
1082 }
1083 }
1084 }
1085
1086 pub fn record_scanner_report(
1087 &self,
1088 task_id: &str,
1089 session_id: &str,
1090 scanner_report: Vec<PermissionAsk>,
1091 ) -> Result<(), String> {
1092 if scanner_report.is_empty() {
1093 return Ok(());
1094 }
1095 let task = self.task_for_session(task_id, session_id).ok_or_else(|| {
1096 "background task not found while recording scanner report".to_string()
1097 })?;
1098 let metadata = {
1099 let mut state = task
1100 .state
1101 .lock()
1102 .map_err(|_| "background task lock poisoned".to_string())?;
1103 state.metadata.scanner_report = scanner_report;
1104 state.metadata.clone()
1105 };
1106 self.persist_task(&task.paths, &metadata)
1107 .map_err(|error| format!("failed to persist scanner report: {error}"))
1108 }
1109
1110 pub fn configure_long_running_reminders(&self, enabled: bool, interval_ms: u64) {
1111 self.inner
1112 .long_running_reminder_enabled
1113 .store(enabled, Ordering::SeqCst);
1114 self.inner
1115 .long_running_reminder_interval_ms
1116 .store(interval_ms, Ordering::SeqCst);
1117 }
1118
1119 #[cfg(unix)]
1120 #[allow(clippy::too_many_arguments)]
1121 pub fn spawn(
1122 &self,
1123 spawn_plan: SpawnPlan,
1124 command: &str,
1125 session_id: String,
1126 workdir: PathBuf,
1127 env: HashMap<String, String>,
1128 timeout: Option<Duration>,
1129 storage_dir: PathBuf,
1130 max_running: usize,
1131 notify_on_completion: bool,
1132 compressed: bool,
1133 project_root: Option<PathBuf>,
1134 ) -> Result<String, String> {
1135 self.start_watchdog();
1136
1137 let running = self.running_count();
1138 if running >= max_running {
1139 #[cfg(unix)]
1140 if let Some(prepared) = spawn_plan.prepared_task() {
1141 let _ = delete_resolved_task(&prepared.resolved_task());
1142 }
1143 return Err(format!(
1144 "background bash task limit exceeded: {running} running (max {max_running})"
1145 ));
1146 }
1147
1148 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1149 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1150 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1151 (spawn_plan.clone(), prepared.resolved_task())
1152 } else {
1153 let task = allocate_task_layout(&storage_dir, &session_id)
1154 .map_err(|error| format!("failed to create background task layout: {error}"))?;
1155 let shell = resolve_posix_shell();
1156 let root = project_root.as_deref().unwrap_or(&workdir);
1157 let environment =
1158 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1159 let prepared = match crate::sandbox_spawn::prepare_task_payload(
1160 &task,
1161 command.as_bytes(),
1162 root,
1163 &workdir,
1164 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1165 &shell,
1166 &environment,
1167 ) {
1168 Ok(prepared) => prepared,
1169 Err(error) => {
1170 let _ = delete_resolved_task(&task);
1171 return Err(error);
1172 }
1173 };
1174 let task = prepared.resolved_task();
1175 (spawn_plan.with_prepared_task(prepared), task)
1176 };
1177 let task_id = task_layout.paths.task_id.clone();
1178 let paths = task_layout.paths.clone();
1179
1180 if self.task(&task_id).is_some() {
1181 let _ = delete_resolved_task(&task_layout);
1182 return Err("background task id collided with a live task".to_string());
1183 }
1184
1185 let mut metadata = PersistedTask::starting(
1186 task_id.clone(),
1187 session_id.clone(),
1188 command.to_string(),
1189 workdir.clone(),
1190 project_root,
1191 timeout_ms,
1192 notify_on_completion,
1193 compressed,
1194 );
1195 #[cfg(unix)]
1199 let capture_pipeline_status = {
1200 let shell = spawn_plan
1201 .host_shell_path()
1202 .map(Path::to_path_buf)
1203 .unwrap_or_else(resolve_posix_shell);
1204 let pipeline = single_top_level_pipeline(command);
1205 let capture = should_capture_pipeline_status(&spawn_plan, pipeline.is_some(), &shell);
1206 if capture {
1207 metadata.pipeline_segments = pipeline
1208 .as_ref()
1209 .map(|pipeline| {
1210 pipeline
1211 .segments
1212 .iter()
1213 .map(|segment| segment.label.clone())
1214 .collect()
1215 })
1216 .unwrap_or_default();
1217 }
1218 capture
1219 };
1220 #[cfg(windows)]
1221 let capture_pipeline_status = false;
1222 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1223 if let Err(error) = write_task_at(&task_layout, &metadata) {
1224 let _ = delete_resolved_task(&task_layout);
1225 return Err(format!(
1226 "failed to persist background task metadata: {error}"
1227 ));
1228 }
1229 self.dual_write_task(&paths, &metadata);
1230
1231 let mut io_handles =
1232 TaskIoHandles::create(&task_layout, BgMode::Pipes, capture_pipeline_status)
1233 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
1234 let child = match spawn_detached_child(
1235 &spawn_plan,
1236 command,
1237 &paths,
1238 &workdir,
1239 &env,
1240 &mut io_handles,
1241 capture_pipeline_status,
1242 ) {
1243 Ok(child) => child,
1244 Err(error) => {
1245 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
1246 let _ = delete_task_bundle(&paths);
1247 return Err(error);
1248 }
1249 };
1250
1251 let child_pid = child.id();
1252 metadata.mark_running(child_pid, child_pid as i32);
1253 self.persist_task(&paths, &metadata)
1254 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1255
1256 let task = Arc::new(BgTask {
1257 task_id: task_id.clone(),
1258 session_id,
1259 paths: paths.clone(),
1260 artifact_root: canonical_artifact_root(&paths),
1261 started: Instant::now(),
1262 last_reminder_at: Mutex::new(None),
1263 terminal_at: Mutex::new(None),
1264 state: Mutex::new(BgTaskState {
1265 metadata,
1266 runtime: TaskRuntime::Piped(Some(child)),
1267 io_handles: Some(io_handles),
1268 detached: false,
1269 child_exit_observed: false,
1270 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
1271 terminal_output_cache: None,
1272 pending_terminal_override: None,
1273 }),
1274 });
1275
1276 self.inner
1277 .tasks
1278 .lock()
1279 .map_err(|_| "background task registry lock poisoned".to_string())?
1280 .insert(task_id.clone(), task);
1281
1282 Ok(task_id)
1283 }
1284
1285 #[allow(clippy::too_many_arguments)]
1286 pub fn spawn_pty(
1287 &self,
1288 spawn_plan: SpawnPlan,
1289 command: &str,
1290 session_id: String,
1291 workdir: PathBuf,
1292 env: HashMap<String, String>,
1293 timeout: Option<Duration>,
1294 storage_dir: PathBuf,
1295 max_running: usize,
1296 notify_on_completion: bool,
1297 compressed: bool,
1298 project_root: Option<PathBuf>,
1299 rows: u16,
1300 cols: u16,
1301 ) -> Result<String, String> {
1302 self.start_watchdog();
1303
1304 let running = self.running_count();
1305 if running >= max_running {
1306 #[cfg(unix)]
1307 if let Some(prepared) = spawn_plan.prepared_task() {
1308 let _ = delete_resolved_task(&prepared.resolved_task());
1309 }
1310 return Err(format!(
1311 "background bash task limit exceeded: {running} running (max {max_running})"
1312 ));
1313 }
1314
1315 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1316 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1317 #[cfg(unix)]
1318 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1319 (spawn_plan.clone(), prepared.resolved_task())
1320 } else {
1321 let task = allocate_task_layout(&storage_dir, &session_id)
1322 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1323 let shell = super::resolved_shell_path(true);
1324 let root = project_root.as_deref().unwrap_or(&workdir);
1325 let environment =
1326 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1327 let prepared = match crate::sandbox_spawn::prepare_task_payload(
1328 &task,
1329 command.as_bytes(),
1330 root,
1331 &workdir,
1332 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1333 &shell,
1334 &environment,
1335 ) {
1336 Ok(prepared) => prepared,
1337 Err(error) => {
1338 let _ = delete_resolved_task(&task);
1339 return Err(error);
1340 }
1341 };
1342 let task = prepared.resolved_task();
1343 (spawn_plan.with_prepared_task(prepared), task)
1344 };
1345 #[cfg(windows)]
1346 let task_layout = allocate_task_layout(&storage_dir, &session_id)
1347 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1348 let task_id = task_layout.paths.task_id.clone();
1349 let paths = task_layout.paths.clone();
1350
1351 let mut metadata = PersistedTask::starting(
1352 task_id.clone(),
1353 session_id.clone(),
1354 command.to_string(),
1355 workdir.clone(),
1356 project_root,
1357 timeout_ms,
1358 notify_on_completion,
1359 compressed,
1360 );
1361 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1362 metadata.mode = BgMode::Pty;
1363 metadata.pty_rows = Some(rows);
1364 metadata.pty_cols = Some(cols);
1365 if let Err(error) = write_task_at(&task_layout, &metadata) {
1366 let _ = delete_resolved_task(&task_layout);
1367 return Err(format!(
1368 "failed to persist background task metadata: {error}"
1369 ));
1370 }
1371 self.dual_write_task(&paths, &metadata);
1372 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pty, false)
1373 .map_err(|error| format!("failed to pre-open PTY output handles: {error}"))?;
1374
1375 let runtime = match spawn_pty_for_command(
1376 &spawn_plan,
1377 &task_id,
1378 &session_id,
1379 command,
1380 &paths,
1381 &workdir,
1382 &env,
1383 rows,
1384 cols,
1385 self.inner.wake_tx.clone(),
1386 &mut io_handles,
1387 ) {
1388 Ok(runtime) => runtime,
1389 Err(error) => {
1390 crate::slog_warn!(
1391 "failed to spawn PTY background bash task {task_id}; deleting partial bundle: {error}"
1392 );
1393 let _ = delete_task_bundle(&paths);
1394 return Err(error);
1395 }
1396 };
1397
1398 if let Some(child_pid) = runtime.child_pid {
1399 metadata.mark_running(child_pid, child_pid as i32);
1400 } else {
1401 metadata.status = BgTaskStatus::Running;
1402 metadata.pgid = None;
1403 }
1404 self.persist_task(&paths, &metadata)
1405 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1406
1407 let task = Arc::new(BgTask {
1408 task_id: task_id.clone(),
1409 session_id,
1410 paths: paths.clone(),
1411 artifact_root: canonical_artifact_root(&paths),
1412 started: Instant::now(),
1413 last_reminder_at: Mutex::new(None),
1414 terminal_at: Mutex::new(None),
1415 state: Mutex::new(BgTaskState {
1416 metadata,
1417 runtime: TaskRuntime::Pty(Some(runtime)),
1418 io_handles: Some(io_handles),
1419 detached: false,
1420 child_exit_observed: false,
1421 buffer: BgBuffer::registered(&paths, BgMode::Pty),
1422 terminal_output_cache: None,
1423 pending_terminal_override: None,
1424 }),
1425 });
1426
1427 self.inner
1428 .tasks
1429 .lock()
1430 .map_err(|_| "background task registry lock poisoned".to_string())?
1431 .insert(task_id.clone(), task);
1432
1433 Ok(task_id)
1434 }
1435
1436 #[cfg(windows)]
1437 #[allow(clippy::too_many_arguments)]
1438 pub fn spawn(
1439 &self,
1440 spawn_plan: SpawnPlan,
1441 command: &str,
1442 session_id: String,
1443 workdir: PathBuf,
1444 env: HashMap<String, String>,
1445 timeout: Option<Duration>,
1446 storage_dir: PathBuf,
1447 max_running: usize,
1448 notify_on_completion: bool,
1449 compressed: bool,
1450 project_root: Option<PathBuf>,
1451 ) -> Result<String, String> {
1452 self.start_watchdog();
1453
1454 let running = self.running_count();
1455 if running >= max_running {
1456 #[cfg(unix)]
1457 if let Some(prepared) = spawn_plan.prepared_task() {
1458 let _ = delete_resolved_task(&prepared.resolved_task());
1459 }
1460 return Err(format!(
1461 "background bash task limit exceeded: {running} running (max {max_running})"
1462 ));
1463 }
1464
1465 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1466 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1467 let task_layout = allocate_task_layout(&storage_dir, &session_id)
1468 .map_err(|error| format!("failed to create background task layout: {error}"))?;
1469 let task_id = task_layout.paths.task_id.clone();
1470 let paths = task_layout.paths.clone();
1471
1472 let mut metadata = PersistedTask::starting(
1473 task_id.clone(),
1474 session_id.clone(),
1475 command.to_string(),
1476 workdir.clone(),
1477 project_root,
1478 timeout_ms,
1479 notify_on_completion,
1480 compressed,
1481 );
1482 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1483 if let Err(error) = write_task_at(&task_layout, &metadata) {
1484 let _ = delete_resolved_task(&task_layout);
1485 return Err(format!(
1486 "failed to persist background task metadata: {error}"
1487 ));
1488 }
1489 self.dual_write_task(&paths, &metadata);
1490 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pipes, false)
1491 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
1492
1493 let child = match spawn_detached_child(
1494 &spawn_plan,
1495 command,
1496 &paths,
1497 &workdir,
1498 &env,
1499 &mut io_handles,
1500 false,
1501 ) {
1502 Ok(child) => child,
1503 Err(error) => {
1504 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
1505 let _ = delete_task_bundle(&paths);
1506 return Err(error);
1507 }
1508 };
1509
1510 let child_pid = child.id();
1511 metadata.status = BgTaskStatus::Running;
1512 metadata.child_pid = Some(child_pid);
1513 metadata.pgid = None;
1514 self.persist_task(&paths, &metadata)
1515 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1516
1517 let task = Arc::new(BgTask {
1518 task_id: task_id.clone(),
1519 session_id,
1520 paths: paths.clone(),
1521 artifact_root: canonical_artifact_root(&paths),
1522 started: Instant::now(),
1523 last_reminder_at: Mutex::new(None),
1524 terminal_at: Mutex::new(None),
1525 state: Mutex::new(BgTaskState {
1526 metadata,
1527 runtime: TaskRuntime::Piped(Some(child)),
1528 io_handles: Some(io_handles),
1529 detached: false,
1530 child_exit_observed: false,
1531 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
1532 terminal_output_cache: None,
1533 pending_terminal_override: None,
1534 }),
1535 });
1536
1537 self.inner
1538 .tasks
1539 .lock()
1540 .map_err(|_| "background task registry lock poisoned".to_string())?
1541 .insert(task_id.clone(), task);
1542
1543 Ok(task_id)
1544 }
1545
1546 pub fn write_pty(
1547 &self,
1548 task_id: &str,
1549 session_id: &str,
1550 input: &[u8],
1551 ) -> Result<usize, String> {
1552 let task = self
1553 .task_for_session(task_id, session_id)
1554 .ok_or_else(|| "task_not_found".to_string())?;
1555
1556 let writer = {
1557 let state = task
1558 .state
1559 .lock()
1560 .map_err(|_| "background task lock poisoned".to_string())?;
1561 if state.metadata.mode != BgMode::Pty {
1562 return Err("task_not_pty".to_string());
1563 }
1564 if state.metadata.status.is_terminal() {
1565 return Err("task_exited".to_string());
1566 }
1567 match &state.runtime {
1568 TaskRuntime::Pty(Some(runtime)) => Arc::clone(&runtime.writer),
1569 TaskRuntime::Pty(None) => return Err("task_exited".to_string()),
1570 TaskRuntime::Piped(_) => return Err("task_not_pty".to_string()),
1571 }
1572 };
1573
1574 let mut writer = writer
1575 .lock()
1576 .map_err(|_| "PTY writer lock poisoned".to_string())?;
1577 writer
1578 .write_all(input)
1579 .map_err(|error| format!("failed to write to PTY: {error}"))?;
1580 writer
1581 .flush()
1582 .map_err(|error| format!("failed to flush PTY writer: {error}"))?;
1583 Ok(input.len())
1584 }
1585
1586 pub fn replay_session(&self, storage_dir: &Path, session_id: &str) -> Result<(), String> {
1587 self.replay_session_inner(storage_dir, session_id, None)
1588 }
1589
1590 pub fn replay_session_for_project(
1591 &self,
1592 storage_dir: &Path,
1593 session_id: &str,
1594 project_root: &Path,
1595 ) -> Result<(), String> {
1596 self.replay_session_inner(storage_dir, session_id, Some(project_root))
1597 }
1598
1599 fn replay_session_inner(
1600 &self,
1601 storage_dir: &Path,
1602 session_id: &str,
1603 project_root: Option<&Path>,
1604 ) -> Result<(), String> {
1605 self.start_watchdog();
1606 if !self.inner.persisted_gc_started.swap(true, Ordering::SeqCst) {
1607 if let Err(error) = self.maybe_gc_persisted(storage_dir) {
1608 crate::slog_warn!("failed to GC persisted background bash tasks: {error}");
1609 }
1610 }
1611
1612 let canonical_project = project_root.map(canonicalized_path);
1613 let tasks = match self.replay_session_from_db(session_id) {
1625 Some(Ok(tasks)) if !tasks.is_empty() => tasks,
1626 Some(Ok(_)) => {
1627 let disk_tasks = self.replay_session_from_disk(storage_dir, session_id)?;
1628 if !disk_tasks.is_empty() {
1629 crate::slog_info!(
1630 "bash task replay: 0 in DB for session {}, {} from disk fallback",
1631 session_id,
1632 disk_tasks.len()
1633 );
1634 }
1635 disk_tasks
1636 }
1637 Some(Err(error)) => {
1638 crate::slog_warn!(
1639 "bash task replay DB lookup failed for session {}; falling back to disk: {}",
1640 session_id,
1641 error
1642 );
1643 self.replay_session_from_disk(storage_dir, session_id)?
1644 }
1645 None => {
1646 self.replay_session_from_disk(storage_dir, session_id)?
1648 }
1649 };
1650
1651 for mut metadata in tasks {
1652 if metadata.session_id != session_id {
1653 continue;
1654 }
1655 if let Some(canonical_project) = canonical_project.as_deref() {
1656 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
1657 if metadata_project.as_deref() != Some(canonical_project) {
1658 continue;
1659 }
1660 }
1661
1662 if validate_task_id(&metadata.task_id).is_err() {
1663 crate::slog_warn!(
1664 "ignoring persisted background task with invalid id {:?}",
1665 metadata.task_id
1666 );
1667 continue;
1668 }
1669 let session_dir = session_tasks_dir(storage_dir, session_id);
1670 let resolved = match resolve_task_layout(&session_dir, &metadata.task_id) {
1671 Ok(task) => task,
1672 Err(error) => {
1673 crate::slog_warn!(
1674 "quarantining unresolved background task {}: {error}",
1675 metadata.task_id
1676 );
1677 let _ = quarantine_task_layout(
1678 storage_dir,
1679 &session_dir,
1680 &metadata.task_id,
1681 "invalid",
1682 );
1683 continue;
1684 }
1685 };
1686 match read_task_at(&resolved) {
1687 Ok(disk) if disk.task_id == metadata.task_id && disk.session_id == session_id => {}
1688 Ok(_) | Err(_) => {
1689 let _ = quarantine_task_layout(
1690 storage_dir,
1691 &session_dir,
1692 &metadata.task_id,
1693 "mismatch",
1694 );
1695 continue;
1696 }
1697 }
1698 let paths = resolved.paths;
1699 match metadata.status {
1700 BgTaskStatus::Starting => {
1701 let completion_was_delivered = metadata.completion_delivered;
1702 metadata.mark_terminal(
1703 BgTaskStatus::Failed,
1704 None,
1705 Some("spawn aborted".to_string()),
1706 );
1707 metadata.completion_delivered |= completion_was_delivered;
1708 let _ = self.persist_task(&paths, &metadata);
1709 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1710 self.insert_rehydrated_task(metadata, paths, true)?;
1711 }
1712 BgTaskStatus::Running | BgTaskStatus::Killing => {
1713 if metadata.mode == BgMode::Pty {
1714 if let Ok(Some(marker)) = read_exit_marker(&paths) {
1715 let completion_was_delivered = metadata.completion_delivered;
1716 metadata = terminal_metadata_from_marker(metadata, marker, None);
1717 metadata.completion_delivered |= completion_was_delivered;
1718 let _ = self.persist_task(&paths, &metadata);
1719 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1720 self.insert_rehydrated_task(metadata, paths, true)?;
1721 } else if metadata.status.is_terminal() {
1722 self.insert_rehydrated_task(metadata, paths, true)?;
1723 } else {
1724 let completion_was_delivered = metadata.completion_delivered;
1725 metadata.mark_terminal(
1726 BgTaskStatus::Killed,
1727 None,
1728 Some("pty_lost_on_bridge_restart".to_string()),
1729 );
1730 metadata.completion_delivered |= completion_was_delivered;
1731 let _ = self.persist_task(&paths, &metadata);
1732 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1733 self.insert_rehydrated_task(metadata, paths, true)?;
1734 }
1735 } else if self.running_metadata_is_stale(&metadata) {
1736 let completion_was_delivered = metadata.completion_delivered;
1737 metadata.mark_terminal(
1738 BgTaskStatus::Killed,
1739 None,
1740 Some("orphaned (>24h)".to_string()),
1741 );
1742 metadata.completion_delivered |= completion_was_delivered;
1743 let _ = write_kill_marker_if_absent(&paths);
1744 let _ = self.persist_task(&paths, &metadata);
1745 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1746 self.insert_rehydrated_task(metadata, paths, true)?;
1747 } else if let Ok(Some(marker)) = read_exit_marker(&paths) {
1748 let reason = (metadata.status == BgTaskStatus::Killing).then(|| {
1749 "recovered from inconsistent killing state on replay".to_string()
1750 });
1751 if reason.is_some() {
1752 crate::slog_warn!("background task {} had killing state with exit marker; preferring marker",
1753 metadata.task_id);
1754 }
1755 let completion_was_delivered = metadata.completion_delivered;
1756 metadata = terminal_metadata_from_marker(metadata, marker, reason);
1757 metadata.completion_delivered |= completion_was_delivered;
1758 let _ = self.persist_task(&paths, &metadata);
1759 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1760 self.insert_rehydrated_task(metadata, paths, true)?;
1761 } else if metadata.status == BgTaskStatus::Killing {
1762 let _ = write_kill_marker_if_absent(&paths);
1763 let completion_was_delivered = metadata.completion_delivered;
1764 metadata.mark_terminal(
1765 BgTaskStatus::Killed,
1766 None,
1767 Some("recovered from inconsistent killing state on replay".to_string()),
1768 );
1769 metadata.completion_delivered |= completion_was_delivered;
1770 let _ = self.persist_task(&paths, &metadata);
1771 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1772 self.insert_rehydrated_task(metadata, paths, true)?;
1773 } else if metadata.child_pid.is_some_and(|pid| !is_process_alive(pid)) {
1774 let completion_was_delivered = metadata.completion_delivered;
1775 metadata.mark_terminal(
1776 BgTaskStatus::Failed,
1777 None,
1778 Some("process exited without exit marker".to_string()),
1779 );
1780 metadata.completion_delivered |= completion_was_delivered;
1781 let _ = self.persist_task(&paths, &metadata);
1782 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1783 self.insert_rehydrated_task(metadata, paths, true)?;
1784 } else {
1785 self.insert_rehydrated_task(metadata, paths, true)?;
1786 }
1787 }
1788 _ if metadata.status.is_terminal() => {
1789 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1795 self.insert_rehydrated_task(metadata, paths, true)?;
1796 }
1797 _ => {}
1798 }
1799 }
1800
1801 Ok(())
1802 }
1803
1804 fn replay_session_from_db(
1805 &self,
1806 session_id: &str,
1807 ) -> Option<Result<Vec<PersistedTask>, String>> {
1808 let pool = self
1809 .inner
1810 .db_pool
1811 .read()
1812 .ok()
1813 .and_then(|slot| slot.clone())?;
1814 let harness = self
1815 .inner
1816 .db_harness
1817 .read()
1818 .ok()
1819 .and_then(|slot| slot.clone())?;
1820 let conn = match pool.lock() {
1821 Ok(conn) => conn,
1822 Err(_) => return Some(Err("db mutex poisoned".to_string())),
1823 };
1824 Some(
1825 crate::db::bash_tasks::list_bash_tasks_for_session(&conn, &harness, session_id)
1826 .map(|rows| rows.into_iter().map(PersistedTask::from).collect())
1827 .map_err(|error| error.to_string()),
1828 )
1829 }
1830
1831 fn replay_session_from_disk(
1832 &self,
1833 storage_dir: &Path,
1834 session_id: &str,
1835 ) -> Result<Vec<PersistedTask>, String> {
1836 let dir = session_tasks_dir(storage_dir, session_id);
1837 if !dir.exists() {
1838 return Ok(Vec::new());
1839 }
1840
1841 let (task_ids, invalid_entries) = discover_task_ids(&dir)
1842 .map_err(|error| format!("failed to discover background task layouts: {error}"))?;
1843 for entry in invalid_entries {
1844 if let Err(error) = quarantine_invalid_entry(storage_dir, &dir, &entry) {
1845 crate::slog_warn!(
1846 "failed to quarantine invalid background task entry {:?}: {error}",
1847 entry
1848 );
1849 }
1850 }
1851
1852 let mut tasks = Vec::new();
1853 for task_id in task_ids {
1854 let task = match resolve_task_layout(&dir, &task_id) {
1855 Ok(task) => task,
1856 Err(error)
1857 if error.kind() == std::io::ErrorKind::NotFound
1858 && uninitialized_layout_is_recent(
1859 &dir,
1860 &task_id,
1861 Duration::from_secs(5 * 60),
1862 )
1863 .unwrap_or(false) =>
1864 {
1865 continue;
1866 }
1867 Err(error) => {
1868 crate::slog_warn!(
1869 "quarantining unresolved background task {task_id} during replay: {error}"
1870 );
1871 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
1872 continue;
1873 }
1874 };
1875 match read_task_at(&task) {
1876 Ok(metadata) if metadata.session_id == session_id => tasks.push(metadata),
1877 Ok(_) => {
1878 crate::slog_warn!(
1879 "quarantining background task {task_id} with mismatched session metadata"
1880 );
1881 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "mismatch");
1882 }
1883 Err(error) => {
1884 crate::slog_warn!(
1885 "quarantining invalid background task metadata {task_id} during replay: {error}"
1886 );
1887 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
1888 }
1889 }
1890 }
1891 Ok(tasks)
1892 }
1893
1894 pub fn register_watch(
1895 &self,
1896 task_id: String,
1897 pattern: WatchPattern,
1898 once: bool,
1899 ) -> Result<String, &'static str> {
1900 let task = self.task(&task_id).ok_or("task_not_found")?;
1901 validate_task_id(&task_id).map_err(|_| "invalid_task_id")?;
1902 let (mode, terminal_at_registration) = task
1903 .state
1904 .lock()
1905 .map(|state| {
1906 (
1907 state.metadata.mode.clone(),
1908 state.metadata.status.is_terminal(),
1909 )
1910 })
1911 .map_err(|_| "background_task_lock_poisoned")?;
1912 let mut stdout = (mode == BgMode::Pipes)
1913 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
1914 .transpose()
1915 .map_err(|_| "artifact_refused")?;
1916 let mut stderr = (mode == BgMode::Pipes)
1917 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
1918 .transpose()
1919 .map_err(|_| "artifact_refused")?;
1920 let mut pty = (mode == BgMode::Pty)
1921 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
1922 .transpose()
1923 .map_err(|_| "artifact_refused")?;
1924
1925 let mut terminal_matches = Vec::new();
1926 let scanned_terminal = terminal_at_registration;
1927 let watch_id = {
1928 let mut registry = self
1929 .inner
1930 .watch_registry
1931 .lock()
1932 .map_err(|_| "watch_registry_poisoned")?;
1933 let watch_id = registry.register(task_id.clone(), pattern.clone(), once)?;
1934 match &mode {
1935 BgMode::Pipes => {
1936 let stdout_key = format!("{task_id}:stdout");
1937 let stderr_key = format!("{task_id}:stderr");
1938 if terminal_at_registration {
1939 registry.set_file_cursor(&stdout_key, 0);
1940 registry.set_file_cursor(&stderr_key, 0);
1941 terminal_matches.extend(registry.scan_file_new_bytes(
1942 &stdout_key,
1943 &task_id,
1944 stdout.as_mut().expect("pipe stdout opened"),
1945 ));
1946 terminal_matches.extend(registry.scan_file_new_bytes(
1947 &stderr_key,
1948 &task_id,
1949 stderr.as_mut().expect("pipe stderr opened"),
1950 ));
1951 } else {
1952 registry.prime_file_cursor(
1953 &stdout_key,
1954 stdout.as_ref().expect("pipe stdout opened"),
1955 );
1956 registry.prime_file_cursor(
1957 &stderr_key,
1958 stderr.as_ref().expect("pipe stderr opened"),
1959 );
1960 }
1961 }
1962 BgMode::Pty => {
1963 let pty_key = format!("{task_id}:pty");
1964 if terminal_at_registration {
1965 registry.set_file_cursor(&pty_key, 0);
1966 terminal_matches.extend(registry.scan_file_new_bytes(
1967 &pty_key,
1968 &task_id,
1969 pty.as_mut().expect("PTY artifact opened"),
1970 ));
1971 } else {
1972 registry.prime_file_cursor(
1973 &pty_key,
1974 pty.as_ref().expect("PTY artifact opened"),
1975 );
1976 }
1977 }
1978 }
1979 watch_id
1980 };
1981
1982 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
1983 self.persist_watch_registration(
1984 &task.session_id,
1985 &task_id,
1986 &watch_id,
1987 &pattern,
1988 once,
1989 stdout_offset,
1990 stderr_offset,
1991 pty_offset,
1992 );
1993
1994 if task.is_terminal() {
1995 if !scanned_terminal {
1996 terminal_matches = {
1997 let mut registry = self
1998 .inner
1999 .watch_registry
2000 .lock()
2001 .map_err(|_| "watch_registry_poisoned")?;
2002 match &mode {
2003 BgMode::Pipes => {
2004 let stdout_key = format!("{task_id}:stdout");
2005 let stderr_key = format!("{task_id}:stderr");
2006 registry.set_file_cursor(&stdout_key, 0);
2007 registry.set_file_cursor(&stderr_key, 0);
2008 let mut matches = registry.scan_file_new_bytes(
2009 &stdout_key,
2010 &task_id,
2011 stdout.as_mut().expect("pipe stdout opened"),
2012 );
2013 matches.extend(registry.scan_file_new_bytes(
2014 &stderr_key,
2015 &task_id,
2016 stderr.as_mut().expect("pipe stderr opened"),
2017 ));
2018 matches
2019 }
2020 BgMode::Pty => {
2021 let pty_key = format!("{task_id}:pty");
2022 registry.set_file_cursor(&pty_key, 0);
2023 registry.scan_file_new_bytes(
2024 &pty_key,
2025 &task_id,
2026 pty.as_mut().expect("PTY artifact opened"),
2027 )
2028 }
2029 }
2030 };
2031 }
2032
2033 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
2034 let (watch_controlled, watch_matched) = self.task_watch_state(&task_id);
2035 if terminal_matches.is_empty() && (!watch_controlled || watch_matched) {
2036 if watch_matched {
2037 let _ = task.set_completion_delivered(true, self);
2038 self.clear_task_watch_state(&task_id);
2039 self.delete_persisted_watches_for_task(&task.session_id, &task_id);
2041 }
2042 return Ok(watch_id);
2043 }
2044
2045 let completion = self
2046 .remove_pending_completion(&task_id)
2047 .or_else(|| self.completion_snapshot_for_task(&task));
2048 if terminal_matches.is_empty() {
2049 if let Some(completion) = completion.as_ref() {
2050 self.emit_bash_watch_exit(completion);
2051 }
2052 } else {
2053 for pattern_match in &terminal_matches {
2054 self.persist_watch_match(
2055 &task.session_id,
2056 &task_id,
2057 pattern_match,
2058 stdout_offset,
2059 stderr_offset,
2060 pty_offset,
2061 );
2062 self.emit_bash_pattern_match(&task.session_id, pattern_match.clone());
2063 }
2064 }
2065 let _ = task.set_completion_delivered(true, self);
2066 self.clear_task_watch_state(&task_id);
2067 self.delete_persisted_watches_for_task(&task.session_id, &task_id);
2069 }
2070
2071 Ok(watch_id)
2072 }
2073
2074 pub fn unregister_watch(&self, task_id: &str, watch_id: &str) {
2075 let session_id = self.task(task_id).map(|task| task.session_id.clone());
2076 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2077 registry.unregister(task_id, watch_id);
2078 }
2079 if let Some(session_id) = session_id {
2080 self.delete_persisted_watch(&session_id, task_id, watch_id);
2081 }
2082 }
2083
2084 pub fn active_watch_count(&self, task_id: &str) -> usize {
2085 self.inner
2086 .watch_registry
2087 .lock()
2088 .map(|registry| registry.active_count(task_id))
2089 .unwrap_or(0)
2090 }
2091
2092 fn task_watch_state(&self, task_id: &str) -> (bool, bool) {
2093 self.inner
2094 .watch_registry
2095 .lock()
2096 .map(|registry| {
2097 (
2098 registry.has_controlled_task(task_id),
2099 registry.has_matched_task(task_id),
2100 )
2101 })
2102 .unwrap_or((false, false))
2103 }
2104
2105 fn task_has_watch_control(&self, task_id: &str) -> bool {
2106 self.inner
2107 .watch_registry
2108 .lock()
2109 .map(|registry| registry.has_controlled_task(task_id))
2110 .unwrap_or(false)
2111 }
2112
2113 fn clear_task_watch_state(&self, task_id: &str) {
2114 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2115 registry.clear_task(task_id);
2116 }
2117 }
2118
2119 pub(crate) fn scan_task_watch_output(&self, task: &Arc<BgTask>) {
2120 let mode = match task.state.lock() {
2121 Ok(state) => state.metadata.mode.clone(),
2122 Err(_) => return,
2123 };
2124 let mut stdout = (mode == BgMode::Pipes)
2125 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
2126 .transpose()
2127 .ok()
2128 .flatten();
2129 let mut stderr = (mode == BgMode::Pipes)
2130 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
2131 .transpose()
2132 .ok()
2133 .flatten();
2134 let mut pty = (mode == BgMode::Pty)
2135 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
2136 .transpose()
2137 .ok()
2138 .flatten();
2139 let mut matches = Vec::new();
2140 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2141 match mode {
2142 BgMode::Pipes => {
2143 let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) else {
2144 return;
2145 };
2146 let stdout_key = format!("{}:stdout", task.task_id);
2147 let stderr_key = format!("{}:stderr", task.task_id);
2148 matches.extend(registry.scan_file_new_bytes(
2149 &stdout_key,
2150 &task.task_id,
2151 stdout,
2152 ));
2153 matches.extend(registry.scan_file_new_bytes(
2154 &stderr_key,
2155 &task.task_id,
2156 stderr,
2157 ));
2158 }
2159 BgMode::Pty => {
2160 let Some(pty) = pty.as_mut() else {
2161 return;
2162 };
2163 let pty_key = format!("{}:pty", task.task_id);
2164 matches.extend(registry.scan_file_new_bytes(&pty_key, &task.task_id, pty));
2165 }
2166 }
2167 }
2168 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
2169 if matches.is_empty() {
2170 if self.task_has_watch_control(&task.task_id) {
2173 self.persist_task_watch_cursors(
2174 &task.session_id,
2175 &task.task_id,
2176 stdout_offset,
2177 stderr_offset,
2178 pty_offset,
2179 );
2180 }
2181 return;
2182 }
2183 for pattern_match in matches {
2184 self.persist_watch_match(
2185 &task.session_id,
2186 &task.task_id,
2187 &pattern_match,
2188 stdout_offset,
2189 stderr_offset,
2190 pty_offset,
2191 );
2192 self.emit_bash_pattern_match(&task.session_id, pattern_match);
2193 }
2194 self.persist_task_watch_cursors(
2195 &task.session_id,
2196 &task.task_id,
2197 stdout_offset,
2198 stderr_offset,
2199 pty_offset,
2200 );
2201 }
2202
2203 pub fn status(
2204 &self,
2205 task_id: &str,
2206 session_id: &str,
2207 project_root: Option<&Path>,
2208 storage_dir: Option<&Path>,
2209 preview_bytes: usize,
2210 ) -> Option<BgTaskSnapshot> {
2211 validate_task_id(task_id).ok()?;
2212 let mut task = self.task_for_session(task_id, session_id);
2213 if task.is_none() {
2214 if let Some(storage_dir) = storage_dir {
2215 let _ = if let Some(project_root) = project_root {
2216 self.replay_session_for_project(storage_dir, session_id, project_root)
2217 } else {
2218 self.replay_session(storage_dir, session_id)
2219 };
2220 task = self.task_for_session(task_id, session_id);
2221 }
2222 }
2223 let Some(task) = task else {
2224 return self.status_relaxed(
2225 task_id,
2226 session_id,
2227 project_root?,
2228 storage_dir?,
2229 preview_bytes,
2230 );
2231 };
2232 let _ = self.poll_task(&task);
2233 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
2234 }
2235
2236 fn status_relaxed_task(
2237 &self,
2238 task_id: &str,
2239 project_root: &Path,
2240 storage_dir: &Path,
2241 ) -> Option<Arc<BgTask>> {
2242 validate_task_id(task_id).ok()?;
2243 let canonical_project = canonicalized_path(project_root);
2244 match self.lookup_relaxed_task_from_db(task_id, project_root) {
2245 Some(Ok(Some(metadata))) => {
2246 if let Some(task) = self.task(task_id) {
2247 let matches_project = task
2248 .state
2249 .lock()
2250 .map(|state| {
2251 state
2252 .metadata
2253 .project_root
2254 .as_deref()
2255 .map(canonicalized_path)
2256 .as_deref()
2257 == Some(canonical_project.as_path())
2258 })
2259 .unwrap_or(false);
2260 return matches_project.then_some(task);
2261 }
2262 let resolved = resolve_task_layout(
2263 &session_tasks_dir(storage_dir, &metadata.session_id),
2264 &metadata.task_id,
2265 )
2266 .ok()?;
2267 let disk = read_task_at(&resolved).ok()?;
2268 if disk.task_id != metadata.task_id || disk.session_id != metadata.session_id {
2269 return None;
2270 }
2271 if self
2272 .insert_rehydrated_task(metadata, resolved.paths, true)
2273 .is_err()
2274 {
2275 return None;
2276 }
2277 return self.task(task_id);
2278 }
2279 Some(Ok(None)) => {
2280 crate::slog_info!(
2281 "bash task relaxed DB miss for {}; falling back to disk",
2282 task_id
2283 );
2284 }
2285 Some(Err(error)) => {
2286 crate::slog_warn!(
2287 "bash task relaxed DB lookup failed for {}; falling back to disk: {}",
2288 task_id,
2289 error
2290 );
2291 }
2292 None => {
2293 crate::slog_info!(
2294 "bash task relaxed DB unavailable for {}; falling back to disk",
2295 task_id
2296 );
2297 }
2298 }
2299 let root = storage_dir.join("bash-tasks");
2300 let entries = fs::read_dir(&root).ok()?;
2301 for entry in entries.flatten() {
2302 let dir = entry.path();
2303 if !dir.is_dir() {
2304 continue;
2305 }
2306 let resolved = match resolve_task_layout(&dir, task_id) {
2307 Ok(task) => task,
2308 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
2309 Err(error) => {
2310 crate::slog_warn!(
2311 "quarantining unresolved background task {task_id} during relaxed lookup: {error}"
2312 );
2313 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
2314 continue;
2315 }
2316 };
2317 let metadata = match read_task_at(&resolved) {
2318 Ok(metadata) => metadata,
2319 Err(error) => {
2320 crate::slog_warn!(
2321 "quarantining invalid background task metadata {task_id} during relaxed lookup: {error}"
2322 );
2323 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
2324 continue;
2325 }
2326 };
2327 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
2328 if metadata_project.as_deref() != Some(canonical_project.as_path()) {
2329 continue;
2330 }
2331 if let Some(task) = self.task(task_id) {
2332 let matches_project = task
2333 .state
2334 .lock()
2335 .map(|state| {
2336 state
2337 .metadata
2338 .project_root
2339 .as_deref()
2340 .map(canonicalized_path)
2341 .as_deref()
2342 == Some(canonical_project.as_path())
2343 })
2344 .unwrap_or(false);
2345 return matches_project.then_some(task);
2346 }
2347 if self
2348 .insert_rehydrated_task(metadata, resolved.paths, true)
2349 .is_err()
2350 {
2351 return None;
2352 }
2353 return self.task(task_id);
2354 }
2355 None
2356 }
2357
2358 fn lookup_relaxed_task_from_db(
2359 &self,
2360 task_id: &str,
2361 project_root: &Path,
2362 ) -> Option<Result<Option<PersistedTask>, String>> {
2363 let pool = self
2364 .inner
2365 .db_pool
2366 .read()
2367 .ok()
2368 .and_then(|slot| slot.clone())?;
2369 let harness = self
2370 .inner
2371 .db_harness
2372 .read()
2373 .ok()
2374 .and_then(|slot| slot.clone())?;
2375 let conn = match pool.lock() {
2376 Ok(conn) => conn,
2377 Err(_) => return Some(Err("db mutex poisoned".to_string())),
2378 };
2379 let project_key = crate::path_identity::project_scope_key(project_root);
2380 Some(
2381 crate::db::bash_tasks::find_bash_task_for_project(
2382 &conn,
2383 &harness,
2384 &project_key,
2385 task_id,
2386 )
2387 .map(|row| row.map(PersistedTask::from))
2388 .map_err(|error| error.to_string()),
2389 )
2390 }
2391
2392 pub(super) fn status_relaxed(
2393 &self,
2394 task_id: &str,
2395 _session_id: &str,
2396 project_root: &Path,
2397 storage_dir: &Path,
2398 preview_bytes: usize,
2399 ) -> Option<BgTaskSnapshot> {
2400 let task = self.status_relaxed_task(task_id, project_root, storage_dir)?;
2401 let _ = self.poll_task(&task);
2402 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
2403 }
2404
2405 pub fn kill_relaxed(
2406 &self,
2407 task_id: &str,
2408 project_root: &Path,
2409 storage_dir: &Path,
2410 ) -> Result<BgTaskSnapshot, String> {
2411 let task = self
2412 .status_relaxed_task(task_id, project_root, storage_dir)
2413 .ok_or_else(|| format!("background task not found: {task_id}"))?;
2414 self.kill_with_status(task_id, &task.session_id, BgTaskStatus::Killed)
2415 }
2416
2417 pub fn maybe_gc_persisted(&self, storage_dir: &Path) -> Result<usize, String> {
2418 #[cfg(test)]
2419 self.inner.persisted_gc_runs.fetch_add(1, Ordering::SeqCst);
2420
2421 let mut deleted = 0usize;
2422
2423 let root = storage_dir.join("bash-tasks");
2424 if root.exists() {
2425 let session_dirs = fs::read_dir(&root).map_err(|e| {
2426 format!(
2427 "failed to read background task root {}: {e}",
2428 root.display()
2429 )
2430 })?;
2431 for session_entry in session_dirs.flatten() {
2432 let session_dir = session_entry.path();
2433 if !session_dir.is_dir() {
2434 continue;
2435 }
2436 let (task_ids, invalid_entries) = match discover_task_ids(&session_dir) {
2437 Ok(discovery) => discovery,
2438 Err(error) => {
2439 crate::slog_warn!(
2440 "failed to discover background task session {}: {error}",
2441 session_dir.display()
2442 );
2443 continue;
2444 }
2445 };
2446 for entry in invalid_entries {
2447 let _ = quarantine_invalid_entry(storage_dir, &session_dir, &entry);
2448 }
2449 for task_id in task_ids {
2450 let resolved = match resolve_task_layout(&session_dir, &task_id) {
2451 Ok(task) => task,
2452 Err(error)
2453 if error.kind() == std::io::ErrorKind::NotFound
2454 && uninitialized_layout_is_recent(
2455 &session_dir,
2456 &task_id,
2457 Duration::from_secs(5 * 60),
2458 )
2459 .unwrap_or(false) =>
2460 {
2461 continue;
2462 }
2463 Err(error) => {
2464 crate::slog_warn!(
2465 "quarantining unresolved background task {task_id}: {error}"
2466 );
2467 quarantine_task_layout(storage_dir, &session_dir, &task_id, "invalid")
2468 .map_err(|error| error.to_string())?;
2469 continue;
2470 }
2471 };
2472 if modified_within(&resolved.paths.json, PERSISTED_GC_GRACE) {
2473 continue;
2474 }
2475 let metadata = match read_task_at(&resolved) {
2476 Ok(metadata) => metadata,
2477 Err(error) => {
2478 crate::slog_warn!(
2479 "quarantining corrupt background task metadata {task_id}: {error}"
2480 );
2481 quarantine_task_layout(storage_dir, &session_dir, &task_id, "corrupt")
2482 .map_err(|error| error.to_string())?;
2483 continue;
2484 }
2485 };
2486 if !(metadata.status.is_terminal() && metadata.completion_delivered) {
2487 continue;
2488 }
2489 match delete_task_bundle(&resolved.paths) {
2490 Ok(()) => {
2491 self.delete_gc_task_from_db(&metadata);
2492 deleted += 1;
2493 log::debug!(
2494 "deleted persisted background task bundle {}",
2495 metadata.task_id
2496 );
2497 }
2498 Err(error) => {
2499 crate::slog_warn!(
2500 "failed to delete background task bundle {}: {error}",
2501 metadata.task_id
2502 );
2503 }
2504 }
2505 }
2506 }
2507 }
2508 gc_quarantine(storage_dir);
2509 Ok(deleted)
2510 }
2511
2512 pub fn list(&self, preview_bytes: usize) -> Vec<BgTaskSnapshot> {
2513 let tasks = self
2514 .inner
2515 .tasks
2516 .lock()
2517 .map(|tasks| tasks.values().cloned().collect::<Vec<_>>())
2518 .unwrap_or_default();
2519 tasks
2520 .into_iter()
2521 .map(|task| {
2522 let _ = self.poll_task(&task);
2523 self.snapshot_with_terminal_cache(&task, preview_bytes)
2524 })
2525 .collect()
2526 }
2527
2528 fn maybe_compress_snapshot(&self, task: &Arc<BgTask>, snapshot: &mut BgTaskSnapshot) {
2534 if !snapshot.info.status.is_terminal() || snapshot.info.mode == BgMode::Pty {
2535 return;
2536 }
2537 if let Some(cache) = self.ensure_terminal_output_cache(task) {
2538 snapshot.output_preview = cache.output_preview;
2539 snapshot.output_truncated = cache.output_truncated;
2540 }
2541 }
2542
2543 pub fn kill(&self, task_id: &str, session_id: &str) -> Result<BgTaskSnapshot, String> {
2544 self.kill_with_status(task_id, session_id, BgTaskStatus::Killed)
2545 }
2546
2547 pub fn kill_running_tasks_for_root(&self, project_root: &Path) -> usize {
2557 let canonical_root = canonicalized_path(project_root);
2558 let targets = self
2559 .inner
2560 .tasks
2561 .lock()
2562 .map(|tasks| {
2563 tasks
2564 .values()
2565 .filter_map(|task| {
2566 let state = task.state.lock().ok()?;
2567 let status = &state.metadata.status;
2568 let running = matches!(status, BgTaskStatus::Running)
2569 || (state.metadata.mode == BgMode::Pty
2570 && matches!(status, BgTaskStatus::Killing));
2571 if !running {
2572 return None;
2573 }
2574 let task_root = state
2575 .metadata
2576 .project_root
2577 .as_deref()
2578 .unwrap_or(&state.metadata.workdir);
2579 (canonicalized_path(task_root) == canonical_root)
2580 .then(|| (task.task_id.clone(), task.session_id.clone()))
2581 })
2582 .collect::<Vec<_>>()
2583 })
2584 .unwrap_or_default();
2585
2586 let mut killed = 0;
2587 for (task_id, session_id) in targets {
2588 match self.kill_with_status_reason(
2589 &task_id,
2590 &session_id,
2591 BgTaskStatus::Killed,
2592 Some(ROOT_RECLAIMED_REASON.to_string()),
2593 ) {
2594 Ok(_) => killed += 1,
2595 Err(error) => crate::slog_warn!(
2596 "failed to terminate background task {task_id} for reclaimed root {}: {error}",
2597 project_root.display()
2598 ),
2599 }
2600 }
2601 killed
2602 }
2603
2604 pub fn promote(&self, task_id: &str, session_id: &str) -> Result<bool, String> {
2605 let task = self
2606 .task_for_session(task_id, session_id)
2607 .ok_or_else(|| format!("background task not found: {task_id}"))?;
2608 let terminal_after_promote = {
2609 let mut state = task
2610 .state
2611 .lock()
2612 .map_err(|_| "background task lock poisoned".to_string())?;
2613 let updated = self
2614 .update_task_metadata(&task.paths, |metadata| {
2615 metadata.notify_on_completion = true;
2616 metadata.completion_delivered = false;
2617 })
2618 .map_err(|e| format!("failed to promote background task: {e}"))?;
2619 state.metadata = updated;
2620 state.metadata.status.is_terminal()
2621 };
2622 if terminal_after_promote {
2623 self.post_terminal_transition(&task, true)?;
2624 }
2625 Ok(true)
2626 }
2627
2628 pub(crate) fn kill_for_timeout(&self, task_id: &str, session_id: &str) -> Result<(), String> {
2629 self.kill_with_status(task_id, session_id, BgTaskStatus::TimedOut)
2630 .map(|_| ())
2631 }
2632
2633 pub fn cleanup_finished(&self, older_than: Duration) {
2634 let cutoff = Instant::now().checked_sub(older_than);
2635 let removable_paths: Vec<(String, TaskPaths)> =
2636 if let Ok(mut tasks) = self.inner.tasks.lock() {
2637 let removable = tasks
2638 .iter()
2639 .filter_map(|(task_id, task)| {
2640 let delivered_terminal = task
2641 .state
2642 .lock()
2643 .map(|state| {
2644 state.metadata.status.is_terminal()
2645 && state.metadata.completion_delivered
2646 })
2647 .unwrap_or(false);
2648 if !delivered_terminal {
2649 return None;
2650 }
2651
2652 let terminal_at = task.terminal_at.lock().ok().and_then(|at| *at);
2653 let expired = match (terminal_at, cutoff) {
2654 (Some(terminal_at), Some(cutoff)) => terminal_at <= cutoff,
2655 (Some(_), None) => true,
2656 (None, _) => false,
2657 };
2658 expired.then(|| task_id.clone())
2659 })
2660 .collect::<Vec<_>>();
2661
2662 removable
2663 .into_iter()
2664 .filter_map(|task_id| {
2665 tasks
2666 .remove(&task_id)
2667 .map(|task| (task_id, task.paths.clone()))
2668 })
2669 .collect()
2670 } else {
2671 Vec::new()
2672 };
2673
2674 for (task_id, paths) in removable_paths {
2675 match delete_task_bundle(&paths) {
2676 Ok(()) => log::debug!("deleted persisted background task bundle {task_id}"),
2677 Err(error) => crate::slog_warn!(
2678 "failed to delete persisted background task bundle {task_id}: {error}"
2679 ),
2680 }
2681 }
2682 }
2683
2684 pub fn drain_completions(&self) -> Vec<BgCompletion> {
2685 self.drain_completions_for_session(None)
2686 }
2687
2688 pub fn drain_completions_for_session(&self, session_id: Option<&str>) -> Vec<BgCompletion> {
2689 let completions = match self.inner.completions.lock() {
2690 Ok(completions) => completions,
2691 Err(_) => return Vec::new(),
2692 };
2693
2694 completions
2695 .iter()
2696 .filter(|completion| completion_matches_session(completion, session_id))
2697 .cloned()
2698 .collect()
2699 }
2700
2701 pub fn has_completions_for_session(&self, session_id: Option<&str>) -> bool {
2702 match self.inner.completions.lock() {
2703 Ok(completions) => completions
2704 .iter()
2705 .any(|completion| completion_matches_session(completion, session_id)),
2706 Err(_) => true,
2710 }
2711 }
2712
2713 pub fn ack_completions_for_session(
2714 &self,
2715 session_id: Option<&str>,
2716 task_ids: &[String],
2717 ) -> Vec<String> {
2718 if task_ids.is_empty() {
2719 return Vec::new();
2720 }
2721 let requested_task_ids = task_ids.iter().map(String::as_str).collect::<HashSet<_>>();
2722 let mut completion_sessions = HashMap::new();
2723 if let Ok(mut completions) = self.inner.completions.lock() {
2724 completions.retain(|completion| {
2725 let session_matches = session_id
2726 .map(|session_id| completion.session_id == session_id)
2727 .unwrap_or(true);
2728 if session_matches && requested_task_ids.contains(completion.task_id.as_str()) {
2729 completion_sessions
2730 .insert(completion.task_id.clone(), completion.session_id.clone());
2731 false
2732 } else {
2733 true
2734 }
2735 });
2736 }
2737
2738 let mut delivered = Vec::new();
2739 for task_id in task_ids {
2740 let task = if let Some(session_id) = session_id {
2741 self.task_for_session(task_id, session_id)
2742 } else if let Some(completion_session_id) = completion_sessions.get(task_id) {
2743 self.task_for_session(task_id, completion_session_id)
2744 } else {
2745 self.task(task_id)
2746 };
2747 if let Some(task) = task {
2748 let terminal = task
2749 .state
2750 .lock()
2751 .map(|state| state.metadata.status.is_terminal())
2752 .unwrap_or(false);
2753 self.ack_persisted_watches_for_task(&task.session_id, task_id, terminal);
2757 if terminal {
2758 self.clear_task_watch_state(task_id);
2759 if task.set_completion_delivered(true, self).is_ok() {
2760 delivered.push(task_id.clone());
2761 }
2762 } else {
2763 self.sync_memory_watches_after_ack(task_id);
2766 delivered.push(task_id.clone());
2767 }
2768 } else if let Some(session_id) = session_id {
2769 self.ack_persisted_watches_for_task(session_id, task_id, true);
2771 delivered.push(task_id.clone());
2772 }
2773 }
2774
2775 delivered
2776 }
2777
2778 fn sync_memory_watches_after_ack(&self, task_id: &str) {
2779 let Some((harness, pool)) = self.db_harness_and_pool() else {
2780 return;
2781 };
2782 let session_id = match self.task(task_id) {
2783 Some(task) => task.session_id.clone(),
2784 None => return,
2785 };
2786 let Ok(conn) = pool.lock() else {
2787 return;
2788 };
2789 let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
2790 &conn,
2791 &harness,
2792 &session_id,
2793 task_id,
2794 ) else {
2795 return;
2796 };
2797 let remaining: HashSet<String> = rows.into_iter().map(|row| row.watch_id).collect();
2798 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2799 registry.retain_watch_ids(task_id, &remaining);
2800 }
2801 }
2802
2803 pub fn pending_completions_for_session(&self, session_id: &str) -> Vec<BgCompletion> {
2804 self.inner
2805 .completions
2806 .lock()
2807 .map(|completions| {
2808 completions
2809 .iter()
2810 .filter(|completion| completion.session_id == session_id)
2811 .cloned()
2812 .collect()
2813 })
2814 .unwrap_or_default()
2815 }
2816
2817 fn remove_pending_completion(&self, task_id: &str) -> Option<BgCompletion> {
2818 let mut completions = self.inner.completions.lock().ok()?;
2819 let idx = completions
2820 .iter()
2821 .position(|completion| completion.task_id == task_id)?;
2822 completions.remove(idx)
2823 }
2824
2825 fn completion_snapshot_for_task(&self, task: &Arc<BgTask>) -> Option<BgCompletion> {
2826 let snapshot = self.snapshot_with_terminal_cache(task, RUNNING_OUTPUT_PREVIEW_BYTES);
2827 if !snapshot.info.status.is_terminal() {
2828 return None;
2829 }
2830 let (output_preview, output_truncated) = if snapshot.info.mode == BgMode::Pty {
2831 (String::new(), false)
2832 } else {
2833 self.ensure_terminal_output_cache(task)
2834 .map(|cache| completion_preview_for_cache(&cache, snapshot.exit_code))
2835 .unwrap_or_else(|| (String::new(), false))
2836 };
2837 Some(BgCompletion {
2838 task_id: snapshot.info.task_id,
2839 session_id: task.session_id.clone(),
2840 status: snapshot.info.status,
2841 exit_code: snapshot.exit_code,
2842 command: snapshot.info.command,
2843 output_preview,
2844 output_truncated,
2845 original_tokens: None,
2846 compressed_tokens: None,
2847 tokens_skipped: false,
2848 status_reason: snapshot.info.status_reason,
2849 })
2850 }
2851
2852 pub fn detach(&self) {
2853 self.inner.shutdown.store(true, Ordering::SeqCst);
2854 if let Ok(mut tasks) = self.inner.tasks.lock() {
2855 for task in tasks.values() {
2856 if let Ok(mut state) = task.state.lock() {
2857 match &mut state.runtime {
2858 TaskRuntime::Piped(child) => *child = None,
2859 TaskRuntime::Pty(runtime) => *runtime = None,
2860 }
2861 state.detached = true;
2862 }
2863 }
2864 tasks.clear();
2865 }
2866 }
2867
2868 pub fn shutdown(&self) {
2869 let tasks = self
2870 .inner
2871 .tasks
2872 .lock()
2873 .map(|tasks| {
2874 tasks
2875 .values()
2876 .map(|task| (task.task_id.clone(), task.session_id.clone()))
2877 .collect::<Vec<_>>()
2878 })
2879 .unwrap_or_default();
2880 for (task_id, session_id) in tasks {
2881 let _ = self.kill(&task_id, &session_id);
2882 }
2883 }
2884
2885 pub(crate) fn poll_task(&self, task: &Arc<BgTask>) -> Result<(), String> {
2886 if let Ok(state) = task.state.lock() {
2887 if let TaskRuntime::Pty(Some(pty)) = &state.runtime {
2888 if !pty.exit_observed.load(Ordering::SeqCst) {
2896 return Ok(());
2897 }
2898 }
2899 }
2900 let marker = match read_exit_marker(&task.paths) {
2901 Ok(Some(marker)) => marker,
2902 Ok(None) => return Ok(()),
2903 Err(error) => return Err(format!("failed to read exit marker: {error}")),
2904 };
2905 self.finalize_from_marker(task, marker, None)
2906 }
2907
2908 pub(crate) fn reap_child(&self, task: &Arc<BgTask>) {
2909 let mut needs_completion = false;
2910 {
2911 let Ok(mut state) = task.state.lock() else {
2912 return;
2913 };
2914 match &mut state.runtime {
2915 TaskRuntime::Piped(child_slot) => {
2916 if let Some(child) = child_slot.as_mut() {
2917 if let Ok(Some(status)) = child.try_wait() {
2918 *child_slot = None;
2919 state.detached = true;
2920 state.child_exit_observed = true;
2921 if let Some(handles) = state.io_handles.as_mut() {
2922 if handles.artifact_len(TaskArtifact::Exit).unwrap_or(1) == 0 {
2923 let marker = status
2924 .code()
2925 .map(|code| code.to_string())
2926 .unwrap_or_else(|| "1".to_string());
2927 let _ = handles.write(TaskArtifact::Exit, marker.as_bytes());
2928 }
2929 }
2930 }
2931 } else if state.detached {
2932 let child_known_dead = state.child_exit_observed
2933 || state
2934 .metadata
2935 .child_pid
2936 .is_some_and(|pid| !is_process_alive(pid));
2937 if child_known_dead {
2938 needs_completion =
2939 self.fail_without_exit_marker_if_needed(task, &mut state);
2940 }
2941 }
2942 }
2943 TaskRuntime::Pty(Some(pty)) => {
2944 if pty.exit_observed.load(Ordering::SeqCst) {
2945 drop(state);
2946 let _ = self.poll_task(task);
2947 return;
2948 }
2949 }
2950 TaskRuntime::Pty(None) => {}
2951 }
2952 }
2953 if needs_completion {
2954 let _ = self.post_terminal_transition(task, true);
2955 }
2956 }
2957
2958 fn fail_without_exit_marker_if_needed(
2959 &self,
2960 task: &Arc<BgTask>,
2961 state: &mut BgTaskState,
2962 ) -> bool {
2963 if state.metadata.status.is_terminal() {
2964 return false;
2965 }
2966 if matches!(read_exit_marker(&task.paths), Ok(Some(_))) {
2967 return false;
2968 }
2969 let watch_controlled = self.task_has_watch_control(&task.task_id);
2970 let updated = self.update_task_metadata(&task.paths, |metadata| {
2971 metadata.mark_terminal(
2972 BgTaskStatus::Failed,
2973 None,
2974 Some("process exited without exit marker".to_string()),
2975 );
2976 if watch_controlled {
2977 metadata.completion_delivered = true;
2978 }
2979 });
2980 if let Ok(metadata) = updated {
2981 state.pending_terminal_override = None;
2982 state.metadata = metadata;
2983 task.mark_terminal_now();
2984 return true;
2985 }
2986 false
2987 }
2988
2989 pub(crate) fn running_tasks(&self) -> Vec<Arc<BgTask>> {
2990 self.inner
2991 .tasks
2992 .lock()
2993 .map(|tasks| {
2994 tasks
2995 .values()
2996 .filter(|task| task.is_running())
2997 .cloned()
2998 .collect()
2999 })
3000 .unwrap_or_default()
3001 }
3002
3003 fn insert_rehydrated_task(
3004 &self,
3005 metadata: PersistedTask,
3006 paths: TaskPaths,
3007 detached: bool,
3008 ) -> Result<(), String> {
3009 let task_id = metadata.task_id.clone();
3010 let session_id = metadata.session_id.clone();
3011 let started = started_instant_from_unix_millis(metadata.started_at);
3012 let suppress_replayed_running_reminder = metadata.status == BgTaskStatus::Running;
3013 let mode = metadata.mode.clone();
3014 let task = Arc::new(BgTask {
3015 task_id: task_id.clone(),
3016 session_id,
3017 paths: paths.clone(),
3018 artifact_root: canonical_artifact_root(&paths),
3019 started,
3020 last_reminder_at: Mutex::new(suppress_replayed_running_reminder.then(Instant::now)),
3021 terminal_at: Mutex::new(metadata.status.is_terminal().then(Instant::now)),
3022 state: Mutex::new(BgTaskState {
3023 metadata,
3024 runtime: if mode == BgMode::Pty {
3025 TaskRuntime::Pty(None)
3026 } else {
3027 TaskRuntime::Piped(None)
3028 },
3029 io_handles: None,
3030 detached,
3031 child_exit_observed: false,
3038 buffer: BgBuffer::registered(&paths, mode.clone()),
3039 terminal_output_cache: None,
3040 pending_terminal_override: None,
3041 }),
3042 });
3043 self.inner
3044 .tasks
3045 .lock()
3046 .map_err(|_| "background task registry lock poisoned".to_string())?
3047 .insert(task_id.clone(), Arc::clone(&task));
3048 self.rearm_persisted_watches(&task);
3052 Ok(())
3053 }
3054
3055 fn rearm_persisted_watches(&self, task: &Arc<BgTask>) {
3056 let Some((harness, pool)) = self.db_harness_and_pool() else {
3057 return;
3058 };
3059 let rows = {
3060 let Ok(conn) = pool.lock() else {
3061 return;
3062 };
3063 match crate::db::bash_watches::list_bash_pattern_watches_for_task(
3064 &conn,
3065 &harness,
3066 &task.session_id,
3067 &task.task_id,
3068 ) {
3069 Ok(rows) if !rows.is_empty() => rows,
3070 _ => return,
3071 }
3072 };
3073
3074 let mode = match task.state.lock() {
3075 Ok(state) => state.metadata.mode.clone(),
3076 Err(_) => return,
3077 };
3078 let terminal = task
3079 .state
3080 .lock()
3081 .map(|state| state.metadata.status.is_terminal())
3082 .unwrap_or(false);
3083 let completion_delivered = task
3084 .state
3085 .lock()
3086 .map(|state| state.metadata.completion_delivered)
3087 .unwrap_or(true);
3088
3089 let mut stdout = (mode == BgMode::Pipes)
3090 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
3091 .transpose()
3092 .ok()
3093 .flatten();
3094 let mut stderr = (mode == BgMode::Pipes)
3095 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
3096 .transpose()
3097 .ok()
3098 .flatten();
3099 let mut pty = (mode == BgMode::Pty)
3100 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
3101 .transpose()
3102 .ok()
3103 .flatten();
3104
3105 let mut pending_to_emit = Vec::new();
3106 let mut gap_matches = Vec::new();
3107 {
3108 let Ok(mut registry) = self.inner.watch_registry.lock() else {
3109 return;
3110 };
3111 let stdout_key = format!("{}:stdout", task.task_id);
3112 let stderr_key = format!("{}:stderr", task.task_id);
3113 let pty_key = format!("{}:pty", task.task_id);
3114
3115 let first = &rows[0];
3117 match mode {
3118 BgMode::Pipes => {
3119 registry.set_file_cursor(&stdout_key, first.stdout_offset.max(0) as u64);
3120 registry.set_file_cursor(&stderr_key, first.stderr_offset.max(0) as u64);
3121 }
3122 BgMode::Pty => {
3123 registry.set_file_cursor(&pty_key, first.pty_offset.max(0) as u64);
3124 }
3125 }
3126
3127 for row in &rows {
3128 let Ok(pattern) = WatchPattern::from_persisted(&row.pattern_kind, &row.pattern)
3129 else {
3130 crate::slog_warn!(
3131 "skipping unreadable persisted watch {}/{}",
3132 row.task_id,
3133 row.watch_id
3134 );
3135 continue;
3136 };
3137 if let Err(error) = registry.restore(
3138 row.watch_id.clone(),
3139 row.task_id.clone(),
3140 pattern,
3141 row.once,
3142 row.scanning,
3143 ) {
3144 crate::slog_warn!(
3145 "failed to restore watch {}/{}: {error}",
3146 row.task_id,
3147 row.watch_id
3148 );
3149 continue;
3150 }
3151 if row.pending_match {
3152 if let (Some(match_text), Some(match_offset), Some(context)) = (
3153 row.match_text.clone(),
3154 row.match_offset,
3155 row.match_context.clone(),
3156 ) {
3157 pending_to_emit.push(PatternMatch {
3158 watch_id: row.watch_id.clone(),
3159 task_id: row.task_id.clone(),
3160 match_text,
3161 match_offset: match_offset.max(0) as u64,
3162 context,
3163 once: row.once,
3164 });
3165 }
3166 }
3167 }
3168
3169 let should_gap_scan =
3173 rows.iter().any(|row| row.scanning) && !pending_to_emit.iter().any(|m| m.once);
3174 if should_gap_scan {
3175 match mode {
3176 BgMode::Pipes => {
3177 if let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) {
3178 gap_matches.extend(registry.scan_file_new_bytes(
3179 &stdout_key,
3180 &task.task_id,
3181 stdout,
3182 ));
3183 gap_matches.extend(registry.scan_file_new_bytes(
3184 &stderr_key,
3185 &task.task_id,
3186 stderr,
3187 ));
3188 }
3189 }
3190 BgMode::Pty => {
3191 if let Some(pty) = pty.as_mut() {
3192 gap_matches.extend(registry.scan_file_new_bytes(
3193 &pty_key,
3194 &task.task_id,
3195 pty,
3196 ));
3197 }
3198 }
3199 }
3200 }
3201 }
3202
3203 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
3204 for pattern_match in &gap_matches {
3205 self.persist_watch_match(
3206 &task.session_id,
3207 &task.task_id,
3208 pattern_match,
3209 stdout_offset,
3210 stderr_offset,
3211 pty_offset,
3212 );
3213 }
3214 if !gap_matches.is_empty() || rows.iter().any(|row| row.scanning) {
3215 self.persist_task_watch_cursors(
3216 &task.session_id,
3217 &task.task_id,
3218 stdout_offset,
3219 stderr_offset,
3220 pty_offset,
3221 );
3222 }
3223
3224 let emitted_pending = !pending_to_emit.is_empty();
3226 let to_emit = if emitted_pending {
3227 pending_to_emit
3228 } else {
3229 gap_matches
3230 };
3231 for pattern_match in to_emit {
3232 self.emit_bash_pattern_match(&task.session_id, pattern_match);
3233 }
3234
3235 if !terminal {
3236 return;
3237 }
3238
3239 let _ = self.remove_pending_completion(&task.task_id);
3242 let (watch_controlled, watch_matched) = self.task_watch_state(&task.task_id);
3243 if !watch_controlled {
3244 return;
3245 }
3246 if watch_matched {
3247 return;
3249 }
3250 if completion_delivered {
3251 self.clear_task_watch_state(&task.task_id);
3253 self.delete_persisted_watches_for_task(&task.session_id, &task.task_id);
3254 return;
3255 }
3256 if let Some(completion) = self.completion_snapshot_for_task(task) {
3257 self.emit_bash_watch_exit(&completion);
3258 }
3259 self.clear_task_watch_state(&task.task_id);
3261 }
3262
3263 fn kill_with_status(
3264 &self,
3265 task_id: &str,
3266 session_id: &str,
3267 terminal_status: BgTaskStatus,
3268 ) -> Result<BgTaskSnapshot, String> {
3269 self.kill_with_status_reason(task_id, session_id, terminal_status, None)
3270 }
3271
3272 fn kill_with_status_reason(
3273 &self,
3274 task_id: &str,
3275 session_id: &str,
3276 terminal_status: BgTaskStatus,
3277 reason: Option<String>,
3278 ) -> Result<BgTaskSnapshot, String> {
3279 let task = self
3280 .task_for_session(task_id, session_id)
3281 .ok_or_else(|| format!("background task not found: {task_id}"))?;
3282 let mut terminalized = false;
3283
3284 {
3285 let mut state = task
3286 .state
3287 .lock()
3288 .map_err(|_| "background task lock poisoned".to_string())?;
3289 if state.metadata.status.is_terminal() {
3290 state.pending_terminal_override = None;
3291 } else if let Ok(Some(marker)) = read_exit_marker(&task.paths) {
3292 state.metadata =
3293 terminal_metadata_from_marker(state.metadata.clone(), marker, reason.clone());
3294 if self.task_has_watch_control(&task.task_id) {
3295 state.metadata.completion_delivered = true;
3296 }
3297 state.pending_terminal_override = None;
3298 task.mark_terminal_now();
3299 match &mut state.runtime {
3300 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
3307 TaskRuntime::Pty(runtime) => *runtime = None,
3308 }
3309 state.detached = true;
3310 self.persist_task(&task.paths, &state.metadata)
3311 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
3312 terminalized = true;
3313 } else {
3314 let was_already_killing = state.metadata.status == BgTaskStatus::Killing;
3315 if !was_already_killing {
3316 state.metadata.status = BgTaskStatus::Killing;
3317 }
3318 if reason.is_some() {
3319 state.metadata.status_reason = reason.clone();
3320 }
3321 if !was_already_killing || reason.is_some() {
3322 self.persist_task(&task.paths, &state.metadata)
3323 .map_err(|e| format!("failed to persist killing state: {e}"))?;
3324 }
3325
3326 #[cfg(unix)]
3327 let pgid = state.metadata.pgid;
3328 #[cfg(windows)]
3329 let child_pid = state.metadata.child_pid;
3330 if !was_already_killing
3331 && state.metadata.mode == BgMode::Pty
3332 && terminal_status == BgTaskStatus::TimedOut
3333 {
3334 state.pending_terminal_override = Some(BgTaskStatus::TimedOut);
3335 }
3336
3337 #[cfg(windows)]
3338 let mut pty_forced_terminal_status: Option<BgTaskStatus> = None;
3339
3340 match &mut state.runtime {
3341 TaskRuntime::Piped(child_slot) => {
3342 #[cfg(unix)]
3343 if let Some(pgid) = pgid {
3344 terminate_pgid(pgid, child_slot.as_mut());
3345 }
3346 #[cfg(windows)]
3347 if let Some(child) = child_slot.as_mut() {
3348 super::process::terminate_process(child);
3349 } else if let Some(pid) = child_pid {
3350 terminate_pid(pid);
3351 }
3352 if let Some(child) = child_slot.as_mut() {
3353 let _ = child.wait();
3354 }
3355 *child_slot = None;
3356 state.detached = true;
3357
3358 if let Some(handles) = state.io_handles.as_mut() {
3359 handles.write(TaskArtifact::Exit, b"killed").map_err(|e| {
3360 format!("failed to write retained kill marker: {e}")
3361 })?;
3362 } else {
3363 write_kill_marker_if_absent(&task.paths)
3364 .map_err(|e| format!("failed to write kill marker: {e}"))?;
3365 }
3366
3367 let exit_code = terminal_exit_code_for_status(&terminal_status);
3368 state
3369 .metadata
3370 .mark_terminal(terminal_status, exit_code, reason.clone());
3371 if self.task_has_watch_control(&task.task_id) {
3372 state.metadata.completion_delivered = true;
3373 }
3374 state.pending_terminal_override = None;
3375 task.mark_terminal_now();
3376 self.persist_task(&task.paths, &state.metadata)
3377 .map_err(|e| format!("failed to persist killed state: {e}"))?;
3378 terminalized = true;
3379 }
3380 TaskRuntime::Pty(Some(pty)) => {
3381 pty.was_killed.store(true, Ordering::SeqCst);
3382 if let Err(error) = pty.killer.kill() {
3383 crate::slog_warn!(
3384 "[pty-kill] {task_id} ChildKiller::kill failed: {error}"
3385 );
3386 }
3387 if let Some(pid) = pty.child_pid {
3388 #[cfg(unix)]
3389 terminate_pgid(pid as i32, None);
3390 #[cfg(windows)]
3391 terminate_pid(pid);
3392 }
3393 drop(pty.master.take());
3394
3395 #[cfg(windows)]
3396 {
3397 let default_status = if terminal_status == BgTaskStatus::TimedOut {
3398 BgTaskStatus::TimedOut
3399 } else {
3400 BgTaskStatus::Killed
3401 };
3402 pty_forced_terminal_status = Some(
3403 state
3404 .pending_terminal_override
3405 .take()
3406 .unwrap_or(default_status),
3407 );
3408 }
3409 }
3410 TaskRuntime::Pty(None) => {}
3411 }
3412
3413 #[cfg(windows)]
3414 if let Some(target_status) = pty_forced_terminal_status {
3415 if !task.paths.exit.exists() {
3416 write_kill_marker_if_absent(&task.paths)
3417 .map_err(|e| format!("failed to write kill marker: {e}"))?;
3418 }
3419
3420 let exit_code = terminal_exit_code_for_status(&target_status);
3421 state
3422 .metadata
3423 .mark_terminal(target_status, exit_code, reason.clone());
3424 if self.task_has_watch_control(&task.task_id) {
3425 state.metadata.completion_delivered = true;
3426 }
3427 state.pending_terminal_override = None;
3428 task.mark_terminal_now();
3429 if let TaskRuntime::Pty(runtime) = &mut state.runtime {
3430 *runtime = None;
3431 }
3432 state.detached = true;
3433 self.persist_task(&task.paths, &state.metadata)
3434 .map_err(|e| format!("failed to persist killed PTY state: {e}"))?;
3435 terminalized = true;
3436 }
3437 }
3438 }
3439
3440 if terminalized {
3441 self.post_terminal_transition(&task, true)?;
3442 }
3443 Ok(self.snapshot_with_terminal_cache(&task, RUNNING_OUTPUT_PREVIEW_BYTES))
3444 }
3445
3446 fn finalize_from_marker(
3447 &self,
3448 task: &Arc<BgTask>,
3449 marker: ExitMarker,
3450 reason: Option<String>,
3451 ) -> Result<(), String> {
3452 let watch_controlled = self.task_has_watch_control(&task.task_id);
3453 let mut pty_reader_done = None;
3454 {
3455 let mut state = task
3456 .state
3457 .lock()
3458 .map_err(|_| "background task lock poisoned".to_string())?;
3459 if state.metadata.status.is_terminal() {
3460 state.pending_terminal_override = None;
3461 return Ok(());
3462 }
3463
3464 let pending_override = state.pending_terminal_override.take();
3465 let is_pty = state.metadata.mode == BgMode::Pty;
3466 let reason = reason.or_else(|| state.metadata.status_reason.clone());
3467 let updated = self
3468 .update_task_metadata(&task.paths, |metadata| {
3469 let mut new_metadata = if is_pty && marker == ExitMarker::Killed {
3470 let mut metadata = metadata.clone();
3471 let target_status = pending_override.unwrap_or(BgTaskStatus::Killed);
3472 let exit_code = terminal_exit_code_for_status(&target_status);
3473 metadata.mark_terminal(target_status, exit_code, reason.clone());
3474 metadata
3475 } else {
3476 terminal_metadata_from_marker(metadata.clone(), marker, reason.clone())
3477 };
3478 if watch_controlled {
3479 new_metadata.completion_delivered = true;
3480 }
3481 *metadata = new_metadata;
3482 })
3483 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
3484 state.metadata = updated;
3485 task.mark_terminal_now();
3486 match &mut state.runtime {
3487 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
3492 TaskRuntime::Pty(runtime) => {
3493 pty_reader_done = runtime
3494 .as_ref()
3495 .map(|runtime| Arc::clone(&runtime.reader_done));
3496 *runtime = None;
3497 }
3498 }
3499 state.detached = true;
3500 }
3501
3502 if let Some(reader_done) = pty_reader_done {
3503 let deadline = Instant::now() + Duration::from_millis(200);
3504 while !reader_done.load(Ordering::SeqCst) && Instant::now() < deadline {
3505 std::thread::sleep(Duration::from_millis(10));
3506 }
3507 }
3508
3509 self.scan_task_watch_output(task);
3512
3513 self.post_terminal_transition(task, true)
3514 }
3515
3516 fn enqueue_completion_if_needed(
3517 &self,
3518 metadata: &PersistedTask,
3519 paths: Option<&TaskPaths>,
3520 emit_frame: bool,
3521 ) {
3522 if metadata.status.is_terminal() && !metadata.completion_delivered {
3523 let cache =
3524 paths.and_then(|paths| self.render_terminal_output_from_paths(metadata, paths));
3525 self.enqueue_completion_from_parts(metadata, None, paths, emit_frame, cache.as_ref());
3526 }
3527 }
3528
3529 fn render_terminal_output_from_paths(
3530 &self,
3531 metadata: &PersistedTask,
3532 paths: &TaskPaths,
3533 ) -> Option<TerminalOutputCache> {
3534 if metadata.mode == BgMode::Pty {
3535 return None;
3536 }
3537 let mut buffer = BgBuffer::registered(paths, BgMode::Pipes);
3538 let disk_truncation = buffer.enforce_terminal_cap();
3539 Some(self.render_terminal_output(metadata, &buffer, disk_truncation, Some(paths)))
3540 }
3541
3542 fn enqueue_completion_from_parts(
3543 &self,
3544 metadata: &PersistedTask,
3545 buffer: Option<&BgBuffer>,
3546 paths: Option<&TaskPaths>,
3547 emit_frame: bool,
3548 terminal_render: Option<&TerminalOutputCache>,
3549 ) {
3550 if !metadata.status.is_terminal() {
3561 return;
3562 }
3563
3564 let owned_buffer = if buffer.is_none() && metadata.mode != BgMode::Pty {
3565 paths.map(|paths| BgBuffer::registered(paths, BgMode::Pipes))
3566 } else {
3567 None
3568 };
3569 let render_buffer = buffer.or(owned_buffer.as_ref());
3570 let owned_render = if terminal_render.is_none() {
3571 render_buffer.map(|buffer| {
3572 let mut capped_buffer = buffer.clone();
3573 let disk_truncation = capped_buffer.enforce_terminal_cap();
3574 self.render_terminal_output(metadata, &capped_buffer, disk_truncation, paths)
3575 })
3576 } else {
3577 None
3578 };
3579 let render = terminal_render.or(owned_render.as_ref());
3580
3581 let (output_preview, output_truncated) = render
3585 .map(|cache| completion_preview_for_cache(cache, metadata.exit_code))
3586 .unwrap_or_else(|| (String::new(), false));
3587
3588 let token_counts = self.completion_token_counts(
3589 metadata,
3590 buffer,
3591 paths,
3592 render.map(|render| render.output_preview.as_str()),
3593 );
3594 let completion = BgCompletion {
3595 task_id: metadata.task_id.clone(),
3596 session_id: metadata.session_id.clone(),
3597 status: metadata.status.clone(),
3598 exit_code: metadata.exit_code,
3599 command: metadata.command.clone(),
3600 output_preview,
3601 output_truncated,
3602 original_tokens: token_counts.original_tokens,
3603 compressed_tokens: token_counts.compressed_tokens,
3604 tokens_skipped: token_counts.tokens_skipped,
3605 status_reason: metadata.status_reason.clone(),
3606 };
3607
3608 self.record_compression_event_if_applicable(metadata, &token_counts);
3619
3620 let (watch_controlled, watch_matched) = self.task_watch_state(&metadata.task_id);
3621 if watch_controlled {
3622 if emit_frame && !watch_matched {
3623 self.emit_bash_watch_exit(&completion);
3624 } else if watch_matched {
3625 if let Some(task) = self.task(&metadata.task_id) {
3630 let _ = task.set_completion_delivered(true, self);
3631 }
3632 }
3633 self.clear_task_watch_state(&metadata.task_id);
3635 return;
3636 }
3637
3638 if metadata.completion_delivered {
3648 return;
3649 }
3650
3651 let pushed = if let Ok(mut completions) = self.inner.completions.lock() {
3654 if completions
3655 .iter()
3656 .any(|existing| existing.task_id == metadata.task_id)
3657 {
3658 false
3659 } else {
3660 completions.push_back(completion.clone());
3661 true
3662 }
3663 } else {
3664 false
3665 };
3666
3667 if pushed && emit_frame {
3668 self.emit_bash_completed(completion);
3669 }
3670 }
3671
3672 fn record_compression_event_if_applicable(
3673 &self,
3674 metadata: &PersistedTask,
3675 token_counts: &CompletionTokenCounts,
3676 ) {
3677 if metadata.mode == BgMode::Pty {
3678 return;
3679 }
3680
3681 let (original_tokens, compressed_tokens, original_bytes, compressed_bytes) = match (
3682 token_counts.original_tokens,
3683 token_counts.compressed_tokens,
3684 token_counts.original_bytes,
3685 token_counts.compressed_bytes,
3686 ) {
3687 (
3688 Some(original_tokens),
3689 Some(compressed_tokens),
3690 Some(original_bytes),
3691 Some(compressed_bytes),
3692 ) => (
3693 original_tokens,
3694 compressed_tokens,
3695 original_bytes,
3696 compressed_bytes,
3697 ),
3698 _ => {
3699 crate::slog_warn!(
3700 "compression event skipped for {}: token counts unavailable (likely spill file missing or unreadable)",
3701 metadata.task_id
3702 );
3703 return;
3704 }
3705 };
3706
3707 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
3708 let Some(pool) = pool else {
3709 crate::slog_warn!(
3710 "compression event skipped for {}: db_pool not initialized — was configure run?",
3711 metadata.task_id
3712 );
3713 return;
3714 };
3715 let harness = self
3716 .inner
3717 .db_harness
3718 .read()
3719 .ok()
3720 .and_then(|slot| slot.clone());
3721 let Some(harness) = harness else {
3722 crate::slog_warn!(
3723 "compression event insert skipped for {}: harness not configured",
3724 metadata.task_id
3725 );
3726 return;
3727 };
3728
3729 let project_root = metadata
3730 .project_root
3731 .as_deref()
3732 .unwrap_or(&metadata.workdir);
3733 let project_key = crate::path_identity::project_scope_key(project_root);
3734 let row = crate::db::compression_events::CompressionEventRow {
3735 harness: &harness,
3736 session_id: Some(&metadata.session_id),
3737 project_key: &project_key,
3738 tool: "bash",
3739 task_id: Some(&metadata.task_id),
3740 command: Some(&metadata.command),
3741 compressor: if metadata.compressed {
3742 "registry"
3743 } else {
3744 "none"
3745 },
3746 original_bytes,
3747 compressed_bytes,
3748 original_tokens,
3749 compressed_tokens,
3750 created_at: unix_millis() as i64,
3751 };
3752
3753 let conn = match pool.lock() {
3754 Ok(conn) => conn,
3755 Err(_) => {
3756 crate::slog_warn!(
3757 "compression event insert failed for {}: db mutex poisoned",
3758 metadata.task_id
3759 );
3760 return;
3761 }
3762 };
3763 match crate::db::compression_events::insert_compression_event(&conn, &row) {
3764 Ok(Some(row_id)) => {
3765 self.inner
3769 .compression_aggregates
3770 .record_successful_insert(&conn, &row, row_id);
3771 crate::slog_debug!(
3775 "compression event recorded for {} (project={}, session={}, {} → {} tokens)",
3776 metadata.task_id,
3777 project_key,
3778 metadata.session_id,
3779 original_tokens,
3780 compressed_tokens
3781 );
3782 }
3783 Ok(None) => {
3784 crate::slog_debug!(
3785 "duplicate compression event ignored for {} (project={}, session={})",
3786 metadata.task_id,
3787 project_key,
3788 metadata.session_id
3789 );
3790 }
3791 Err(error) => {
3792 crate::slog_warn!(
3793 "compression event insert failed for {}: {}",
3794 metadata.task_id,
3795 error
3796 );
3797 }
3798 }
3799 }
3800
3801 fn emit_bash_pattern_match(&self, session_id: &str, pattern_match: PatternMatch) {
3802 let Ok(progress_sender) = self
3803 .inner
3804 .progress_sender
3805 .lock()
3806 .map(|sender| sender.clone())
3807 else {
3808 return;
3809 };
3810 if let Some(sender) = progress_sender.as_ref() {
3811 sender(PushFrame::BashPatternMatch(BashPatternMatchFrame::new(
3812 pattern_match.task_id,
3813 session_id.to_string(),
3814 pattern_match.watch_id,
3815 pattern_match.match_text,
3816 pattern_match.match_offset,
3817 pattern_match.context,
3818 pattern_match.once,
3819 )));
3820 }
3821 }
3822
3823 fn emit_bash_watch_exit(&self, completion: &BgCompletion) {
3824 let Ok(progress_sender) = self
3825 .inner
3826 .progress_sender
3827 .lock()
3828 .map(|sender| sender.clone())
3829 else {
3830 return;
3831 };
3832 let Some(sender) = progress_sender.as_ref() else {
3833 return;
3834 };
3835 let status = completion_status_text(&completion.status, completion.exit_code);
3836 let preview = completion.output_preview.trim_end();
3837 let context = if preview.is_empty() {
3838 format!("task {} exited ({status})", completion.task_id)
3839 } else {
3840 format!(
3841 "task {} exited ({status})
3842{preview}",
3843 completion.task_id
3844 )
3845 };
3846 sender(PushFrame::BashPatternMatch(
3847 BashPatternMatchFrame::task_exit(
3848 completion.task_id.clone(),
3849 completion.session_id.clone(),
3850 format!("exited ({status})"),
3851 context,
3852 ),
3853 ));
3854 }
3855
3856 fn emit_bash_completed(&self, completion: BgCompletion) {
3857 let Ok(progress_sender) = self
3858 .inner
3859 .progress_sender
3860 .lock()
3861 .map(|sender| sender.clone())
3862 else {
3863 return;
3864 };
3865 let Some(sender) = progress_sender.as_ref() else {
3866 return;
3867 };
3868 let mut frame = BashCompletedFrame::new(
3876 completion.task_id,
3877 completion.session_id,
3878 completion.status,
3879 completion.exit_code,
3880 completion.command,
3881 completion.output_preview,
3882 completion.output_truncated,
3883 completion.original_tokens,
3884 completion.compressed_tokens,
3885 completion.tokens_skipped,
3886 );
3887 frame.status_reason = completion.status_reason;
3888 sender(PushFrame::BashCompleted(frame));
3889 }
3890
3891 fn completion_token_counts(
3892 &self,
3893 metadata: &PersistedTask,
3894 buffer: Option<&BgBuffer>,
3895 paths: Option<&TaskPaths>,
3896 rendered_output: Option<&str>,
3897 ) -> CompletionTokenCounts {
3898 if metadata.mode == BgMode::Pty {
3899 return CompletionTokenCounts::skipped();
3900 }
3901
3902 let raw = match buffer {
3903 Some(buffer) => buffer.read_for_token_count(TOKENIZE_CAP_BYTES_PER_STREAM),
3904 None => paths
3905 .map(|paths| {
3906 read_for_token_count_from_disk(metadata, paths, TOKENIZE_CAP_BYTES_PER_STREAM)
3907 })
3908 .unwrap_or(TokenCountInput::Skipped),
3909 };
3910
3911 let TokenCountInput::Text(raw_output) = raw else {
3912 return CompletionTokenCounts::skipped();
3913 };
3914
3915 let original_tokens = token_count_u32(&raw_output);
3916 let original_bytes = raw_output.len() as i64;
3917 let compressed_output = rendered_output.unwrap_or(&raw_output);
3918 let compressed_tokens = token_count_u32(compressed_output);
3919 let compressed_bytes = compressed_output.len() as i64;
3920 CompletionTokenCounts {
3921 original_tokens: Some(original_tokens),
3922 compressed_tokens: Some(compressed_tokens),
3923 original_bytes: Some(original_bytes),
3924 compressed_bytes: Some(compressed_bytes),
3925 tokens_skipped: false,
3926 }
3927 }
3928
3929 pub(crate) fn maybe_emit_long_running_reminder(&self, task: &Arc<BgTask>) {
3930 if !self
3931 .inner
3932 .long_running_reminder_enabled
3933 .load(Ordering::SeqCst)
3934 {
3935 return;
3936 }
3937 let interval_ms = self
3938 .inner
3939 .long_running_reminder_interval_ms
3940 .load(Ordering::SeqCst);
3941 if interval_ms == 0 {
3942 return;
3943 }
3944 let interval = Duration::from_millis(interval_ms);
3945 let now = Instant::now();
3946 let Ok(mut last_reminder_at) = task.last_reminder_at.lock() else {
3947 return;
3948 };
3949 let since = last_reminder_at.unwrap_or(task.started);
3950 if now.duration_since(since) < interval {
3951 return;
3952 }
3953 let command = task
3954 .state
3955 .lock()
3956 .map(|state| state.metadata.command.clone())
3957 .unwrap_or_default();
3958 *last_reminder_at = Some(now);
3959 self.emit_bash_long_running(BashLongRunningFrame::new(
3960 task.task_id.clone(),
3961 task.session_id.clone(),
3962 command,
3963 task.started.elapsed().as_millis() as u64,
3964 ));
3965 }
3966
3967 fn emit_bash_long_running(&self, frame: BashLongRunningFrame) {
3968 let Ok(progress_sender) = self
3969 .inner
3970 .progress_sender
3971 .lock()
3972 .map(|sender| sender.clone())
3973 else {
3974 return;
3975 };
3976 if let Some(sender) = progress_sender.as_ref() {
3977 sender(PushFrame::BashLongRunning(frame));
3978 }
3979 }
3980
3981 fn task(&self, task_id: &str) -> Option<Arc<BgTask>> {
3982 validate_task_id(task_id).ok()?;
3983 self.inner
3984 .tasks
3985 .lock()
3986 .ok()
3987 .and_then(|tasks| tasks.get(task_id).cloned())
3988 }
3989
3990 fn task_for_session(&self, task_id: &str, session_id: &str) -> Option<Arc<BgTask>> {
3991 self.task(task_id)
3992 .filter(|task| task.session_id == session_id)
3993 }
3994
3995 pub fn try_health_counts(&self) -> Option<BgTaskHealthCounts> {
3996 let running = self
3997 .inner
3998 .tasks
3999 .try_lock()
4000 .ok()
4001 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())?;
4002 let pending_completions = self.inner.completions.try_lock().ok().map(|q| q.len())?;
4003 Some(BgTaskHealthCounts {
4004 running,
4005 pending_completions,
4006 })
4007 }
4008
4009 pub fn estimated_memory(&self) -> crate::memory::MemoryEstimate {
4013 let tasks = match self.inner.tasks.try_lock() {
4014 Ok(tasks) => tasks.values().cloned().collect::<Vec<_>>(),
4015 Err(_) => return crate::memory::MemoryEstimate::busy(),
4016 };
4017 let mut bytes = 0u64;
4018 let mut terminal_output_caches = 0usize;
4019 let mut sessions = HashSet::new();
4020 for task in &tasks {
4021 sessions.insert(task.session_id.clone());
4022 let state = match task.state.try_lock() {
4023 Ok(state) => state,
4024 Err(_) => return crate::memory::MemoryEstimate::busy(),
4025 };
4026 if let Some(cache) = state.terminal_output_cache.as_ref() {
4027 terminal_output_caches = terminal_output_caches.saturating_add(1);
4028 bytes = bytes.saturating_add(terminal_output_cache_estimated_bytes(cache));
4029 }
4030 }
4031 let completion_count = match self.inner.completions.try_lock() {
4032 Ok(completions) => {
4033 for completion in completions.iter() {
4034 sessions.insert(completion.session_id.clone());
4035 bytes = bytes.saturating_add(completion_estimated_bytes(completion));
4036 }
4037 completions.len()
4038 }
4039 Err(_) => return crate::memory::MemoryEstimate::busy(),
4040 };
4041
4042 crate::memory::MemoryEstimate::estimated(bytes)
4043 .count("tasks", tasks.len())
4044 .count("sessions", sessions.len())
4045 .count("terminal_output_caches", terminal_output_caches)
4046 .count("completion_caches", completion_count)
4047 .count_u64("output_ring_bytes", 0)
4048 }
4049
4050 fn running_count(&self) -> usize {
4051 self.inner
4052 .tasks
4053 .lock()
4054 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())
4055 .unwrap_or(0)
4056 }
4057
4058 fn start_watchdog(&self) {
4059 if !self.inner.watchdog_started.swap(true, Ordering::SeqCst) {
4060 super::watchdog::start(self.clone());
4061 }
4062 }
4063
4064 fn running_metadata_is_stale(&self, metadata: &PersistedTask) -> bool {
4065 unix_millis().saturating_sub(metadata.started_at) > STALE_RUNNING_AFTER.as_millis() as u64
4066 }
4067
4068 #[cfg(test)]
4069 pub fn task_json_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
4070 self.task_for_session(task_id, session_id)
4071 .map(|task| task.paths.json.clone())
4072 }
4073
4074 #[cfg(test)]
4075 pub fn task_exit_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
4076 self.task_for_session(task_id, session_id)
4077 .map(|task| task.paths.exit.clone())
4078 }
4079}
4080
4081#[cfg(unix)]
4082fn should_capture_pipeline_status(
4083 spawn_plan: &SpawnPlan,
4084 has_pipeline: bool,
4085 shell: &Path,
4086) -> bool {
4087 if spawn_plan.is_native_launcher() {
4088 return false;
4091 }
4092 has_pipeline && super::process::pipeline_shell_kind(shell).is_some()
4093}
4094
4095fn canonical_artifact_root(paths: &TaskPaths) -> PathBuf {
4096 fs::canonicalize(&paths.io_dir).unwrap_or_else(|_| paths.io_dir.clone())
4097}
4098
4099fn append_pipeline_warning(
4104 cache: &mut TerminalOutputCache,
4105 metadata: &PersistedTask,
4106 paths: Option<&TaskPaths>,
4107) {
4108 if metadata.exit_code != Some(0) {
4109 return;
4110 }
4111 let Some(paths) = paths else {
4112 return;
4113 };
4114 if metadata.pipeline_segments.len() < 2 {
4115 return;
4116 }
4117 let Ok(mut status_file) = open_task_artifact(paths, TaskArtifact::PipelineStatus) else {
4118 return;
4119 };
4120 let Ok(status_bytes) = status_file.read_all() else {
4121 return;
4122 };
4123 let Some(statuses) = String::from_utf8_lossy(&status_bytes)
4124 .lines()
4125 .map(|line| line.trim().parse::<i32>().ok())
4126 .collect::<Option<Vec<_>>>()
4127 else {
4128 return;
4129 };
4130 if statuses.len() != metadata.pipeline_segments.len() {
4131 return;
4132 }
4133 let Some((failing_index, failing_code)) = statuses
4134 .iter()
4135 .enumerate()
4136 .take(statuses.len().saturating_sub(1))
4137 .find(|(_, code)| **code != 0)
4138 .map(|(index, code)| (index, *code))
4139 else {
4140 return;
4141 };
4142 let Some(final_segment) = metadata.pipeline_segments.last() else {
4143 return;
4144 };
4145 let failing_segment = &metadata.pipeline_segments[failing_index];
4146 let footer = format!(
4147 "note: `{}` (segment {} of {}) exited {}; the pipeline's exit code is `{}`'s.",
4148 failing_segment,
4149 failing_index + 1,
4150 metadata.pipeline_segments.len(),
4151 failing_code,
4152 final_segment,
4153 );
4154 if cache.output_preview.trim().is_empty() {
4155 cache.output_preview = footer;
4156 } else {
4157 cache.output_preview = format!("{}\n{}", cache.output_preview.trim_end(), footer,);
4158 }
4159}
4160
4161fn render_compressed_with_recovery(
4162 buffer: &BgBuffer,
4163 mut compressed: CompressionResult,
4164 input_truncated: bool,
4165 disk_truncation: DiskTruncation,
4166 artifact_access: ArtifactRecoveryAccess,
4167) -> TerminalOutputCache {
4168 let had_trailing_newline = compressed.text.ends_with('\n');
4176 let mut text = strip_plain_truncation_marker_lines(&compressed.text)
4177 .trim_end()
4178 .to_string();
4179 if had_trailing_newline && !text.is_empty() {
4180 text.push('\n');
4181 }
4182 compressed.text = text;
4183
4184 let output_path = buffer.output_path().map(|path| path.display().to_string());
4185 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
4186 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
4187 let mut recovery = RecoveryContext {
4188 dropped_by_class: compressed.dropped_by_class,
4189 had_inner_drop: compressed.had_inner_drop,
4190 offset_hint_eligible: compressed.offset_hint_eligible,
4191 offset_start_line: compressed.offset_start_line,
4192 byte_truncated: input_truncated,
4193 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
4194 output_path: output_path.clone(),
4195 stderr_path: stderr_path.clone(),
4196 include_stderr_path,
4197 artifact_access: artifact_access.clone(),
4198 };
4199
4200 let (output_preview, output_truncated) =
4201 render_body_with_recovery_marker(&compressed.text, &mut recovery);
4202 TerminalOutputCache {
4203 output_preview,
4204 output_truncated,
4205 kind: TerminalOutputKind::Compressed,
4206 output_path,
4207 stderr_path,
4208 artifact_access,
4209 recovery: Some(recovery),
4210 }
4211}
4212
4213fn render_body_with_recovery_marker(body: &str, recovery: &mut RecoveryContext) -> (String, bool) {
4214 render_body_with_recovery_marker_at_cap(
4215 body,
4216 recovery,
4217 FINAL_OUTPUT_CAP_BYTES,
4218 cap_final_output,
4219 cap_final_output_with_marker,
4220 )
4221}
4222
4223fn render_raw_body_with_recovery_marker(
4224 body: &str,
4225 recovery: &mut RecoveryContext,
4226) -> (String, bool) {
4227 render_body_with_recovery_marker_at_cap(
4228 body,
4229 recovery,
4230 RAW_PASSTHROUGH_CAP_BYTES,
4231 |input| {
4232 super::output::cap_head_tail(
4233 input,
4234 RAW_PASSTHROUGH_CAP_BYTES,
4235 RAW_PASSTHROUGH_HEAD_BYTES,
4236 RAW_PASSTHROUGH_TAIL_BYTES,
4237 )
4238 },
4239 |input, marker| {
4240 super::output::cap_head_tail_with_marker(
4241 input,
4242 RAW_PASSTHROUGH_CAP_BYTES,
4243 RAW_PASSTHROUGH_HEAD_BYTES,
4244 RAW_PASSTHROUGH_TAIL_BYTES,
4245 marker,
4246 )
4247 },
4248 )
4249}
4250
4251fn render_body_with_recovery_marker_at_cap<F, G>(
4252 body: &str,
4253 recovery: &mut RecoveryContext,
4254 cap_bytes: usize,
4255 cap_plain: F,
4256 cap_with_marker: G,
4257) -> (String, bool)
4258where
4259 F: Fn(&str) -> super::output::CappedText,
4260 G: Fn(&str, &str) -> super::output::CappedText,
4261{
4262 let needs_marker = recovery.has_visible_drop();
4263 if body.len() > cap_bytes {
4264 recovery.byte_truncated = true;
4265 if let Some(marker) = recovery_marker(recovery) {
4266 let capped = cap_with_marker(body, &marker);
4267 return (capped.text, true);
4268 }
4269 let capped = cap_plain(body);
4270 return (capped.text, capped.truncated || needs_marker);
4271 }
4272
4273 if !needs_marker {
4274 return (body.to_string(), false);
4275 }
4276
4277 let Some(marker) = recovery_marker(recovery) else {
4278 return (body.to_string(), true);
4279 };
4280 let with_marker = append_recovery_marker(body, &marker);
4281 if with_marker.len() <= cap_bytes {
4282 return (with_marker, true);
4283 }
4284
4285 recovery.byte_truncated = true;
4286 let marker = recovery_marker(recovery).unwrap_or(marker);
4287 let capped = cap_with_marker(body, &marker);
4288 (capped.text, true)
4289}
4290
4291fn append_recovery_marker(body: &str, marker: &str) -> String {
4292 if body.is_empty() {
4293 return marker.to_string();
4294 }
4295 let mut output = body.trim_end().to_string();
4296 output.push('\n');
4297 output.push_str(marker);
4298 output
4299}
4300
4301fn recovery_marker(recovery: &RecoveryContext) -> Option<String> {
4302 let mut parts = Vec::new();
4303 for (class, count) in &recovery.dropped_by_class {
4304 let label = if *count == 1 {
4305 class.singular()
4306 } else {
4307 class.plural()
4308 };
4309 parts.push(format!("+{count} more {label}"));
4310 }
4311 if recovery.byte_truncated {
4312 parts.push("truncated output".to_string());
4313 }
4314 let disk_truncated_prefix_bytes = recovery.disk_truncated_prefix_bytes;
4315 if disk_truncated_prefix_bytes > 0 {
4316 parts.push(format!(
4317 "truncated {disk_truncated_prefix_bytes} bytes from saved output prefix"
4318 ));
4319 } else if recovery.had_inner_drop && parts.is_empty() {
4320 parts.push("omitted output".to_string());
4321 }
4322
4323 if parts.is_empty() {
4324 return None;
4325 }
4326
4327 let hint = recovery_hint(recovery);
4328 Some(format!("[{}; {hint}]", parts.join(", ")))
4329}
4330
4331fn bash_status_recovery_hint(access: &ArtifactRecoveryAccess) -> String {
4332 let task_id = serde_json::to_string(&access.task_id)
4333 .unwrap_or_else(|_| format!("\"{}\"", access.task_id));
4334 format!("use bash_status({{taskId: {task_id}}})")
4335}
4336
4337fn recovery_hint(recovery: &RecoveryContext) -> String {
4338 if !recovery.artifact_access.readable {
4339 return bash_status_recovery_hint(&recovery.artifact_access);
4340 }
4341
4342 if recovery.offset_hint_eligible
4346 && !recovery.byte_truncated
4347 && recovery.dropped_by_class.is_empty()
4348 && !recovery.include_stderr_path
4349 {
4350 if let (Some(path), Some(line)) =
4351 (recovery.output_path.as_deref(), recovery.offset_start_line)
4352 {
4353 return format!("see remaining: tail -n +{line} {}", quote_path(path));
4354 }
4355 }
4356
4357 let mut paths = Vec::new();
4358 if let Some(path) = recovery.output_path.as_deref() {
4359 paths.push(path);
4360 }
4361 if recovery.include_stderr_path {
4362 if let Some(path) = recovery.stderr_path.as_deref() {
4363 if !paths.contains(&path) {
4364 paths.push(path);
4365 }
4366 }
4367 }
4368
4369 if paths.is_empty() {
4370 return "full output unavailable".to_string();
4371 }
4372
4373 let reads = paths
4374 .into_iter()
4375 .map(|path| format!("read {}", quote_path(path)))
4376 .collect::<Vec<_>>()
4377 .join(" and ");
4378 if recovery.disk_truncated_prefix_bytes > 0 {
4379 format!("retained output: {reads}")
4380 } else {
4381 format!("full output: {reads}")
4382 }
4383}
4384
4385fn strip_plain_truncation_marker_lines(input: &str) -> String {
4386 input
4387 .lines()
4388 .filter(|line| !is_plain_truncation_marker(line.trim()))
4389 .collect::<Vec<_>>()
4390 .join("\n")
4391}
4392
4393fn strip_recovery_marker_lines(input: &str) -> String {
4394 input
4395 .lines()
4396 .filter(|line| !is_recovery_marker(line.trim()))
4397 .collect::<Vec<_>>()
4398 .join("\n")
4399}
4400
4401fn is_plain_truncation_marker(line: &str) -> bool {
4402 let Some(rest) = line.strip_prefix("...<truncated ") else {
4403 return false;
4404 };
4405 let Some(bytes) = rest.strip_suffix(" bytes>...") else {
4406 return false;
4407 };
4408 !bytes.is_empty() && bytes.chars().all(|ch| ch.is_ascii_digit())
4409}
4410
4411fn is_recovery_marker(line: &str) -> bool {
4412 line.starts_with('[')
4413 && line.ends_with(']')
4414 && (line.contains("full output: read ")
4415 || line.contains("retained output: read ")
4416 || line.contains("see remaining: tail -n +")
4417 || line.contains("use bash_status({taskId:")
4418 || line.contains("full output unavailable"))
4419}
4420
4421fn structured_output_pointer(
4422 total_bytes: u64,
4423 output_path: &str,
4424 truncated_prefix_bytes: u64,
4425 artifact_access: &ArtifactRecoveryAccess,
4426) -> String {
4427 if artifact_access.readable {
4428 return if truncated_prefix_bytes > 0 {
4429 retained_json_output_pointer(total_bytes, output_path, truncated_prefix_bytes)
4430 } else {
4431 json_output_pointer(total_bytes, output_path)
4432 };
4433 }
4434
4435 let kb = total_bytes.div_ceil(1024);
4436 let hint = bash_status_recovery_hint(artifact_access);
4437 if truncated_prefix_bytes > 0 {
4438 format!(
4439 "[JSON output {kb} KB; truncated {truncated_prefix_bytes} bytes from saved output prefix; retained output: {hint}]"
4440 )
4441 } else {
4442 format!("[JSON output {kb} KB; full output: {hint}]")
4443 }
4444}
4445
4446fn render_structured_output(
4447 command: &str,
4448 buffer: &BgBuffer,
4449 disk_truncation: DiskTruncation,
4450 artifact_access: ArtifactRecoveryAccess,
4451) -> Option<TerminalOutputCache> {
4452 if !is_gh_structured_command(command) {
4453 return None;
4454 }
4455
4456 let output_path = buffer
4457 .output_path()
4458 .map(|path| path.display().to_string())?;
4459 let stdout_bytes = buffer.stream_len(StreamKind::Stdout);
4460 if stdout_bytes == 0 {
4461 return None;
4462 }
4463
4464 if stdout_bytes > STRUCTURED_OUTPUT_CAP_BYTES as u64 {
4465 if !stream_starts_like_json(buffer, StreamKind::Stdout) {
4466 return None;
4467 }
4468 let output_preview = structured_output_pointer(
4469 stdout_bytes,
4470 &output_path,
4471 disk_truncation.total_prefix_bytes(),
4472 &artifact_access,
4473 );
4474 return Some(TerminalOutputCache {
4475 output_preview,
4476 output_truncated: true,
4477 kind: TerminalOutputKind::Structured,
4478 output_path: Some(output_path),
4479 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
4480 artifact_access,
4481 recovery: None,
4482 });
4483 }
4484
4485 let stdout = buffer.read_stream_bounded(StreamKind::Stdout, STRUCTURED_OUTPUT_CAP_BYTES);
4486 if stdout.truncated || !is_structured_body(&stdout.text) {
4487 return None;
4488 }
4489
4490 Some(TerminalOutputCache {
4491 output_preview: stdout.text,
4492 output_truncated: false,
4493 kind: TerminalOutputKind::Structured,
4494 output_path: Some(output_path),
4495 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
4496 artifact_access,
4497 recovery: None,
4498 })
4499}
4500
4501fn render_raw_passthrough(
4502 buffer: &BgBuffer,
4503 disk_truncation: DiskTruncation,
4504 artifact_access: ArtifactRecoveryAccess,
4505) -> TerminalOutputCache {
4506 let raw = buffer.read_combined_head_tail(
4507 RAW_PASSTHROUGH_CAP_BYTES,
4508 RAW_PASSTHROUGH_HEAD_BYTES,
4509 RAW_PASSTHROUGH_TAIL_BYTES,
4510 );
4511 let output_path = buffer.output_path().map(|path| path.display().to_string());
4512 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
4513 if !raw.truncated && disk_truncation.total_prefix_bytes() == 0 {
4514 return TerminalOutputCache {
4515 output_preview: raw.text,
4516 output_truncated: false,
4517 kind: TerminalOutputKind::Raw,
4518 output_path,
4519 stderr_path,
4520 artifact_access,
4521 recovery: None,
4522 };
4523 }
4524
4525 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
4526 let mut recovery = RecoveryContext {
4527 dropped_by_class: BTreeMap::new(),
4528 had_inner_drop: false,
4529 offset_hint_eligible: false,
4530 offset_start_line: None,
4531 byte_truncated: raw.truncated,
4532 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
4533 output_path: output_path.clone(),
4534 stderr_path: stderr_path.clone(),
4535 include_stderr_path,
4536 artifact_access: artifact_access.clone(),
4537 };
4538 let (output_preview, output_truncated) =
4539 render_raw_body_with_recovery_marker(&raw.text, &mut recovery);
4540 TerminalOutputCache {
4541 output_preview,
4542 output_truncated,
4543 kind: TerminalOutputKind::Raw,
4544 output_path,
4545 stderr_path,
4546 artifact_access,
4547 recovery: Some(recovery),
4548 }
4549}
4550
4551fn completion_preview_for_cache(
4552 cache: &TerminalOutputCache,
4553 exit_code: Option<i32>,
4554) -> (String, bool) {
4555 let exit_ok = exit_code == Some(0);
4558 let threshold = completion_preview_threshold(exit_ok);
4559 if cache.kind == TerminalOutputKind::Structured && cache.output_preview.len() > threshold {
4560 if let Some(path) = cache.output_path.as_deref() {
4561 return (
4562 structured_output_pointer(
4563 cache.output_preview.len() as u64,
4564 path,
4565 0,
4566 &cache.artifact_access,
4567 ),
4568 true,
4569 );
4570 }
4571 return (cache.output_preview.clone(), cache.output_truncated);
4572 }
4573
4574 if let Some(recovery) = cache.recovery.as_ref() {
4575 if cache.output_preview.len() <= threshold {
4576 return (cache.output_preview.clone(), cache.output_truncated);
4577 }
4578 let body = strip_recovery_marker_lines(&cache.output_preview);
4579 let mut completion_recovery = recovery.clone();
4580 completion_recovery.byte_truncated = true;
4581 if let Some(marker) = recovery_marker(&completion_recovery) {
4582 let capped = cap_completion_output_with_marker(&body, &marker, exit_ok);
4583 return (capped.text, true);
4584 }
4585 }
4586
4587 let capped = cap_completion_output(&cache.output_preview, exit_ok);
4588 (capped.text, cache.output_truncated || capped.truncated)
4589}
4590
4591fn is_gh_structured_command(command: &str) -> bool {
4592 let Some(normalized) = crate::compress::plain_command_for_structured_output(command) else {
4593 return false;
4594 };
4595 let tokens = shell_words_for_flags(&normalized);
4596 let Some(head) = tokens.first() else {
4597 return false;
4598 };
4599 let head_name = Path::new(head)
4600 .file_name()
4601 .and_then(|name| name.to_str())
4602 .unwrap_or(head);
4603 if !(head_name == "gh" || head_name.eq_ignore_ascii_case("gh.exe")) {
4604 return false;
4605 }
4606 tokens.iter().any(|token| {
4607 matches!(token.as_str(), "--json" | "--jq" | "--template")
4608 || token.starts_with("--json=")
4609 || token.starts_with("--jq=")
4610 || token.starts_with("--template=")
4611 })
4612}
4613
4614fn shell_words_for_flags(command: &str) -> Vec<String> {
4615 let mut words = Vec::new();
4616 let mut current = String::new();
4617 let mut in_single = false;
4618 let mut in_double = false;
4619 let mut escaped = false;
4620
4621 for ch in command.chars() {
4622 if escaped {
4623 current.push(ch);
4624 escaped = false;
4625 continue;
4626 }
4627 if ch == '\\' && !in_single {
4628 escaped = true;
4629 continue;
4630 }
4631 if ch == '\'' && !in_double {
4632 in_single = !in_single;
4633 continue;
4634 }
4635 if ch == '"' && !in_single {
4636 in_double = !in_double;
4637 continue;
4638 }
4639 if ch.is_whitespace() && !in_single && !in_double {
4640 if !current.is_empty() {
4641 words.push(std::mem::take(&mut current));
4642 }
4643 continue;
4644 }
4645 if matches!(ch, ';' | '&' | '|') && !in_single && !in_double {
4646 if !current.is_empty() {
4647 words.push(std::mem::take(&mut current));
4648 }
4649 continue;
4650 }
4651 current.push(ch);
4652 }
4653 if !current.is_empty() {
4654 words.push(current);
4655 }
4656 words
4657}
4658
4659fn is_structured_body(body: &str) -> bool {
4660 let trimmed = body.trim();
4661 if trimmed.is_empty() {
4662 return false;
4663 }
4664 if serde_json::from_str::<serde_json::Value>(trimmed).is_ok() {
4665 return true;
4666 }
4667
4668 let mut saw_line = false;
4669 for line in trimmed
4670 .lines()
4671 .map(str::trim)
4672 .filter(|line| !line.is_empty())
4673 {
4674 saw_line = true;
4675 if serde_json::from_str::<serde_json::Value>(line).is_err() {
4676 return false;
4677 }
4678 }
4679 saw_line
4680}
4681
4682fn stream_starts_like_json(buffer: &BgBuffer, stream: StreamKind) -> bool {
4683 buffer
4684 .read_stream_bounded(stream, 512)
4685 .text
4686 .chars()
4687 .find(|ch| !ch.is_whitespace())
4688 .is_some_and(|ch| matches!(ch, '{' | '[' | '"' | '-' | '0'..='9' | 't' | 'f' | 'n'))
4689}
4690
4691struct CompletionTokenCounts {
4692 original_tokens: Option<u32>,
4693 compressed_tokens: Option<u32>,
4694 original_bytes: Option<i64>,
4695 compressed_bytes: Option<i64>,
4696 tokens_skipped: bool,
4697}
4698
4699impl CompletionTokenCounts {
4700 fn skipped() -> Self {
4701 Self {
4702 original_tokens: None,
4703 compressed_tokens: None,
4704 original_bytes: None,
4705 compressed_bytes: None,
4706 tokens_skipped: true,
4707 }
4708 }
4709}
4710
4711fn completion_status_text(status: &BgTaskStatus, exit_code: Option<i32>) -> String {
4712 match status {
4713 BgTaskStatus::TimedOut => "timed out".to_string(),
4714 BgTaskStatus::Killed => "killed".to_string(),
4715 _ => exit_code
4716 .map(|code| format!("exit {code}"))
4717 .unwrap_or_else(|| format!("{status:?}").to_lowercase()),
4718 }
4719}
4720
4721fn token_count_u32(text: &str) -> u32 {
4722 aft_tokenizer::count_tokens(text)
4723 .try_into()
4724 .unwrap_or(u32::MAX)
4725}
4726
4727impl Default for BgTaskRegistry {
4728 fn default() -> Self {
4729 Self::new(Arc::new(Mutex::new(None)))
4730 }
4731}
4732
4733fn modified_within(path: &Path, grace: Duration) -> bool {
4734 fs::metadata(path)
4735 .and_then(|metadata| metadata.modified())
4736 .ok()
4737 .and_then(|modified| SystemTime::now().duration_since(modified).ok())
4738 .map(|age| age < grace)
4739 .unwrap_or(false)
4740}
4741
4742fn canonicalized_path(path: &Path) -> PathBuf {
4743 fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf())
4744}
4745
4746fn started_instant_from_unix_millis(started_at: u64) -> Instant {
4747 let now_ms = SystemTime::now()
4748 .duration_since(UNIX_EPOCH)
4749 .ok()
4750 .map(|duration| duration.as_millis() as u64)
4751 .unwrap_or(started_at);
4752 let elapsed_ms = now_ms.saturating_sub(started_at);
4753 Instant::now()
4754 .checked_sub(Duration::from_millis(elapsed_ms))
4755 .unwrap_or_else(Instant::now)
4756}
4757
4758fn gc_quarantine(storage_dir: &Path) {
4759 let quarantine_root = storage_dir.join("bash-tasks-quarantine");
4760 let Ok(session_dirs) = fs::read_dir(&quarantine_root) else {
4761 return;
4762 };
4763 for session_entry in session_dirs.flatten() {
4764 let session_quarantine_dir = session_entry.path();
4765 if !session_quarantine_dir.is_dir() {
4766 continue;
4767 }
4768 let entries = match fs::read_dir(&session_quarantine_dir) {
4769 Ok(entries) => entries,
4770 Err(error) => {
4771 crate::slog_warn!(
4772 "failed to read background task quarantine dir {}: {error}",
4773 session_quarantine_dir.display()
4774 );
4775 continue;
4776 }
4777 };
4778 for entry in entries.flatten() {
4779 let path = entry.path();
4780 if modified_within(&path, QUARANTINE_GC_GRACE) {
4781 continue;
4782 }
4783 let result = if path.is_dir() {
4784 fs::remove_dir_all(&path)
4785 } else {
4786 fs::remove_file(&path)
4787 };
4788 match result {
4789 Ok(()) => log::debug!(
4790 "deleted old background task quarantine entry {}",
4791 path.display()
4792 ),
4793 Err(error) => crate::slog_warn!(
4794 "failed to delete old background task quarantine entry {}: {error}",
4795 path.display()
4796 ),
4797 }
4798 }
4799 let _ = fs::remove_dir(&session_quarantine_dir);
4800 }
4801 let _ = fs::remove_dir(&quarantine_root);
4802}
4803
4804fn read_for_token_count_from_disk(
4805 metadata: &PersistedTask,
4806 paths: &TaskPaths,
4807 max_bytes_per_stream: usize,
4808) -> TokenCountInput {
4809 if metadata.mode == BgMode::Pty {
4810 return TokenCountInput::Skipped;
4811 }
4812 let stdout = read_file_tail_capped(paths, TaskArtifact::Stdout, max_bytes_per_stream);
4819 let stderr = read_file_tail_capped(paths, TaskArtifact::Stderr, max_bytes_per_stream);
4820 match (stdout, stderr) {
4821 (Ok(stdout), Ok(stderr)) => TokenCountInput::Text(combine_streams(
4822 String::from_utf8_lossy(&stdout).as_ref(),
4823 String::from_utf8_lossy(&stderr).as_ref(),
4824 )),
4825 (Ok(stdout), Err(_)) => TokenCountInput::Text(combine_streams(
4826 String::from_utf8_lossy(&stdout).as_ref(),
4827 "",
4828 )),
4829 (Err(_), Ok(stderr)) => TokenCountInput::Text(combine_streams(
4830 "",
4831 String::from_utf8_lossy(&stderr).as_ref(),
4832 )),
4833 (Err(_), Err(_)) => TokenCountInput::Skipped,
4834 }
4835}
4836
4837fn read_file_tail_capped(
4838 paths: &TaskPaths,
4839 artifact: TaskArtifact,
4840 max_bytes: usize,
4841) -> std::io::Result<Vec<u8>> {
4842 let mut file = open_task_artifact(paths, artifact)?;
4843 file.tail(max_bytes).map(|(bytes, _)| bytes)
4844}
4845
4846impl BgTask {
4847 fn snapshot(&self, preview_bytes: usize) -> BgTaskSnapshot {
4848 let state = self
4849 .state
4850 .lock()
4851 .unwrap_or_else(|poison| poison.into_inner());
4852 self.snapshot_locked(&state, preview_bytes)
4853 }
4854
4855 fn snapshot_locked(&self, state: &BgTaskState, preview_bytes: usize) -> BgTaskSnapshot {
4856 let metadata = &state.metadata;
4857 let duration_ms = metadata.duration_ms.or_else(|| {
4858 metadata
4859 .status
4860 .is_terminal()
4861 .then(|| self.started.elapsed().as_millis() as u64)
4862 });
4863 let (output_preview, output_truncated) = if metadata.mode == BgMode::Pty {
4864 (String::new(), false)
4865 } else if metadata.status.is_terminal() {
4866 state
4867 .terminal_output_cache
4868 .as_ref()
4869 .map(|cache| (cache.output_preview.clone(), cache.output_truncated))
4870 .unwrap_or_else(|| (String::new(), false))
4871 } else if preview_bytes == 0 {
4872 (String::new(), false)
4873 } else {
4874 state.buffer.read_tail(preview_bytes)
4875 };
4876 BgTaskSnapshot {
4877 info: BgTaskInfo {
4878 task_id: self.task_id.clone(),
4879 status: metadata.status.clone(),
4880 command: metadata.command.clone(),
4881 mode: metadata.mode.clone(),
4882 started_at: metadata.started_at,
4883 duration_ms,
4884 status_reason: metadata.status_reason.clone(),
4885 },
4886 exit_code: metadata.exit_code,
4887 child_pid: metadata.child_pid,
4888 workdir: metadata.workdir.display().to_string(),
4889 output_preview,
4890 output_truncated,
4891 output_path: state
4892 .buffer
4893 .output_path()
4894 .map(|path| path.display().to_string()),
4895 stderr_path: state
4896 .buffer
4897 .stderr_path()
4898 .map(|path| path.display().to_string()),
4899 pty_rows: (metadata.mode == BgMode::Pty).then_some(metadata.pty_rows.unwrap_or(24)),
4900 pty_cols: (metadata.mode == BgMode::Pty).then_some(metadata.pty_cols.unwrap_or(80)),
4901 pty_screen: None,
4902 scanner_report: metadata.scanner_report.clone(),
4903 sandbox_native: metadata.sandbox_native,
4904 sandbox_unavailable: metadata.sandbox_native
4905 && open_task_artifact(&self.paths, TaskArtifact::SandboxUnavailable)
4906 .and_then(|mut file| file.read_all())
4907 .is_ok_and(|bytes| bytes == b"sandbox_unavailable"),
4908 }
4909 }
4910
4911 pub(crate) fn is_running(&self) -> bool {
4912 self.state
4913 .lock()
4914 .map(|state| {
4915 state.metadata.status == BgTaskStatus::Running
4916 || (state.metadata.mode == BgMode::Pty
4917 && state.metadata.status == BgTaskStatus::Killing)
4918 })
4919 .unwrap_or(false)
4920 }
4921
4922 fn is_terminal(&self) -> bool {
4923 self.state
4924 .lock()
4925 .map(|state| state.metadata.status.is_terminal())
4926 .unwrap_or(false)
4927 }
4928
4929 fn mark_terminal_now(&self) {
4930 if let Ok(mut terminal_at) = self.terminal_at.lock() {
4931 if terminal_at.is_none() {
4932 *terminal_at = Some(Instant::now());
4933 }
4934 }
4935 }
4936
4937 fn set_completion_delivered(
4938 &self,
4939 delivered: bool,
4940 registry: &BgTaskRegistry,
4941 ) -> Result<(), String> {
4942 let mut state = self
4943 .state
4944 .lock()
4945 .map_err(|_| "background task lock poisoned".to_string())?;
4946 let updated = registry
4947 .update_task_metadata(&self.paths, |metadata| {
4948 metadata.completion_delivered = delivered;
4949 })
4950 .map_err(|e| format!("failed to update completion delivery: {e}"))?;
4951 state.metadata = updated;
4952 Ok(())
4953 }
4954}
4955
4956#[cfg(unix)]
4977fn reap_piped_child(child_slot: &mut Option<Child>) {
4978 if let Some(mut child) = child_slot.take() {
4979 if matches!(child.try_wait(), Ok(None)) {
4980 let _ = child.wait();
4981 }
4982 }
4983}
4984
4985#[cfg(windows)]
4990fn reap_piped_child(child_slot: &mut Option<Child>) {
4991 *child_slot = None;
4992}
4993
4994fn terminal_metadata_from_marker(
4995 mut metadata: PersistedTask,
4996 marker: ExitMarker,
4997 reason: Option<String>,
4998) -> PersistedTask {
4999 match marker {
5000 ExitMarker::Code(code) => {
5001 let status = if code == 0 {
5002 BgTaskStatus::Completed
5003 } else {
5004 BgTaskStatus::Failed
5005 };
5006 metadata.mark_terminal(status, Some(code), reason);
5007 }
5008 ExitMarker::Killed => metadata.mark_terminal(
5009 BgTaskStatus::Killed,
5010 terminal_exit_code_for_status(&BgTaskStatus::Killed),
5011 reason,
5012 ),
5013 }
5014 metadata
5015}
5016
5017fn terminal_exit_code_for_status(status: &BgTaskStatus) -> Option<i32> {
5018 match status {
5019 BgTaskStatus::TimedOut => Some(124),
5020 BgTaskStatus::Killed => Some(137),
5021 _ => None,
5022 }
5023}
5024
5025fn attach_sandbox_metadata(metadata: &mut PersistedTask, spawn_plan: &SpawnPlan) {
5026 metadata.sandbox_native = spawn_plan.is_native_launcher();
5027 metadata.sandbox_temp_dir = spawn_plan.temp_dir().map(Path::to_path_buf);
5028}
5029
5030#[cfg(unix)]
5031pub(crate) fn resolve_posix_shell() -> PathBuf {
5032 static POSIX_SHELL: OnceLock<PathBuf> = OnceLock::new();
5033 POSIX_SHELL
5034 .get_or_init(|| {
5035 std::env::var_os("BASH")
5036 .filter(|value| !value.is_empty())
5037 .map(PathBuf::from)
5038 .filter(|path| path.exists())
5039 .or_else(|| which::which("bash").ok())
5040 .or_else(|| which::which("zsh").ok())
5041 .unwrap_or_else(|| PathBuf::from("/bin/sh"))
5042 })
5043 .clone()
5044}
5045
5046#[cfg(windows)]
5047fn detached_shell_command_for(
5048 shell: crate::windows_shell::WindowsShell,
5049 command: &str,
5050 exit_path: &Path,
5051 paths: &TaskPaths,
5052 creation_flags: u32,
5053) -> Result<Command, String> {
5054 use crate::windows_shell::WindowsShell;
5055 let wrapper_body = shell.wrapper_script_bytes(command, exit_path);
5068 let wrapper_ext = match shell {
5069 WindowsShell::Pwsh | WindowsShell::Powershell => "ps1",
5070 WindowsShell::Cmd => "bat",
5071 WindowsShell::Posix(_) => "sh",
5075 };
5076 let wrapper_path = paths.dir.join(format!(
5077 "{}.{}",
5078 paths
5079 .json
5080 .file_stem()
5081 .and_then(|s| s.to_str())
5082 .unwrap_or("wrapper"),
5083 wrapper_ext
5084 ));
5085 fs::write(&wrapper_path, wrapper_body)
5086 .map_err(|e| format!("failed to write background bash wrapper script: {e}"))?;
5087
5088 let mut cmd = Command::new(shell.binary().as_ref());
5089 match shell {
5090 WindowsShell::Pwsh | WindowsShell::Powershell => {
5091 cmd.args([
5094 "-NoLogo",
5095 "-NoProfile",
5096 "-NonInteractive",
5097 "-ExecutionPolicy",
5098 "Bypass",
5099 "-File",
5100 ]);
5101 cmd.arg(&wrapper_path);
5102 }
5103 WindowsShell::Cmd => {
5104 cmd.args(["/D", "/C"]);
5111 cmd.arg(&wrapper_path);
5112 }
5113 WindowsShell::Posix(_) => {
5114 cmd.arg(&wrapper_path);
5119 }
5120 }
5121
5122 cmd.creation_flags(creation_flags);
5126 Ok(cmd)
5127}
5128
5129fn spawn_detached_child(
5145 spawn_plan: &SpawnPlan,
5146 command: &str,
5147 paths: &TaskPaths,
5148 workdir: &Path,
5149 env: &HashMap<String, String>,
5150 io_handles: &mut TaskIoHandles,
5151 capture_pipeline_status: bool,
5152) -> Result<std::process::Child, String> {
5153 #[cfg(windows)]
5154 let _ = capture_pipeline_status;
5155 #[cfg(not(windows))]
5156 let _ = command;
5157 #[cfg(not(windows))]
5158 {
5159 use std::os::fd::AsRawFd;
5160
5161 let stdout = io_handles
5162 .clone_file(TaskArtifact::Stdout)
5163 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
5164 let stderr = io_handles
5165 .clone_file(TaskArtifact::Stderr)
5166 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
5167 let prepared = spawn_plan
5168 .prepared_task()
5169 .ok_or_else(|| "background task payload was not prepared".to_string())?;
5170 let payload = prepared.invocation()?;
5171 let exit = io_handles
5172 .inheritable_file(TaskArtifact::Exit)
5173 .map_err(|e| format!("failed to inherit exit marker handle: {e}"))?;
5174 let failure = io_handles
5175 .inheritable_file(TaskArtifact::SandboxUnavailable)
5176 .map_err(|e| format!("failed to inherit sandbox failure marker handle: {e}"))?;
5177 let pipeline_status = capture_pipeline_status
5178 .then(|| io_handles.inheritable_file(TaskArtifact::PipelineStatus))
5179 .transpose()
5180 .map_err(|e| format!("failed to inherit pipeline status handle: {e}"))?;
5181 let shell = spawn_plan
5182 .host_shell_path()
5183 .map(Path::to_path_buf)
5184 .unwrap_or_else(resolve_posix_shell);
5185 let pipeline_shell = super::process::pipeline_shell_kind(&shell).unwrap_or("");
5186 let pipeline_status_fd = if capture_pipeline_status {
5187 crate::sandbox_spawn::CHILD_PIPE_STATUS_FD.to_string()
5188 } else {
5189 String::new()
5190 };
5191 let args = vec![
5192 OsString::from("-c"),
5193 payload.wrapper_text.clone(),
5194 OsString::from("aft-payload-wrapper"),
5195 shell.as_os_str().to_os_string(),
5196 payload.command_text.clone(),
5197 OsString::from(crate::sandbox_spawn::CHILD_EXIT_FD.to_string()),
5198 OsString::from(pipeline_status_fd),
5199 OsString::from(pipeline_shell),
5200 ];
5201 let (mut child_command, profile_handle) = crate::sandbox_spawn::detached_command_for_plan(
5202 spawn_plan,
5203 std::ffi::OsStr::new("/bin/sh"),
5204 &args,
5205 &paths.json,
5206 crate::sandbox_spawn::CHILD_EXIT_FD,
5207 crate::sandbox_spawn::CHILD_FAILURE_FD,
5208 )?;
5209 crate::sandbox_spawn::apply_marker_fd_allowlist(
5210 &mut child_command,
5211 exit.as_raw_fd(),
5212 failure.as_raw_fd(),
5213 pipeline_status.as_ref().map(|file| file.as_raw_fd()),
5214 )?;
5215 child_command
5216 .current_dir(workdir)
5217 .envs(env)
5218 .stdin(Stdio::null())
5219 .stdout(Stdio::from(stdout))
5220 .stderr(Stdio::from(stderr));
5221 crate::sandbox_spawn::apply_sandbox_environment(spawn_plan, &mut child_command, env);
5222 let child = child_command
5223 .spawn()
5224 .map_err(|e| format!("failed to spawn background bash command: {e}"));
5225 drop((payload, exit, failure, pipeline_status, profile_handle));
5226 child
5227 }
5228 #[cfg(windows)]
5229 {
5230 use crate::windows_shell::shell_candidates;
5231 match spawn_plan {
5232 SpawnPlan::Unsandboxed | SpawnPlan::Host { .. } => {}
5233 SpawnPlan::Refused { code, .. } => return Err((*code).to_string()),
5234 SpawnPlan::Launcher { .. } => return Err("sandbox_unavailable".to_string()),
5235 }
5236 let candidates: Vec<crate::windows_shell::WindowsShell> = shell_candidates();
5247 const FLAG_CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200;
5260 const FLAG_CREATE_BREAKAWAY_FROM_JOB: u32 = 0x0100_0000;
5261 const FLAG_CREATE_NO_WINDOW: u32 = 0x0800_0000;
5262 let with_breakaway =
5263 FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP | FLAG_CREATE_BREAKAWAY_FROM_JOB;
5264 let without_breakaway = FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP;
5265 let mut last_error: Option<String> = None;
5266 for (idx, shell) in candidates.iter().enumerate() {
5267 for &flags in &[with_breakaway, without_breakaway] {
5271 let stdout = io_handles
5274 .clone_file(TaskArtifact::Stdout)
5275 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
5276 let stderr = io_handles
5277 .clone_file(TaskArtifact::Stderr)
5278 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
5279 let mut cmd =
5280 detached_shell_command_for(shell.clone(), command, &paths.exit, paths, flags)?;
5281 cmd.current_dir(workdir)
5282 .envs(env)
5283 .stdin(Stdio::null())
5284 .stdout(Stdio::from(stdout))
5285 .stderr(Stdio::from(stderr));
5286 match cmd.spawn() {
5287 Ok(child) => {
5288 if idx > 0 {
5289 crate::slog_warn!("background bash spawn fell back to {} after {} earlier candidate(s) failed; \
5290 the cached PATH probe disagreed with runtime spawn — likely PATH \
5291 inheritance, antivirus / AppLocker / Defender ASR, or sandbox policy.",
5292 shell.binary(),
5293 idx);
5294 }
5295 if flags == without_breakaway {
5296 crate::slog_warn!(
5297 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected \
5298 (likely a restrictive Job Object — CI sandbox or MDM policy). \
5299 Spawned without breakaway; the bg task will be torn down if the \
5300 AFT process group is killed."
5301 );
5302 }
5303 return Ok(child);
5304 }
5305 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
5306 crate::slog_warn!("background bash spawn: {} returned NotFound at runtime — trying next candidate",
5307 shell.binary());
5308 last_error = Some(format!("{}: {e}", shell.binary()));
5309 break;
5312 }
5313 Err(e) if flags == with_breakaway && e.raw_os_error() == Some(5) => {
5314 crate::slog_warn!(
5316 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected with \
5317 Access Denied — retrying {} without breakaway",
5318 shell.binary()
5319 );
5320 last_error = Some(format!("{}: {e}", shell.binary()));
5321 continue;
5322 }
5323 Err(e) => {
5324 return Err(format!(
5325 "failed to spawn background bash command via {}: {e}",
5326 shell.binary()
5327 ));
5328 }
5329 }
5330 }
5331 }
5332 Err(format!(
5333 "failed to spawn background bash command: no Windows shell could be spawned. \
5334 Last error: {}. PATH-probed candidates: {:?}",
5335 last_error.unwrap_or_else(|| "no candidates were attempted".to_string()),
5336 candidates.iter().map(|s| s.binary()).collect::<Vec<_>>()
5337 ))
5338 }
5339}
5340
5341#[cfg(test)]
5342fn random_slug() -> String {
5343 let mut bytes = [0u8; 8];
5351 getrandom::fill(&mut bytes).unwrap_or_else(|_| {
5353 let t = SystemTime::now()
5355 .duration_since(UNIX_EPOCH)
5356 .map(|d| d.as_nanos() as u64)
5357 .unwrap_or(0);
5358 let p = u64::from(std::process::id());
5359 bytes.copy_from_slice(&(t ^ p.rotate_left(32)).to_le_bytes());
5360 });
5361 let hex: String = bytes.iter().map(|b| format!("{b:02x}")).collect();
5363 format!("bash-{hex}")
5364}
5365
5366#[cfg(test)]
5367mod tests {
5368 use std::collections::HashMap;
5369 use std::fs;
5370 use std::io::Write;
5371 #[cfg(unix)]
5372 use std::os::unix::fs::PermissionsExt;
5373 use std::sync::atomic::{AtomicBool, AtomicUsize};
5374 use std::sync::{Arc, Mutex};
5375 use std::time::{Duration, Instant, SystemTime};
5376
5377 use super::*;
5378 use crate::bash_background::persistence::{read_task, task_paths, write_task};
5379
5380 #[cfg(unix)]
5381 const QUICK_SUCCESS_COMMAND: &str = "true";
5382 #[cfg(windows)]
5383 const QUICK_SUCCESS_COMMAND: &str = "cmd /c exit 0";
5384
5385 #[cfg(unix)]
5386 const LONG_RUNNING_COMMAND: &str = "sleep 5";
5387
5388 #[cfg(unix)]
5389 #[test]
5390 fn launcher_plans_disable_pipeline_status_capture() {
5391 let launcher = SpawnPlan::launcher_for_test(
5392 crate::sandbox_profile::SandboxProfile {
5393 v: crate::sandbox_profile::SANDBOX_PROFILE_VERSION,
5394 writable_roots: Vec::new(),
5395 write_deny: Vec::new(),
5396 write_deny_nested: Vec::new(),
5397 read_allow: Vec::new(),
5398 read_deny: Vec::new(),
5399 socket_deny: Vec::new(),
5400 cache_roots: Vec::new(),
5401 temp_dir: PathBuf::from("/tmp/aft-test-sandbox"),
5402 },
5403 PathBuf::from("/bin/true"),
5404 );
5405 assert!(!should_capture_pipeline_status(
5406 &launcher,
5407 true,
5408 Path::new("/bin/bash")
5409 ));
5410 assert!(should_capture_pipeline_status(
5411 &SpawnPlan::Unsandboxed,
5412 true,
5413 Path::new("/bin/bash")
5414 ));
5415 }
5416
5417 #[cfg(windows)]
5418 const LONG_RUNNING_COMMAND: &str = "cmd /c timeout /t 5 /nobreak > nul";
5419
5420 #[test]
5421 fn bash_memory_estimate_is_zero_when_empty_and_nonzero_for_completion_cache() {
5422 let registry = BgTaskRegistry::default();
5423 assert_eq!(registry.estimated_memory().estimated_bytes, Some(0));
5424 registry
5425 .inner
5426 .completions
5427 .lock()
5428 .unwrap()
5429 .push_back(BgCompletion {
5430 task_id: "bash-memory".to_string(),
5431 session_id: "session-memory".to_string(),
5432 status: BgTaskStatus::Completed,
5433 exit_code: Some(0),
5434 command: "printf memory".to_string(),
5435 output_preview: "resident completion output".to_string(),
5436 output_truncated: false,
5437 original_tokens: None,
5438 compressed_tokens: None,
5439 tokens_skipped: false,
5440 status_reason: None,
5441 });
5442 let estimate = registry.estimated_memory();
5443 assert!(estimate.estimated_bytes.unwrap() > 0);
5444 assert_eq!(estimate.counts["completion_caches"], 1);
5445 assert_eq!(estimate.counts["sessions"], 1);
5446 }
5447
5448 #[test]
5449 fn gh_structured_detection_rejects_piped_commands() {
5450 assert!(is_gh_structured_command(
5451 "gh issue list --json number,title"
5452 ));
5453 assert!(is_gh_structured_command(
5454 "cd repo && gh issue list --json number,title"
5455 ));
5456
5457 assert!(!is_gh_structured_command(
5458 "gh issue list --json number,title | jq '.[]'"
5459 ));
5460 assert!(!is_gh_structured_command(
5461 "gh issue list --json number,title |"
5462 ));
5463 }
5464
5465 fn insert_terminal_piped_task(
5466 registry: &BgTaskRegistry,
5467 dir: &tempfile::TempDir,
5468 command: &str,
5469 stdout: &str,
5470 stderr: &str,
5471 compressed: bool,
5472 ) -> (String, Arc<BgTask>) {
5473 let task_id = random_slug();
5474 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5475 fs::create_dir_all(&paths.dir).unwrap();
5476 fs::write(&paths.stdout, stdout).unwrap();
5477 fs::write(&paths.stderr, stderr).unwrap();
5478 let mut metadata = PersistedTask::starting(
5479 task_id.clone(),
5480 "session".to_string(),
5481 command.to_string(),
5482 dir.path().to_path_buf(),
5483 Some(dir.path().to_path_buf()),
5484 Some(30_000),
5485 true,
5486 compressed,
5487 );
5488 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
5489 write_task(&paths.json, &metadata).unwrap();
5490 registry
5491 .insert_rehydrated_task(metadata, paths, true)
5492 .expect("insert terminal task");
5493 let task = registry.task_for_session(&task_id, "session").unwrap();
5494 (task_id, task)
5495 }
5496
5497 #[test]
5498 fn bash_zero_preview_running_status_skips_output_read_while_explicit_preview_reads() {
5499 let registry = BgTaskRegistry::default();
5500 let dir = tempfile::tempdir().unwrap();
5501 let task_id = random_slug();
5502 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5503 fs::create_dir_all(&paths.dir).unwrap();
5504 fs::write(&paths.stdout, "live output\n").unwrap();
5505 fs::write(&paths.stderr, "").unwrap();
5506 let stdout_path = paths.stdout.clone();
5507 let mut metadata = PersistedTask::starting(
5508 task_id.clone(),
5509 "session".to_string(),
5510 "sleep 60".to_string(),
5511 dir.path().to_path_buf(),
5512 Some(dir.path().to_path_buf()),
5513 Some(30_000),
5514 true,
5515 false,
5516 );
5517 metadata.status = BgTaskStatus::Running;
5518 write_task(&paths.json, &metadata).unwrap();
5519 registry
5520 .insert_rehydrated_task(metadata, paths, false)
5521 .expect("insert running task");
5522
5523 crate::bash_background::buffer::reset_tail_read_count(&stdout_path);
5524 for _ in 0..5 {
5525 let snapshot = registry
5526 .status(&task_id, "session", Some(dir.path()), Some(dir.path()), 0)
5527 .expect("running snapshot");
5528 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
5529 assert!(snapshot.output_preview.is_empty());
5530 }
5531 assert_eq!(
5532 crate::bash_background::buffer::tail_read_count(&stdout_path),
5533 0
5534 );
5535
5536 let snapshot = registry
5537 .status(
5538 &task_id,
5539 "session",
5540 Some(dir.path()),
5541 Some(dir.path()),
5542 RUNNING_OUTPUT_PREVIEW_BYTES,
5543 )
5544 .expect("explicit running snapshot");
5545 assert_eq!(snapshot.output_preview, "live output\n");
5546 assert_eq!(
5547 crate::bash_background::buffer::tail_read_count(&stdout_path),
5548 1
5549 );
5550 }
5551
5552 #[test]
5553 fn artifact_read_capability_requires_exact_canonical_path_and_session() {
5554 let registry = BgTaskRegistry::default();
5555 let dir = tempfile::tempdir().unwrap();
5556 let (_task_id, task) = insert_terminal_piped_task(
5557 ®istry,
5558 &dir,
5559 "printf output",
5560 "stdout\n",
5561 "stderr\n",
5562 true,
5563 );
5564 fs::write(&task.paths.exit, "0\n").unwrap();
5565
5566 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stdout));
5567 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stderr));
5568 assert!(registry.is_session_owned_artifact_path("session", &task.paths.exit));
5569 assert!(!registry.is_session_owned_artifact_path("different-session", &task.paths.stdout));
5570 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.json));
5571
5572 let unregistered = task.paths.dir.join("unregistered-output");
5573 fs::write(&unregistered, "not a task artifact\n").unwrap();
5574 assert!(!registry.is_session_owned_artifact_path("session", &unregistered));
5575 }
5576
5577 #[cfg(unix)]
5578 #[test]
5579 fn artifact_directory_symlink_does_not_create_a_prefix_exception() {
5580 let registry = BgTaskRegistry::default();
5581 let dir = tempfile::tempdir().unwrap();
5582 let project = dir.path().join("project");
5583 fs::create_dir_all(&project).unwrap();
5584 let (_task_id, task) =
5585 insert_terminal_piped_task(®istry, &dir, "printf output", "stdout\n", "", true);
5586 let link = project.join("task-artifacts");
5587 std::os::unix::fs::symlink(&task.paths.dir, &link).unwrap();
5588 let unregistered = task.paths.dir.join("unregistered-output");
5589 fs::write(&unregistered, "not registered\n").unwrap();
5590
5591 assert!(!registry.is_session_owned_artifact_path("session", &link));
5592 assert!(
5593 !registry.is_session_owned_artifact_path("session", &link.join("unregistered-output"))
5594 );
5595 assert!(registry.is_session_owned_artifact_path(
5596 "session",
5597 &link.join(task.paths.stdout.file_name().unwrap())
5598 ));
5599
5600 let outside = dir.path().join("outside-secret");
5601 fs::write(&outside, "must stay private\n").unwrap();
5602 fs::remove_file(&task.paths.stdout).unwrap();
5603 std::os::unix::fs::symlink(&outside, &task.paths.stdout).unwrap();
5604 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.stdout));
5605 }
5606
5607 #[test]
5608 fn recovery_footer_uses_bash_status_when_artifact_is_not_registered() {
5609 let registry = BgTaskRegistry::default();
5610 let dir = tempfile::tempdir().unwrap();
5611 let task_id = "bash-1111111111111111";
5612 let paths = task_paths(dir.path(), "session", task_id).unwrap();
5613 fs::create_dir_all(&paths.dir).unwrap();
5614 fs::write(
5615 &paths.stdout,
5616 format!("{}tail\n", "output-line\n".repeat(2_000)),
5617 )
5618 .unwrap();
5619 fs::write(&paths.stderr, "").unwrap();
5620 let mut metadata = PersistedTask::starting(
5621 task_id.to_string(),
5622 "session".to_string(),
5623 "printf output".to_string(),
5624 dir.path().to_path_buf(),
5625 Some(dir.path().to_path_buf()),
5626 Some(30_000),
5627 true,
5628 true,
5629 );
5630 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
5631 write_task(&paths.json, &metadata).unwrap();
5632
5633 let cache = registry
5634 .render_terminal_output_from_paths(&metadata, &paths)
5635 .expect("terminal render");
5636
5637 assert!(cache
5638 .output_preview
5639 .contains("use bash_status({taskId: \"bash-1111111111111111\"})"));
5640 assert!(!cache.output_preview.contains("full output: read "));
5641 }
5642
5643 fn insert_terminal_pty_task(
5644 registry: &BgTaskRegistry,
5645 dir: &tempfile::TempDir,
5646 pty_output: &str,
5647 ) -> (String, Arc<BgTask>) {
5648 let task_id = random_slug();
5649 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5650 fs::create_dir_all(&paths.dir).unwrap();
5651 fs::write(&paths.pty, pty_output).unwrap();
5652 let mut metadata = PersistedTask::starting(
5653 task_id.clone(),
5654 "session".to_string(),
5655 "python".to_string(),
5656 dir.path().to_path_buf(),
5657 Some(dir.path().to_path_buf()),
5658 Some(30_000),
5659 true,
5660 true,
5661 );
5662 metadata.mode = BgMode::Pty;
5663 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
5664 write_task(&paths.json, &metadata).unwrap();
5665 registry
5666 .insert_rehydrated_task(metadata, paths, true)
5667 .expect("insert terminal pty task");
5668 let task = registry.task_for_session(&task_id, "session").unwrap();
5669 (task_id, task)
5670 }
5671
5672 #[cfg(unix)]
5673 fn wait_for_terminal_snapshot(
5674 registry: &BgTaskRegistry,
5675 task_id: &str,
5676 session_id: &str,
5677 project: &Path,
5678 storage: &Path,
5679 ) -> BgTaskSnapshot {
5680 let started = Instant::now();
5681 loop {
5682 let snapshot = registry
5683 .status(task_id, session_id, Some(project), Some(storage), 4096)
5684 .expect("spawned task should be visible to status");
5685 if snapshot.info.status.is_terminal() {
5686 return snapshot;
5687 }
5688 assert!(
5689 started.elapsed() < Duration::from_secs(10),
5690 "timed out waiting for task {task_id} to finish; last status={:?}",
5691 snapshot.info.status
5692 );
5693 std::thread::sleep(Duration::from_millis(50));
5694 }
5695 }
5696
5697 fn write_running_project_task(storage: &Path, project: &Path, session: &str, task_id: &str) {
5698 let paths = task_paths(storage, session, task_id).unwrap();
5699 let mut metadata = PersistedTask::starting(
5700 task_id.to_string(),
5701 session.to_string(),
5702 "sleep 60".to_string(),
5703 project.to_path_buf(),
5704 Some(project.to_path_buf()),
5705 Some(30_000),
5706 true,
5707 true,
5708 );
5709 metadata.status = BgTaskStatus::Running;
5710 write_task(&paths.json, &metadata).unwrap();
5711 fs::write(&paths.stdout, "still running\n").unwrap();
5712 fs::write(&paths.stderr, "").unwrap();
5713 }
5714
5715 #[test]
5716 fn status_replay_filters_same_session_by_project_root() {
5717 let project_a = tempfile::tempdir().unwrap();
5718 let project_b = tempfile::tempdir().unwrap();
5719 let storage = tempfile::tempdir().unwrap();
5720 let session = "shared-session";
5721 let task_id = "bash-2222222222222222";
5722 write_running_project_task(storage.path(), project_a.path(), session, task_id);
5723
5724 let actor_b = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5725 assert!(actor_b
5726 .status(
5727 task_id,
5728 session,
5729 Some(project_b.path()),
5730 Some(storage.path()),
5731 1024,
5732 )
5733 .is_none());
5734 assert!(actor_b.task_for_session(task_id, session).is_none());
5735
5736 let actor_a = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5737 let snapshot = actor_a
5738 .status(
5739 task_id,
5740 session,
5741 Some(project_a.path()),
5742 Some(storage.path()),
5743 1024,
5744 )
5745 .expect("owning project should replay its task");
5746 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
5747 }
5748
5749 #[cfg(unix)]
5750 #[test]
5751 fn multiline_pipeline_stdout_persists_all_lines_after_terminal_status() {
5752 let cases = [
5753 (
5754 "long-first",
5755 "sleep 0.5; printf 'one\\n' | cat\nprintf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
5756 vec!["one", "1", "three"],
5757 ),
5758 (
5759 "short-first",
5760 "printf 'one\\n' | cat\nsleep 0.2; printf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
5761 vec!["one", "1", "three"],
5762 ),
5763 (
5764 "failing-middle",
5765 "sleep 0.2; printf 'one\\n' | cat\nfalse; printf 'after-false\\n' | cat\nprintf 'three\\n' | cat",
5766 vec!["one", "after-false", "three"],
5767 ),
5768 ];
5769
5770 for (name, command, expected_lines) in cases {
5771 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5772 let dir = tempfile::tempdir().unwrap();
5773 let session_id = format!("session-{name}");
5774 let task_id = registry
5775 .spawn(
5776 SpawnPlan::Unsandboxed,
5777 command,
5778 session_id.clone(),
5779 dir.path().to_path_buf(),
5780 HashMap::new(),
5781 Some(Duration::from_secs(30)),
5782 dir.path().to_path_buf(),
5783 10,
5784 true,
5785 true,
5786 Some(dir.path().to_path_buf()),
5787 )
5788 .unwrap();
5789
5790 let snapshot = wait_for_terminal_snapshot(
5791 ®istry,
5792 &task_id,
5793 &session_id,
5794 dir.path(),
5795 dir.path(),
5796 );
5797 assert_eq!(
5798 snapshot.info.status,
5799 BgTaskStatus::Completed,
5800 "{name}: task should complete; snapshot={snapshot:?}"
5801 );
5802 assert_eq!(
5803 snapshot.exit_code,
5804 Some(0),
5805 "{name}: script should use the final command's exit code"
5806 );
5807
5808 let stdout = String::from_utf8(
5809 registry
5810 .read_artifact(&task_id, &session_id, TaskArtifact::Stdout)
5811 .expect("read validated stdout artifact"),
5812 )
5813 .expect("stdout is UTF-8");
5814 let lines: Vec<&str> = stdout.lines().collect();
5815 assert_eq!(
5816 lines, expected_lines,
5817 "{name}: raw stdout artifact must include every newline-separated command's output"
5818 );
5819 }
5820 }
5821
5822 #[test]
5823 fn recognizes_all_recovery_marker_forms() {
5824 assert!(is_recovery_marker(
5825 "[truncated output; full output: read \"/tmp/out\"]"
5826 ));
5827 assert!(is_recovery_marker(
5828 "[omitted output; see remaining: tail -n +42 \"/tmp/out\"]"
5829 ));
5830 assert!(is_recovery_marker(
5831 "[truncated output; full output unavailable]"
5832 ));
5833 assert!(is_recovery_marker(
5834 r#"[truncated 123 bytes from saved output prefix; retained output: read "/tmp/out"]"#
5835 ));
5836 }
5837
5838 #[test]
5839 fn recovery_marker_reports_disk_prefix_truncation_as_retained_output() {
5840 let recovery = RecoveryContext {
5841 dropped_by_class: BTreeMap::new(),
5842 had_inner_drop: false,
5843 offset_hint_eligible: false,
5844 offset_start_line: None,
5845 byte_truncated: false,
5846 disk_truncated_prefix_bytes: 4096,
5847 output_path: Some("/tmp/stdout".to_string()),
5848 stderr_path: None,
5849 include_stderr_path: false,
5850 artifact_access: ArtifactRecoveryAccess {
5851 task_id: "bash-test".to_string(),
5852 readable: true,
5853 },
5854 };
5855
5856 let marker = recovery_marker(&recovery).expect("disk truncation must emit marker");
5857
5858 assert!(marker.contains("truncated 4096 bytes from saved output prefix"));
5859 assert!(marker.contains(r#"retained output: read "/tmp/stdout""#));
5860 assert!(!marker.contains("full output: read"));
5861 }
5862
5863 #[test]
5864 fn killed_exit_marker_sets_nonzero_sentinel_exit_code() {
5865 let metadata = PersistedTask::starting(
5866 "task".to_string(),
5867 "session".to_string(),
5868 "cargo test".to_string(),
5869 PathBuf::from("/tmp"),
5870 None,
5871 None,
5872 true,
5873 true,
5874 );
5875
5876 let terminal = terminal_metadata_from_marker(metadata, ExitMarker::Killed, None);
5877
5878 assert_eq!(terminal.status, BgTaskStatus::Killed);
5879 assert_eq!(terminal.exit_code, Some(137));
5880 }
5881
5882 #[test]
5883 fn terminal_status_polls_use_cached_render_once_and_off_lock() {
5884 let registry = BgTaskRegistry::default();
5885 let dir = tempfile::tempdir().unwrap();
5886 let (_task_id, task) = insert_terminal_piped_task(
5887 ®istry,
5888 &dir,
5889 "custom-tool --verbose",
5890 &"stdout line\n".repeat(200_000),
5891 "",
5892 true,
5893 );
5894 let calls = Arc::new(AtomicUsize::new(0));
5895 let saw_unlocked_state = Arc::new(AtomicBool::new(false));
5896 let task_holder = Arc::new(Mutex::new(Some(Arc::clone(&task))));
5897 let calls_for_closure = Arc::clone(&calls);
5898 let unlocked_for_closure = Arc::clone(&saw_unlocked_state);
5899 let task_for_closure = Arc::clone(&task_holder);
5900 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
5901 calls_for_closure.fetch_add(1, Ordering::SeqCst);
5902 if let Some(task) = task_for_closure.lock().unwrap().as_ref() {
5903 if task.state.try_lock().is_ok() {
5904 unlocked_for_closure.store(true, Ordering::SeqCst);
5905 }
5906 }
5907 CompressionResult::new(format!("compressed {} bytes", output.len()))
5908 });
5909
5910 let first = registry
5911 .status(
5912 &task.task_id,
5913 "session",
5914 None,
5915 Some(dir.path()),
5916 RUNNING_OUTPUT_PREVIEW_BYTES,
5917 )
5918 .unwrap();
5919 let second = registry
5920 .status(
5921 &task.task_id,
5922 "session",
5923 None,
5924 Some(dir.path()),
5925 RUNNING_OUTPUT_PREVIEW_BYTES,
5926 )
5927 .unwrap();
5928 let listed = registry.list(RUNNING_OUTPUT_PREVIEW_BYTES);
5929
5930 assert_eq!(
5931 calls.load(Ordering::SeqCst),
5932 1,
5933 "terminal render must be cached"
5934 );
5935 assert!(
5936 saw_unlocked_state.load(Ordering::SeqCst),
5937 "compressor must run after releasing the task state lock"
5938 );
5939 assert!(first.output_preview.starts_with("compressed "));
5940 assert_eq!(second.output_preview, first.output_preview);
5941 assert_eq!(listed[0].output_preview, first.output_preview);
5942 }
5943
5944 #[test]
5945 fn completion_preview_success_keeps_tail_only() {
5946 let registry = BgTaskRegistry::default();
5951 let dir = tempfile::tempdir().unwrap();
5952 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
5953 let (_task_id, task) =
5954 insert_terminal_piped_task(®istry, &dir, "cat big.log", &output, "", false);
5955
5956 registry.post_terminal_transition(&task, true).unwrap();
5957 let completions = registry.drain_completions_for_session(Some("session"));
5958 assert_eq!(completions.len(), 1);
5959 let preview = &completions[0].output_preview;
5960 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
5961 assert!(!preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
5962 assert!(completions[0].output_truncated);
5963 }
5964
5965 #[test]
5966 fn completion_preview_failure_keeps_head_and_tail() {
5967 let registry = BgTaskRegistry::default();
5970 let dir = tempfile::tempdir().unwrap();
5971 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
5972 let task_id = random_slug();
5973 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5974 fs::create_dir_all(&paths.dir).unwrap();
5975 fs::write(&paths.stdout, &output).unwrap();
5976 fs::write(&paths.stderr, "").unwrap();
5977 let mut metadata = PersistedTask::starting(
5978 task_id.clone(),
5979 "session".to_string(),
5980 "cat big.log".to_string(),
5981 dir.path().to_path_buf(),
5982 Some(dir.path().to_path_buf()),
5983 Some(30_000),
5984 true,
5985 false,
5986 );
5987 metadata.mark_terminal(BgTaskStatus::Failed, Some(1), None);
5988 write_task(&paths.json, &metadata).unwrap();
5989 registry
5990 .insert_rehydrated_task(metadata, paths, true)
5991 .expect("insert terminal task");
5992 let task = registry.task_for_session(&task_id, "session").unwrap();
5993
5994 registry.post_terminal_transition(&task, true).unwrap();
5995 let completions = registry.drain_completions_for_session(Some("session"));
5996 assert_eq!(completions.len(), 1);
5997 let preview = &completions[0].output_preview;
5998 assert!(preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
5999 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
6000 }
6001
6002 #[test]
6003 fn has_completions_for_session_matches_pending_delivery() {
6004 let registry = BgTaskRegistry::default();
6005 assert!(!registry.has_completions_for_session(Some("session")));
6006 assert!(!registry.has_completions_for_session(None));
6007
6008 let dir = tempfile::tempdir().unwrap();
6009 let (_task_id, task) =
6010 insert_terminal_piped_task(®istry, &dir, QUICK_SUCCESS_COMMAND, "done\n", "", false);
6011 registry.post_terminal_transition(&task, true).unwrap();
6012
6013 assert!(registry.has_completions_for_session(Some("session")));
6014 assert!(registry.has_completions_for_session(None));
6015 assert!(!registry.has_completions_for_session(Some("other-session")));
6016
6017 let completions = registry.drain_completions_for_session(Some("session"));
6018 assert_eq!(completions.len(), 1);
6019 assert_eq!(completions[0].task_id, task.task_id);
6020 }
6021
6022 #[test]
6023 fn structured_gh_json_survives_intact_and_ignores_stderr() {
6024 let registry = BgTaskRegistry::default();
6025 let dir = tempfile::tempdir().unwrap();
6026 let calls = Arc::new(AtomicUsize::new(0));
6027 let calls_for_closure = Arc::clone(&calls);
6028 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
6029 calls_for_closure.fetch_add(1, Ordering::SeqCst);
6030 CompressionResult::new(output)
6031 });
6032 let (task_id, _task) = insert_terminal_piped_task(
6033 ®istry,
6034 &dir,
6035 "gh pr view 123 --json body",
6036 "{\"body\":\"hello\"}",
6037 "warning: stderr must not join json",
6038 true,
6039 );
6040
6041 let snapshot = registry
6042 .status(
6043 &task_id,
6044 "session",
6045 None,
6046 Some(dir.path()),
6047 RUNNING_OUTPUT_PREVIEW_BYTES,
6048 )
6049 .unwrap();
6050
6051 assert_eq!(snapshot.output_preview, "{\"body\":\"hello\"}");
6052 assert!(!snapshot.output_preview.contains("warning"));
6053 assert!(!snapshot.output_truncated);
6054 assert_eq!(
6055 calls.load(Ordering::SeqCst),
6056 0,
6057 "structured JSON bypasses compression"
6058 );
6059 }
6060
6061 #[test]
6062 fn registry_emits_single_recovery_marker_for_class_drops() {
6063 let registry = BgTaskRegistry::default();
6064 let dir = tempfile::tempdir().unwrap();
6065 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
6066 let mut dropped = BTreeMap::new();
6067 dropped.insert(DropClass::Error, 18);
6068 dropped.insert(DropClass::Warning, 6);
6069 CompressionResult::with_class_drops("kept diagnostic", dropped)
6070 });
6071 let (task_id, task) =
6072 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
6073
6074 let snapshot = registry
6075 .status(
6076 &task_id,
6077 "session",
6078 None,
6079 Some(dir.path()),
6080 RUNNING_OUTPUT_PREVIEW_BYTES,
6081 )
6082 .unwrap();
6083
6084 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
6085 assert!(snapshot.output_preview.contains("+18 more errors"));
6086 assert!(snapshot.output_preview.contains("+6 more warnings"));
6087 assert!(snapshot
6088 .output_preview
6089 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
6090 assert!(!snapshot.output_preview.contains("tail -n +"));
6091 assert!(snapshot.output_truncated);
6092 }
6093
6094 #[test]
6095 fn registry_marker_reports_semantic_and_byte_drops_once() {
6096 let registry = BgTaskRegistry::default();
6097 let dir = tempfile::tempdir().unwrap();
6098 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
6099 let mut dropped = BTreeMap::new();
6100 dropped.insert(DropClass::Error, 1);
6101 CompressionResult::with_class_drops(
6102 format!("HEAD-SIGNAL\n{}TAIL-SIGNAL", "middle\n".repeat(8_000)),
6103 dropped,
6104 )
6105 });
6106 let (task_id, _task) =
6107 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
6108
6109 let snapshot = registry
6110 .status(
6111 &task_id,
6112 "session",
6113 None,
6114 Some(dir.path()),
6115 RUNNING_OUTPUT_PREVIEW_BYTES,
6116 )
6117 .unwrap();
6118
6119 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
6120 assert!(snapshot.output_preview.contains("+1 more error"));
6121 assert!(snapshot.output_preview.contains("truncated output"));
6122 assert!(snapshot.output_preview.contains("HEAD-SIGNAL"));
6123 assert!(snapshot.output_preview.contains("TAIL-SIGNAL"));
6124 assert!(!snapshot.output_preview.contains("...<truncated"));
6125 assert!(snapshot.output_truncated);
6126 }
6127
6128 #[test]
6129 fn cargo_stderr_class_drops_name_both_capture_paths() {
6130 let registry = BgTaskRegistry::default();
6131 let dir = tempfile::tempdir().unwrap();
6132 let filter_registry = crate::compress::toml_filter::FilterRegistry::default();
6133 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
6134 crate::compress::compress_with_registry_exit_code(
6135 command,
6136 &output,
6137 exit_code,
6138 &filter_registry,
6139 )
6140 });
6141 let stderr = (0..22)
6142 .map(|index| {
6143 format!(
6144 "error: cargo failure {index}\n --> src/lib.rs:{}:1\n |\n{} | boom\n",
6145 index + 1,
6146 index + 1
6147 )
6148 })
6149 .collect::<Vec<_>>()
6150 .join("\n");
6151 let (task_id, task) = insert_terminal_piped_task(
6152 ®istry,
6153 &dir,
6154 "cargo check",
6155 "Finished dev [unoptimized] target(s) in 0.01s\n",
6156 &stderr,
6157 true,
6158 );
6159
6160 let snapshot = registry
6161 .status(
6162 &task_id,
6163 "session",
6164 None,
6165 Some(dir.path()),
6166 RUNNING_OUTPUT_PREVIEW_BYTES,
6167 )
6168 .unwrap();
6169
6170 assert!(snapshot.output_preview.contains("+2 more errors"));
6171 assert!(snapshot
6172 .output_preview
6173 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
6174 assert!(snapshot
6175 .output_preview
6176 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
6177 assert!(!snapshot.output_preview.contains("tail -n +"));
6178 }
6179
6180 #[test]
6181 fn over_ceiling_structured_json_uses_pointer_not_partial_json() {
6182 let registry = BgTaskRegistry::default();
6183 let dir = tempfile::tempdir().unwrap();
6184 let body = format!("{{\"body\":\"{}\"}}", "x".repeat(60 * 1024));
6185 let (task_id, task) = insert_terminal_piped_task(
6186 ®istry,
6187 &dir,
6188 "cd /repo && gh pr view 123 --json body",
6189 &body,
6190 "",
6191 true,
6192 );
6193
6194 let snapshot = registry
6195 .status(
6196 &task_id,
6197 "session",
6198 None,
6199 Some(dir.path()),
6200 RUNNING_OUTPUT_PREVIEW_BYTES,
6201 )
6202 .unwrap();
6203
6204 assert!(snapshot.output_preview.starts_with("[JSON output "));
6205 assert!(snapshot
6206 .output_preview
6207 .contains(&task.paths.stdout.display().to_string()));
6208 assert!(!snapshot.output_preview.contains(&"x".repeat(1024)));
6209 assert!(snapshot.output_truncated);
6210 }
6211
6212 #[test]
6213 fn toml_strip_tail_cap_uses_full_output_hint_not_offset_hint() {
6214 let registry = BgTaskRegistry::default();
6215 let dir = tempfile::tempdir().unwrap();
6216 let filter_registry = crate::compress::toml_filter::build_registry(
6217 crate::compress::builtin_filters::ALL,
6218 None,
6219 None,
6220 );
6221 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
6222 crate::compress::compress_with_registry_exit_code(
6223 command,
6224 &output,
6225 exit_code,
6226 &filter_registry,
6227 )
6228 });
6229 let stdout = format!(
6230 "make[1]: Entering directory `/tmp`\n{}",
6231 (0..100)
6232 .map(|index| format!("compile line {index}"))
6233 .collect::<Vec<_>>()
6234 .join("\n")
6235 );
6236 let (task_id, task) =
6237 insert_terminal_piped_task(®istry, &dir, "make all", &stdout, "", true);
6238
6239 let snapshot = registry
6240 .status(
6241 &task_id,
6242 "session",
6243 None,
6244 Some(dir.path()),
6245 RUNNING_OUTPUT_PREVIEW_BYTES,
6246 )
6247 .unwrap();
6248
6249 assert!(snapshot.output_preview.contains("compile line 99"));
6250 assert!(snapshot.output_preview.contains(&format!(
6251 "full output: read \"{}\"",
6252 task.paths.stdout.display()
6253 )));
6254 assert!(!snapshot
6255 .output_preview
6256 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
6257 assert!(!snapshot.output_preview.contains("tail -n +"));
6258 }
6259
6260 #[test]
6261 fn compressed_false_raw_passthrough_uses_wider_head_tail_cap() {
6262 let registry = BgTaskRegistry::default();
6263 let dir = tempfile::tempdir().unwrap();
6264 let output = format!("RAW-HEAD\n{}RAW-TAIL\n", "raw-middle\n".repeat(8_000));
6265 let (task_id, task) =
6266 insert_terminal_piped_task(®istry, &dir, "cat raw.log", &output, "RAW-ERR\n", false);
6267
6268 let snapshot = registry
6269 .status(
6270 &task_id,
6271 "session",
6272 None,
6273 Some(dir.path()),
6274 RUNNING_OUTPUT_PREVIEW_BYTES,
6275 )
6276 .unwrap();
6277
6278 assert!(snapshot.output_preview.contains("RAW-HEAD"));
6279 assert!(snapshot.output_preview.contains("RAW-TAIL"));
6280 assert!(snapshot.output_preview.contains("truncated output"));
6281 assert!(snapshot
6282 .output_preview
6283 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
6284 assert!(snapshot
6285 .output_preview
6286 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
6287 assert!(!snapshot.output_preview.contains("tail -n +"));
6288 assert!(snapshot.output_preview.len() > 16 * 1024);
6289 assert!(snapshot.output_truncated);
6290 }
6291
6292 #[test]
6293 fn pty_terminal_snapshot_bypasses_line_compression() {
6294 let registry = BgTaskRegistry::default();
6295 let dir = tempfile::tempdir().unwrap();
6296 let calls = Arc::new(AtomicUsize::new(0));
6297 let calls_for_closure = Arc::clone(&calls);
6298 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
6299 calls_for_closure.fetch_add(1, Ordering::SeqCst);
6300 CompressionResult::new(output)
6301 });
6302 let (task_id, _task) = insert_terminal_pty_task(®istry, &dir, "raw\u{1b}[31m pty bytes");
6303
6304 let snapshot = registry
6305 .status(
6306 &task_id,
6307 "session",
6308 None,
6309 Some(dir.path()),
6310 RUNNING_OUTPUT_PREVIEW_BYTES,
6311 )
6312 .unwrap();
6313
6314 assert_eq!(snapshot.info.mode, BgMode::Pty);
6315 assert_eq!(snapshot.output_preview, "");
6316 assert_eq!(calls.load(Ordering::SeqCst), 0);
6317 }
6318
6319 #[test]
6320 fn pty_dimensions_are_persisted_and_returned_in_snapshot() {
6321 let registry = BgTaskRegistry::default();
6322 let dir = tempfile::tempdir().unwrap();
6323 let task_id = registry
6324 .spawn_pty(
6325 SpawnPlan::Unsandboxed,
6326 QUICK_SUCCESS_COMMAND,
6327 "session".to_string(),
6328 dir.path().to_path_buf(),
6329 HashMap::new(),
6330 Some(Duration::from_secs(30)),
6331 dir.path().to_path_buf(),
6332 10,
6333 true,
6334 false,
6335 Some(dir.path().to_path_buf()),
6336 50,
6337 120,
6338 )
6339 .unwrap();
6340
6341 let resolved =
6342 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
6343 let metadata = read_task_at(&resolved).unwrap();
6344 assert_eq!(
6345 metadata.schema_version,
6346 crate::bash_background::persistence::SCHEMA_VERSION
6347 );
6348 assert_eq!(metadata.mode, BgMode::Pty);
6349 assert_eq!(metadata.pty_rows, Some(50));
6350 assert_eq!(metadata.pty_cols, Some(120));
6351
6352 let snapshot = registry
6353 .status(&task_id, "session", None, Some(dir.path()), 1024)
6354 .unwrap();
6355 assert_eq!(snapshot.pty_rows, Some(50));
6356 assert_eq!(snapshot.pty_cols, Some(120));
6357 }
6358
6359 fn spawn_dead_child() -> std::process::Child {
6364 #[cfg(unix)]
6365 let mut cmd = std::process::Command::new("true");
6366 #[cfg(windows)]
6367 let mut cmd = {
6368 let mut c = std::process::Command::new("cmd");
6369 c.args(["/c", "exit", "0"]);
6370 c
6371 };
6372 cmd.stdin(std::process::Stdio::null());
6373 cmd.stdout(std::process::Stdio::null());
6374 cmd.stderr(std::process::Stdio::null());
6375 let mut child = cmd.spawn().expect("spawn replacement child for reap test");
6376 let started = Instant::now();
6385 loop {
6386 match child.try_wait() {
6387 Ok(Some(_)) => break,
6388 Ok(None) => {
6389 if started.elapsed() > Duration::from_secs(5) {
6390 panic!("dead-child stand-in did not exit within 5s");
6391 }
6392 std::thread::sleep(Duration::from_millis(10));
6393 }
6394 Err(error) => panic!("dead-child try_wait failed: {error}"),
6395 }
6396 }
6397 child
6398 }
6399
6400 #[test]
6401 fn ack_marks_delivered_even_when_completion_was_already_consumed_locally() {
6402 let registry = BgTaskRegistry::default();
6403 let dir = tempfile::tempdir().unwrap();
6404 let task_id = registry
6405 .spawn(
6406 SpawnPlan::Unsandboxed,
6407 LONG_RUNNING_COMMAND,
6408 "session".to_string(),
6409 dir.path().to_path_buf(),
6410 HashMap::new(),
6411 Some(Duration::from_secs(30)),
6412 dir.path().to_path_buf(),
6413 10,
6414 true,
6415 false,
6416 Some(dir.path().to_path_buf()),
6417 )
6418 .unwrap();
6419 registry
6420 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6421 .unwrap();
6422 assert_eq!(
6423 registry
6424 .drain_completions_for_session(Some("session"))
6425 .len(),
6426 1
6427 );
6428
6429 registry.inner.completions.lock().unwrap().clear();
6432
6433 assert_eq!(
6434 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
6435 vec![task_id.clone()]
6436 );
6437 assert!(registry
6438 .drain_completions_for_session(Some("session"))
6439 .is_empty());
6440
6441 let resolved =
6442 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
6443 let metadata = read_task_at(&resolved).unwrap();
6444 assert!(metadata.completion_delivered);
6445
6446 let replayed = BgTaskRegistry::default();
6447 replayed
6448 .replay_session_inner(dir.path(), "session", None)
6449 .unwrap();
6450 assert!(replayed
6451 .drain_completions_for_session(Some("session"))
6452 .is_empty());
6453 }
6454
6455 #[test]
6456 fn reclaimed_root_kills_running_task_and_persists_reason() {
6457 let registry = BgTaskRegistry::default();
6458 let root = tempfile::tempdir().unwrap();
6459 let storage = tempfile::tempdir().unwrap();
6460 let task_id = registry
6461 .spawn(
6462 SpawnPlan::Unsandboxed,
6463 LONG_RUNNING_COMMAND,
6464 "session".to_string(),
6465 root.path().to_path_buf(),
6466 HashMap::new(),
6467 Some(Duration::from_secs(30)),
6468 storage.path().to_path_buf(),
6469 10,
6470 true,
6471 false,
6472 Some(root.path().to_path_buf()),
6473 )
6474 .unwrap();
6475 let pid = registry
6476 .status(
6477 &task_id,
6478 "session",
6479 Some(root.path()),
6480 Some(storage.path()),
6481 0,
6482 )
6483 .unwrap()
6484 .child_pid
6485 .unwrap();
6486 assert!(is_process_alive(pid));
6487
6488 assert_eq!(registry.kill_running_tasks_for_root(root.path()), 1);
6489 let deadline = Instant::now() + Duration::from_secs(5);
6490 while is_process_alive(pid) {
6491 assert!(
6492 Instant::now() < deadline,
6493 "reclaimed task process survived kill"
6494 );
6495 std::thread::sleep(Duration::from_millis(20));
6496 }
6497
6498 let snapshot = registry
6499 .status(
6500 &task_id,
6501 "session",
6502 Some(root.path()),
6503 Some(storage.path()),
6504 0,
6505 )
6506 .unwrap();
6507 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
6508 assert_eq!(
6509 snapshot.info.status_reason.as_deref(),
6510 Some(ROOT_RECLAIMED_REASON)
6511 );
6512 let persisted = read_task(
6513 ®istry
6514 .task_json_path(&task_id, "session")
6515 .expect("reclaimed task metadata path"),
6516 )
6517 .expect("persisted reclaimed task");
6518 assert_eq!(
6519 persisted.status_reason.as_deref(),
6520 Some(ROOT_RECLAIMED_REASON)
6521 );
6522 let completion = registry
6523 .drain_completions_for_session(Some("session"))
6524 .pop()
6525 .expect("reclaimed task completion");
6526 assert_eq!(
6527 completion.status_reason.as_deref(),
6528 Some(ROOT_RECLAIMED_REASON)
6529 );
6530 registry.detach();
6531 }
6532
6533 #[test]
6534 fn reclaimed_root_kills_pty_task_and_preserves_reason() {
6535 let registry = BgTaskRegistry::default();
6536 let root = tempfile::tempdir().unwrap();
6537 let storage = tempfile::tempdir().unwrap();
6538 let command = if cfg!(windows) {
6539 "Start-Sleep -Seconds 30"
6540 } else {
6541 "sleep 30"
6542 };
6543 let task_id = registry
6544 .spawn_pty(
6545 SpawnPlan::Unsandboxed,
6546 command,
6547 "session".to_string(),
6548 root.path().to_path_buf(),
6549 HashMap::new(),
6550 Some(Duration::from_secs(60)),
6551 storage.path().to_path_buf(),
6552 10,
6553 true,
6554 false,
6555 Some(root.path().to_path_buf()),
6556 24,
6557 80,
6558 )
6559 .unwrap();
6560 let pid = registry
6561 .status(
6562 &task_id,
6563 "session",
6564 Some(root.path()),
6565 Some(storage.path()),
6566 0,
6567 )
6568 .unwrap()
6569 .child_pid
6570 .unwrap();
6571 assert!(is_process_alive(pid));
6572
6573 assert_eq!(registry.kill_running_tasks_for_root(root.path()), 1);
6574 let deadline = Instant::now() + Duration::from_secs(10);
6575 loop {
6576 let snapshot = registry
6577 .status(
6578 &task_id,
6579 "session",
6580 Some(root.path()),
6581 Some(storage.path()),
6582 0,
6583 )
6584 .unwrap();
6585 if snapshot.info.status.is_terminal() {
6586 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
6587 assert_eq!(
6588 snapshot.info.status_reason.as_deref(),
6589 Some(ROOT_RECLAIMED_REASON)
6590 );
6591 break;
6592 }
6593 assert!(
6594 Instant::now() < deadline,
6595 "reclaimed PTY task did not terminate"
6596 );
6597 std::thread::sleep(Duration::from_millis(20));
6598 }
6599 assert!(!is_process_alive(pid));
6600 let completion = loop {
6605 if let Some(completion) = registry
6606 .drain_completions_for_session(Some("session"))
6607 .pop()
6608 {
6609 break completion;
6610 }
6611 assert!(
6612 Instant::now() < deadline,
6613 "reclaimed PTY completion never arrived"
6614 );
6615 std::thread::sleep(Duration::from_millis(20));
6616 };
6617 assert_eq!(
6618 completion.status_reason.as_deref(),
6619 Some(ROOT_RECLAIMED_REASON)
6620 );
6621 registry.detach();
6622 }
6623
6624 #[test]
6625 fn register_watch_rejects_unknown_task() {
6626 let registry = BgTaskRegistry::default();
6627
6628 let result = registry.register_watch(
6629 "missing-task".to_string(),
6630 WatchPattern::Substring("READY".into()),
6631 true,
6632 );
6633
6634 assert_eq!(result, Err("task_not_found"));
6635 }
6636
6637 #[test]
6638 fn register_watch_on_terminal_task_scans_existing_output() {
6639 let frames = Arc::new(Mutex::new(Vec::new()));
6640 let captured = Arc::clone(&frames);
6641 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
6642 captured.lock().unwrap().push(frame);
6643 })
6644 as Box<dyn Fn(PushFrame) + Send + Sync>);
6645 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
6646 let dir = tempfile::tempdir().unwrap();
6647 let task_id = registry
6648 .spawn(
6649 SpawnPlan::Unsandboxed,
6650 LONG_RUNNING_COMMAND,
6651 "session".to_string(),
6652 dir.path().to_path_buf(),
6653 HashMap::new(),
6654 Some(Duration::from_secs(30)),
6655 dir.path().to_path_buf(),
6656 10,
6657 true,
6658 false,
6659 Some(dir.path().to_path_buf()),
6660 )
6661 .unwrap();
6662 registry
6663 .inner
6664 .shutdown
6665 .store(true, std::sync::atomic::Ordering::SeqCst);
6666 let task = registry.task_for_session(&task_id, "session").unwrap();
6667 std::fs::write(&task.paths.stdout, "READY\n").unwrap();
6668 registry
6669 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6670 .unwrap();
6671 frames.lock().unwrap().clear();
6672 registry.inner.completions.lock().unwrap().clear();
6673
6674 registry
6675 .register_watch(
6676 task_id.clone(),
6677 WatchPattern::Substring("READY".into()),
6678 true,
6679 )
6680 .unwrap();
6681
6682 let frames = frames.lock().unwrap();
6683 let frame = frames
6684 .iter()
6685 .find_map(|frame| match frame {
6686 PushFrame::BashPatternMatch(frame) => Some(frame),
6687 _ => None,
6688 })
6689 .expect("terminal watch registration should emit pattern frame");
6690 assert_eq!(frame.reason, "pattern_match");
6691 assert_eq!(frame.task_id, task_id);
6692 assert_eq!(frame.session_id, "session");
6693 assert_eq!(frame.match_text, "READY");
6694 assert_eq!(frame.match_offset, 0);
6695 assert_eq!(registry.active_watch_count(&frame.task_id), 0);
6696 let metadata = read_task(&task.paths.json).unwrap();
6697 assert!(metadata.completion_delivered);
6698 }
6699
6700 #[test]
6701 fn cleanup_finished_removes_terminal_tasks_older_than_threshold() {
6702 let registry = BgTaskRegistry::default();
6703 let dir = tempfile::tempdir().unwrap();
6704 let task_id = registry
6705 .spawn(
6706 SpawnPlan::Unsandboxed,
6707 QUICK_SUCCESS_COMMAND,
6708 "session".to_string(),
6709 dir.path().to_path_buf(),
6710 HashMap::new(),
6711 Some(Duration::from_secs(30)),
6712 dir.path().to_path_buf(),
6713 10,
6714 true,
6715 false,
6716 Some(dir.path().to_path_buf()),
6717 )
6718 .unwrap();
6719 registry
6720 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6721 .unwrap();
6722 let completions = registry.drain_completions_for_session(Some("session"));
6723 assert_eq!(completions.len(), 1);
6724 assert_eq!(
6725 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
6726 vec![task_id.clone()]
6727 );
6728
6729 registry.cleanup_finished(Duration::ZERO);
6730
6731 assert!(registry.inner.tasks.lock().unwrap().is_empty());
6732 }
6733
6734 #[test]
6735 fn cleanup_finished_retains_undelivered_terminals() {
6736 let registry = BgTaskRegistry::default();
6737 let dir = tempfile::tempdir().unwrap();
6738 let task_id = registry
6739 .spawn(
6740 SpawnPlan::Unsandboxed,
6741 QUICK_SUCCESS_COMMAND,
6742 "session".to_string(),
6743 dir.path().to_path_buf(),
6744 HashMap::new(),
6745 Some(Duration::from_secs(30)),
6746 dir.path().to_path_buf(),
6747 10,
6748 true,
6749 false,
6750 Some(dir.path().to_path_buf()),
6751 )
6752 .unwrap();
6753 registry
6754 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6755 .unwrap();
6756
6757 registry.cleanup_finished(Duration::ZERO);
6758
6759 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
6760 }
6761
6762 #[test]
6770 fn reap_child_marks_failed_when_child_exits_without_exit_marker() {
6771 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6772 let dir = tempfile::tempdir().unwrap();
6773 let task_id = registry
6774 .spawn(
6775 SpawnPlan::Unsandboxed,
6776 QUICK_SUCCESS_COMMAND,
6777 "session".to_string(),
6778 dir.path().to_path_buf(),
6779 HashMap::new(),
6780 Some(Duration::from_secs(30)),
6781 dir.path().to_path_buf(),
6782 10,
6783 true,
6784 false,
6785 Some(dir.path().to_path_buf()),
6786 )
6787 .unwrap();
6788
6789 let task = registry.task_for_session(&task_id, "session").unwrap();
6790
6791 let started = Instant::now();
6796 loop {
6797 let exited = {
6798 let mut state = task.state.lock().unwrap();
6799 match &mut state.runtime {
6800 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
6801 _ => true,
6802 }
6803 };
6804 if exited {
6805 break;
6806 }
6807 assert!(
6808 started.elapsed() < Duration::from_secs(5),
6809 "child should exit quickly"
6810 );
6811 std::thread::sleep(Duration::from_millis(20));
6812 }
6813
6814 registry
6822 .inner
6823 .shutdown
6824 .store(true, std::sync::atomic::Ordering::SeqCst);
6825 std::thread::sleep(Duration::from_millis(550));
6829
6830 let _ = std::fs::remove_file(&task.paths.exit);
6833
6834 {
6849 let mut state = task.state.lock().unwrap();
6850 state.metadata.status = BgTaskStatus::Running;
6851 state.metadata.status_reason = None;
6852 state.metadata.exit_code = None;
6853 state.metadata.finished_at = None;
6854 state.metadata.duration_ms = None;
6855 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
6858 .expect("persist reset Running metadata for reap_child test");
6859 if matches!(state.runtime, TaskRuntime::Piped(None)) {
6863 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
6864 }
6865 }
6866 *task.terminal_at.lock().unwrap() = None;
6869
6870 assert!(
6873 task.is_running(),
6874 "precondition: metadata.status == Running"
6875 );
6876 assert!(
6877 !task.paths.exit.exists(),
6878 "precondition: exit marker absent"
6879 );
6880
6881 registry.reap_child(&task);
6886
6887 {
6888 let state = task.state.lock().unwrap();
6889 assert_eq!(
6890 state.metadata.status,
6891 BgTaskStatus::Running,
6892 "first reap must leave status Running while waiting one pass for marker"
6893 );
6894 assert_eq!(
6895 state.metadata.status_reason, None,
6896 "first reap must not record a failure reason"
6897 );
6898 assert!(
6899 matches!(state.runtime, TaskRuntime::Piped(None)),
6900 "child handle must be released after first reap"
6901 );
6902 assert!(
6903 state.detached,
6904 "task must be marked detached after first reap"
6905 );
6906 }
6907
6908 registry.reap_child(&task);
6912
6913 let state = task.state.lock().unwrap();
6914 assert!(
6915 state.metadata.status.is_terminal(),
6916 "second reap must transition to terminal when PID dead and no marker. Got status={:?}",
6917 state.metadata.status
6918 );
6919 assert_eq!(
6920 state.metadata.status,
6921 BgTaskStatus::Failed,
6922 "must specifically be Failed (not Killed): status={:?}",
6923 state.metadata.status
6924 );
6925 assert_eq!(
6926 state.metadata.status_reason.as_deref(),
6927 Some("process exited without exit marker"),
6928 "reason must match replay path's wording: {:?}",
6929 state.metadata.status_reason
6930 );
6931 assert!(
6932 matches!(state.runtime, TaskRuntime::Piped(None)),
6933 "child handle must stay released after second reap"
6934 );
6935 assert!(
6936 state.detached,
6937 "task must remain detached after second reap"
6938 );
6939 }
6940
6941 #[test]
6946 fn reap_child_preserves_running_when_exit_marker_exists() {
6947 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6948 let dir = tempfile::tempdir().unwrap();
6949 let task_id = registry
6950 .spawn(
6951 SpawnPlan::Unsandboxed,
6952 QUICK_SUCCESS_COMMAND,
6953 "session".to_string(),
6954 dir.path().to_path_buf(),
6955 HashMap::new(),
6956 Some(Duration::from_secs(30)),
6957 dir.path().to_path_buf(),
6958 10,
6959 true,
6960 false,
6961 Some(dir.path().to_path_buf()),
6962 )
6963 .unwrap();
6964
6965 let task = registry.task_for_session(&task_id, "session").unwrap();
6966
6967 let started = Instant::now();
6970 loop {
6971 let exited = {
6972 let mut state = task.state.lock().unwrap();
6973 match &mut state.runtime {
6974 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
6975 _ => true,
6976 }
6977 };
6978 if exited && task.paths.exit.exists() {
6979 break;
6980 }
6981 assert!(
6982 started.elapsed() < Duration::from_secs(5),
6983 "child should exit and write marker quickly"
6984 );
6985 std::thread::sleep(Duration::from_millis(20));
6986 }
6987
6988 registry
6994 .inner
6995 .shutdown
6996 .store(true, std::sync::atomic::Ordering::SeqCst);
6997 std::thread::sleep(Duration::from_millis(550));
6998
6999 {
7005 let mut state = task.state.lock().unwrap();
7006 state.metadata.status = BgTaskStatus::Running;
7007 state.metadata.status_reason = None;
7008 if matches!(state.runtime, TaskRuntime::Piped(None)) {
7009 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
7010 }
7011 }
7012 *task.terminal_at.lock().unwrap() = None;
7013 if !task.paths.exit.exists() {
7016 std::fs::write(&task.paths.exit, "0").expect("write replacement exit marker");
7017 }
7018
7019 registry.reap_child(&task);
7023
7024 let state = task.state.lock().unwrap();
7025 assert!(
7026 matches!(state.runtime, TaskRuntime::Piped(None)),
7027 "child handle still released even when marker exists"
7028 );
7029 assert!(
7030 state.detached,
7031 "task still marked detached even when marker exists"
7032 );
7033 assert_eq!(
7038 state.metadata.status,
7039 BgTaskStatus::Running,
7040 "reap_child must defer to poll_task when marker exists"
7041 );
7042 }
7043
7044 #[cfg(unix)]
7048 fn pid_stat(pid: u32) -> Option<String> {
7049 let output = std::process::Command::new("ps")
7050 .args(["-o", "stat=", "-p", &pid.to_string()])
7051 .output()
7052 .ok()?;
7053 if !output.status.success() {
7054 return None;
7055 }
7056 let stat = String::from_utf8_lossy(&output.stdout).trim().to_string();
7057 if stat.is_empty() {
7058 None
7059 } else {
7060 Some(stat)
7061 }
7062 }
7063
7064 #[cfg(unix)]
7066 fn is_zombie(pid: u32) -> bool {
7067 pid_stat(pid).is_some_and(|stat| stat.starts_with('Z'))
7068 }
7069
7070 #[cfg(unix)]
7076 fn spawn_unreaped_zombie() -> std::process::Child {
7077 let child = std::process::Command::new("true")
7078 .stdin(std::process::Stdio::null())
7079 .stdout(std::process::Stdio::null())
7080 .stderr(std::process::Stdio::null())
7081 .spawn()
7082 .expect("spawn zombie stand-in");
7083 let pid = child.id();
7084 let started = Instant::now();
7085 while !is_zombie(pid) {
7086 assert!(
7087 started.elapsed() < Duration::from_secs(5),
7088 "stand-in child should become a zombie within 5s"
7089 );
7090 std::thread::sleep(Duration::from_millis(10));
7091 }
7092 child
7094 }
7095
7096 #[cfg(unix)]
7106 #[test]
7107 fn finalize_from_marker_reaps_child_no_zombie() {
7108 use std::sync::atomic::Ordering;
7109
7110 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7111 let dir = tempfile::tempdir().unwrap();
7112 let task_id = registry
7113 .spawn(
7114 SpawnPlan::Unsandboxed,
7115 QUICK_SUCCESS_COMMAND,
7116 "session".to_string(),
7117 dir.path().to_path_buf(),
7118 HashMap::new(),
7119 Some(Duration::from_secs(30)),
7120 dir.path().to_path_buf(),
7121 10,
7122 true,
7123 false,
7124 Some(dir.path().to_path_buf()),
7125 )
7126 .unwrap();
7127
7128 registry.inner.shutdown.store(true, Ordering::SeqCst);
7132 std::thread::sleep(Duration::from_millis(550));
7133
7134 let task = registry.task_for_session(&task_id, "session").unwrap();
7135
7136 let started = Instant::now();
7140 while !task.paths.exit.exists() {
7141 assert!(
7142 started.elapsed() < Duration::from_secs(5),
7143 "exit marker should land quickly for `true`"
7144 );
7145 std::thread::sleep(Duration::from_millis(20));
7146 }
7147
7148 let zombie_pid;
7154 {
7155 let mut state = task.state.lock().unwrap();
7156 state.metadata.status = BgTaskStatus::Running;
7157 state.metadata.status_reason = None;
7158 state.metadata.exit_code = None;
7159 state.metadata.finished_at = None;
7160 state.metadata.duration_ms = None;
7161 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
7162 .expect("persist reset Running metadata");
7163 let zombie = spawn_unreaped_zombie();
7164 zombie_pid = zombie.id();
7165 state.runtime = TaskRuntime::Piped(Some(zombie));
7166 }
7167 *task.terminal_at.lock().unwrap() = None;
7168
7169 assert!(
7171 is_zombie(zombie_pid),
7172 "precondition: stand-in child {zombie_pid} must be a zombie before finalize"
7173 );
7174
7175 registry.poll_task(&task).unwrap();
7178
7179 {
7180 let state = task.state.lock().unwrap();
7181 assert!(
7182 matches!(state.runtime, TaskRuntime::Piped(None)),
7183 "child handle must be released after marker finalize"
7184 );
7185 assert!(
7186 state.metadata.status.is_terminal(),
7187 "task must be terminal after marker finalize: {:?}",
7188 state.metadata.status
7189 );
7190 }
7191
7192 assert!(
7195 !is_zombie(zombie_pid),
7196 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
7197 after the exit-marker terminal transition"
7198 );
7199 }
7200
7201 #[cfg(unix)]
7205 #[test]
7206 fn kill_with_existing_marker_reaps_child_no_zombie() {
7207 use std::sync::atomic::Ordering;
7208
7209 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7210 let dir = tempfile::tempdir().unwrap();
7211 let task_id = registry
7212 .spawn(
7213 SpawnPlan::Unsandboxed,
7214 QUICK_SUCCESS_COMMAND,
7215 "session".to_string(),
7216 dir.path().to_path_buf(),
7217 HashMap::new(),
7218 Some(Duration::from_secs(30)),
7219 dir.path().to_path_buf(),
7220 10,
7221 true,
7222 false,
7223 Some(dir.path().to_path_buf()),
7224 )
7225 .unwrap();
7226
7227 registry.inner.shutdown.store(true, Ordering::SeqCst);
7228 std::thread::sleep(Duration::from_millis(550));
7229
7230 let task = registry.task_for_session(&task_id, "session").unwrap();
7231
7232 let started = Instant::now();
7233 while !task.paths.exit.exists() {
7234 assert!(
7235 started.elapsed() < Duration::from_secs(5),
7236 "exit marker should land quickly for `true`"
7237 );
7238 std::thread::sleep(Duration::from_millis(20));
7239 }
7240
7241 let zombie_pid;
7242 {
7243 let mut state = task.state.lock().unwrap();
7244 state.metadata.status = BgTaskStatus::Running;
7245 state.metadata.status_reason = None;
7246 state.metadata.exit_code = None;
7247 state.metadata.finished_at = None;
7248 state.metadata.duration_ms = None;
7249 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
7250 .expect("persist reset Running metadata");
7251 let zombie = spawn_unreaped_zombie();
7252 zombie_pid = zombie.id();
7253 state.runtime = TaskRuntime::Piped(Some(zombie));
7254 }
7255 *task.terminal_at.lock().unwrap() = None;
7256
7257 assert!(
7258 is_zombie(zombie_pid),
7259 "precondition: stand-in child {zombie_pid} must be a zombie before kill"
7260 );
7261
7262 registry
7264 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
7265 .expect("kill should succeed");
7266
7267 {
7268 let state = task.state.lock().unwrap();
7269 assert!(
7270 matches!(state.runtime, TaskRuntime::Piped(None)),
7271 "child handle must be released after marker-aware kill"
7272 );
7273 assert!(state.metadata.status.is_terminal());
7274 }
7275
7276 assert!(
7277 !is_zombie(zombie_pid),
7278 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
7279 after a marker-aware kill"
7280 );
7281 }
7282
7283 #[test]
7284 fn cleanup_finished_keeps_running_tasks() {
7285 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7286 let dir = tempfile::tempdir().unwrap();
7287 let task_id = registry
7288 .spawn(
7289 SpawnPlan::Unsandboxed,
7290 LONG_RUNNING_COMMAND,
7291 "session".to_string(),
7292 dir.path().to_path_buf(),
7293 HashMap::new(),
7294 Some(Duration::from_secs(30)),
7295 dir.path().to_path_buf(),
7296 10,
7297 true,
7298 false,
7299 Some(dir.path().to_path_buf()),
7300 )
7301 .unwrap();
7302
7303 registry.cleanup_finished(Duration::ZERO);
7304
7305 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
7306 let _ = registry.kill(&task_id, "session");
7307 }
7308
7309 #[cfg(unix)]
7310 #[test]
7311 fn rehydrating_sandboxed_task_never_respawns_persisted_command() {
7312 let project = tempfile::tempdir().unwrap();
7313 let storage = tempfile::tempdir().unwrap();
7314 let sandbox_temp = storage.path().join("sandbox-temp");
7315 fs::create_dir(&sandbox_temp).unwrap();
7316 let launcher_script = project.path().join("sandbox-launch");
7317 let launcher = PathBuf::from("/bin/sh");
7318 fs::write(
7319 &launcher_script,
7320 "while [ \"$#\" -gt 0 ]; do\n if [ \"$1\" = -- ]; then\n shift\n exec \"$@\"\n fi\n shift\ndone\nexit 78\n",
7321 )
7322 .unwrap();
7323 let mut permissions = fs::metadata(&launcher_script).unwrap().permissions();
7324 permissions.set_mode(0o700);
7325 fs::set_permissions(&launcher_script, permissions).unwrap();
7326
7327 let profile = crate::sandbox_profile::SandboxProfile::build(
7328 vec![project.path().to_path_buf()],
7329 Vec::new(),
7330 Vec::new(),
7331 Vec::new(),
7332 Vec::new(),
7333 Vec::new(),
7334 Vec::new(),
7335 sandbox_temp,
7336 )
7337 .unwrap();
7338 let plan = SpawnPlan::launcher_for_test(profile, launcher);
7339 let spawn_marker = project.path().join("spawn-count");
7340 let stop_marker = project.path().join("stop-command");
7341 let quote =
7342 |path: &Path| format!("'{}'", path.display().to_string().replace('\'', "'\\''"));
7343 let command = format!(
7344 "printf 'spawn\\n' >> {}; while [ ! -e {} ]; do sleep 0.05; done",
7345 quote(&spawn_marker),
7346 quote(&stop_marker)
7347 );
7348
7349 let original = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7350 let task_id = original
7351 .spawn(
7352 plan,
7353 &command,
7354 "sandbox-rehydrate".to_string(),
7355 project.path().to_path_buf(),
7356 HashMap::new(),
7357 Some(Duration::from_secs(30)),
7358 storage.path().to_path_buf(),
7359 10,
7360 true,
7361 false,
7362 Some(project.path().to_path_buf()),
7363 )
7364 .unwrap();
7365 let started = Instant::now();
7366 while !spawn_marker.exists() {
7367 assert!(
7368 started.elapsed() < Duration::from_secs(20),
7369 "original sandboxed task did not start"
7370 );
7371 std::thread::sleep(Duration::from_millis(10));
7372 }
7373 original.detach();
7374
7375 let restarted = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7376 restarted
7377 .replay_session(storage.path(), "sandbox-rehydrate")
7378 .unwrap();
7379 let replayed = restarted
7380 .status(
7381 &task_id,
7382 "sandbox-rehydrate",
7383 Some(project.path()),
7384 Some(storage.path()),
7385 4096,
7386 )
7387 .expect("rehydrated sandbox task");
7388 assert_eq!(replayed.info.status, BgTaskStatus::Running);
7389 assert!(replayed.sandbox_native);
7390
7391 std::thread::sleep(Duration::from_millis(650));
7392 assert_eq!(
7393 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
7394 1,
7395 "registry replay must observe the persisted process without spawning its command"
7396 );
7397
7398 fs::write(&stop_marker, "stop").unwrap();
7399 let terminal = wait_for_terminal_snapshot(
7400 &restarted,
7401 &task_id,
7402 "sandbox-rehydrate",
7403 project.path(),
7404 storage.path(),
7405 );
7406 assert_eq!(terminal.info.status, BgTaskStatus::Completed);
7407 assert_eq!(
7408 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
7409 1
7410 );
7411 restarted.detach();
7412 }
7413
7414 #[cfg(windows)]
7415 fn wait_for_file(path: &Path) -> String {
7416 let started = Instant::now();
7423 loop {
7424 if let Ok(content) = fs::read_to_string(path) {
7425 if !content.trim().is_empty() {
7426 return content;
7427 }
7428 }
7429 assert!(
7430 started.elapsed() < Duration::from_secs(30),
7431 "timed out waiting for non-empty {}",
7432 path.display()
7433 );
7434 std::thread::sleep(Duration::from_millis(100));
7435 }
7436 }
7437
7438 #[cfg(windows)]
7439 fn spawn_windows_registry_command(
7440 command: &str,
7441 ) -> (BgTaskRegistry, tempfile::TempDir, String) {
7442 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7443 let dir = tempfile::tempdir().unwrap();
7444 let task_id = registry
7445 .spawn(
7446 SpawnPlan::Unsandboxed,
7447 command,
7448 "session".to_string(),
7449 dir.path().to_path_buf(),
7450 HashMap::new(),
7451 Some(Duration::from_secs(30)),
7452 dir.path().to_path_buf(),
7453 10,
7454 false,
7455 false,
7456 Some(dir.path().to_path_buf()),
7457 )
7458 .unwrap();
7459 (registry, dir, task_id)
7460 }
7461
7462 #[cfg(windows)]
7463 #[test]
7464 fn windows_spawn_writes_exit_marker_for_zero_exit() {
7465 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 0");
7466 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
7467
7468 let content = wait_for_file(&exit_path);
7469
7470 assert_eq!(content.trim(), "0");
7471 }
7472
7473 #[cfg(windows)]
7474 #[test]
7475 fn windows_spawn_writes_exit_marker_for_nonzero_exit() {
7476 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 42");
7477 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
7478
7479 let content = wait_for_file(&exit_path);
7480
7481 assert_eq!(content.trim(), "42");
7482 }
7483
7484 #[cfg(windows)]
7485 #[test]
7486 fn windows_spawn_captures_stdout_to_disk() {
7487 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c echo hello");
7488 let task = registry.task_for_session(&task_id, "session").unwrap();
7489 let stdout_path = task.paths.stdout.clone();
7490 let exit_path = task.paths.exit.clone();
7491
7492 let _ = wait_for_file(&exit_path);
7493 let stdout = fs::read_to_string(stdout_path).expect("read stdout");
7494
7495 assert!(stdout.contains("hello"), "stdout was {stdout:?}");
7496 }
7497
7498 #[cfg(windows)]
7499 #[test]
7500 fn windows_spawn_uses_pwsh_when_available() {
7501 let candidates = crate::windows_shell::shell_candidates_with(
7505 |binary| match binary {
7506 "pwsh.exe" => Some(std::path::PathBuf::from(r"C:\pwsh\pwsh.exe")),
7507 "powershell.exe" => Some(std::path::PathBuf::from(r"C:\ps\powershell.exe")),
7508 _ => None,
7509 },
7510 || None,
7511 );
7512 let shell = candidates.first().expect("at least one candidate").clone();
7513 assert_eq!(shell, crate::windows_shell::WindowsShell::Pwsh);
7514 assert_eq!(shell.binary().as_ref(), "pwsh.exe");
7515 }
7516
7517 #[cfg(windows)]
7520 #[test]
7521 fn windows_shell_cmd_wrapper_writes_marker_via_temp_rename() {
7522 let exit_path = Path::new(r"C:\Temp\bash-test.exit");
7523 let script =
7524 crate::windows_shell::WindowsShell::Cmd.wrapper_script("cmd /c exit 42", exit_path);
7525
7526 assert!(
7527 script.contains("set CODE=%ERRORLEVEL%"),
7528 "wrapper must capture the child exit code: {script}"
7529 );
7530 assert!(
7531 script.contains("exit /B %CODE%"),
7532 "wrapper must propagate the child exit code: {script}"
7533 );
7534 assert!(
7539 script.contains("bash-test.exit"),
7540 "wrapper must target the exit marker path: {script}"
7541 );
7542 assert!(
7543 script.contains("move /Y"),
7544 "wrapper must write the marker atomically via temp-file + rename: {script}"
7545 );
7546 }
7547
7548 #[cfg(windows)]
7554 #[test]
7555 fn windows_shell_cmd_bg_command_uses_minimal_cmd_flags() {
7556 use crate::windows_shell::WindowsShell;
7557 let cmd = WindowsShell::Cmd.bg_command("echo wrapped");
7558 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
7559 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
7560 assert_eq!(
7561 args_strs,
7562 vec!["/D", "/S", "/C", "echo wrapped"],
7563 "Cmd::bg_command must prepend /D /S /C"
7564 );
7565 }
7566
7567 #[cfg(windows)]
7570 #[test]
7571 fn windows_shell_pwsh_bg_command_uses_standard_args() {
7572 use crate::windows_shell::WindowsShell;
7573 let cmd = WindowsShell::Pwsh.bg_command("Get-Date");
7574 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
7575 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
7576 assert!(
7577 args_strs.contains(&"-Command"),
7578 "Pwsh::bg_command must use -Command: {args_strs:?}"
7579 );
7580 assert!(
7581 args_strs.contains(&"Get-Date"),
7582 "Pwsh::bg_command must include the user command body"
7583 );
7584 }
7585
7586 fn registry_with_db_and_frames(
7587 storage: &Path,
7588 ) -> (
7589 BgTaskRegistry,
7590 Arc<Mutex<Connection>>,
7591 Arc<Mutex<Vec<PushFrame>>>,
7592 ) {
7593 let frames = Arc::new(Mutex::new(Vec::new()));
7594 let captured = Arc::clone(&frames);
7595 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
7596 captured.lock().unwrap().push(frame);
7597 })
7598 as Box<dyn Fn(PushFrame) + Send + Sync>);
7599 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
7600 registry.set_harness(Harness::Opencode);
7601 let conn = crate::db::open(&storage.join("aft.db")).expect("open test DB");
7602 let shared = Arc::new(Mutex::new(conn));
7603 registry.set_db_pool(shared.clone());
7604 (registry, shared, frames)
7605 }
7606
7607 fn pattern_match_frames(frames: &Mutex<Vec<PushFrame>>) -> Vec<BashPatternMatchFrame> {
7608 frames
7609 .lock()
7610 .unwrap()
7611 .iter()
7612 .filter_map(|frame| match frame {
7613 PushFrame::BashPatternMatch(frame) => Some(frame.clone()),
7614 _ => None,
7615 })
7616 .collect()
7617 }
7618
7619 #[test]
7620 fn pattern_watch_survives_registry_teardown_and_rehydrate() {
7621 let dir = tempfile::tempdir().unwrap();
7622 let storage = dir.path();
7623 let (registry, _db, frames) = registry_with_db_and_frames(storage);
7624 let task_id = registry
7625 .spawn(
7626 SpawnPlan::Unsandboxed,
7627 LONG_RUNNING_COMMAND,
7628 "session".to_string(),
7629 storage.to_path_buf(),
7630 HashMap::new(),
7631 Some(Duration::from_secs(30)),
7632 storage.to_path_buf(),
7633 10,
7634 true,
7635 false,
7636 Some(storage.to_path_buf()),
7637 )
7638 .unwrap();
7639 registry
7640 .register_watch(
7641 task_id.clone(),
7642 WatchPattern::Substring("READY".into()),
7643 true,
7644 )
7645 .unwrap();
7646 let task = registry.task_for_session(&task_id, "session").unwrap();
7647 registry.clear_task_watch_state(&task_id);
7649 assert_eq!(registry.active_watch_count(&task_id), 0);
7650
7651 std::fs::OpenOptions::new()
7652 .append(true)
7653 .open(&task.paths.stdout)
7654 .unwrap()
7655 .write_all(b"READY\n")
7656 .unwrap();
7657 frames.lock().unwrap().clear();
7658
7659 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
7660 registry
7662 .inner
7663 .shutdown
7664 .store(true, std::sync::atomic::Ordering::SeqCst);
7665 replayed
7666 .replay_session_inner(storage, "session", None)
7667 .unwrap();
7668
7669 let matches = pattern_match_frames(&replay_frames);
7670 assert!(
7671 matches.iter().any(|frame| {
7672 frame.task_id == task_id
7673 && frame.reason == "pattern_match"
7674 && frame.match_text == "READY"
7675 }),
7676 "rehydrate should deliver gap match: {matches:?}"
7677 );
7678 }
7679
7680 #[test]
7681 fn pattern_watch_gap_match_between_teardown_and_rehydrate_delivers_once() {
7682 let dir = tempfile::tempdir().unwrap();
7683 let storage = dir.path();
7684 let (registry, _db, frames) = registry_with_db_and_frames(storage);
7685 let task_id = registry
7686 .spawn(
7687 SpawnPlan::Unsandboxed,
7688 LONG_RUNNING_COMMAND,
7689 "session".to_string(),
7690 storage.to_path_buf(),
7691 HashMap::new(),
7692 Some(Duration::from_secs(30)),
7693 storage.to_path_buf(),
7694 10,
7695 true,
7696 false,
7697 Some(storage.to_path_buf()),
7698 )
7699 .unwrap();
7700 registry
7701 .register_watch(
7702 task_id.clone(),
7703 WatchPattern::Substring("GAP-HIT".into()),
7704 true,
7705 )
7706 .unwrap();
7707 let task = registry.task_for_session(&task_id, "session").unwrap();
7708 let cursor_before = registry.watch_stream_cursors(&task_id).0;
7709 registry.clear_task_watch_state(&task_id);
7710
7711 std::fs::OpenOptions::new()
7713 .append(true)
7714 .open(&task.paths.stdout)
7715 .unwrap()
7716 .write_all(b"prefix GAP-HIT suffix\n")
7717 .unwrap();
7718 frames.lock().unwrap().clear();
7719
7720 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
7721 registry
7722 .inner
7723 .shutdown
7724 .store(true, std::sync::atomic::Ordering::SeqCst);
7725 replayed
7726 .replay_session_inner(storage, "session", None)
7727 .unwrap();
7728
7729 let matches: Vec<_> = pattern_match_frames(&replay_frames)
7730 .into_iter()
7731 .filter(|frame| frame.task_id == task_id && frame.match_text.contains("GAP-HIT"))
7732 .collect();
7733 assert_eq!(
7734 matches.len(),
7735 1,
7736 "gap match must deliver exactly once: {matches:?}"
7737 );
7738 assert!(
7739 matches[0].match_offset >= cursor_before,
7740 "match offset should be at/after the persisted cursor ({cursor_before}), got {}",
7741 matches[0].match_offset
7742 );
7743 }
7744
7745 #[test]
7746 fn pattern_watch_acked_match_does_not_redeliver_after_restart() {
7747 let dir = tempfile::tempdir().unwrap();
7748 let storage = dir.path();
7749 let (registry, db, frames) = registry_with_db_and_frames(storage);
7750 let task_id = registry
7751 .spawn(
7752 SpawnPlan::Unsandboxed,
7753 LONG_RUNNING_COMMAND,
7754 "session".to_string(),
7755 storage.to_path_buf(),
7756 HashMap::new(),
7757 Some(Duration::from_secs(30)),
7758 storage.to_path_buf(),
7759 10,
7760 true,
7761 false,
7762 Some(storage.to_path_buf()),
7763 )
7764 .unwrap();
7765 registry
7766 .register_watch(
7767 task_id.clone(),
7768 WatchPattern::Substring("READY".into()),
7769 true,
7770 )
7771 .unwrap();
7772 let task = registry.task_for_session(&task_id, "session").unwrap();
7773 std::fs::OpenOptions::new()
7774 .append(true)
7775 .open(&task.paths.stdout)
7776 .unwrap()
7777 .write_all(b"READY\n")
7778 .unwrap();
7779 registry.scan_task_watch_output(&task);
7780 let delivered = pattern_match_frames(&frames);
7781 assert!(
7782 delivered
7783 .iter()
7784 .any(|frame| frame.task_id == task_id && frame.match_text == "READY"),
7785 "live path should deliver match: {delivered:?}"
7786 );
7787 assert!(registry
7789 .ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id))
7790 .contains(&task_id));
7791 {
7792 let conn = db.lock().unwrap();
7793 let rows = crate::db::bash_watches::list_bash_pattern_watches_for_task(
7794 &conn, "opencode", "session", &task_id,
7795 )
7796 .unwrap();
7797 assert!(
7798 rows.is_empty(),
7799 "acked once-watch rows must be deleted: {rows:?}"
7800 );
7801 }
7802
7803 frames.lock().unwrap().clear();
7804 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
7805 registry
7806 .inner
7807 .shutdown
7808 .store(true, std::sync::atomic::Ordering::SeqCst);
7809 replayed
7810 .replay_session_inner(storage, "session", None)
7811 .unwrap();
7812 let matches = pattern_match_frames(&replay_frames)
7813 .into_iter()
7814 .filter(|frame| frame.task_id == task_id)
7815 .collect::<Vec<_>>();
7816 assert!(
7817 matches.is_empty(),
7818 "acked match must not re-deliver after restart: {matches:?}"
7819 );
7820 }
7821
7822 #[test]
7823 fn pattern_watch_rows_are_removed_when_task_is_gc_deleted() {
7824 let dir = tempfile::tempdir().unwrap();
7825 let storage = dir.path();
7826 let (registry, db, _frames) = registry_with_db_and_frames(storage);
7827 let task_id = "bash-aaaaaaaaaaaaaaaa";
7828 let paths = task_paths(storage, "session", task_id).unwrap();
7829 let mut metadata = PersistedTask::starting(
7830 task_id.to_string(),
7831 "session".to_string(),
7832 "true".to_string(),
7833 storage.to_path_buf(),
7834 Some(storage.to_path_buf()),
7835 None,
7836 true,
7837 true,
7838 );
7839 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
7840 metadata.completion_delivered = true;
7841 write_task(&paths.json, &metadata).unwrap();
7842 {
7843 let conn = db.lock().unwrap();
7844 crate::db::bash_tasks::upsert_bash_task(
7845 &conn,
7846 &metadata.to_bash_task_row("opencode", &paths).unwrap(),
7847 )
7848 .unwrap();
7849 crate::db::bash_watches::upsert_bash_pattern_watch(
7850 &conn,
7851 &BashPatternWatchRow {
7852 harness: "opencode".into(),
7853 session_id: "session".into(),
7854 task_id: task_id.into(),
7855 watch_id: "watch-00000001".into(),
7856 pattern_kind: "substring".into(),
7857 pattern: "x".into(),
7858 once: true,
7859 created_at: 1,
7860 stdout_offset: 0,
7861 stderr_offset: 0,
7862 pty_offset: 0,
7863 scanning: true,
7864 pending_match: false,
7865 match_text: None,
7866 match_offset: None,
7867 match_context: None,
7868 },
7869 )
7870 .unwrap();
7871 }
7872 let old = SystemTime::now()
7873 .checked_sub(Duration::from_secs(25 * 60 * 60))
7874 .unwrap();
7875 filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
7876
7877 let deleted = registry.maybe_gc_persisted(storage).unwrap();
7878 assert!(
7879 deleted >= 1,
7880 "expected GC to delete the terminal task bundle"
7881 );
7882 let conn = db.lock().unwrap();
7883 let watches = crate::db::bash_watches::list_bash_pattern_watches_for_task(
7884 &conn, "opencode", "session", task_id,
7885 )
7886 .unwrap();
7887 assert!(
7888 watches.is_empty(),
7889 "task GC must remove watch rows: {watches:?}"
7890 );
7891 }
7892}