1use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
2#[cfg(not(windows))]
3use std::ffi::OsString;
4use std::fs;
5use std::io::Write;
6use std::path::{Path, PathBuf};
7#[cfg(windows)]
8use std::process::Command;
9use std::process::{Child, Stdio};
10use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
11#[cfg(unix)]
12use std::sync::OnceLock;
13use std::sync::{Arc, Mutex, RwLock};
14use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
15
16use rusqlite::Connection;
17use serde::Serialize;
18
19use crate::bash_permissions::PermissionAsk;
20use crate::compress::caps::DropClass;
21#[cfg(unix)]
22use crate::compress::single_top_level_pipeline;
23use crate::compress::CompressionResult;
24use crate::context::SharedProgressSender;
25use crate::db::compression_events::CompressionAggregateCache;
26use crate::harness::Harness;
27use crate::protocol::{BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, PushFrame};
28use crate::sandbox_spawn::SpawnPlan;
29
30#[cfg(windows)]
31use std::os::windows::process::CommandExt;
32
33use super::buffer::{combine_streams, BgBuffer, DiskTruncation, StreamKind, TokenCountInput};
34use super::output::{
35 cap_completion_output, cap_completion_output_with_marker, cap_final_output,
36 cap_final_output_with_marker, completion_preview_threshold, json_output_pointer, quote_path,
37 retained_json_output_pointer, COMPRESS_INPUT_CAP_BYTES, COMPRESS_INPUT_HEAD_BYTES,
38 COMPRESS_INPUT_TAIL_BYTES, FINAL_OUTPUT_CAP_BYTES, RAW_PASSTHROUGH_CAP_BYTES,
39 RAW_PASSTHROUGH_HEAD_BYTES, RAW_PASSTHROUGH_TAIL_BYTES, RUNNING_OUTPUT_PREVIEW_BYTES,
40 STRUCTURED_OUTPUT_CAP_BYTES,
41};
42use super::persistence::{
43 allocate_task_layout, delete_resolved_task, delete_task_bundle, discover_task_ids,
44 open_task_artifact, quarantine_invalid_entry, quarantine_task_layout, read_exit_marker,
45 read_task_at, resolve_task_layout, session_tasks_dir, uninitialized_layout_is_recent,
46 unix_millis, update_task_at, validate_task_id, write_kill_marker_if_absent, write_task_at,
47 BgMode, ExitMarker, PersistedTask, TaskArtifact, TaskIoHandles, TaskPaths,
48};
49#[cfg(unix)]
50use super::process::terminate_pgid;
51#[cfg(windows)]
52use super::process::terminate_pid;
53use super::process::{is_process_alive, is_recorded_process_alive};
54use super::pty_process::spawn_pty_for_command;
55use super::pty_runtime::PtyRuntime;
56use super::watches::{
57 PatternMatch, WatchPattern, WatchRegistry, WATCH_TARGET_ERASED_CONTEXT,
58 WATCH_TARGET_ERASED_TEXT,
59};
60use super::{BgTaskInfo, BgTaskStatus};
61use crate::db::bash_tasks::BashTaskRow;
62use crate::db::bash_watches::BashPatternWatchRow;
63const DEFAULT_BG_TIMEOUT: Duration = Duration::from_secs(30 * 60);
66const PERSISTED_GC_GRACE: Duration = Duration::from_secs(24 * 60 * 60);
67const QUARANTINE_GC_GRACE: Duration = Duration::from_secs(30 * 24 * 60 * 60);
68
69const TOKENIZE_CAP_BYTES_PER_STREAM: usize = 128 * 1024;
70pub const ROOT_RECLAIMED_REASON: &str = "root_reclaimed";
71
72#[derive(Debug, Clone, Serialize)]
73pub struct BgCompletion {
74 pub task_id: String,
75 #[serde(skip_serializing)]
78 pub session_id: String,
79 pub status: BgTaskStatus,
80 pub exit_code: Option<i32>,
81 pub command: String,
82 #[serde(default, skip_serializing_if = "String::is_empty")]
88 pub output_preview: String,
89 #[serde(default, skip_serializing_if = "is_false")]
94 pub output_truncated: bool,
95 #[serde(default, skip_serializing_if = "Option::is_none")]
98 pub original_tokens: Option<u32>,
99 #[serde(default, skip_serializing_if = "Option::is_none")]
102 pub compressed_tokens: Option<u32>,
103 #[serde(default, skip_serializing_if = "is_false")]
105 pub tokens_skipped: bool,
106 #[serde(default, skip_serializing_if = "Option::is_none")]
107 pub status_reason: Option<String>,
108}
109
110fn is_false(v: &bool) -> bool {
111 !*v
112}
113
114#[derive(Debug, Clone, Serialize)]
115pub struct BgTaskSnapshot {
116 #[serde(flatten)]
117 pub info: BgTaskInfo,
118 pub exit_code: Option<i32>,
119 pub child_pid: Option<u32>,
120 pub workdir: String,
121 pub output_preview: String,
122 pub output_truncated: bool,
123 pub output_path: Option<String>,
124 pub stderr_path: Option<String>,
125 #[serde(skip_serializing_if = "Option::is_none")]
126 pub pty_rows: Option<u16>,
127 #[serde(skip_serializing_if = "Option::is_none")]
128 pub pty_cols: Option<u16>,
129 #[serde(skip_serializing_if = "Option::is_none")]
130 pub pty_screen: Option<String>,
131 #[serde(default, skip_serializing_if = "Vec::is_empty")]
132 pub scanner_report: Vec<PermissionAsk>,
133 #[serde(default, skip_serializing_if = "is_false")]
134 pub sandbox_native: bool,
135 #[serde(default, skip_serializing_if = "is_false")]
136 pub sandbox_unavailable: bool,
137}
138
139#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
140pub struct BgTaskHealthCounts {
141 pub running: usize,
142 pub pending_completions: usize,
143}
144
145#[derive(Debug, Clone, Copy, PartialEq, Eq)]
146enum TerminalOutputKind {
147 Compressed,
148 Raw,
149 Structured,
150}
151
152#[derive(Debug, Clone, PartialEq, Eq)]
153struct TerminalOutputCache {
154 output_preview: String,
155 output_truncated: bool,
156 kind: TerminalOutputKind,
157 output_path: Option<String>,
158 stderr_path: Option<String>,
159 artifact_access: ArtifactRecoveryAccess,
160 recovery: Option<RecoveryContext>,
161}
162
163#[derive(Debug, Clone, PartialEq, Eq)]
164struct ArtifactRecoveryAccess {
165 task_id: String,
166 readable: bool,
167}
168
169#[derive(Debug, Clone, PartialEq, Eq)]
170struct RecoveryContext {
171 dropped_by_class: BTreeMap<DropClass, usize>,
172 had_inner_drop: bool,
173 offset_hint_eligible: bool,
174 offset_start_line: Option<usize>,
175 byte_truncated: bool,
176 disk_truncated_prefix_bytes: u64,
177 output_path: Option<String>,
178 stderr_path: Option<String>,
179 include_stderr_path: bool,
180 artifact_access: ArtifactRecoveryAccess,
181}
182
183fn optional_string_bytes(value: Option<&String>) -> u64 {
184 value
185 .map(|value| crate::memory::usize_to_u64(value.len()))
186 .unwrap_or(0)
187}
188
189fn terminal_output_cache_estimated_bytes(cache: &TerminalOutputCache) -> u64 {
190 let recovery_bytes = cache
191 .recovery
192 .as_ref()
193 .map(|recovery| {
194 crate::memory::usize_to_u64(recovery.dropped_by_class.len())
195 .saturating_mul(
196 (std::mem::size_of::<DropClass>() + std::mem::size_of::<usize>()) as u64,
197 )
198 .saturating_add(optional_string_bytes(recovery.output_path.as_ref()))
199 .saturating_add(optional_string_bytes(recovery.stderr_path.as_ref()))
200 .saturating_add(crate::memory::usize_to_u64(
201 recovery.artifact_access.task_id.len(),
202 ))
203 })
204 .unwrap_or(0);
205 (std::mem::size_of::<TerminalOutputCache>() as u64)
206 .saturating_add(crate::memory::usize_to_u64(cache.output_preview.len()))
207 .saturating_add(optional_string_bytes(cache.output_path.as_ref()))
208 .saturating_add(optional_string_bytes(cache.stderr_path.as_ref()))
209 .saturating_add(crate::memory::usize_to_u64(
210 cache.artifact_access.task_id.len(),
211 ))
212 .saturating_add(recovery_bytes)
213}
214
215fn completion_estimated_bytes(completion: &BgCompletion) -> u64 {
216 (std::mem::size_of::<BgCompletion>() as u64)
217 .saturating_add(crate::memory::usize_to_u64(completion.task_id.len()))
218 .saturating_add(crate::memory::usize_to_u64(completion.session_id.len()))
219 .saturating_add(crate::memory::usize_to_u64(completion.command.len()))
220 .saturating_add(crate::memory::usize_to_u64(completion.output_preview.len()))
221}
222
223impl RecoveryContext {
224 fn has_visible_drop(&self) -> bool {
225 self.byte_truncated
226 || self.disk_truncated_prefix_bytes > 0
227 || self.had_inner_drop
228 || !self.dropped_by_class.is_empty()
229 }
230}
231
232#[derive(Clone)]
233pub struct BgTaskRegistry {
234 pub(crate) inner: Arc<RegistryInner>,
235}
236
237pub(crate) struct RegistryInner {
238 pub(crate) tasks: Mutex<HashMap<String, Arc<BgTask>>>,
239 pub(crate) completions: Mutex<VecDeque<BgCompletion>>,
240 pub(crate) progress_sender: SharedProgressSender,
241 watchdog_started: AtomicBool,
242 pub(crate) shutdown: AtomicBool,
243 pub(crate) long_running_reminder_enabled: AtomicBool,
244 pub(crate) long_running_reminder_interval_ms: AtomicU64,
245 persisted_gc_started: AtomicBool,
246 #[cfg(test)]
247 persisted_gc_runs: AtomicU64,
248 pub(crate) compressor:
254 Mutex<Option<Box<dyn Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync>>>,
255 pub(crate) db_pool: RwLock<Option<Arc<Mutex<Connection>>>>,
256 pub(crate) db_harness: RwLock<Option<String>>,
257 pub(crate) compression_aggregates: Arc<CompressionAggregateCache>,
258 pub(crate) wake_tx: crossbeam_channel::Sender<()>,
259 pub(crate) wake_rx: crossbeam_channel::Receiver<()>,
260 pub(crate) watch_registry: Mutex<WatchRegistry>,
261 wait_detach_sessions: Mutex<HashSet<String>>,
262 active_wait_sessions: Mutex<HashMap<String, usize>>,
263 wait_registered_tasks: Mutex<HashMap<String, HashSet<String>>>,
264}
265
266pub(crate) struct BgTask {
267 pub(crate) task_id: String,
268 pub(crate) session_id: String,
269 delivery_session_id: String,
270 pub(crate) paths: TaskPaths,
271 artifact_root: PathBuf,
272 pub(crate) started: Instant,
273 pub(crate) last_reminder_at: Mutex<Option<Instant>>,
274 pub(crate) terminal_at: Mutex<Option<Instant>>,
275 pub(crate) state: Mutex<BgTaskState>,
276}
277
278pub(crate) enum TaskRuntime {
279 Piped(Option<Child>),
280 Pty(Option<PtyRuntime>),
281}
282
283pub(crate) struct BgTaskState {
284 pub(crate) metadata: PersistedTask,
285 pub(crate) runtime: TaskRuntime,
286 pub(crate) io_handles: Option<TaskIoHandles>,
289 pub(crate) detached: bool,
290 pub(crate) child_exit_observed: bool,
301 pub(crate) buffer: BgBuffer,
302 terminal_output_cache: Option<TerminalOutputCache>,
303 pub(crate) pending_terminal_override: Option<BgTaskStatus>,
305}
306
307fn completion_matches_session(completion: &BgCompletion, session_id: Option<&str>) -> bool {
308 session_id
309 .map(|session_id| completion.session_id == session_id)
310 .unwrap_or(true)
311}
312
313impl BgTaskRegistry {
314 pub fn new(progress_sender: SharedProgressSender) -> Self {
315 let (wake_tx, wake_rx) = crossbeam_channel::bounded(1);
316 Self {
317 inner: Arc::new(RegistryInner {
318 tasks: Mutex::new(HashMap::new()),
319 completions: Mutex::new(VecDeque::new()),
320 progress_sender,
321 watchdog_started: AtomicBool::new(false),
322 shutdown: AtomicBool::new(false),
323 long_running_reminder_enabled: AtomicBool::new(true),
324 long_running_reminder_interval_ms: AtomicU64::new(600_000),
325 persisted_gc_started: AtomicBool::new(false),
326 #[cfg(test)]
327 persisted_gc_runs: AtomicU64::new(0),
328 compressor: Mutex::new(None),
329 db_pool: RwLock::new(None),
330 db_harness: RwLock::new(None),
331 compression_aggregates: Arc::new(CompressionAggregateCache::default()),
332 wake_tx,
333 wake_rx,
334 watch_registry: Mutex::new(WatchRegistry::default()),
335 wait_detach_sessions: Mutex::new(HashSet::new()),
336 active_wait_sessions: Mutex::new(HashMap::new()),
337 wait_registered_tasks: Mutex::new(HashMap::new()),
338 }),
339 }
340 }
341
342 pub fn is_session_owned_artifact_path(&self, session_id: &str, path: &Path) -> bool {
350 let Ok(requested) = fs::canonicalize(path) else {
351 return false;
352 };
353 let Ok(tasks) = self.inner.tasks.lock() else {
354 return false;
355 };
356
357 tasks.values().any(|task| {
358 task.session_id == session_id
359 && [
360 &task.paths.stdout,
361 &task.paths.stderr,
362 &task.paths.exit,
363 &task.paths.pty,
364 ]
365 .into_iter()
366 .filter_map(|known| known.file_name())
367 .any(|name| task.artifact_root.join(name) == requested)
368 })
369 }
370
371 pub fn read_artifact_path(
372 &self,
373 session_id: &str,
374 path: &Path,
375 ) -> Option<Result<Vec<u8>, String>> {
376 let requested = fs::canonicalize(path).ok()?;
377 let tasks = self.inner.tasks.lock().ok()?;
378 let (task, artifact) = tasks.values().find_map(|task| {
379 if task.session_id != session_id {
380 return None;
381 }
382 TaskArtifact::ALL.into_iter().find_map(|artifact| {
383 let expected = task
384 .paths
385 .artifact_path(artifact)
386 .file_name()
387 .map(|name| task.artifact_root.join(name));
388 (expected.as_deref() == Some(requested.as_path()))
389 .then(|| (Arc::clone(task), artifact))
390 })
391 })?;
392 drop(tasks);
393 Some(self.read_artifact(&task.task_id, session_id, artifact))
394 }
395
396 pub fn read_artifact_range(
397 &self,
398 task_id: &str,
399 session_id: &str,
400 artifact: TaskArtifact,
401 offset: u64,
402 ) -> Result<(Vec<u8>, u64), String> {
403 validate_task_id(task_id).map_err(|error| error.to_string())?;
404 let task = self
405 .task_for_session(task_id, session_id)
406 .ok_or_else(|| "task_not_found".to_string())?;
407 let mut file = open_task_artifact(&task.paths, artifact)
408 .map_err(|error| format!("artifact_refused: {error}"))?;
409 let len = file
410 .len()
411 .map_err(|error| format!("artifact_refused: {error}"))?;
412 let start = offset.min(len);
413 let bytes = file
414 .read_range(start, len.saturating_sub(start))
415 .map_err(|error| format!("artifact_refused: {error}"))?;
416 Ok((bytes, len))
417 }
418
419 pub fn read_artifact(
420 &self,
421 task_id: &str,
422 session_id: &str,
423 artifact: TaskArtifact,
424 ) -> Result<Vec<u8>, String> {
425 validate_task_id(task_id).map_err(|error| error.to_string())?;
426 let task = self
427 .task_for_session(task_id, session_id)
428 .ok_or_else(|| "task_not_found".to_string())?;
429 let mut file = open_task_artifact(&task.paths, artifact)
430 .map_err(|error| format!("artifact_refused: {error}"))?;
431 file.read_all()
432 .map_err(|error| format!("artifact_refused: {error}"))
433 }
434
435 pub fn set_harness(&self, harness: Harness) {
436 if let Ok(mut slot) = self.inner.db_harness.write() {
437 *slot = Some(harness.storage_segment());
438 }
439 }
440
441 pub fn set_db_pool(&self, conn: Arc<Mutex<Connection>>) {
442 if let Ok(mut slot) = self.inner.db_pool.write() {
443 *slot = Some(conn);
444 }
445 self.inner.compression_aggregates.clear();
446 }
447
448 pub fn clear_db_pool(&self) {
449 if let Ok(mut slot) = self.inner.db_pool.write() {
450 *slot = None;
451 }
452 self.inner.compression_aggregates.clear();
453 }
454
455 pub(crate) fn compression_aggregate_cache(&self) -> Arc<CompressionAggregateCache> {
456 Arc::clone(&self.inner.compression_aggregates)
457 }
458
459 pub fn register_foreground_task(&self, session_id: &str, task_id: &str) {
460 if let Ok(mut tasks) = self.inner.wait_registered_tasks.lock() {
461 tasks
462 .entry(session_id.to_string())
463 .or_default()
464 .insert(task_id.to_string());
465 }
466 }
467
468 pub fn begin_wait_mode_session(&self, session_id: &str, task_id: &str) {
469 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
470 *active.entry(session_id.to_string()).or_insert(0) += 1;
471 }
472 self.register_foreground_task(session_id, task_id);
473 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
474 detach.remove(session_id);
475 }
476 }
477
478 pub fn unregister_foreground_task(&self, session_id: &str, task_id: &str) {
479 if let Ok(mut tasks) = self.inner.wait_registered_tasks.lock() {
480 if let Some(session_tasks) = tasks.get_mut(session_id) {
481 session_tasks.remove(task_id);
482 if session_tasks.is_empty() {
483 tasks.remove(session_id);
484 }
485 }
486 }
487 }
488
489 pub fn end_wait_mode_session(&self, session_id: &str, task_id: &str) {
490 let no_active_wait = if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
491 match active.get_mut(session_id) {
492 Some(count) if *count > 1 => *count -= 1,
493 Some(_) => {
494 active.remove(session_id);
495 }
496 None => {}
497 }
498 !active.contains_key(session_id)
499 } else {
500 false
501 };
502 self.unregister_foreground_task(session_id, task_id);
503 if no_active_wait {
504 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
505 detach.remove(session_id);
506 }
507 }
508 }
509
510 pub fn abort_inflight(&self, session_id: &str) -> Result<usize, String> {
514 let task_ids = self
515 .inner
516 .wait_registered_tasks
517 .lock()
518 .map(|mut tasks| tasks.remove(session_id).unwrap_or_default())
519 .map_err(|_| "wait registration lock poisoned".to_string())?;
520 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
521 active.remove(session_id);
522 }
523 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
524 detach.remove(session_id);
525 }
526
527 let mut killed = 0;
528 for task_id in task_ids {
529 let Some(task) = self.task_for_session(&task_id, session_id) else {
530 continue;
531 };
532 let is_terminal = task
533 .state
534 .lock()
535 .map(|state| state.metadata.status.is_terminal())
536 .map_err(|_| "background task lock poisoned".to_string())?;
537 if is_terminal {
538 continue;
539 }
540 let snapshot = self.kill_with_status_reason(
541 &task_id,
542 session_id,
543 BgTaskStatus::Killed,
544 Some("call_aborted".to_string()),
545 )?;
546 if snapshot.info.status == BgTaskStatus::Killed
547 && snapshot.info.status_reason.as_deref() == Some("call_aborted")
548 {
549 killed += 1;
550 }
551 }
552 Ok(killed)
553 }
554
555 pub fn signal_wait_mode_detach(&self, session_id: &str) -> bool {
556 let is_waiting = self
557 .inner
558 .active_wait_sessions
559 .lock()
560 .map(|active| active.get(session_id).copied().unwrap_or(0) > 0)
561 .unwrap_or(false);
562 if !is_waiting {
563 return false;
564 }
565 self.inner
566 .wait_detach_sessions
567 .lock()
568 .map(|mut detach| detach.insert(session_id.to_string()))
569 .unwrap_or(false)
570 }
571
572 pub fn active_wait_session_count(&self) -> usize {
575 self.inner
576 .active_wait_sessions
577 .lock()
578 .map(|active| active.len())
579 .unwrap_or(0)
580 }
581
582 pub fn take_wait_mode_detach(&self, session_id: &str) -> bool {
583 self.inner
584 .wait_detach_sessions
585 .lock()
586 .map(|mut detach| detach.remove(session_id))
587 .unwrap_or(false)
588 }
589
590 pub fn set_compressor<F>(&self, compressor: F)
595 where
596 F: Fn(&str, String) -> CompressionResult + Send + Sync + 'static,
597 {
598 self.set_compressor_with_exit_code(move |command, output, _exit_code| {
599 compressor(command, output)
600 });
601 }
602
603 pub fn set_compressor_with_exit_code<F>(&self, compressor: F)
604 where
605 F: Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync + 'static,
606 {
607 if let Ok(mut slot) = self.inner.compressor.lock() {
608 *slot = Some(Box::new(compressor));
609 }
610 }
611
612 pub(crate) fn compress_output(
615 &self,
616 command: &str,
617 output: String,
618 exit_code: Option<i32>,
619 ) -> CompressionResult {
620 let Ok(slot) = self.inner.compressor.lock() else {
621 return CompressionResult::new(output);
622 };
623 match slot.as_ref() {
624 Some(compressor) => compressor(command, output, exit_code),
625 None => CompressionResult::new(output),
626 }
627 }
628
629 fn ensure_terminal_output_cache(&self, task: &Arc<BgTask>) -> Option<TerminalOutputCache> {
630 let (metadata, buffer) = {
631 let state = task.state.lock().ok()?;
632 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
633 return None;
634 }
635 if let Some(cache) = state.terminal_output_cache.clone() {
636 return Some(cache);
637 }
638 (state.metadata.clone(), state.buffer.clone())
639 };
640
641 let mut cap_buffer = buffer.clone();
642 let disk_truncation = cap_buffer.enforce_terminal_cap();
643 let cache =
644 self.render_terminal_output(&metadata, &cap_buffer, disk_truncation, Some(&task.paths));
645 let mut state = task.state.lock().ok()?;
646 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
647 return None;
648 }
649 if let Some(existing) = state.terminal_output_cache.clone() {
650 return Some(existing);
651 }
652 state.terminal_output_cache = Some(cache.clone());
653 Some(cache)
654 }
655
656 fn render_terminal_output(
657 &self,
658 metadata: &PersistedTask,
659 buffer: &BgBuffer,
660 disk_truncation: DiskTruncation,
661 paths: Option<&TaskPaths>,
662 ) -> TerminalOutputCache {
663 let output_readable = buffer
664 .output_path()
665 .is_some_and(|path| self.is_session_owned_artifact_path(&metadata.session_id, &path));
666 let stderr_readable = buffer
667 .stderr_path()
668 .map(|path| self.is_session_owned_artifact_path(&metadata.session_id, path))
669 .unwrap_or(true);
670 let artifact_access = ArtifactRecoveryAccess {
671 task_id: metadata.task_id.clone(),
672 readable: output_readable && stderr_readable,
673 };
674
675 if metadata.mode == BgMode::Pty {
676 return TerminalOutputCache {
677 output_preview: String::new(),
678 output_truncated: false,
679 kind: TerminalOutputKind::Raw,
680 output_path: buffer.output_path().map(|path| path.display().to_string()),
681 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
682 artifact_access,
683 recovery: None,
684 };
685 }
686
687 let mut rendered = if let Some(structured) = render_structured_output(
688 &metadata.command,
689 buffer,
690 disk_truncation,
691 artifact_access.clone(),
692 ) {
693 structured
694 } else if !metadata.compressed {
695 render_raw_passthrough(buffer, disk_truncation, artifact_access)
696 } else {
697 let raw = buffer.read_combined_head_tail(
698 COMPRESS_INPUT_CAP_BYTES,
699 COMPRESS_INPUT_HEAD_BYTES,
700 COMPRESS_INPUT_TAIL_BYTES,
701 );
702 let compressed = self.compress_output(&metadata.command, raw.text, metadata.exit_code);
703 render_compressed_with_recovery(
704 buffer,
705 compressed,
706 raw.truncated,
707 disk_truncation,
708 artifact_access,
709 )
710 };
711 normalize_piped_display_output(&mut rendered.output_preview);
712 append_pipeline_warning(&mut rendered, metadata, paths);
713 rendered
714 }
715
716 fn snapshot_with_terminal_cache(
717 &self,
718 task: &Arc<BgTask>,
719 preview_bytes: usize,
720 ) -> BgTaskSnapshot {
721 let mut snapshot = task.snapshot(preview_bytes);
722 self.maybe_compress_snapshot(task, &mut snapshot);
723 snapshot
724 }
725
726 fn post_terminal_transition(&self, task: &Arc<BgTask>, emit_frame: bool) -> Result<(), String> {
727 let (metadata, buffer) = {
728 let state = task
729 .state
730 .lock()
731 .map_err(|_| "background task lock poisoned".to_string())?;
732 if !state.metadata.status.is_terminal() {
733 return Ok(());
734 }
735 (state.metadata.clone(), state.buffer.clone())
736 };
737
738 let cache = self.ensure_terminal_output_cache(task);
739 self.enqueue_completion_from_parts(
740 &metadata,
741 Some(&buffer),
742 None,
743 emit_frame,
744 cache.as_ref(),
745 );
746 self.retarget_pending_completion(&metadata.task_id, &task.delivery_session_id);
747 Ok(())
748 }
749
750 fn persist_task(&self, paths: &TaskPaths, metadata: &PersistedTask) -> std::io::Result<()> {
751 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
752 write_task_at(&task, metadata)?;
753 self.dual_write_task(paths, metadata);
754 Ok(())
755 }
756
757 fn update_task_metadata<F>(
758 &self,
759 paths: &TaskPaths,
760 update: F,
761 ) -> std::io::Result<PersistedTask>
762 where
763 F: FnOnce(&mut PersistedTask),
764 {
765 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
766 let metadata = update_task_at(&task, update)?;
767 self.dual_write_task(paths, &metadata);
768 Ok(metadata)
769 }
770
771 fn dual_write_task(&self, paths: &TaskPaths, metadata: &PersistedTask) {
772 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
773 let Some(pool) = pool else {
774 return;
775 };
776 let harness = self
777 .inner
778 .db_harness
779 .read()
780 .ok()
781 .and_then(|slot| slot.clone());
782 let Some(harness) = harness else {
783 crate::slog_warn!(
784 "dual-write bash_task to DB skipped for {}: harness not configured",
785 metadata.task_id
786 );
787 return;
788 };
789 let row = match metadata.to_bash_task_row(&harness, paths) {
790 Ok(row) => row,
791 Err(error) => {
792 crate::slog_warn!(
793 "dual-write bash_task to DB failed for {}: {}",
794 metadata.task_id,
795 error
796 );
797 return;
798 }
799 };
800 let conn = match pool.lock() {
801 Ok(conn) => conn,
802 Err(_) => {
803 crate::slog_warn!(
804 "dual-write bash_task to DB failed for {}: db mutex poisoned",
805 metadata.task_id
806 );
807 return;
808 }
809 };
810 if let Err(error) = crate::db::bash_tasks::upsert_bash_task(&conn, &row) {
811 crate::slog_warn!(
812 "dual-write bash_task to DB failed for {}: {}",
813 metadata.task_id,
814 error
815 );
816 }
817 }
818
819 fn delete_gc_task_from_db(&self, metadata: &PersistedTask) {
820 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
821 let Some(pool) = pool else {
822 return;
823 };
824 let harness = self
825 .inner
826 .db_harness
827 .read()
828 .ok()
829 .and_then(|slot| slot.clone());
830 let Some(harness) = harness else {
831 crate::slog_warn!(
832 "GC bash_task DB delete skipped for {}: harness not configured",
833 metadata.task_id
834 );
835 return;
836 };
837 let conn = match pool.lock() {
838 Ok(conn) => conn,
839 Err(_) => {
840 crate::slog_warn!(
841 "GC bash_task DB delete failed for {}: db mutex poisoned",
842 metadata.task_id
843 );
844 return;
845 }
846 };
847 if let Err(error) = crate::db::bash_tasks::delete_delivered_terminal_bash_task(
848 &conn,
849 &harness,
850 &metadata.session_id,
851 &metadata.task_id,
852 "persisted_gc_delivered_terminal",
853 ) {
854 crate::slog_warn!(
855 "GC bash_task DB delete failed for {}: {}",
856 metadata.task_id,
857 error
858 );
859 }
860 }
861
862 fn persisted_task_process_is_alive(metadata: &PersistedTask) -> bool {
863 let child_pid = metadata.child_pid;
864 let group_leader = metadata.pgid.and_then(|pid| u32::try_from(pid).ok());
865 child_pid
866 .into_iter()
867 .chain(group_leader)
868 .any(|pid| is_recorded_process_alive(pid, metadata.started_at))
869 }
870
871 fn db_has_live_process_for_task(&self, task_id: &str) -> bool {
872 let Some((harness, pool)) = self.db_harness_and_pool() else {
873 return false;
874 };
875 let Ok(conn) = pool.lock() else {
876 return false;
877 };
878 crate::db::bash_tasks::list_bash_tasks_by_id(&conn, &harness, task_id)
879 .map(|rows| {
880 rows.into_iter().any(|row| {
881 let started_at = u64::try_from(row.started_at).unwrap_or_default();
882 row.pid
883 .and_then(|pid| u32::try_from(pid).ok())
884 .into_iter()
885 .chain(row.pgid.and_then(|pid| u32::try_from(pid).ok()))
886 .any(|pid| is_recorded_process_alive(pid, started_at))
887 })
888 })
889 .unwrap_or(false)
890 }
891
892 fn db_harness_and_pool(&self) -> Option<(String, Arc<Mutex<Connection>>)> {
893 let pool = self
894 .inner
895 .db_pool
896 .read()
897 .ok()
898 .and_then(|slot| slot.clone())?;
899 let harness = self
900 .inner
901 .db_harness
902 .read()
903 .ok()
904 .and_then(|slot| slot.clone())?;
905 Some((harness, pool))
906 }
907
908 fn redeliver_pending_watches_for_session(&self, session_id: &str) -> usize {
909 let Some((harness, pool)) = self.db_harness_and_pool() else {
910 return 0;
911 };
912 let rows = {
913 let Ok(conn) = pool.lock() else {
914 return 0;
915 };
916 match crate::db::bash_watches::list_bash_pattern_watches_for_session(
917 &conn, &harness, session_id,
918 ) {
919 Ok(rows) => rows,
920 Err(error) => {
921 crate::slog_warn!(
922 "failed to load pending bash watches for session {session_id}: {error}"
923 );
924 return 0;
925 }
926 }
927 };
928 let mut delivered = 0;
929 for row in rows.into_iter().filter(|row| row.pending_match) {
930 let Some(match_text) = row.match_text else {
931 crate::slog_warn!(
932 "pending bash watch {}/{} has no match text",
933 row.task_id,
934 row.watch_id
935 );
936 continue;
937 };
938 let context = row.match_context.unwrap_or_else(|| match_text.clone());
939 if match_text == WATCH_TARGET_ERASED_TEXT {
940 self.emit_bash_watch_erased(session_id, &row.task_id, &row.watch_id);
941 } else {
942 self.emit_bash_pattern_match(
943 session_id,
944 PatternMatch {
945 watch_id: row.watch_id,
946 task_id: row.task_id,
947 match_text,
948 match_offset: row.match_offset.unwrap_or_default().max(0) as u64,
949 context,
950 once: row.once,
951 },
952 );
953 }
954 delivered += 1;
955 }
956 delivered
957 }
958
959 fn terminal_db_status_for_session(
960 &self,
961 session_id: &str,
962 task_id: &str,
963 storage_dir: &Path,
964 ) -> Option<BgTaskSnapshot> {
965 let (harness, pool) = self.db_harness_and_pool()?;
966 let conn = pool.lock().ok()?;
967 let row =
968 crate::db::bash_tasks::get_bash_task(&conn, &harness, session_id, task_id).ok()??;
969 if !task_bundle_is_absent(storage_dir, &row.session_id, &row.task_id) {
970 return None;
971 }
972 let metadata = PersistedTask::from(row.clone());
973 metadata
974 .status
975 .is_terminal()
976 .then(|| terminal_db_row_snapshot(row, metadata))
977 }
978
979 pub fn has_erased_watch_reference(&self, task_id: &str) -> bool {
980 let Some((harness, pool)) = self.db_harness_and_pool() else {
981 return false;
982 };
983 let Ok(conn) = pool.lock() else {
984 return false;
985 };
986 let watched =
987 crate::db::bash_watches::list_bash_pattern_watches_by_task_id(&conn, &harness, task_id)
988 .map(|rows| !rows.is_empty())
989 .unwrap_or(false);
990 if !watched {
991 return false;
992 }
993 crate::db::bash_tasks::list_bash_tasks_by_id(&conn, &harness, task_id)
994 .map(|rows| rows.is_empty())
995 .unwrap_or(false)
996 }
997
998 pub(crate) fn evaluate_erased_watch_targets(&self) {
999 let Some((harness, pool)) = self.db_harness_and_pool() else {
1000 return;
1001 };
1002 let notifications = {
1003 let Ok(conn) = pool.lock() else {
1004 return;
1005 };
1006 let rows = match crate::db::bash_watches::list_bash_pattern_watches(&conn, &harness) {
1007 Ok(rows) => rows,
1008 Err(error) => {
1009 crate::slog_warn!("failed to inspect bash watch targets: {error}");
1010 return;
1011 }
1012 };
1013 let mut notifications = Vec::new();
1014 for mut row in rows {
1015 let task_exists = match crate::db::bash_tasks::get_bash_task(
1016 &conn,
1017 &harness,
1018 &row.session_id,
1019 &row.task_id,
1020 ) {
1021 Ok(task) => task.is_some(),
1022 Err(error) => {
1023 crate::slog_warn!(
1024 "failed to inspect bash watch target {}: {error}",
1025 row.task_id
1026 );
1027 continue;
1028 }
1029 };
1030 if task_exists {
1031 continue;
1032 }
1033 let already_tombstoned = !row.scanning
1034 && row.pending_match
1035 && row.match_text.as_deref() == Some(WATCH_TARGET_ERASED_TEXT);
1036 if !already_tombstoned {
1037 row.scanning = false;
1038 row.pending_match = true;
1039 row.match_text = Some(WATCH_TARGET_ERASED_TEXT.to_string());
1040 row.match_offset = Some(0);
1041 row.match_context = Some(WATCH_TARGET_ERASED_CONTEXT.to_string());
1042 if let Err(error) =
1043 crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row)
1044 {
1045 crate::slog_warn!(
1046 "failed to terminalize erased bash watch {}/{}: {error}",
1047 row.task_id,
1048 row.watch_id
1049 );
1050 continue;
1051 }
1052 }
1053 notifications.push((row.session_id, row.task_id, row.watch_id));
1054 }
1055 notifications
1056 };
1057
1058 for (session_id, task_id, watch_id) in notifications {
1059 let should_emit = self
1060 .inner
1061 .watch_registry
1062 .lock()
1063 .map(|mut registry| registry.terminalize_erased_task(&task_id, &watch_id))
1064 .unwrap_or(false);
1065 if should_emit {
1066 self.emit_bash_watch_erased(&session_id, &task_id, &watch_id);
1067 }
1068 }
1069 }
1070
1071 fn persist_watch_registration(
1072 &self,
1073 session_id: &str,
1074 task_id: &str,
1075 watch_id: &str,
1076 pattern: &WatchPattern,
1077 once: bool,
1078 stdout_offset: u64,
1079 stderr_offset: u64,
1080 pty_offset: u64,
1081 ) {
1082 let Some((harness, pool)) = self.db_harness_and_pool() else {
1083 return;
1084 };
1085 let Ok(conn) = pool.lock() else {
1086 return;
1087 };
1088 let row = BashPatternWatchRow {
1089 harness,
1090 session_id: session_id.to_string(),
1091 task_id: task_id.to_string(),
1092 watch_id: watch_id.to_string(),
1093 pattern_kind: pattern.kind_name().to_string(),
1094 pattern: pattern.pattern_text().to_string(),
1095 once,
1096 created_at: unix_millis() as i64,
1097 stdout_offset: stdout_offset as i64,
1098 stderr_offset: stderr_offset as i64,
1099 pty_offset: pty_offset as i64,
1100 scanning: true,
1101 pending_match: false,
1102 match_text: None,
1103 match_offset: None,
1104 match_context: None,
1105 };
1106 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
1107 crate::slog_warn!(
1108 "persist bash_pattern_watch failed for {task_id}/{watch_id}: {error}"
1109 );
1110 }
1111 }
1112
1113 fn delete_persisted_watch(&self, session_id: &str, task_id: &str, watch_id: &str) {
1114 let Some((harness, pool)) = self.db_harness_and_pool() else {
1115 return;
1116 };
1117 let Ok(conn) = pool.lock() else {
1118 return;
1119 };
1120 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watch(
1121 &conn, &harness, session_id, task_id, watch_id,
1122 ) {
1123 crate::slog_warn!("delete bash_pattern_watch failed for {task_id}/{watch_id}: {error}");
1124 }
1125 }
1126
1127 fn delete_persisted_watches_for_task(&self, session_id: &str, task_id: &str) {
1128 let Some((harness, pool)) = self.db_harness_and_pool() else {
1129 return;
1130 };
1131 let Ok(conn) = pool.lock() else {
1132 return;
1133 };
1134 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
1135 &conn, &harness, session_id, task_id,
1136 ) {
1137 crate::slog_warn!("delete bash_pattern_watches for {task_id} failed: {error}");
1138 }
1139 }
1140
1141 fn persist_watch_match(
1142 &self,
1143 session_id: &str,
1144 task_id: &str,
1145 pattern_match: &PatternMatch,
1146 stdout_offset: u64,
1147 stderr_offset: u64,
1148 pty_offset: u64,
1149 ) {
1150 let Some((harness, pool)) = self.db_harness_and_pool() else {
1151 return;
1152 };
1153 let Ok(conn) = pool.lock() else {
1154 return;
1155 };
1156 let Ok(Some(mut row)) = crate::db::bash_watches::get_bash_pattern_watch(
1157 &conn,
1158 &harness,
1159 session_id,
1160 task_id,
1161 &pattern_match.watch_id,
1162 ) else {
1163 return;
1164 };
1165 row.stdout_offset = stdout_offset as i64;
1166 row.stderr_offset = stderr_offset as i64;
1167 row.pty_offset = pty_offset as i64;
1168 row.pending_match = true;
1169 row.match_text = Some(pattern_match.match_text.clone());
1170 row.match_offset = Some(pattern_match.match_offset as i64);
1171 row.match_context = Some(pattern_match.context.clone());
1172 if pattern_match.once {
1173 row.scanning = false;
1176 }
1177 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
1178 crate::slog_warn!(
1179 "persist bash_pattern_watch match failed for {}/{}: {error}",
1180 task_id,
1181 pattern_match.watch_id
1182 );
1183 }
1184 }
1185
1186 fn persist_task_watch_cursors(
1187 &self,
1188 session_id: &str,
1189 task_id: &str,
1190 stdout_offset: u64,
1191 stderr_offset: u64,
1192 pty_offset: u64,
1193 ) {
1194 let Some((harness, pool)) = self.db_harness_and_pool() else {
1195 return;
1196 };
1197 let Ok(conn) = pool.lock() else {
1198 return;
1199 };
1200 if let Err(error) = crate::db::bash_watches::update_watch_offsets_for_task(
1201 &conn,
1202 &harness,
1203 session_id,
1204 task_id,
1205 stdout_offset as i64,
1206 stderr_offset as i64,
1207 pty_offset as i64,
1208 ) {
1209 crate::slog_warn!("persist bash_pattern_watch cursors failed for {task_id}: {error}");
1210 }
1211 }
1212
1213 fn watch_stream_cursors(&self, task_id: &str) -> (u64, u64, u64) {
1214 let Ok(registry) = self.inner.watch_registry.lock() else {
1215 return (0, 0, 0);
1216 };
1217 let stdout = registry
1218 .file_cursor(&format!("{task_id}:stdout"))
1219 .unwrap_or(0);
1220 let stderr = registry
1221 .file_cursor(&format!("{task_id}:stderr"))
1222 .unwrap_or(0);
1223 let pty = registry.file_cursor(&format!("{task_id}:pty")).unwrap_or(0);
1224 (stdout, stderr, pty)
1225 }
1226
1227 fn ack_persisted_watches_for_task(&self, session_id: &str, task_id: &str, task_terminal: bool) {
1230 let Some((harness, pool)) = self.db_harness_and_pool() else {
1231 return;
1232 };
1233 let Ok(conn) = pool.lock() else {
1234 return;
1235 };
1236 if task_terminal {
1237 let _ = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
1238 &conn, &harness, session_id, task_id,
1239 );
1240 return;
1241 }
1242 let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
1243 &conn, &harness, session_id, task_id,
1244 ) else {
1245 return;
1246 };
1247 for mut row in rows {
1248 if row.once && (!row.scanning || row.pending_match) {
1249 let _ = crate::db::bash_watches::delete_bash_pattern_watch(
1250 &conn,
1251 &harness,
1252 session_id,
1253 task_id,
1254 &row.watch_id,
1255 );
1256 continue;
1257 }
1258 if row.pending_match {
1259 row.pending_match = false;
1260 row.match_text = None;
1261 row.match_offset = None;
1262 row.match_context = None;
1263 let _ = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row);
1264 }
1265 }
1266 }
1267
1268 pub fn record_scanner_report(
1269 &self,
1270 task_id: &str,
1271 session_id: &str,
1272 scanner_report: Vec<PermissionAsk>,
1273 ) -> Result<(), String> {
1274 if scanner_report.is_empty() {
1275 return Ok(());
1276 }
1277 let task = self.task_for_session(task_id, session_id).ok_or_else(|| {
1278 "background task not found while recording scanner report".to_string()
1279 })?;
1280 let metadata = {
1281 let mut state = task
1282 .state
1283 .lock()
1284 .map_err(|_| "background task lock poisoned".to_string())?;
1285 state.metadata.scanner_report = scanner_report;
1286 state.metadata.clone()
1287 };
1288 self.persist_task(&task.paths, &metadata)
1289 .map_err(|error| format!("failed to persist scanner report: {error}"))
1290 }
1291
1292 pub fn configure_long_running_reminders(&self, enabled: bool, interval_ms: u64) {
1293 self.inner
1294 .long_running_reminder_enabled
1295 .store(enabled, Ordering::SeqCst);
1296 self.inner
1297 .long_running_reminder_interval_ms
1298 .store(interval_ms, Ordering::SeqCst);
1299 }
1300
1301 #[cfg(unix)]
1302 #[allow(clippy::too_many_arguments)]
1303 pub fn spawn(
1304 &self,
1305 spawn_plan: SpawnPlan,
1306 command: &str,
1307 session_id: String,
1308 workdir: PathBuf,
1309 env: HashMap<String, String>,
1310 timeout: Option<Duration>,
1311 storage_dir: PathBuf,
1312 max_running: usize,
1313 notify_on_completion: bool,
1314 compressed: bool,
1315 project_root: Option<PathBuf>,
1316 ) -> Result<String, String> {
1317 self.spawn_with_shell(
1318 spawn_plan,
1319 command,
1320 super::BashShell::Bash,
1321 resolve_posix_shell(),
1322 session_id,
1323 workdir,
1324 env,
1325 timeout,
1326 storage_dir,
1327 max_running,
1328 notify_on_completion,
1329 compressed,
1330 project_root,
1331 )
1332 }
1333
1334 #[cfg(unix)]
1335 #[allow(clippy::too_many_arguments)]
1336 pub fn spawn_with_shell(
1337 &self,
1338 spawn_plan: SpawnPlan,
1339 command: &str,
1340 shell: super::BashShell,
1341 shell_path: PathBuf,
1342 session_id: String,
1343 workdir: PathBuf,
1344 env: HashMap<String, String>,
1345 timeout: Option<Duration>,
1346 storage_dir: PathBuf,
1347 max_running: usize,
1348 notify_on_completion: bool,
1349 compressed: bool,
1350 project_root: Option<PathBuf>,
1351 ) -> Result<String, String> {
1352 self.start_watchdog();
1353
1354 let running = self.running_count();
1355 if running >= max_running {
1356 #[cfg(unix)]
1357 if let Some(prepared) = spawn_plan.prepared_task() {
1358 let _ = delete_resolved_task(&prepared.resolved_task());
1359 }
1360 return Err(format!(
1361 "background bash task limit exceeded: {running} running (max {max_running})"
1362 ));
1363 }
1364
1365 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1366 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1367 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1368 (spawn_plan.clone(), prepared.resolved_task())
1369 } else {
1370 let task = allocate_task_layout(&storage_dir, &session_id)
1371 .map_err(|error| format!("failed to create background task layout: {error}"))?;
1372 let root = project_root.as_deref().unwrap_or(&workdir);
1373 let environment =
1374 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1375 let prepared = match crate::sandbox_spawn::prepare_task_payload(
1376 &task,
1377 command.as_bytes(),
1378 root,
1379 &workdir,
1380 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1381 &shell_path,
1382 &environment,
1383 ) {
1384 Ok(prepared) => prepared,
1385 Err(error) => {
1386 let _ = delete_resolved_task(&task);
1387 return Err(error);
1388 }
1389 };
1390 let task = prepared.resolved_task();
1391 (spawn_plan.with_prepared_task(prepared), task)
1392 };
1393 let task_id = task_layout.paths.task_id.clone();
1394 let paths = task_layout.paths.clone();
1395
1396 if self.task(&task_id).is_some() {
1397 let _ = delete_resolved_task(&task_layout);
1398 return Err("background task id collided with a live task".to_string());
1399 }
1400
1401 let mut metadata = PersistedTask::starting(
1402 task_id.clone(),
1403 session_id.clone(),
1404 command.to_string(),
1405 workdir.clone(),
1406 project_root,
1407 timeout_ms,
1408 notify_on_completion,
1409 compressed,
1410 );
1411 #[cfg(unix)]
1415 let capture_pipeline_status = {
1416 let pipeline = single_top_level_pipeline(command);
1417 let capture = !shell.is_powershell()
1418 && should_capture_pipeline_status(&spawn_plan, pipeline.is_some(), &shell_path);
1419 if capture {
1420 metadata.pipeline_segments = pipeline
1421 .as_ref()
1422 .map(|pipeline| {
1423 pipeline
1424 .segments
1425 .iter()
1426 .map(|segment| segment.label.clone())
1427 .collect()
1428 })
1429 .unwrap_or_default();
1430 }
1431 capture
1432 };
1433 #[cfg(windows)]
1434 let capture_pipeline_status = false;
1435 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1436 if let Err(error) = write_task_at(&task_layout, &metadata) {
1437 let _ = delete_resolved_task(&task_layout);
1438 return Err(format!(
1439 "failed to persist background task metadata: {error}"
1440 ));
1441 }
1442 self.dual_write_task(&paths, &metadata);
1443
1444 let mut io_handles =
1445 TaskIoHandles::create(&task_layout, BgMode::Pipes, capture_pipeline_status)
1446 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
1447 let child = match spawn_detached_child(
1448 &spawn_plan,
1449 command,
1450 shell,
1451 &shell_path,
1452 &paths,
1453 &workdir,
1454 &env,
1455 &mut io_handles,
1456 capture_pipeline_status,
1457 ) {
1458 Ok(child) => child,
1459 Err(error) => {
1460 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
1461 let _ = delete_task_bundle(&paths);
1462 return Err(error);
1463 }
1464 };
1465
1466 let child_pid = child.id();
1467 metadata.mark_running(child_pid, child_pid as i32);
1468 self.persist_task(&paths, &metadata)
1469 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1470
1471 let task = Arc::new(BgTask {
1472 task_id: task_id.clone(),
1473 delivery_session_id: session_id.clone(),
1474 session_id,
1475 paths: paths.clone(),
1476 artifact_root: canonical_artifact_root(&paths),
1477 started: Instant::now(),
1478 last_reminder_at: Mutex::new(None),
1479 terminal_at: Mutex::new(None),
1480 state: Mutex::new(BgTaskState {
1481 metadata,
1482 runtime: TaskRuntime::Piped(Some(child)),
1483 io_handles: Some(io_handles),
1484 detached: false,
1485 child_exit_observed: false,
1486 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
1487 terminal_output_cache: None,
1488 pending_terminal_override: None,
1489 }),
1490 });
1491
1492 self.inner
1493 .tasks
1494 .lock()
1495 .map_err(|_| "background task registry lock poisoned".to_string())?
1496 .insert(task_id.clone(), task);
1497
1498 Ok(task_id)
1499 }
1500
1501 #[allow(clippy::too_many_arguments)]
1502 pub fn spawn_pty(
1503 &self,
1504 spawn_plan: SpawnPlan,
1505 command: &str,
1506 session_id: String,
1507 workdir: PathBuf,
1508 env: HashMap<String, String>,
1509 timeout: Option<Duration>,
1510 storage_dir: PathBuf,
1511 max_running: usize,
1512 notify_on_completion: bool,
1513 compressed: bool,
1514 project_root: Option<PathBuf>,
1515 rows: u16,
1516 cols: u16,
1517 ) -> Result<String, String> {
1518 self.spawn_pty_with_shell(
1519 spawn_plan,
1520 command,
1521 super::BashShell::Bash,
1522 super::resolve_shell_path(true, super::BashShell::Bash)
1523 .expect("POSIX shell must resolve for bash PTY"),
1524 session_id,
1525 workdir,
1526 env,
1527 timeout,
1528 storage_dir,
1529 max_running,
1530 notify_on_completion,
1531 compressed,
1532 project_root,
1533 rows,
1534 cols,
1535 )
1536 }
1537
1538 #[allow(clippy::too_many_arguments)]
1539 pub fn spawn_pty_with_shell(
1540 &self,
1541 spawn_plan: SpawnPlan,
1542 command: &str,
1543 shell: super::BashShell,
1544 shell_path: PathBuf,
1545 session_id: String,
1546 workdir: PathBuf,
1547 env: HashMap<String, String>,
1548 timeout: Option<Duration>,
1549 storage_dir: PathBuf,
1550 max_running: usize,
1551 notify_on_completion: bool,
1552 compressed: bool,
1553 project_root: Option<PathBuf>,
1554 rows: u16,
1555 cols: u16,
1556 ) -> Result<String, String> {
1557 self.start_watchdog();
1558
1559 let running = self.running_count();
1560 if running >= max_running {
1561 #[cfg(unix)]
1562 if let Some(prepared) = spawn_plan.prepared_task() {
1563 let _ = delete_resolved_task(&prepared.resolved_task());
1564 }
1565 return Err(format!(
1566 "background bash task limit exceeded: {running} running (max {max_running})"
1567 ));
1568 }
1569
1570 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1571 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1572 #[cfg(unix)]
1573 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1574 (spawn_plan.clone(), prepared.resolved_task())
1575 } else {
1576 let task = allocate_task_layout(&storage_dir, &session_id)
1577 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1578 let root = project_root.as_deref().unwrap_or(&workdir);
1579 let environment =
1580 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1581 let prepared = match crate::sandbox_spawn::prepare_task_payload(
1582 &task,
1583 command.as_bytes(),
1584 root,
1585 &workdir,
1586 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1587 &shell_path,
1588 &environment,
1589 ) {
1590 Ok(prepared) => prepared,
1591 Err(error) => {
1592 let _ = delete_resolved_task(&task);
1593 return Err(error);
1594 }
1595 };
1596 let task = prepared.resolved_task();
1597 (spawn_plan.with_prepared_task(prepared), task)
1598 };
1599 #[cfg(windows)]
1600 let task_layout = allocate_task_layout(&storage_dir, &session_id)
1601 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1602 let task_id = task_layout.paths.task_id.clone();
1603 let paths = task_layout.paths.clone();
1604
1605 let mut metadata = PersistedTask::starting(
1606 task_id.clone(),
1607 session_id.clone(),
1608 command.to_string(),
1609 workdir.clone(),
1610 project_root,
1611 timeout_ms,
1612 notify_on_completion,
1613 compressed,
1614 );
1615 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1616 metadata.mode = BgMode::Pty;
1617 metadata.pty_rows = Some(rows);
1618 metadata.pty_cols = Some(cols);
1619 if let Err(error) = write_task_at(&task_layout, &metadata) {
1620 let _ = delete_resolved_task(&task_layout);
1621 return Err(format!(
1622 "failed to persist background task metadata: {error}"
1623 ));
1624 }
1625 self.dual_write_task(&paths, &metadata);
1626 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pty, false)
1627 .map_err(|error| format!("failed to pre-open PTY output handles: {error}"))?;
1628
1629 let runtime = match spawn_pty_for_command(
1630 &spawn_plan,
1631 &task_id,
1632 &session_id,
1633 command,
1634 shell,
1635 &shell_path,
1636 &paths,
1637 &workdir,
1638 &env,
1639 rows,
1640 cols,
1641 self.inner.wake_tx.clone(),
1642 &mut io_handles,
1643 ) {
1644 Ok(runtime) => runtime,
1645 Err(error) => {
1646 crate::slog_warn!(
1647 "failed to spawn PTY background bash task {task_id}; deleting partial bundle: {error}"
1648 );
1649 let _ = delete_task_bundle(&paths);
1650 return Err(error);
1651 }
1652 };
1653
1654 if let Some(child_pid) = runtime.child_pid {
1655 metadata.mark_running(child_pid, child_pid as i32);
1656 } else {
1657 metadata.status = BgTaskStatus::Running;
1658 metadata.pgid = None;
1659 }
1660 self.persist_task(&paths, &metadata)
1661 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1662
1663 let task = Arc::new(BgTask {
1664 task_id: task_id.clone(),
1665 delivery_session_id: session_id.clone(),
1666 session_id,
1667 paths: paths.clone(),
1668 artifact_root: canonical_artifact_root(&paths),
1669 started: Instant::now(),
1670 last_reminder_at: Mutex::new(None),
1671 terminal_at: Mutex::new(None),
1672 state: Mutex::new(BgTaskState {
1673 metadata,
1674 runtime: TaskRuntime::Pty(Some(runtime)),
1675 io_handles: Some(io_handles),
1676 detached: false,
1677 child_exit_observed: false,
1678 buffer: BgBuffer::registered(&paths, BgMode::Pty),
1679 terminal_output_cache: None,
1680 pending_terminal_override: None,
1681 }),
1682 });
1683
1684 self.inner
1685 .tasks
1686 .lock()
1687 .map_err(|_| "background task registry lock poisoned".to_string())?
1688 .insert(task_id.clone(), task);
1689
1690 Ok(task_id)
1691 }
1692
1693 #[cfg(windows)]
1694 #[allow(clippy::too_many_arguments)]
1695 pub fn spawn(
1696 &self,
1697 spawn_plan: SpawnPlan,
1698 command: &str,
1699 session_id: String,
1700 workdir: PathBuf,
1701 env: HashMap<String, String>,
1702 timeout: Option<Duration>,
1703 storage_dir: PathBuf,
1704 max_running: usize,
1705 notify_on_completion: bool,
1706 compressed: bool,
1707 project_root: Option<PathBuf>,
1708 ) -> Result<String, String> {
1709 self.spawn_with_shell(
1710 spawn_plan,
1711 command,
1712 super::BashShell::Bash,
1713 PathBuf::from("cmd.exe"),
1714 session_id,
1715 workdir,
1716 env,
1717 timeout,
1718 storage_dir,
1719 max_running,
1720 notify_on_completion,
1721 compressed,
1722 project_root,
1723 )
1724 }
1725
1726 #[cfg(windows)]
1727 #[allow(clippy::too_many_arguments)]
1728 pub fn spawn_with_shell(
1729 &self,
1730 spawn_plan: SpawnPlan,
1731 command: &str,
1732 shell: super::BashShell,
1733 shell_path: PathBuf,
1734 session_id: String,
1735 workdir: PathBuf,
1736 env: HashMap<String, String>,
1737 timeout: Option<Duration>,
1738 storage_dir: PathBuf,
1739 max_running: usize,
1740 notify_on_completion: bool,
1741 compressed: bool,
1742 project_root: Option<PathBuf>,
1743 ) -> Result<String, String> {
1744 self.start_watchdog();
1745
1746 let running = self.running_count();
1747 if running >= max_running {
1748 #[cfg(unix)]
1749 if let Some(prepared) = spawn_plan.prepared_task() {
1750 let _ = delete_resolved_task(&prepared.resolved_task());
1751 }
1752 return Err(format!(
1753 "background bash task limit exceeded: {running} running (max {max_running})"
1754 ));
1755 }
1756
1757 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1758 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1759 let task_layout = allocate_task_layout(&storage_dir, &session_id)
1760 .map_err(|error| format!("failed to create background task layout: {error}"))?;
1761 let task_id = task_layout.paths.task_id.clone();
1762 let paths = task_layout.paths.clone();
1763
1764 let mut metadata = PersistedTask::starting(
1765 task_id.clone(),
1766 session_id.clone(),
1767 command.to_string(),
1768 workdir.clone(),
1769 project_root,
1770 timeout_ms,
1771 notify_on_completion,
1772 compressed,
1773 );
1774 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1775 if let Err(error) = write_task_at(&task_layout, &metadata) {
1776 let _ = delete_resolved_task(&task_layout);
1777 return Err(format!(
1778 "failed to persist background task metadata: {error}"
1779 ));
1780 }
1781 self.dual_write_task(&paths, &metadata);
1782 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pipes, false)
1783 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
1784
1785 let child = match spawn_detached_child(
1786 &spawn_plan,
1787 command,
1788 shell,
1789 &shell_path,
1790 &paths,
1791 &workdir,
1792 &env,
1793 &mut io_handles,
1794 false,
1795 ) {
1796 Ok(child) => child,
1797 Err(error) => {
1798 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
1799 let _ = delete_task_bundle(&paths);
1800 return Err(error);
1801 }
1802 };
1803
1804 let child_pid = child.id();
1805 metadata.status = BgTaskStatus::Running;
1806 metadata.child_pid = Some(child_pid);
1807 metadata.pgid = None;
1808 self.persist_task(&paths, &metadata)
1809 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1810
1811 let task = Arc::new(BgTask {
1812 task_id: task_id.clone(),
1813 delivery_session_id: session_id.clone(),
1814 session_id,
1815 paths: paths.clone(),
1816 artifact_root: canonical_artifact_root(&paths),
1817 started: Instant::now(),
1818 last_reminder_at: Mutex::new(None),
1819 terminal_at: Mutex::new(None),
1820 state: Mutex::new(BgTaskState {
1821 metadata,
1822 runtime: TaskRuntime::Piped(Some(child)),
1823 io_handles: Some(io_handles),
1824 detached: false,
1825 child_exit_observed: false,
1826 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
1827 terminal_output_cache: None,
1828 pending_terminal_override: None,
1829 }),
1830 });
1831
1832 self.inner
1833 .tasks
1834 .lock()
1835 .map_err(|_| "background task registry lock poisoned".to_string())?
1836 .insert(task_id.clone(), task);
1837
1838 Ok(task_id)
1839 }
1840
1841 pub fn write_pty(
1842 &self,
1843 task_id: &str,
1844 session_id: &str,
1845 input: &[u8],
1846 ) -> Result<usize, String> {
1847 let task = self
1848 .task_for_session(task_id, session_id)
1849 .ok_or_else(|| "task_not_found".to_string())?;
1850
1851 let writer = {
1852 let state = task
1853 .state
1854 .lock()
1855 .map_err(|_| "background task lock poisoned".to_string())?;
1856 if state.metadata.mode != BgMode::Pty {
1857 return Err("task_not_pty".to_string());
1858 }
1859 if state.metadata.status.is_terminal() {
1860 return Err("task_exited".to_string());
1861 }
1862 match &state.runtime {
1863 TaskRuntime::Pty(Some(runtime)) => Arc::clone(&runtime.writer),
1864 TaskRuntime::Pty(None) => return Err("task_exited".to_string()),
1865 TaskRuntime::Piped(_) => return Err("task_not_pty".to_string()),
1866 }
1867 };
1868
1869 let mut writer = writer
1870 .lock()
1871 .map_err(|_| "PTY writer lock poisoned".to_string())?;
1872 writer
1873 .write_all(input)
1874 .map_err(|error| format!("failed to write to PTY: {error}"))?;
1875 writer
1876 .flush()
1877 .map_err(|error| format!("failed to flush PTY writer: {error}"))?;
1878 Ok(input.len())
1879 }
1880
1881 pub fn replay_session(&self, storage_dir: &Path, session_id: &str) -> Result<(), String> {
1882 self.replay_session_inner(storage_dir, session_id, None)
1883 }
1884
1885 pub fn replay_session_for_project(
1886 &self,
1887 storage_dir: &Path,
1888 session_id: &str,
1889 project_root: &Path,
1890 ) -> Result<(), String> {
1891 self.replay_session_inner(storage_dir, session_id, Some(project_root))
1892 }
1893
1894 fn replay_session_inner(
1895 &self,
1896 storage_dir: &Path,
1897 session_id: &str,
1898 project_root: Option<&Path>,
1899 ) -> Result<(), String> {
1900 self.start_watchdog();
1901 if !self.inner.persisted_gc_started.swap(true, Ordering::SeqCst) {
1902 if let Err(error) = self.maybe_gc_persisted(storage_dir) {
1903 crate::slog_warn!("failed to GC persisted background bash tasks: {error}");
1904 }
1905 }
1906
1907 let canonical_project = project_root.map(canonicalized_path);
1908 let tasks = match self.replay_session_from_db(session_id, project_root) {
1920 Some(Ok(tasks)) if !tasks.is_empty() => tasks,
1921 Some(Ok(_)) => {
1922 let disk_tasks = self.replay_session_from_disk(storage_dir, session_id)?;
1923 if !disk_tasks.is_empty() {
1924 crate::slog_info!(
1925 "bash task replay: 0 in DB for session {}, {} from disk fallback",
1926 session_id,
1927 disk_tasks.len()
1928 );
1929 }
1930 disk_tasks
1931 }
1932 Some(Err(error)) => {
1933 crate::slog_warn!(
1934 "bash task replay DB lookup failed for session {}; falling back to disk: {}",
1935 session_id,
1936 error
1937 );
1938 self.replay_session_from_disk(storage_dir, session_id)?
1939 }
1940 None => {
1941 self.replay_session_from_disk(storage_dir, session_id)?
1943 }
1944 };
1945
1946 for mut metadata in tasks {
1947 if project_root.is_none() && metadata.session_id != session_id {
1948 continue;
1949 }
1950 if let Some(canonical_project) = canonical_project.as_deref() {
1951 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
1952 if metadata_project.as_deref() != Some(canonical_project) {
1953 continue;
1954 }
1955 }
1956
1957 if validate_task_id(&metadata.task_id).is_err() {
1958 crate::slog_warn!(
1959 "ignoring persisted background task with invalid id {:?}",
1960 metadata.task_id
1961 );
1962 continue;
1963 }
1964 if self.task(&metadata.task_id).is_some() {
1968 continue;
1969 }
1970 let session_dir = session_tasks_dir(storage_dir, &metadata.session_id);
1971 let resolved = match resolve_task_layout(&session_dir, &metadata.task_id) {
1972 Ok(task) => task,
1973 Err(error) => {
1974 if Self::persisted_task_process_is_alive(&metadata) {
1975 crate::slog_warn!(
1976 "refusing to quarantine unresolved live background task {}: {error}",
1977 metadata.task_id
1978 );
1979 continue;
1980 }
1981 crate::slog_warn!(
1982 "quarantining unresolved background task {}: {error}",
1983 metadata.task_id
1984 );
1985 let _ = quarantine_task_layout(
1986 storage_dir,
1987 &session_dir,
1988 &metadata.task_id,
1989 "invalid",
1990 );
1991 continue;
1992 }
1993 };
1994 match read_task_at(&resolved) {
1995 Ok(disk)
1996 if disk.task_id == metadata.task_id
1997 && disk.session_id == metadata.session_id => {}
1998 Ok(_) | Err(_) => {
1999 if Self::persisted_task_process_is_alive(&metadata) {
2000 crate::slog_warn!(
2001 "refusing to quarantine mismatched live background task {}",
2002 metadata.task_id
2003 );
2004 continue;
2005 }
2006 let _ = quarantine_task_layout(
2007 storage_dir,
2008 &session_dir,
2009 &metadata.task_id,
2010 "mismatch",
2011 );
2012 continue;
2013 }
2014 }
2015 let paths = resolved.paths;
2016 let replay_task_id = metadata.task_id.clone();
2017 let delivery_session_id = (metadata.session_id != session_id).then_some(session_id);
2018 match metadata.status {
2019 BgTaskStatus::Starting => {
2020 let completion_was_delivered = metadata.completion_delivered;
2021 metadata.mark_terminal(
2022 BgTaskStatus::Failed,
2023 None,
2024 Some("spawn aborted".to_string()),
2025 );
2026 metadata.completion_delivered |= completion_was_delivered;
2027 let _ = self.persist_task(&paths, &metadata);
2028 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
2029 self.insert_rehydrated_task(metadata, paths, true, delivery_session_id)?;
2030 }
2031 BgTaskStatus::Running | BgTaskStatus::Killing => {
2032 if metadata.mode == BgMode::Pty {
2033 if let Ok(Some(marker)) = read_exit_marker(&paths) {
2034 let completion_was_delivered = metadata.completion_delivered;
2035 metadata = terminal_metadata_from_marker(metadata, marker, None);
2036 metadata.completion_delivered |= completion_was_delivered;
2037 let _ = self.persist_task(&paths, &metadata);
2038 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
2039 self.insert_rehydrated_task(
2040 metadata,
2041 paths,
2042 true,
2043 delivery_session_id,
2044 )?;
2045 } else if metadata.status.is_terminal() {
2046 self.insert_rehydrated_task(
2047 metadata,
2048 paths,
2049 true,
2050 delivery_session_id,
2051 )?;
2052 } else {
2053 let completion_was_delivered = metadata.completion_delivered;
2054 metadata.mark_terminal(
2055 BgTaskStatus::Killed,
2056 None,
2057 Some("pty_lost_on_bridge_restart".to_string()),
2058 );
2059 metadata.completion_delivered |= completion_was_delivered;
2060 let _ = self.persist_task(&paths, &metadata);
2061 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
2062 self.insert_rehydrated_task(
2063 metadata,
2064 paths,
2065 true,
2066 delivery_session_id,
2067 )?;
2068 }
2069 } else if let Ok(Some(marker)) = read_exit_marker(&paths) {
2070 let reason = (metadata.status == BgTaskStatus::Killing).then(|| {
2071 "recovered from inconsistent killing state on replay".to_string()
2072 });
2073 if reason.is_some() {
2074 crate::slog_warn!("background task {} had killing state with exit marker; preferring marker",
2075 metadata.task_id);
2076 }
2077 let completion_was_delivered = metadata.completion_delivered;
2078 metadata = terminal_metadata_from_marker(metadata, marker, reason);
2079 metadata.completion_delivered |= completion_was_delivered;
2080 let _ = self.persist_task(&paths, &metadata);
2081 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
2082 self.insert_rehydrated_task(metadata, paths, true, delivery_session_id)?;
2083 } else if metadata.status == BgTaskStatus::Killing {
2084 let _ = write_kill_marker_if_absent(&paths);
2085 let completion_was_delivered = metadata.completion_delivered;
2086 metadata.mark_terminal(
2087 BgTaskStatus::Killed,
2088 None,
2089 Some("recovered from inconsistent killing state on replay".to_string()),
2090 );
2091 metadata.completion_delivered |= completion_was_delivered;
2092 let _ = self.persist_task(&paths, &metadata);
2093 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
2094 self.insert_rehydrated_task(metadata, paths, true, delivery_session_id)?;
2095 } else if Self::persisted_task_process_is_alive(&metadata) {
2096 self.insert_rehydrated_task(metadata, paths, true, delivery_session_id)?;
2097 } else {
2098 let completion_was_delivered = metadata.completion_delivered;
2099 metadata.mark_terminal(
2100 BgTaskStatus::FateUnknown,
2101 None,
2102 Some(restart_fate_unknown_reason(&metadata, &paths)),
2103 );
2104 metadata.completion_delivered |= completion_was_delivered;
2105 let _ = self.persist_task(&paths, &metadata);
2106 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
2107 self.insert_rehydrated_task(metadata, paths, true, delivery_session_id)?;
2108 }
2109 }
2110 _ if metadata.status.is_terminal() => {
2111 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
2117 self.insert_rehydrated_task(metadata, paths, true, delivery_session_id)?;
2118 }
2119 _ => {}
2120 }
2121 self.retarget_pending_completion(&replay_task_id, session_id);
2122 }
2123
2124 Ok(())
2125 }
2126
2127 fn replay_session_from_db(
2128 &self,
2129 session_id: &str,
2130 project_root: Option<&Path>,
2131 ) -> Option<Result<Vec<PersistedTask>, String>> {
2132 let pool = self
2133 .inner
2134 .db_pool
2135 .read()
2136 .ok()
2137 .and_then(|slot| slot.clone())?;
2138 let harness = self
2139 .inner
2140 .db_harness
2141 .read()
2142 .ok()
2143 .and_then(|slot| slot.clone())?;
2144 let conn = match pool.lock() {
2145 Ok(conn) => conn,
2146 Err(_) => return Some(Err("db mutex poisoned".to_string())),
2147 };
2148 let rows = if let Some(project_root) = project_root {
2149 let project_key = crate::path_identity::project_scope_key(project_root);
2150 crate::db::bash_tasks::list_replayable_bash_tasks_for_project(
2151 &conn,
2152 &harness,
2153 &project_key,
2154 )
2155 } else {
2156 crate::db::bash_tasks::list_bash_tasks_for_session(&conn, &harness, session_id)
2157 };
2158 Some(
2159 rows.map(|rows| rows.into_iter().map(PersistedTask::from).collect())
2160 .map_err(|error| error.to_string()),
2161 )
2162 }
2163
2164 fn replay_session_from_disk(
2165 &self,
2166 storage_dir: &Path,
2167 session_id: &str,
2168 ) -> Result<Vec<PersistedTask>, String> {
2169 let dir = session_tasks_dir(storage_dir, session_id);
2170 if !dir.exists() {
2171 return Ok(Vec::new());
2172 }
2173
2174 let (task_ids, invalid_entries) = discover_task_ids(&dir)
2175 .map_err(|error| format!("failed to discover background task layouts: {error}"))?;
2176 for entry in invalid_entries {
2177 if let Err(error) = quarantine_invalid_entry(storage_dir, &dir, &entry) {
2178 crate::slog_warn!(
2179 "failed to quarantine invalid background task entry {:?}: {error}",
2180 entry
2181 );
2182 }
2183 }
2184
2185 let mut tasks = Vec::new();
2186 for task_id in task_ids {
2187 let task = match resolve_task_layout(&dir, &task_id) {
2188 Ok(task) => task,
2189 Err(error)
2190 if error.kind() == std::io::ErrorKind::NotFound
2191 && uninitialized_layout_is_recent(
2192 &dir,
2193 &task_id,
2194 Duration::from_secs(5 * 60),
2195 )
2196 .unwrap_or(false) =>
2197 {
2198 continue;
2199 }
2200 Err(error) => {
2201 if self.db_has_live_process_for_task(&task_id) {
2202 crate::slog_warn!(
2203 "refusing to quarantine unresolved live background task {task_id} during replay: {error}"
2204 );
2205 continue;
2206 }
2207 crate::slog_warn!(
2208 "quarantining unresolved background task {task_id} during replay: {error}"
2209 );
2210 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
2211 continue;
2212 }
2213 };
2214 match read_task_at(&task) {
2215 Ok(metadata) if metadata.session_id == session_id => tasks.push(metadata),
2216 Ok(_) => {
2217 crate::slog_warn!(
2218 "quarantining background task {task_id} with mismatched session metadata"
2219 );
2220 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "mismatch");
2221 }
2222 Err(error) => {
2223 if self.db_has_live_process_for_task(&task_id) {
2224 crate::slog_warn!(
2225 "refusing to quarantine unreadable live background task {task_id} during replay: {error}"
2226 );
2227 continue;
2228 }
2229 crate::slog_warn!(
2230 "quarantining invalid background task metadata {task_id} during replay: {error}"
2231 );
2232 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
2233 }
2234 }
2235 }
2236 Ok(tasks)
2237 }
2238
2239 pub fn register_watch(
2240 &self,
2241 task_id: String,
2242 pattern: WatchPattern,
2243 once: bool,
2244 ) -> Result<String, &'static str> {
2245 let task = self.task(&task_id).ok_or("task_not_found")?;
2246 validate_task_id(&task_id).map_err(|_| "invalid_task_id")?;
2247 let (mode, terminal_at_registration) = task
2248 .state
2249 .lock()
2250 .map(|state| {
2251 (
2252 state.metadata.mode.clone(),
2253 state.metadata.status.is_terminal(),
2254 )
2255 })
2256 .map_err(|_| "background_task_lock_poisoned")?;
2257 let mut stdout = (mode == BgMode::Pipes)
2258 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
2259 .transpose()
2260 .map_err(|_| "artifact_refused")?;
2261 let mut stderr = (mode == BgMode::Pipes)
2262 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
2263 .transpose()
2264 .map_err(|_| "artifact_refused")?;
2265 let mut pty = (mode == BgMode::Pty)
2266 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
2267 .transpose()
2268 .map_err(|_| "artifact_refused")?;
2269
2270 let mut terminal_matches = Vec::new();
2271 let scanned_terminal = terminal_at_registration;
2272 let watch_id = {
2273 let mut registry = self
2274 .inner
2275 .watch_registry
2276 .lock()
2277 .map_err(|_| "watch_registry_poisoned")?;
2278 let watch_id = registry.register(task_id.clone(), pattern.clone(), once)?;
2279 match &mode {
2280 BgMode::Pipes => {
2281 let stdout_key = format!("{task_id}:stdout");
2282 let stderr_key = format!("{task_id}:stderr");
2283 if terminal_at_registration {
2284 registry.set_file_cursor(&stdout_key, 0);
2285 registry.set_file_cursor(&stderr_key, 0);
2286 terminal_matches.extend(registry.scan_file_new_bytes(
2287 &stdout_key,
2288 &task_id,
2289 stdout.as_mut().expect("pipe stdout opened"),
2290 ));
2291 terminal_matches.extend(registry.scan_file_new_bytes(
2292 &stderr_key,
2293 &task_id,
2294 stderr.as_mut().expect("pipe stderr opened"),
2295 ));
2296 } else {
2297 registry.prime_file_cursor(
2298 &stdout_key,
2299 stdout.as_ref().expect("pipe stdout opened"),
2300 );
2301 registry.prime_file_cursor(
2302 &stderr_key,
2303 stderr.as_ref().expect("pipe stderr opened"),
2304 );
2305 }
2306 }
2307 BgMode::Pty => {
2308 let pty_key = format!("{task_id}:pty");
2309 if terminal_at_registration {
2310 registry.set_file_cursor(&pty_key, 0);
2311 terminal_matches.extend(registry.scan_file_new_bytes(
2312 &pty_key,
2313 &task_id,
2314 pty.as_mut().expect("PTY artifact opened"),
2315 ));
2316 } else {
2317 registry.prime_file_cursor(
2318 &pty_key,
2319 pty.as_ref().expect("PTY artifact opened"),
2320 );
2321 }
2322 }
2323 }
2324 watch_id
2325 };
2326
2327 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
2328 self.persist_watch_registration(
2329 &task.session_id,
2330 &task_id,
2331 &watch_id,
2332 &pattern,
2333 once,
2334 stdout_offset,
2335 stderr_offset,
2336 pty_offset,
2337 );
2338
2339 if task.is_terminal() {
2340 if !scanned_terminal {
2341 terminal_matches = {
2342 let mut registry = self
2343 .inner
2344 .watch_registry
2345 .lock()
2346 .map_err(|_| "watch_registry_poisoned")?;
2347 match &mode {
2348 BgMode::Pipes => {
2349 let stdout_key = format!("{task_id}:stdout");
2350 let stderr_key = format!("{task_id}:stderr");
2351 registry.set_file_cursor(&stdout_key, 0);
2352 registry.set_file_cursor(&stderr_key, 0);
2353 let mut matches = registry.scan_file_new_bytes(
2354 &stdout_key,
2355 &task_id,
2356 stdout.as_mut().expect("pipe stdout opened"),
2357 );
2358 matches.extend(registry.scan_file_new_bytes(
2359 &stderr_key,
2360 &task_id,
2361 stderr.as_mut().expect("pipe stderr opened"),
2362 ));
2363 matches
2364 }
2365 BgMode::Pty => {
2366 let pty_key = format!("{task_id}:pty");
2367 registry.set_file_cursor(&pty_key, 0);
2368 registry.scan_file_new_bytes(
2369 &pty_key,
2370 &task_id,
2371 pty.as_mut().expect("PTY artifact opened"),
2372 )
2373 }
2374 }
2375 };
2376 }
2377
2378 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
2379 let (watch_controlled, watch_matched) = self.task_watch_state(&task_id);
2380 if terminal_matches.is_empty() && (!watch_controlled || watch_matched) {
2381 if watch_matched {
2382 let _ = task.set_completion_delivered(true, self);
2383 self.clear_task_watch_state(&task_id);
2384 self.delete_persisted_watches_for_task(&task.session_id, &task_id);
2386 }
2387 return Ok(watch_id);
2388 }
2389
2390 let completion = self
2391 .remove_pending_completion(&task_id)
2392 .or_else(|| self.completion_snapshot_for_task(&task));
2393 if terminal_matches.is_empty() {
2394 if let Some(completion) = completion.as_ref() {
2395 self.emit_bash_watch_exit(completion);
2396 }
2397 } else {
2398 for pattern_match in &terminal_matches {
2399 self.persist_watch_match(
2400 &task.session_id,
2401 &task_id,
2402 pattern_match,
2403 stdout_offset,
2404 stderr_offset,
2405 pty_offset,
2406 );
2407 self.emit_bash_pattern_match(&task.delivery_session_id, pattern_match.clone());
2408 }
2409 }
2410 let _ = task.set_completion_delivered(true, self);
2411 self.clear_task_watch_state(&task_id);
2412 self.delete_persisted_watches_for_task(&task.session_id, &task_id);
2414 }
2415
2416 Ok(watch_id)
2417 }
2418
2419 pub fn unregister_watch(&self, task_id: &str, watch_id: &str) {
2420 let session_id = self.task(task_id).map(|task| task.session_id.clone());
2421 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2422 registry.unregister(task_id, watch_id);
2423 }
2424 if let Some(session_id) = session_id {
2425 self.delete_persisted_watch(&session_id, task_id, watch_id);
2426 }
2427 }
2428
2429 pub fn active_watch_count(&self, task_id: &str) -> usize {
2430 self.inner
2431 .watch_registry
2432 .lock()
2433 .map(|registry| registry.active_count(task_id))
2434 .unwrap_or(0)
2435 }
2436
2437 fn task_watch_state(&self, task_id: &str) -> (bool, bool) {
2438 self.inner
2439 .watch_registry
2440 .lock()
2441 .map(|registry| {
2442 (
2443 registry.has_controlled_task(task_id),
2444 registry.has_matched_task(task_id),
2445 )
2446 })
2447 .unwrap_or((false, false))
2448 }
2449
2450 fn task_has_watch_control(&self, task_id: &str) -> bool {
2451 self.inner
2452 .watch_registry
2453 .lock()
2454 .map(|registry| registry.has_controlled_task(task_id))
2455 .unwrap_or(false)
2456 }
2457
2458 fn clear_task_watch_state(&self, task_id: &str) {
2459 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2460 registry.clear_task(task_id);
2461 }
2462 }
2463
2464 pub(crate) fn scan_task_watch_output(&self, task: &Arc<BgTask>) {
2465 let mode = match task.state.lock() {
2466 Ok(state) => state.metadata.mode.clone(),
2467 Err(_) => return,
2468 };
2469 let mut stdout = (mode == BgMode::Pipes)
2470 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
2471 .transpose()
2472 .ok()
2473 .flatten();
2474 let mut stderr = (mode == BgMode::Pipes)
2475 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
2476 .transpose()
2477 .ok()
2478 .flatten();
2479 let mut pty = (mode == BgMode::Pty)
2480 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
2481 .transpose()
2482 .ok()
2483 .flatten();
2484 let mut matches = Vec::new();
2485 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2486 match mode {
2487 BgMode::Pipes => {
2488 let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) else {
2489 return;
2490 };
2491 let stdout_key = format!("{}:stdout", task.task_id);
2492 let stderr_key = format!("{}:stderr", task.task_id);
2493 matches.extend(registry.scan_file_new_bytes(
2494 &stdout_key,
2495 &task.task_id,
2496 stdout,
2497 ));
2498 matches.extend(registry.scan_file_new_bytes(
2499 &stderr_key,
2500 &task.task_id,
2501 stderr,
2502 ));
2503 }
2504 BgMode::Pty => {
2505 let Some(pty) = pty.as_mut() else {
2506 return;
2507 };
2508 let pty_key = format!("{}:pty", task.task_id);
2509 matches.extend(registry.scan_file_new_bytes(&pty_key, &task.task_id, pty));
2510 }
2511 }
2512 }
2513 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
2514 if matches.is_empty() {
2515 if self.task_has_watch_control(&task.task_id) {
2518 self.persist_task_watch_cursors(
2519 &task.session_id,
2520 &task.task_id,
2521 stdout_offset,
2522 stderr_offset,
2523 pty_offset,
2524 );
2525 }
2526 return;
2527 }
2528 for pattern_match in matches {
2529 self.persist_watch_match(
2530 &task.session_id,
2531 &task.task_id,
2532 &pattern_match,
2533 stdout_offset,
2534 stderr_offset,
2535 pty_offset,
2536 );
2537 self.emit_bash_pattern_match(&task.delivery_session_id, pattern_match);
2538 }
2539 self.persist_task_watch_cursors(
2540 &task.session_id,
2541 &task.task_id,
2542 stdout_offset,
2543 stderr_offset,
2544 pty_offset,
2545 );
2546 }
2547
2548 pub fn status(
2549 &self,
2550 task_id: &str,
2551 session_id: &str,
2552 project_root: Option<&Path>,
2553 storage_dir: Option<&Path>,
2554 preview_bytes: usize,
2555 ) -> Option<BgTaskSnapshot> {
2556 validate_task_id(task_id).ok()?;
2557 let terminal_db_fallback_allowed = storage_dir
2558 .is_some_and(|storage_dir| task_bundle_is_absent(storage_dir, session_id, task_id));
2559 let mut task = self.task_for_session(task_id, session_id);
2560 if task.is_none() {
2561 if let Some(storage_dir) = storage_dir {
2562 let _ = if let Some(project_root) = project_root {
2563 self.replay_session_for_project(storage_dir, session_id, project_root)
2564 } else {
2565 self.replay_session(storage_dir, session_id)
2566 };
2567 task = self.task_for_session(task_id, session_id);
2568 }
2569 }
2570 let Some(task) = task else {
2571 if terminal_db_fallback_allowed {
2572 if let Some(snapshot) = storage_dir.and_then(|storage_dir| {
2573 self.terminal_db_status_for_session(session_id, task_id, storage_dir)
2574 }) {
2575 return Some(snapshot);
2576 }
2577 }
2578 return self.status_relaxed(
2579 task_id,
2580 session_id,
2581 project_root?,
2582 storage_dir?,
2583 preview_bytes,
2584 terminal_db_fallback_allowed,
2585 );
2586 };
2587 let _ = self.poll_task(&task);
2588 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
2589 }
2590
2591 fn status_relaxed_task(
2592 &self,
2593 task_id: &str,
2594 project_root: &Path,
2595 storage_dir: &Path,
2596 ) -> Option<Arc<BgTask>> {
2597 validate_task_id(task_id).ok()?;
2598 let canonical_project = canonicalized_path(project_root);
2599 match self.lookup_relaxed_task_from_db(task_id, project_root) {
2600 Some(Ok(Some(row))) => {
2601 let metadata = PersistedTask::from(row);
2602 if let Some(task) = self.task(task_id) {
2603 let matches_project = task
2604 .state
2605 .lock()
2606 .map(|state| {
2607 state
2608 .metadata
2609 .project_root
2610 .as_deref()
2611 .map(canonicalized_path)
2612 .as_deref()
2613 == Some(canonical_project.as_path())
2614 })
2615 .unwrap_or(false);
2616 return matches_project.then_some(task);
2617 }
2618 let resolved = resolve_task_layout(
2619 &session_tasks_dir(storage_dir, &metadata.session_id),
2620 &metadata.task_id,
2621 )
2622 .ok()?;
2623 let disk = read_task_at(&resolved).ok()?;
2624 if disk.task_id != metadata.task_id || disk.session_id != metadata.session_id {
2625 return None;
2626 }
2627 if self
2628 .insert_rehydrated_task(metadata, resolved.paths, true, None)
2629 .is_err()
2630 {
2631 return None;
2632 }
2633 return self.task(task_id);
2634 }
2635 Some(Ok(None)) => {
2636 crate::slog_info!(
2637 "bash task relaxed DB miss for {}; falling back to disk",
2638 task_id
2639 );
2640 }
2641 Some(Err(error)) => {
2642 crate::slog_warn!(
2643 "bash task relaxed DB lookup failed for {}; falling back to disk: {}",
2644 task_id,
2645 error
2646 );
2647 }
2648 None => {
2649 crate::slog_info!(
2650 "bash task relaxed DB unavailable for {}; falling back to disk",
2651 task_id
2652 );
2653 }
2654 }
2655 let root = storage_dir.join("bash-tasks");
2656 let entries = fs::read_dir(&root).ok()?;
2657 for entry in entries.flatten() {
2658 let dir = entry.path();
2659 if !dir.is_dir() {
2660 continue;
2661 }
2662 let resolved = match resolve_task_layout(&dir, task_id) {
2663 Ok(task) => task,
2664 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
2665 Err(error) => {
2666 if self.db_has_live_process_for_task(task_id) {
2667 crate::slog_warn!(
2668 "refusing to quarantine unresolved live background task {task_id} during relaxed lookup: {error}"
2669 );
2670 continue;
2671 }
2672 crate::slog_warn!(
2673 "quarantining unresolved background task {task_id} during relaxed lookup: {error}"
2674 );
2675 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
2676 continue;
2677 }
2678 };
2679 let metadata = match read_task_at(&resolved) {
2680 Ok(metadata) => metadata,
2681 Err(error) => {
2682 if self.db_has_live_process_for_task(task_id) {
2683 crate::slog_warn!(
2684 "refusing to quarantine unreadable live background task {task_id} during relaxed lookup: {error}"
2685 );
2686 continue;
2687 }
2688 crate::slog_warn!(
2689 "quarantining invalid background task metadata {task_id} during relaxed lookup: {error}"
2690 );
2691 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
2692 continue;
2693 }
2694 };
2695 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
2696 if metadata_project.as_deref() != Some(canonical_project.as_path()) {
2697 continue;
2698 }
2699 if let Some(task) = self.task(task_id) {
2700 let matches_project = task
2701 .state
2702 .lock()
2703 .map(|state| {
2704 state
2705 .metadata
2706 .project_root
2707 .as_deref()
2708 .map(canonicalized_path)
2709 .as_deref()
2710 == Some(canonical_project.as_path())
2711 })
2712 .unwrap_or(false);
2713 return matches_project.then_some(task);
2714 }
2715 if self
2716 .insert_rehydrated_task(metadata, resolved.paths, true, None)
2717 .is_err()
2718 {
2719 return None;
2720 }
2721 return self.task(task_id);
2722 }
2723 None
2724 }
2725
2726 fn lookup_relaxed_task_from_db(
2727 &self,
2728 task_id: &str,
2729 project_root: &Path,
2730 ) -> Option<Result<Option<BashTaskRow>, String>> {
2731 let pool = self
2732 .inner
2733 .db_pool
2734 .read()
2735 .ok()
2736 .and_then(|slot| slot.clone())?;
2737 let harness = self
2738 .inner
2739 .db_harness
2740 .read()
2741 .ok()
2742 .and_then(|slot| slot.clone())?;
2743 let conn = match pool.lock() {
2744 Ok(conn) => conn,
2745 Err(_) => return Some(Err("db mutex poisoned".to_string())),
2746 };
2747 let project_key = crate::path_identity::project_scope_key(project_root);
2748 Some(
2749 crate::db::bash_tasks::find_bash_task_for_project(
2750 &conn,
2751 &harness,
2752 &project_key,
2753 task_id,
2754 )
2755 .map_err(|error| error.to_string()),
2756 )
2757 }
2758
2759 pub(super) fn status_relaxed(
2760 &self,
2761 task_id: &str,
2762 _session_id: &str,
2763 project_root: &Path,
2764 storage_dir: &Path,
2765 preview_bytes: usize,
2766 allow_terminal_db_fallback: bool,
2767 ) -> Option<BgTaskSnapshot> {
2768 let fallback_row = if allow_terminal_db_fallback {
2769 self.lookup_relaxed_task_from_db(task_id, project_root)
2770 } else {
2771 None
2772 }
2773 .and_then(Result::ok)
2774 .flatten()
2775 .filter(|row| task_bundle_is_absent(storage_dir, &row.session_id, &row.task_id));
2776 if let Some(task) = self.status_relaxed_task(task_id, project_root, storage_dir) {
2777 let _ = self.poll_task(&task);
2778 return Some(self.snapshot_with_terminal_cache(&task, preview_bytes));
2779 }
2780 let row = fallback_row?;
2781 let metadata = PersistedTask::from(row.clone());
2782 metadata
2783 .status
2784 .is_terminal()
2785 .then(|| terminal_db_row_snapshot(row, metadata))
2786 }
2787
2788 pub fn kill_relaxed(
2789 &self,
2790 task_id: &str,
2791 project_root: &Path,
2792 storage_dir: &Path,
2793 ) -> Result<BgTaskSnapshot, String> {
2794 let task = self
2795 .status_relaxed_task(task_id, project_root, storage_dir)
2796 .ok_or_else(|| format!("background task not found: {task_id}"))?;
2797 self.kill_with_status(task_id, &task.session_id, BgTaskStatus::Killed)
2798 }
2799
2800 pub fn maybe_gc_persisted(&self, storage_dir: &Path) -> Result<usize, String> {
2801 #[cfg(test)]
2802 self.inner.persisted_gc_runs.fetch_add(1, Ordering::SeqCst);
2803
2804 let mut deleted = 0usize;
2805
2806 let root = storage_dir.join("bash-tasks");
2807 if root.exists() {
2808 let session_dirs = fs::read_dir(&root).map_err(|e| {
2809 format!(
2810 "failed to read background task root {}: {e}",
2811 root.display()
2812 )
2813 })?;
2814 for session_entry in session_dirs.flatten() {
2815 let session_dir = session_entry.path();
2816 if !session_dir.is_dir() {
2817 continue;
2818 }
2819 let (task_ids, invalid_entries) = match discover_task_ids(&session_dir) {
2820 Ok(discovery) => discovery,
2821 Err(error) => {
2822 crate::slog_warn!(
2823 "failed to discover background task session {}: {error}",
2824 session_dir.display()
2825 );
2826 continue;
2827 }
2828 };
2829 for entry in invalid_entries {
2830 let _ = quarantine_invalid_entry(storage_dir, &session_dir, &entry);
2831 }
2832 for task_id in task_ids {
2833 let resolved = match resolve_task_layout(&session_dir, &task_id) {
2834 Ok(task) => task,
2835 Err(error)
2836 if error.kind() == std::io::ErrorKind::NotFound
2837 && uninitialized_layout_is_recent(
2838 &session_dir,
2839 &task_id,
2840 Duration::from_secs(5 * 60),
2841 )
2842 .unwrap_or(false) =>
2843 {
2844 continue;
2845 }
2846 Err(error) => {
2847 if self.db_has_live_process_for_task(&task_id) {
2848 crate::slog_warn!(
2849 "refusing to quarantine unresolved live background task {task_id} during GC: {error}"
2850 );
2851 continue;
2852 }
2853 crate::slog_warn!(
2854 "quarantining unresolved background task {task_id}: {error}"
2855 );
2856 quarantine_task_layout(storage_dir, &session_dir, &task_id, "invalid")
2857 .map_err(|error| error.to_string())?;
2858 continue;
2859 }
2860 };
2861 if modified_within(&resolved.paths.json, PERSISTED_GC_GRACE) {
2862 continue;
2863 }
2864 let metadata = match read_task_at(&resolved) {
2865 Ok(metadata) => metadata,
2866 Err(error) => {
2867 if self.db_has_live_process_for_task(&task_id) {
2868 crate::slog_warn!(
2869 "refusing to quarantine unreadable live background task {task_id} during GC: {error}"
2870 );
2871 continue;
2872 }
2873 crate::slog_warn!(
2874 "quarantining corrupt background task metadata {task_id}: {error}"
2875 );
2876 quarantine_task_layout(storage_dir, &session_dir, &task_id, "corrupt")
2877 .map_err(|error| error.to_string())?;
2878 continue;
2879 }
2880 };
2881 if !(metadata.status.is_terminal() && metadata.completion_delivered) {
2882 continue;
2883 }
2884 if Self::persisted_task_process_is_alive(&metadata)
2885 || self.db_has_live_process_for_task(&task_id)
2886 {
2887 crate::slog_warn!(
2888 "refusing to delete terminal background task bundle {task_id}: recorded process is still alive"
2889 );
2890 continue;
2891 }
2892 match delete_task_bundle(&resolved.paths) {
2893 Ok(()) => {
2894 self.delete_gc_task_from_db(&metadata);
2895 self.evaluate_erased_watch_targets();
2896 deleted += 1;
2897 log::debug!(
2898 "deleted persisted background task bundle {}",
2899 metadata.task_id
2900 );
2901 }
2902 Err(error) => {
2903 crate::slog_warn!(
2904 "failed to delete background task bundle {}: {error}",
2905 metadata.task_id
2906 );
2907 }
2908 }
2909 }
2910 }
2911 }
2912 gc_quarantine(storage_dir);
2913 Ok(deleted)
2914 }
2915
2916 pub fn list(&self, preview_bytes: usize) -> Vec<BgTaskSnapshot> {
2917 let tasks = self
2918 .inner
2919 .tasks
2920 .lock()
2921 .map(|tasks| tasks.values().cloned().collect::<Vec<_>>())
2922 .unwrap_or_default();
2923 tasks
2924 .into_iter()
2925 .map(|task| {
2926 let _ = self.poll_task(&task);
2927 self.snapshot_with_terminal_cache(&task, preview_bytes)
2928 })
2929 .collect()
2930 }
2931
2932 fn maybe_compress_snapshot(&self, task: &Arc<BgTask>, snapshot: &mut BgTaskSnapshot) {
2938 if !snapshot.info.status.is_terminal() || snapshot.info.mode == BgMode::Pty {
2939 return;
2940 }
2941 if let Some(cache) = self.ensure_terminal_output_cache(task) {
2942 snapshot.output_preview = cache.output_preview;
2943 snapshot.output_truncated = cache.output_truncated;
2944 }
2945 }
2946
2947 pub fn kill(&self, task_id: &str, session_id: &str) -> Result<BgTaskSnapshot, String> {
2948 self.kill_with_status(task_id, session_id, BgTaskStatus::Killed)
2949 }
2950
2951 pub fn kill_running_tasks_for_root(&self, project_root: &Path) -> usize {
2961 let canonical_root = canonicalized_path(project_root);
2962 let targets = self
2963 .inner
2964 .tasks
2965 .lock()
2966 .map(|tasks| {
2967 tasks
2968 .values()
2969 .filter_map(|task| {
2970 let state = task.state.lock().ok()?;
2971 let status = &state.metadata.status;
2972 let running = matches!(status, BgTaskStatus::Running)
2973 || (state.metadata.mode == BgMode::Pty
2974 && matches!(status, BgTaskStatus::Killing));
2975 if !running {
2976 return None;
2977 }
2978 let task_root = state
2979 .metadata
2980 .project_root
2981 .as_deref()
2982 .unwrap_or(&state.metadata.workdir);
2983 (canonicalized_path(task_root) == canonical_root)
2984 .then(|| (task.task_id.clone(), task.session_id.clone()))
2985 })
2986 .collect::<Vec<_>>()
2987 })
2988 .unwrap_or_default();
2989
2990 let mut killed = 0;
2991 for (task_id, session_id) in targets {
2992 match self.kill_with_status_reason(
2993 &task_id,
2994 &session_id,
2995 BgTaskStatus::Killed,
2996 Some(ROOT_RECLAIMED_REASON.to_string()),
2997 ) {
2998 Ok(_) => killed += 1,
2999 Err(error) => crate::slog_warn!(
3000 "failed to terminate background task {task_id} for reclaimed root {}: {error}",
3001 project_root.display()
3002 ),
3003 }
3004 }
3005 killed
3006 }
3007
3008 pub fn promote(&self, task_id: &str, session_id: &str) -> Result<bool, String> {
3009 let task = self
3010 .task_for_session(task_id, session_id)
3011 .ok_or_else(|| format!("background task not found: {task_id}"))?;
3012 let terminal_after_promote = {
3013 let mut state = task
3014 .state
3015 .lock()
3016 .map_err(|_| "background task lock poisoned".to_string())?;
3017 let updated = self
3018 .update_task_metadata(&task.paths, |metadata| {
3019 metadata.notify_on_completion = true;
3020 metadata.completion_delivered = false;
3021 })
3022 .map_err(|e| format!("failed to promote background task: {e}"))?;
3023 state.metadata = updated;
3024 state.metadata.status.is_terminal()
3025 };
3026 if terminal_after_promote {
3027 self.post_terminal_transition(&task, true)?;
3028 }
3029 Ok(true)
3030 }
3031
3032 pub(crate) fn kill_for_timeout(&self, task_id: &str, session_id: &str) -> Result<(), String> {
3033 self.kill_with_status(task_id, session_id, BgTaskStatus::TimedOut)
3034 .map(|_| ())
3035 }
3036
3037 pub fn cleanup_finished(&self, older_than: Duration) {
3038 let cutoff = Instant::now().checked_sub(older_than);
3039 let removable_paths: Vec<(String, TaskPaths)> =
3040 if let Ok(mut tasks) = self.inner.tasks.lock() {
3041 let removable = tasks
3042 .iter()
3043 .filter_map(|(task_id, task)| {
3044 let delivered_terminal = task
3045 .state
3046 .lock()
3047 .map(|state| {
3048 state.metadata.status.is_terminal()
3049 && state.metadata.completion_delivered
3050 })
3051 .unwrap_or(false);
3052 if !delivered_terminal {
3053 return None;
3054 }
3055
3056 let terminal_at = task.terminal_at.lock().ok().and_then(|at| *at);
3057 let expired = match (terminal_at, cutoff) {
3058 (Some(terminal_at), Some(cutoff)) => terminal_at <= cutoff,
3059 (Some(_), None) => true,
3060 (None, _) => false,
3061 };
3062 expired.then(|| task_id.clone())
3063 })
3064 .collect::<Vec<_>>();
3065
3066 removable
3067 .into_iter()
3068 .filter_map(|task_id| {
3069 tasks
3070 .remove(&task_id)
3071 .map(|task| (task_id, task.paths.clone()))
3072 })
3073 .collect()
3074 } else {
3075 Vec::new()
3076 };
3077
3078 for (task_id, paths) in removable_paths {
3079 match delete_task_bundle(&paths) {
3080 Ok(()) => log::debug!("deleted persisted background task bundle {task_id}"),
3081 Err(error) => crate::slog_warn!(
3082 "failed to delete persisted background task bundle {task_id}: {error}"
3083 ),
3084 }
3085 }
3086 }
3087
3088 pub fn drain_completions(&self) -> Vec<BgCompletion> {
3089 self.drain_completions_for_session(None)
3090 }
3091
3092 pub fn drain_completions_for_session(&self, session_id: Option<&str>) -> Vec<BgCompletion> {
3093 if let Some(session_id) = session_id {
3094 self.redeliver_pending_watches_for_session(session_id);
3095 }
3096 let completions = match self.inner.completions.lock() {
3097 Ok(completions) => completions,
3098 Err(_) => return Vec::new(),
3099 };
3100
3101 completions
3102 .iter()
3103 .filter(|completion| completion_matches_session(completion, session_id))
3104 .cloned()
3105 .collect()
3106 }
3107
3108 pub fn has_completions_for_session(&self, session_id: Option<&str>) -> bool {
3109 match self.inner.completions.lock() {
3110 Ok(completions) => completions
3111 .iter()
3112 .any(|completion| completion_matches_session(completion, session_id)),
3113 Err(_) => true,
3117 }
3118 }
3119
3120 pub fn ack_completions_for_session(
3121 &self,
3122 session_id: Option<&str>,
3123 task_ids: &[String],
3124 ) -> Vec<String> {
3125 if task_ids.is_empty() {
3126 return Vec::new();
3127 }
3128 let requested_task_ids = task_ids.iter().map(String::as_str).collect::<HashSet<_>>();
3129 let mut completion_sessions = HashMap::new();
3130 if let Ok(mut completions) = self.inner.completions.lock() {
3131 completions.retain(|completion| {
3132 let session_matches = session_id
3133 .map(|session_id| completion.session_id == session_id)
3134 .unwrap_or(true);
3135 if session_matches && requested_task_ids.contains(completion.task_id.as_str()) {
3136 completion_sessions
3137 .insert(completion.task_id.clone(), completion.session_id.clone());
3138 false
3139 } else {
3140 true
3141 }
3142 });
3143 }
3144
3145 let mut delivered = Vec::new();
3146 for task_id in task_ids {
3147 if self.has_erased_watch_reference(task_id) {
3148 if let Some((harness, pool)) = self.db_harness_and_pool() {
3149 if let Ok(conn) = pool.lock() {
3150 if let Ok(rows) =
3151 crate::db::bash_watches::list_bash_pattern_watches_by_task_id(
3152 &conn, &harness, task_id,
3153 )
3154 {
3155 for row in rows {
3156 let _ = crate::db::bash_watches::delete_bash_pattern_watch(
3157 &conn,
3158 &harness,
3159 &row.session_id,
3160 task_id,
3161 &row.watch_id,
3162 );
3163 }
3164 }
3165 }
3166 }
3167 self.clear_task_watch_state(task_id);
3168 if let Ok(mut registry) = self.inner.watch_registry.lock() {
3169 registry.forget_erased_task(task_id);
3170 }
3171 delivered.push(task_id.clone());
3172 continue;
3173 }
3174 let task = if let Some(session_id) = session_id {
3175 self.task_for_session(task_id, session_id)
3176 .or_else(|| {
3177 self.task(task_id)
3178 .filter(|task| task.delivery_session_id == session_id)
3179 })
3180 .or_else(|| {
3181 completion_sessions
3182 .contains_key(task_id)
3183 .then(|| self.task(task_id))
3184 .flatten()
3185 })
3186 } else if let Some(completion_session_id) = completion_sessions.get(task_id) {
3187 self.task_for_session(task_id, completion_session_id)
3188 .or_else(|| self.task(task_id))
3189 } else {
3190 self.task(task_id)
3191 };
3192 if let Some(task) = task {
3193 let terminal = task
3194 .state
3195 .lock()
3196 .map(|state| state.metadata.status.is_terminal())
3197 .unwrap_or(false);
3198 self.ack_persisted_watches_for_task(&task.session_id, task_id, terminal);
3202 if terminal {
3203 self.clear_task_watch_state(task_id);
3204 if task.set_completion_delivered(true, self).is_ok() {
3205 delivered.push(task_id.clone());
3206 }
3207 } else {
3208 self.sync_memory_watches_after_ack(task_id);
3211 delivered.push(task_id.clone());
3212 }
3213 } else if let Some(session_id) = session_id {
3214 self.ack_persisted_watches_for_task(session_id, task_id, true);
3216 delivered.push(task_id.clone());
3217 }
3218 }
3219
3220 delivered
3221 }
3222
3223 fn sync_memory_watches_after_ack(&self, task_id: &str) {
3224 let Some((harness, pool)) = self.db_harness_and_pool() else {
3225 return;
3226 };
3227 let session_id = match self.task(task_id) {
3228 Some(task) => task.session_id.clone(),
3229 None => return,
3230 };
3231 let Ok(conn) = pool.lock() else {
3232 return;
3233 };
3234 let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
3235 &conn,
3236 &harness,
3237 &session_id,
3238 task_id,
3239 ) else {
3240 return;
3241 };
3242 let remaining: HashSet<String> = rows.into_iter().map(|row| row.watch_id).collect();
3243 if let Ok(mut registry) = self.inner.watch_registry.lock() {
3244 registry.retain_watch_ids(task_id, &remaining);
3245 }
3246 }
3247
3248 pub fn pending_completions_for_session(&self, session_id: &str) -> Vec<BgCompletion> {
3249 self.inner
3250 .completions
3251 .lock()
3252 .map(|completions| {
3253 completions
3254 .iter()
3255 .filter(|completion| completion.session_id == session_id)
3256 .cloned()
3257 .collect()
3258 })
3259 .unwrap_or_default()
3260 }
3261
3262 fn remove_pending_completion(&self, task_id: &str) -> Option<BgCompletion> {
3263 let mut completions = self.inner.completions.lock().ok()?;
3264 let idx = completions
3265 .iter()
3266 .position(|completion| completion.task_id == task_id)?;
3267 completions.remove(idx)
3268 }
3269
3270 fn retarget_pending_completion(&self, task_id: &str, session_id: &str) {
3271 if let Ok(mut completions) = self.inner.completions.lock() {
3272 if let Some(completion) = completions
3273 .iter_mut()
3274 .find(|completion| completion.task_id == task_id)
3275 {
3276 completion.session_id = session_id.to_string();
3277 }
3278 }
3279 }
3280
3281 fn completion_snapshot_for_task(&self, task: &Arc<BgTask>) -> Option<BgCompletion> {
3282 let snapshot = self.snapshot_with_terminal_cache(task, RUNNING_OUTPUT_PREVIEW_BYTES);
3283 if !snapshot.info.status.is_terminal() {
3284 return None;
3285 }
3286 let (output_preview, output_truncated) = if snapshot.info.mode == BgMode::Pty {
3287 (String::new(), false)
3288 } else {
3289 self.ensure_terminal_output_cache(task)
3290 .map(|cache| completion_preview_for_cache(&cache, snapshot.exit_code))
3291 .unwrap_or_else(|| (String::new(), false))
3292 };
3293 Some(BgCompletion {
3294 task_id: snapshot.info.task_id,
3295 session_id: task.delivery_session_id.clone(),
3296 status: snapshot.info.status,
3297 exit_code: snapshot.exit_code,
3298 command: snapshot.info.command,
3299 output_preview,
3300 output_truncated,
3301 original_tokens: None,
3302 compressed_tokens: None,
3303 tokens_skipped: false,
3304 status_reason: snapshot.info.status_reason,
3305 })
3306 }
3307
3308 pub fn detach(&self) {
3309 self.inner.shutdown.store(true, Ordering::SeqCst);
3310 if let Ok(mut tasks) = self.inner.tasks.lock() {
3311 for task in tasks.values() {
3312 if let Ok(mut state) = task.state.lock() {
3313 match &mut state.runtime {
3314 TaskRuntime::Piped(child) => *child = None,
3315 TaskRuntime::Pty(runtime) => *runtime = None,
3316 }
3317 state.detached = true;
3318 }
3319 }
3320 tasks.clear();
3321 }
3322 }
3323
3324 pub fn shutdown(&self) {
3325 let tasks = self
3326 .inner
3327 .tasks
3328 .lock()
3329 .map(|tasks| {
3330 tasks
3331 .values()
3332 .map(|task| (task.task_id.clone(), task.session_id.clone()))
3333 .collect::<Vec<_>>()
3334 })
3335 .unwrap_or_default();
3336 for (task_id, session_id) in tasks {
3337 let _ = self.kill(&task_id, &session_id);
3338 }
3339 }
3340
3341 pub(crate) fn poll_task(&self, task: &Arc<BgTask>) -> Result<(), String> {
3342 if let Ok(state) = task.state.lock() {
3343 if let TaskRuntime::Pty(Some(pty)) = &state.runtime {
3344 if !pty.exit_observed.load(Ordering::SeqCst) {
3352 return Ok(());
3353 }
3354 }
3355 }
3356 let marker = match read_exit_marker(&task.paths) {
3357 Ok(Some(marker)) => marker,
3358 Ok(None) => return Ok(()),
3359 Err(error) => return Err(format!("failed to read exit marker: {error}")),
3360 };
3361 self.finalize_from_marker(task, marker, None)
3362 }
3363
3364 pub(crate) fn reap_child(&self, task: &Arc<BgTask>) {
3365 let mut needs_completion = false;
3366 {
3367 let Ok(mut state) = task.state.lock() else {
3368 return;
3369 };
3370 match &mut state.runtime {
3371 TaskRuntime::Piped(child_slot) => {
3372 if let Some(child) = child_slot.as_mut() {
3373 if let Ok(Some(status)) = child.try_wait() {
3374 *child_slot = None;
3375 state.detached = true;
3376 state.child_exit_observed = true;
3377 if let Some(handles) = state.io_handles.as_mut() {
3378 if handles.artifact_len(TaskArtifact::Exit).unwrap_or(1) == 0 {
3379 let marker = status
3380 .code()
3381 .map(|code| code.to_string())
3382 .unwrap_or_else(|| "1".to_string());
3383 let _ = handles.write(TaskArtifact::Exit, marker.as_bytes());
3384 }
3385 }
3386 }
3387 } else if state.detached {
3388 let child_known_dead = state.child_exit_observed
3389 || state
3390 .metadata
3391 .child_pid
3392 .is_some_and(|pid| !is_process_alive(pid));
3393 if child_known_dead {
3394 needs_completion =
3395 self.fail_without_exit_marker_if_needed(task, &mut state);
3396 }
3397 }
3398 }
3399 TaskRuntime::Pty(Some(pty)) => {
3400 if pty.exit_observed.load(Ordering::SeqCst) {
3401 drop(state);
3402 let _ = self.poll_task(task);
3403 return;
3404 }
3405 }
3406 TaskRuntime::Pty(None) => {}
3407 }
3408 }
3409 if needs_completion {
3410 let _ = self.post_terminal_transition(task, true);
3411 }
3412 }
3413
3414 fn fail_without_exit_marker_if_needed(
3415 &self,
3416 task: &Arc<BgTask>,
3417 state: &mut BgTaskState,
3418 ) -> bool {
3419 if state.metadata.status.is_terminal() {
3420 return false;
3421 }
3422 if matches!(read_exit_marker(&task.paths), Ok(Some(_))) {
3423 return false;
3424 }
3425 let watch_controlled = self.task_has_watch_control(&task.task_id);
3426 let child_exit_observed = state.child_exit_observed;
3427 let updated = self.update_task_metadata(&task.paths, |metadata| {
3428 let (status, reason) = if child_exit_observed {
3429 (
3430 BgTaskStatus::Failed,
3431 "process exited without exit marker".to_string(),
3432 )
3433 } else {
3434 (
3435 BgTaskStatus::FateUnknown,
3436 restart_fate_unknown_reason(metadata, &task.paths),
3437 )
3438 };
3439 metadata.mark_terminal(status, None, Some(reason));
3440 if watch_controlled {
3441 metadata.completion_delivered = true;
3442 }
3443 });
3444 if let Ok(metadata) = updated {
3445 state.pending_terminal_override = None;
3446 state.metadata = metadata;
3447 task.mark_terminal_now();
3448 return true;
3449 }
3450 false
3451 }
3452
3453 pub(crate) fn running_tasks(&self) -> Vec<Arc<BgTask>> {
3454 self.inner
3455 .tasks
3456 .lock()
3457 .map(|tasks| {
3458 tasks
3459 .values()
3460 .filter(|task| task.is_running())
3461 .cloned()
3462 .collect()
3463 })
3464 .unwrap_or_default()
3465 }
3466
3467 fn insert_rehydrated_task(
3468 &self,
3469 metadata: PersistedTask,
3470 paths: TaskPaths,
3471 detached: bool,
3472 delivery_session_id: Option<&str>,
3473 ) -> Result<(), String> {
3474 let task_id = metadata.task_id.clone();
3475 let session_id = metadata.session_id.clone();
3476 let started = started_instant_from_unix_millis(metadata.started_at);
3477 let suppress_replayed_running_reminder = metadata.status == BgTaskStatus::Running;
3478 let mode = metadata.mode.clone();
3479 let task = Arc::new(BgTask {
3480 task_id: task_id.clone(),
3481 delivery_session_id: delivery_session_id.unwrap_or(&session_id).to_string(),
3482 session_id,
3483 paths: paths.clone(),
3484 artifact_root: canonical_artifact_root(&paths),
3485 started,
3486 last_reminder_at: Mutex::new(suppress_replayed_running_reminder.then(Instant::now)),
3487 terminal_at: Mutex::new(metadata.status.is_terminal().then(Instant::now)),
3488 state: Mutex::new(BgTaskState {
3489 metadata,
3490 runtime: if mode == BgMode::Pty {
3491 TaskRuntime::Pty(None)
3492 } else {
3493 TaskRuntime::Piped(None)
3494 },
3495 io_handles: None,
3496 detached,
3497 child_exit_observed: false,
3504 buffer: BgBuffer::registered(&paths, mode.clone()),
3505 terminal_output_cache: None,
3506 pending_terminal_override: None,
3507 }),
3508 });
3509 self.inner
3510 .tasks
3511 .lock()
3512 .map_err(|_| "background task registry lock poisoned".to_string())?
3513 .insert(task_id.clone(), Arc::clone(&task));
3514 self.rearm_persisted_watches(&task);
3518 Ok(())
3519 }
3520
3521 fn rearm_persisted_watches(&self, task: &Arc<BgTask>) {
3522 let Some((harness, pool)) = self.db_harness_and_pool() else {
3523 return;
3524 };
3525 let rows = {
3526 let Ok(conn) = pool.lock() else {
3527 return;
3528 };
3529 match crate::db::bash_watches::list_bash_pattern_watches_for_task(
3530 &conn,
3531 &harness,
3532 &task.session_id,
3533 &task.task_id,
3534 ) {
3535 Ok(rows) if !rows.is_empty() => rows,
3536 _ => return,
3537 }
3538 };
3539
3540 let mode = match task.state.lock() {
3541 Ok(state) => state.metadata.mode.clone(),
3542 Err(_) => return,
3543 };
3544 let terminal = task
3545 .state
3546 .lock()
3547 .map(|state| state.metadata.status.is_terminal())
3548 .unwrap_or(false);
3549 let completion_delivered = task
3550 .state
3551 .lock()
3552 .map(|state| state.metadata.completion_delivered)
3553 .unwrap_or(true);
3554
3555 let mut stdout = (mode == BgMode::Pipes)
3556 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
3557 .transpose()
3558 .ok()
3559 .flatten();
3560 let mut stderr = (mode == BgMode::Pipes)
3561 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
3562 .transpose()
3563 .ok()
3564 .flatten();
3565 let mut pty = (mode == BgMode::Pty)
3566 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
3567 .transpose()
3568 .ok()
3569 .flatten();
3570
3571 let mut pending_to_emit = Vec::new();
3572 let mut gap_matches = Vec::new();
3573 {
3574 let Ok(mut registry) = self.inner.watch_registry.lock() else {
3575 return;
3576 };
3577 let stdout_key = format!("{}:stdout", task.task_id);
3578 let stderr_key = format!("{}:stderr", task.task_id);
3579 let pty_key = format!("{}:pty", task.task_id);
3580
3581 let first = &rows[0];
3583 match mode {
3584 BgMode::Pipes => {
3585 registry.set_file_cursor(&stdout_key, first.stdout_offset.max(0) as u64);
3586 registry.set_file_cursor(&stderr_key, first.stderr_offset.max(0) as u64);
3587 }
3588 BgMode::Pty => {
3589 registry.set_file_cursor(&pty_key, first.pty_offset.max(0) as u64);
3590 }
3591 }
3592
3593 for row in &rows {
3594 let Ok(pattern) = WatchPattern::from_persisted(&row.pattern_kind, &row.pattern)
3595 else {
3596 crate::slog_warn!(
3597 "skipping unreadable persisted watch {}/{}",
3598 row.task_id,
3599 row.watch_id
3600 );
3601 continue;
3602 };
3603 if let Err(error) = registry.restore(
3604 row.watch_id.clone(),
3605 row.task_id.clone(),
3606 pattern,
3607 row.once,
3608 row.scanning,
3609 ) {
3610 crate::slog_warn!(
3611 "failed to restore watch {}/{}: {error}",
3612 row.task_id,
3613 row.watch_id
3614 );
3615 continue;
3616 }
3617 if row.pending_match {
3618 if let (Some(match_text), Some(match_offset), Some(context)) = (
3619 row.match_text.clone(),
3620 row.match_offset,
3621 row.match_context.clone(),
3622 ) {
3623 pending_to_emit.push(PatternMatch {
3624 watch_id: row.watch_id.clone(),
3625 task_id: row.task_id.clone(),
3626 match_text,
3627 match_offset: match_offset.max(0) as u64,
3628 context,
3629 once: row.once,
3630 });
3631 }
3632 }
3633 }
3634
3635 let should_gap_scan =
3639 rows.iter().any(|row| row.scanning) && !pending_to_emit.iter().any(|m| m.once);
3640 if should_gap_scan {
3641 match mode {
3642 BgMode::Pipes => {
3643 if let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) {
3644 gap_matches.extend(registry.scan_file_new_bytes(
3645 &stdout_key,
3646 &task.task_id,
3647 stdout,
3648 ));
3649 gap_matches.extend(registry.scan_file_new_bytes(
3650 &stderr_key,
3651 &task.task_id,
3652 stderr,
3653 ));
3654 }
3655 }
3656 BgMode::Pty => {
3657 if let Some(pty) = pty.as_mut() {
3658 gap_matches.extend(registry.scan_file_new_bytes(
3659 &pty_key,
3660 &task.task_id,
3661 pty,
3662 ));
3663 }
3664 }
3665 }
3666 }
3667 }
3668
3669 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
3670 for pattern_match in &gap_matches {
3671 self.persist_watch_match(
3672 &task.session_id,
3673 &task.task_id,
3674 pattern_match,
3675 stdout_offset,
3676 stderr_offset,
3677 pty_offset,
3678 );
3679 }
3680 if !gap_matches.is_empty() || rows.iter().any(|row| row.scanning) {
3681 self.persist_task_watch_cursors(
3682 &task.session_id,
3683 &task.task_id,
3684 stdout_offset,
3685 stderr_offset,
3686 pty_offset,
3687 );
3688 }
3689
3690 let emitted_pending = !pending_to_emit.is_empty();
3692 let to_emit = if emitted_pending {
3693 pending_to_emit
3694 } else {
3695 gap_matches
3696 };
3697 for pattern_match in to_emit {
3698 self.emit_bash_pattern_match(&task.delivery_session_id, pattern_match);
3699 }
3700
3701 if !terminal {
3702 return;
3703 }
3704
3705 let _ = self.remove_pending_completion(&task.task_id);
3708 let (watch_controlled, watch_matched) = self.task_watch_state(&task.task_id);
3709 if !watch_controlled {
3710 return;
3711 }
3712 if watch_matched {
3713 return;
3715 }
3716 if completion_delivered {
3717 self.clear_task_watch_state(&task.task_id);
3719 self.delete_persisted_watches_for_task(&task.session_id, &task.task_id);
3720 return;
3721 }
3722 if let Some(completion) = self.completion_snapshot_for_task(task) {
3723 self.emit_bash_watch_exit(&completion);
3724 }
3725 self.clear_task_watch_state(&task.task_id);
3727 }
3728
3729 fn kill_with_status(
3730 &self,
3731 task_id: &str,
3732 session_id: &str,
3733 terminal_status: BgTaskStatus,
3734 ) -> Result<BgTaskSnapshot, String> {
3735 self.kill_with_status_reason(task_id, session_id, terminal_status, None)
3736 }
3737
3738 fn kill_with_status_reason(
3739 &self,
3740 task_id: &str,
3741 session_id: &str,
3742 terminal_status: BgTaskStatus,
3743 reason: Option<String>,
3744 ) -> Result<BgTaskSnapshot, String> {
3745 let task = self
3746 .task_for_session(task_id, session_id)
3747 .ok_or_else(|| format!("background task not found: {task_id}"))?;
3748 let mut terminalized = false;
3749
3750 {
3751 let mut state = task
3752 .state
3753 .lock()
3754 .map_err(|_| "background task lock poisoned".to_string())?;
3755 if state.metadata.status.is_terminal() {
3756 state.pending_terminal_override = None;
3757 } else if let Ok(Some(marker)) = read_exit_marker(&task.paths) {
3758 state.metadata =
3759 terminal_metadata_from_marker(state.metadata.clone(), marker, reason.clone());
3760 if self.task_has_watch_control(&task.task_id) {
3761 state.metadata.completion_delivered = true;
3762 }
3763 state.pending_terminal_override = None;
3764 task.mark_terminal_now();
3765 match &mut state.runtime {
3766 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
3773 TaskRuntime::Pty(runtime) => *runtime = None,
3774 }
3775 state.detached = true;
3776 self.persist_task(&task.paths, &state.metadata)
3777 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
3778 terminalized = true;
3779 } else {
3780 let was_already_killing = state.metadata.status == BgTaskStatus::Killing;
3781 if !was_already_killing {
3782 state.metadata.status = BgTaskStatus::Killing;
3783 }
3784 if reason.is_some() {
3785 state.metadata.status_reason = reason.clone();
3786 }
3787 if !was_already_killing || reason.is_some() {
3788 self.persist_task(&task.paths, &state.metadata)
3789 .map_err(|e| format!("failed to persist killing state: {e}"))?;
3790 }
3791
3792 #[cfg(unix)]
3793 let pgid = state.metadata.pgid;
3794 #[cfg(windows)]
3795 let child_pid = state.metadata.child_pid;
3796 if !was_already_killing
3797 && state.metadata.mode == BgMode::Pty
3798 && terminal_status == BgTaskStatus::TimedOut
3799 {
3800 state.pending_terminal_override = Some(BgTaskStatus::TimedOut);
3801 }
3802
3803 #[cfg(windows)]
3804 let mut pty_forced_terminal_status: Option<BgTaskStatus> = None;
3805
3806 match &mut state.runtime {
3807 TaskRuntime::Piped(child_slot) => {
3808 #[cfg(unix)]
3809 if let Some(pgid) = pgid {
3810 terminate_pgid(pgid, child_slot.as_mut());
3811 }
3812 #[cfg(windows)]
3813 if let Some(child) = child_slot.as_mut() {
3814 super::process::terminate_process(child);
3815 } else if let Some(pid) = child_pid {
3816 terminate_pid(pid);
3817 }
3818 if let Some(child) = child_slot.as_mut() {
3819 let _ = child.wait();
3820 }
3821 *child_slot = None;
3822 state.detached = true;
3823
3824 if let Some(handles) = state.io_handles.as_mut() {
3825 handles.write(TaskArtifact::Exit, b"killed").map_err(|e| {
3826 format!("failed to write retained kill marker: {e}")
3827 })?;
3828 } else {
3829 write_kill_marker_if_absent(&task.paths)
3830 .map_err(|e| format!("failed to write kill marker: {e}"))?;
3831 }
3832
3833 let exit_code = terminal_exit_code_for_status(&terminal_status);
3834 state
3835 .metadata
3836 .mark_terminal(terminal_status, exit_code, reason.clone());
3837 if self.task_has_watch_control(&task.task_id) {
3838 state.metadata.completion_delivered = true;
3839 }
3840 state.pending_terminal_override = None;
3841 task.mark_terminal_now();
3842 self.persist_task(&task.paths, &state.metadata)
3843 .map_err(|e| format!("failed to persist killed state: {e}"))?;
3844 terminalized = true;
3845 }
3846 TaskRuntime::Pty(Some(pty)) => {
3847 pty.was_killed.store(true, Ordering::SeqCst);
3848 if let Err(error) = pty.killer.kill() {
3849 crate::slog_warn!(
3850 "[pty-kill] {task_id} ChildKiller::kill failed: {error}"
3851 );
3852 }
3853 if let Some(pid) = pty.child_pid {
3854 #[cfg(unix)]
3855 terminate_pgid(pid as i32, None);
3856 #[cfg(windows)]
3857 terminate_pid(pid);
3858 }
3859 drop(pty.master.take());
3860
3861 #[cfg(windows)]
3862 {
3863 let default_status = if terminal_status == BgTaskStatus::TimedOut {
3864 BgTaskStatus::TimedOut
3865 } else {
3866 BgTaskStatus::Killed
3867 };
3868 pty_forced_terminal_status = Some(
3869 state
3870 .pending_terminal_override
3871 .take()
3872 .unwrap_or(default_status),
3873 );
3874 }
3875 }
3876 TaskRuntime::Pty(None) => {}
3877 }
3878
3879 #[cfg(windows)]
3880 if let Some(target_status) = pty_forced_terminal_status {
3881 if !task.paths.exit.exists() {
3882 write_kill_marker_if_absent(&task.paths)
3883 .map_err(|e| format!("failed to write kill marker: {e}"))?;
3884 }
3885
3886 let exit_code = terminal_exit_code_for_status(&target_status);
3887 state
3888 .metadata
3889 .mark_terminal(target_status, exit_code, reason.clone());
3890 if self.task_has_watch_control(&task.task_id) {
3891 state.metadata.completion_delivered = true;
3892 }
3893 state.pending_terminal_override = None;
3894 task.mark_terminal_now();
3895 if let TaskRuntime::Pty(runtime) = &mut state.runtime {
3896 *runtime = None;
3897 }
3898 state.detached = true;
3899 self.persist_task(&task.paths, &state.metadata)
3900 .map_err(|e| format!("failed to persist killed PTY state: {e}"))?;
3901 terminalized = true;
3902 }
3903 }
3904 }
3905
3906 if terminalized {
3907 self.post_terminal_transition(&task, true)?;
3908 }
3909 Ok(self.snapshot_with_terminal_cache(&task, RUNNING_OUTPUT_PREVIEW_BYTES))
3910 }
3911
3912 fn finalize_from_marker(
3913 &self,
3914 task: &Arc<BgTask>,
3915 marker: ExitMarker,
3916 reason: Option<String>,
3917 ) -> Result<(), String> {
3918 let watch_controlled = self.task_has_watch_control(&task.task_id);
3919 let mut pty_reader_done = None;
3920 {
3921 let mut state = task
3922 .state
3923 .lock()
3924 .map_err(|_| "background task lock poisoned".to_string())?;
3925 if state.metadata.status.is_terminal() {
3926 state.pending_terminal_override = None;
3927 return Ok(());
3928 }
3929
3930 let pending_override = state.pending_terminal_override.take();
3931 let is_pty = state.metadata.mode == BgMode::Pty;
3932 let reason = reason.or_else(|| state.metadata.status_reason.clone());
3933 let updated = self
3934 .update_task_metadata(&task.paths, |metadata| {
3935 let mut new_metadata = if is_pty && marker == ExitMarker::Killed {
3936 let mut metadata = metadata.clone();
3937 let target_status = pending_override.unwrap_or(BgTaskStatus::Killed);
3938 let exit_code = terminal_exit_code_for_status(&target_status);
3939 metadata.mark_terminal(target_status, exit_code, reason.clone());
3940 metadata
3941 } else {
3942 terminal_metadata_from_marker(metadata.clone(), marker, reason.clone())
3943 };
3944 if watch_controlled {
3945 new_metadata.completion_delivered = true;
3946 }
3947 *metadata = new_metadata;
3948 })
3949 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
3950 state.metadata = updated;
3951 task.mark_terminal_now();
3952 match &mut state.runtime {
3953 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
3958 TaskRuntime::Pty(runtime) => {
3959 pty_reader_done = runtime
3960 .as_ref()
3961 .map(|runtime| Arc::clone(&runtime.reader_done));
3962 *runtime = None;
3963 }
3964 }
3965 state.detached = true;
3966 }
3967
3968 if let Some(reader_done) = pty_reader_done {
3969 let deadline = Instant::now() + Duration::from_millis(200);
3970 while !reader_done.load(Ordering::SeqCst) && Instant::now() < deadline {
3971 std::thread::sleep(Duration::from_millis(10));
3972 }
3973 }
3974
3975 self.scan_task_watch_output(task);
3978
3979 self.post_terminal_transition(task, true)
3980 }
3981
3982 fn enqueue_completion_if_needed(
3983 &self,
3984 metadata: &PersistedTask,
3985 paths: Option<&TaskPaths>,
3986 emit_frame: bool,
3987 ) {
3988 if metadata.status.is_terminal() && !metadata.completion_delivered {
3989 let cache =
3990 paths.and_then(|paths| self.render_terminal_output_from_paths(metadata, paths));
3991 self.enqueue_completion_from_parts(metadata, None, paths, emit_frame, cache.as_ref());
3992 }
3993 }
3994
3995 fn render_terminal_output_from_paths(
3996 &self,
3997 metadata: &PersistedTask,
3998 paths: &TaskPaths,
3999 ) -> Option<TerminalOutputCache> {
4000 if metadata.mode == BgMode::Pty {
4001 return None;
4002 }
4003 let mut buffer = BgBuffer::registered(paths, BgMode::Pipes);
4004 let disk_truncation = buffer.enforce_terminal_cap();
4005 Some(self.render_terminal_output(metadata, &buffer, disk_truncation, Some(paths)))
4006 }
4007
4008 fn enqueue_completion_from_parts(
4009 &self,
4010 metadata: &PersistedTask,
4011 buffer: Option<&BgBuffer>,
4012 paths: Option<&TaskPaths>,
4013 emit_frame: bool,
4014 terminal_render: Option<&TerminalOutputCache>,
4015 ) {
4016 if !metadata.status.is_terminal() {
4027 return;
4028 }
4029
4030 let owned_buffer = if buffer.is_none() && metadata.mode != BgMode::Pty {
4031 paths.map(|paths| BgBuffer::registered(paths, BgMode::Pipes))
4032 } else {
4033 None
4034 };
4035 let render_buffer = buffer.or(owned_buffer.as_ref());
4036 let owned_render = if terminal_render.is_none() {
4037 render_buffer.map(|buffer| {
4038 let mut capped_buffer = buffer.clone();
4039 let disk_truncation = capped_buffer.enforce_terminal_cap();
4040 self.render_terminal_output(metadata, &capped_buffer, disk_truncation, paths)
4041 })
4042 } else {
4043 None
4044 };
4045 let render = terminal_render.or(owned_render.as_ref());
4046
4047 let (mut output_preview, output_truncated) = render
4051 .map(|cache| completion_preview_for_cache(cache, metadata.exit_code))
4052 .unwrap_or_else(|| (String::new(), false));
4053 if metadata.status == BgTaskStatus::FateUnknown {
4054 if let Some(reason) = metadata.status_reason.as_deref() {
4055 output_preview = if output_preview.is_empty() {
4056 reason.to_string()
4057 } else {
4058 format!("{reason}\n{output_preview}")
4059 };
4060 }
4061 }
4062
4063 let token_counts = self.completion_token_counts(
4064 metadata,
4065 buffer,
4066 paths,
4067 render.map(|render| render.output_preview.as_str()),
4068 );
4069 let completion = BgCompletion {
4070 task_id: metadata.task_id.clone(),
4071 session_id: metadata.session_id.clone(),
4072 status: metadata.status.clone(),
4073 exit_code: metadata.exit_code,
4074 command: metadata.command.clone(),
4075 output_preview,
4076 output_truncated,
4077 original_tokens: token_counts.original_tokens,
4078 compressed_tokens: token_counts.compressed_tokens,
4079 tokens_skipped: token_counts.tokens_skipped,
4080 status_reason: metadata.status_reason.clone(),
4081 };
4082
4083 self.record_compression_event_if_applicable(metadata, &token_counts);
4094
4095 let (watch_controlled, watch_matched) = self.task_watch_state(&metadata.task_id);
4096 if watch_controlled {
4097 if emit_frame && !watch_matched {
4098 self.emit_bash_watch_exit(&completion);
4099 } else if watch_matched {
4100 if let Some(task) = self.task(&metadata.task_id) {
4105 let _ = task.set_completion_delivered(true, self);
4106 }
4107 }
4108 self.clear_task_watch_state(&metadata.task_id);
4110 return;
4111 }
4112
4113 if metadata.completion_delivered {
4123 return;
4124 }
4125
4126 let pushed = if let Ok(mut completions) = self.inner.completions.lock() {
4129 if completions
4130 .iter()
4131 .any(|existing| existing.task_id == metadata.task_id)
4132 {
4133 false
4134 } else {
4135 completions.push_back(completion.clone());
4136 true
4137 }
4138 } else {
4139 false
4140 };
4141
4142 if pushed && emit_frame {
4143 self.emit_bash_completed(completion);
4144 }
4145 }
4146
4147 fn record_compression_event_if_applicable(
4148 &self,
4149 metadata: &PersistedTask,
4150 token_counts: &CompletionTokenCounts,
4151 ) {
4152 if metadata.mode == BgMode::Pty {
4153 return;
4154 }
4155
4156 let (original_tokens, compressed_tokens, original_bytes, compressed_bytes) = match (
4157 token_counts.original_tokens,
4158 token_counts.compressed_tokens,
4159 token_counts.original_bytes,
4160 token_counts.compressed_bytes,
4161 ) {
4162 (
4163 Some(original_tokens),
4164 Some(compressed_tokens),
4165 Some(original_bytes),
4166 Some(compressed_bytes),
4167 ) => (
4168 original_tokens,
4169 compressed_tokens,
4170 original_bytes,
4171 compressed_bytes,
4172 ),
4173 _ => {
4174 crate::slog_warn!(
4175 "compression event skipped for {}: token counts unavailable (likely spill file missing or unreadable)",
4176 metadata.task_id
4177 );
4178 return;
4179 }
4180 };
4181
4182 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
4183 let Some(pool) = pool else {
4184 crate::slog_warn!(
4185 "compression event skipped for {}: db_pool not initialized — was configure run?",
4186 metadata.task_id
4187 );
4188 return;
4189 };
4190 let harness = self
4191 .inner
4192 .db_harness
4193 .read()
4194 .ok()
4195 .and_then(|slot| slot.clone());
4196 let Some(harness) = harness else {
4197 crate::slog_warn!(
4198 "compression event insert skipped for {}: harness not configured",
4199 metadata.task_id
4200 );
4201 return;
4202 };
4203
4204 let project_root = metadata
4205 .project_root
4206 .as_deref()
4207 .unwrap_or(&metadata.workdir);
4208 let project_key = crate::path_identity::project_scope_key(project_root);
4209 let row = crate::db::compression_events::CompressionEventRow {
4210 harness: &harness,
4211 session_id: Some(&metadata.session_id),
4212 project_key: &project_key,
4213 tool: "bash",
4214 task_id: Some(&metadata.task_id),
4215 command: Some(&metadata.command),
4216 compressor: if metadata.compressed {
4217 "registry"
4218 } else {
4219 "none"
4220 },
4221 original_bytes,
4222 compressed_bytes,
4223 original_tokens,
4224 compressed_tokens,
4225 created_at: unix_millis() as i64,
4226 };
4227
4228 let conn = match pool.lock() {
4229 Ok(conn) => conn,
4230 Err(_) => {
4231 crate::slog_warn!(
4232 "compression event insert failed for {}: db mutex poisoned",
4233 metadata.task_id
4234 );
4235 return;
4236 }
4237 };
4238 match crate::db::compression_events::insert_compression_event(&conn, &row) {
4239 Ok(Some(row_id)) => {
4240 self.inner
4244 .compression_aggregates
4245 .record_successful_insert(&conn, &row, row_id);
4246 crate::slog_debug!(
4250 "compression event recorded for {} (project={}, session={}, {} → {} tokens)",
4251 metadata.task_id,
4252 project_key,
4253 metadata.session_id,
4254 original_tokens,
4255 compressed_tokens
4256 );
4257 }
4258 Ok(None) => {
4259 crate::slog_debug!(
4260 "duplicate compression event ignored for {} (project={}, session={})",
4261 metadata.task_id,
4262 project_key,
4263 metadata.session_id
4264 );
4265 }
4266 Err(error) => {
4267 crate::slog_warn!(
4268 "compression event insert failed for {}: {}",
4269 metadata.task_id,
4270 error
4271 );
4272 }
4273 }
4274 }
4275
4276 fn emit_bash_pattern_match(&self, session_id: &str, pattern_match: PatternMatch) {
4277 let Ok(progress_sender) = self
4278 .inner
4279 .progress_sender
4280 .lock()
4281 .map(|sender| sender.clone())
4282 else {
4283 return;
4284 };
4285 if let Some(sender) = progress_sender.as_ref() {
4286 sender(PushFrame::BashPatternMatch(BashPatternMatchFrame::new(
4287 pattern_match.task_id,
4288 session_id.to_string(),
4289 pattern_match.watch_id,
4290 pattern_match.match_text,
4291 pattern_match.match_offset,
4292 pattern_match.context,
4293 pattern_match.once,
4294 )));
4295 }
4296 }
4297
4298 fn emit_bash_watch_erased(&self, session_id: &str, task_id: &str, watch_id: &str) {
4299 let Ok(progress_sender) = self
4300 .inner
4301 .progress_sender
4302 .lock()
4303 .map(|sender| sender.clone())
4304 else {
4305 return;
4306 };
4307 let Some(sender) = progress_sender.as_ref() else {
4308 return;
4309 };
4310 sender(PushFrame::BashPatternMatch(
4311 BashPatternMatchFrame::watch_target_erased(
4312 task_id,
4313 session_id,
4314 watch_id,
4315 WATCH_TARGET_ERASED_TEXT,
4316 WATCH_TARGET_ERASED_CONTEXT,
4317 ),
4318 ));
4319 }
4320
4321 fn emit_bash_watch_exit(&self, completion: &BgCompletion) {
4322 let Ok(progress_sender) = self
4323 .inner
4324 .progress_sender
4325 .lock()
4326 .map(|sender| sender.clone())
4327 else {
4328 return;
4329 };
4330 let Some(sender) = progress_sender.as_ref() else {
4331 return;
4332 };
4333 let status = completion_status_text(&completion.status, completion.exit_code);
4334 let preview = completion.output_preview.trim_end();
4335 let context = if preview.is_empty() {
4336 format!("task {} exited ({status})", completion.task_id)
4337 } else {
4338 format!(
4339 "task {} exited ({status})
4340{preview}",
4341 completion.task_id
4342 )
4343 };
4344 sender(PushFrame::BashPatternMatch(
4345 BashPatternMatchFrame::task_exit(
4346 completion.task_id.clone(),
4347 completion.session_id.clone(),
4348 format!("exited ({status})"),
4349 context,
4350 ),
4351 ));
4352 }
4353
4354 fn emit_bash_completed(&self, completion: BgCompletion) {
4355 let Ok(progress_sender) = self
4356 .inner
4357 .progress_sender
4358 .lock()
4359 .map(|sender| sender.clone())
4360 else {
4361 return;
4362 };
4363 let Some(sender) = progress_sender.as_ref() else {
4364 return;
4365 };
4366 let mut frame = BashCompletedFrame::new(
4374 completion.task_id,
4375 completion.session_id,
4376 completion.status,
4377 completion.exit_code,
4378 completion.command,
4379 completion.output_preview,
4380 completion.output_truncated,
4381 completion.original_tokens,
4382 completion.compressed_tokens,
4383 completion.tokens_skipped,
4384 );
4385 frame.status_reason = completion.status_reason;
4386 sender(PushFrame::BashCompleted(frame));
4387 }
4388
4389 fn completion_token_counts(
4390 &self,
4391 metadata: &PersistedTask,
4392 buffer: Option<&BgBuffer>,
4393 paths: Option<&TaskPaths>,
4394 rendered_output: Option<&str>,
4395 ) -> CompletionTokenCounts {
4396 if metadata.mode == BgMode::Pty {
4397 return CompletionTokenCounts::skipped();
4398 }
4399
4400 let raw = match buffer {
4401 Some(buffer) => buffer.read_for_token_count(TOKENIZE_CAP_BYTES_PER_STREAM),
4402 None => paths
4403 .map(|paths| {
4404 read_for_token_count_from_disk(metadata, paths, TOKENIZE_CAP_BYTES_PER_STREAM)
4405 })
4406 .unwrap_or(TokenCountInput::Skipped),
4407 };
4408
4409 let TokenCountInput::Text(raw_output) = raw else {
4410 return CompletionTokenCounts::skipped();
4411 };
4412
4413 let original_tokens = token_count_u32(&raw_output);
4414 let original_bytes = raw_output.len() as i64;
4415 let compressed_output = rendered_output.unwrap_or(&raw_output);
4416 let compressed_tokens = token_count_u32(compressed_output);
4417 let compressed_bytes = compressed_output.len() as i64;
4418 CompletionTokenCounts {
4419 original_tokens: Some(original_tokens),
4420 compressed_tokens: Some(compressed_tokens),
4421 original_bytes: Some(original_bytes),
4422 compressed_bytes: Some(compressed_bytes),
4423 tokens_skipped: false,
4424 }
4425 }
4426
4427 pub(crate) fn maybe_emit_long_running_reminder(&self, task: &Arc<BgTask>) {
4428 if !self
4429 .inner
4430 .long_running_reminder_enabled
4431 .load(Ordering::SeqCst)
4432 {
4433 return;
4434 }
4435 let interval_ms = self
4436 .inner
4437 .long_running_reminder_interval_ms
4438 .load(Ordering::SeqCst);
4439 if interval_ms == 0 {
4440 return;
4441 }
4442 let interval = Duration::from_millis(interval_ms);
4443 let now = Instant::now();
4444 let Ok(mut last_reminder_at) = task.last_reminder_at.lock() else {
4445 return;
4446 };
4447 let since = last_reminder_at.unwrap_or(task.started);
4448 if now.duration_since(since) < interval {
4449 return;
4450 }
4451 let command = task
4452 .state
4453 .lock()
4454 .map(|state| state.metadata.command.clone())
4455 .unwrap_or_default();
4456 *last_reminder_at = Some(now);
4457 self.emit_bash_long_running(BashLongRunningFrame::new(
4458 task.task_id.clone(),
4459 task.session_id.clone(),
4460 command,
4461 task.started.elapsed().as_millis() as u64,
4462 ));
4463 }
4464
4465 fn emit_bash_long_running(&self, frame: BashLongRunningFrame) {
4466 let Ok(progress_sender) = self
4467 .inner
4468 .progress_sender
4469 .lock()
4470 .map(|sender| sender.clone())
4471 else {
4472 return;
4473 };
4474 if let Some(sender) = progress_sender.as_ref() {
4475 sender(PushFrame::BashLongRunning(frame));
4476 }
4477 }
4478
4479 fn task(&self, task_id: &str) -> Option<Arc<BgTask>> {
4480 validate_task_id(task_id).ok()?;
4481 self.inner
4482 .tasks
4483 .lock()
4484 .ok()
4485 .and_then(|tasks| tasks.get(task_id).cloned())
4486 }
4487
4488 fn task_for_session(&self, task_id: &str, session_id: &str) -> Option<Arc<BgTask>> {
4489 self.task(task_id)
4490 .filter(|task| task.session_id == session_id)
4491 }
4492
4493 pub fn try_health_counts(&self) -> Option<BgTaskHealthCounts> {
4494 let running = self
4495 .inner
4496 .tasks
4497 .try_lock()
4498 .ok()
4499 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())?;
4500 let pending_completions = self.inner.completions.try_lock().ok().map(|q| q.len())?;
4501 Some(BgTaskHealthCounts {
4502 running,
4503 pending_completions,
4504 })
4505 }
4506
4507 pub fn estimated_memory(&self) -> crate::memory::MemoryEstimate {
4511 let tasks = match self.inner.tasks.try_lock() {
4512 Ok(tasks) => tasks.values().cloned().collect::<Vec<_>>(),
4513 Err(_) => return crate::memory::MemoryEstimate::busy(),
4514 };
4515 let mut bytes = 0u64;
4516 let mut terminal_output_caches = 0usize;
4517 let mut sessions = HashSet::new();
4518 for task in &tasks {
4519 sessions.insert(task.session_id.clone());
4520 let state = match task.state.try_lock() {
4521 Ok(state) => state,
4522 Err(_) => return crate::memory::MemoryEstimate::busy(),
4523 };
4524 if let Some(cache) = state.terminal_output_cache.as_ref() {
4525 terminal_output_caches = terminal_output_caches.saturating_add(1);
4526 bytes = bytes.saturating_add(terminal_output_cache_estimated_bytes(cache));
4527 }
4528 }
4529 let completion_count = match self.inner.completions.try_lock() {
4530 Ok(completions) => {
4531 for completion in completions.iter() {
4532 sessions.insert(completion.session_id.clone());
4533 bytes = bytes.saturating_add(completion_estimated_bytes(completion));
4534 }
4535 completions.len()
4536 }
4537 Err(_) => return crate::memory::MemoryEstimate::busy(),
4538 };
4539
4540 crate::memory::MemoryEstimate::estimated(bytes)
4541 .count("tasks", tasks.len())
4542 .count("sessions", sessions.len())
4543 .count("terminal_output_caches", terminal_output_caches)
4544 .count("completion_caches", completion_count)
4545 .count_u64("output_ring_bytes", 0)
4546 }
4547
4548 fn running_count(&self) -> usize {
4549 self.inner
4550 .tasks
4551 .lock()
4552 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())
4553 .unwrap_or(0)
4554 }
4555
4556 fn start_watchdog(&self) {
4557 if !self.inner.watchdog_started.swap(true, Ordering::SeqCst) {
4558 super::watchdog::start(self.clone());
4559 }
4560 }
4561
4562 #[cfg(test)]
4563 pub fn task_json_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
4564 self.task_for_session(task_id, session_id)
4565 .map(|task| task.paths.json.clone())
4566 }
4567
4568 #[cfg(test)]
4569 pub fn task_exit_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
4570 self.task_for_session(task_id, session_id)
4571 .map(|task| task.paths.exit.clone())
4572 }
4573}
4574
4575#[cfg(unix)]
4576fn should_capture_pipeline_status(
4577 spawn_plan: &SpawnPlan,
4578 has_pipeline: bool,
4579 shell: &Path,
4580) -> bool {
4581 if spawn_plan.is_native_launcher() {
4582 return false;
4585 }
4586 has_pipeline && super::process::pipeline_shell_kind(shell).is_some()
4587}
4588
4589fn canonical_artifact_root(paths: &TaskPaths) -> PathBuf {
4590 fs::canonicalize(&paths.io_dir).unwrap_or_else(|_| paths.io_dir.clone())
4591}
4592
4593fn restart_fate_unknown_reason(metadata: &PersistedTask, paths: &TaskPaths) -> String {
4594 let output = match metadata.mode {
4595 BgMode::Pipes => &paths.stdout,
4596 BgMode::Pty => &paths.pty,
4597 };
4598 format!(
4599 "task {}: daemon restarted, process fate unknown, last output at {}",
4600 metadata.task_id,
4601 output.display()
4602 )
4603}
4604
4605fn append_pipeline_warning(
4610 cache: &mut TerminalOutputCache,
4611 metadata: &PersistedTask,
4612 paths: Option<&TaskPaths>,
4613) {
4614 if metadata.exit_code != Some(0) {
4615 return;
4616 }
4617 let Some(paths) = paths else {
4618 return;
4619 };
4620 if metadata.pipeline_segments.len() < 2 {
4621 return;
4622 }
4623 let Ok(mut status_file) = open_task_artifact(paths, TaskArtifact::PipelineStatus) else {
4624 return;
4625 };
4626 let Ok(status_bytes) = status_file.read_all() else {
4627 return;
4628 };
4629 let Some(statuses) = String::from_utf8_lossy(&status_bytes)
4630 .lines()
4631 .map(|line| line.trim().parse::<i32>().ok())
4632 .collect::<Option<Vec<_>>>()
4633 else {
4634 return;
4635 };
4636 if statuses.len() != metadata.pipeline_segments.len() {
4637 return;
4638 }
4639 let Some((failing_index, failing_code)) = statuses
4640 .iter()
4641 .enumerate()
4642 .take(statuses.len().saturating_sub(1))
4643 .find(|(_, code)| **code != 0)
4644 .map(|(index, code)| (index, *code))
4645 else {
4646 return;
4647 };
4648 let Some(final_segment) = metadata.pipeline_segments.last() else {
4649 return;
4650 };
4651 let failing_segment = &metadata.pipeline_segments[failing_index];
4652 let footer = format!(
4653 "note: `{}` (segment {} of {}) exited {}; the pipeline's exit code is `{}`'s.",
4654 failing_segment,
4655 failing_index + 1,
4656 metadata.pipeline_segments.len(),
4657 failing_code,
4658 final_segment,
4659 );
4660 if cache.output_preview.trim().is_empty() {
4661 cache.output_preview = footer;
4662 } else {
4663 cache.output_preview = format!("{}\n{}", cache.output_preview.trim_end(), footer,);
4664 }
4665}
4666
4667fn normalize_piped_display_output(text: &mut String) {
4670 if !text.contains('\r') {
4671 return;
4672 }
4673
4674 let mut rendered = String::with_capacity(text.len());
4675 let mut line = Vec::new();
4676 let mut column = 0;
4677 let mut chars = text.chars().peekable();
4678
4679 while let Some(character) = chars.next() {
4680 match character {
4681 '\r' if chars.peek() == Some(&'\n') => {
4682 chars.next();
4683 for character in &line {
4684 rendered.push(*character);
4685 }
4686 rendered.push('\n');
4687 line.clear();
4688 column = 0;
4689 }
4690 '\r' => column = 0,
4691 '\n' => {
4692 for character in &line {
4693 rendered.push(*character);
4694 }
4695 rendered.push('\n');
4696 line.clear();
4697 column = 0;
4698 }
4699 character => {
4700 if column < line.len() {
4701 line[column] = character;
4702 } else {
4703 line.resize(column, ' ');
4704 line.push(character);
4705 }
4706 column += 1;
4707 }
4708 }
4709 }
4710
4711 for character in &line {
4712 rendered.push(*character);
4713 }
4714 *text = rendered;
4715}
4716
4717fn render_compressed_with_recovery(
4718 buffer: &BgBuffer,
4719 mut compressed: CompressionResult,
4720 input_truncated: bool,
4721 disk_truncation: DiskTruncation,
4722 artifact_access: ArtifactRecoveryAccess,
4723) -> TerminalOutputCache {
4724 let had_trailing_newline = compressed.text.ends_with('\n');
4732 let mut text = strip_plain_truncation_marker_lines(&compressed.text)
4733 .trim_end()
4734 .to_string();
4735 if had_trailing_newline && !text.is_empty() {
4736 text.push('\n');
4737 }
4738 compressed.text = text;
4739
4740 let output_path = buffer.output_path().map(|path| path.display().to_string());
4741 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
4742 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
4743 let mut recovery = RecoveryContext {
4744 dropped_by_class: compressed.dropped_by_class,
4745 had_inner_drop: compressed.had_inner_drop,
4746 offset_hint_eligible: compressed.offset_hint_eligible,
4747 offset_start_line: compressed.offset_start_line,
4748 byte_truncated: input_truncated,
4749 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
4750 output_path: output_path.clone(),
4751 stderr_path: stderr_path.clone(),
4752 include_stderr_path,
4753 artifact_access: artifact_access.clone(),
4754 };
4755
4756 let (output_preview, output_truncated) =
4757 render_body_with_recovery_marker(&compressed.text, &mut recovery);
4758 TerminalOutputCache {
4759 output_preview,
4760 output_truncated,
4761 kind: TerminalOutputKind::Compressed,
4762 output_path,
4763 stderr_path,
4764 artifact_access,
4765 recovery: Some(recovery),
4766 }
4767}
4768
4769fn render_body_with_recovery_marker(body: &str, recovery: &mut RecoveryContext) -> (String, bool) {
4770 render_body_with_recovery_marker_at_cap(
4771 body,
4772 recovery,
4773 FINAL_OUTPUT_CAP_BYTES,
4774 cap_final_output,
4775 cap_final_output_with_marker,
4776 )
4777}
4778
4779fn render_raw_body_with_recovery_marker(
4780 body: &str,
4781 recovery: &mut RecoveryContext,
4782) -> (String, bool) {
4783 render_body_with_recovery_marker_at_cap(
4784 body,
4785 recovery,
4786 RAW_PASSTHROUGH_CAP_BYTES,
4787 |input| {
4788 super::output::cap_head_tail(
4789 input,
4790 RAW_PASSTHROUGH_CAP_BYTES,
4791 RAW_PASSTHROUGH_HEAD_BYTES,
4792 RAW_PASSTHROUGH_TAIL_BYTES,
4793 )
4794 },
4795 |input, marker| {
4796 super::output::cap_head_tail_with_marker(
4797 input,
4798 RAW_PASSTHROUGH_CAP_BYTES,
4799 RAW_PASSTHROUGH_HEAD_BYTES,
4800 RAW_PASSTHROUGH_TAIL_BYTES,
4801 marker,
4802 )
4803 },
4804 )
4805}
4806
4807fn render_body_with_recovery_marker_at_cap<F, G>(
4808 body: &str,
4809 recovery: &mut RecoveryContext,
4810 cap_bytes: usize,
4811 cap_plain: F,
4812 cap_with_marker: G,
4813) -> (String, bool)
4814where
4815 F: Fn(&str) -> super::output::CappedText,
4816 G: Fn(&str, &str) -> super::output::CappedText,
4817{
4818 let needs_marker = recovery.has_visible_drop();
4819 if body.len() > cap_bytes {
4820 recovery.byte_truncated = true;
4821 if let Some(marker) = recovery_marker(recovery) {
4822 let capped = cap_with_marker(body, &marker);
4823 return (capped.text, true);
4824 }
4825 let capped = cap_plain(body);
4826 return (capped.text, capped.truncated || needs_marker);
4827 }
4828
4829 if !needs_marker {
4830 return (body.to_string(), false);
4831 }
4832
4833 let Some(marker) = recovery_marker(recovery) else {
4834 return (body.to_string(), true);
4835 };
4836 let with_marker = append_recovery_marker(body, &marker);
4837 if with_marker.len() <= cap_bytes {
4838 return (with_marker, true);
4839 }
4840
4841 recovery.byte_truncated = true;
4842 let marker = recovery_marker(recovery).unwrap_or(marker);
4843 let capped = cap_with_marker(body, &marker);
4844 (capped.text, true)
4845}
4846
4847fn append_recovery_marker(body: &str, marker: &str) -> String {
4848 if body.is_empty() {
4849 return marker.to_string();
4850 }
4851 let mut output = body.trim_end().to_string();
4852 output.push('\n');
4853 output.push_str(marker);
4854 output
4855}
4856
4857fn recovery_marker(recovery: &RecoveryContext) -> Option<String> {
4858 let mut parts = Vec::new();
4859 for (class, count) in &recovery.dropped_by_class {
4860 let label = if *count == 1 {
4861 class.singular()
4862 } else {
4863 class.plural()
4864 };
4865 parts.push(format!("+{count} more {label}"));
4866 }
4867 if recovery.byte_truncated {
4868 parts.push("truncated output".to_string());
4869 }
4870 let disk_truncated_prefix_bytes = recovery.disk_truncated_prefix_bytes;
4871 if disk_truncated_prefix_bytes > 0 {
4872 parts.push(format!(
4873 "truncated {disk_truncated_prefix_bytes} bytes from saved output prefix"
4874 ));
4875 } else if recovery.had_inner_drop && parts.is_empty() {
4876 parts.push("omitted output".to_string());
4877 }
4878
4879 if parts.is_empty() {
4880 return None;
4881 }
4882
4883 let hint = recovery_hint(recovery);
4884 Some(format!("[{}; {hint}]", parts.join(", ")))
4885}
4886
4887fn bash_status_recovery_hint(access: &ArtifactRecoveryAccess) -> String {
4888 let task_id = serde_json::to_string(&access.task_id)
4889 .unwrap_or_else(|_| format!("\"{}\"", access.task_id));
4890 format!("use bash_status({{taskId: {task_id}}})")
4891}
4892
4893fn recovery_hint(recovery: &RecoveryContext) -> String {
4894 if !recovery.artifact_access.readable {
4895 return bash_status_recovery_hint(&recovery.artifact_access);
4896 }
4897
4898 if recovery.offset_hint_eligible
4902 && !recovery.byte_truncated
4903 && recovery.dropped_by_class.is_empty()
4904 && !recovery.include_stderr_path
4905 {
4906 if let (Some(path), Some(line)) =
4907 (recovery.output_path.as_deref(), recovery.offset_start_line)
4908 {
4909 return format!("see remaining: tail -n +{line} {}", quote_path(path));
4910 }
4911 }
4912
4913 let mut paths = Vec::new();
4914 if let Some(path) = recovery.output_path.as_deref() {
4915 paths.push(path);
4916 }
4917 if recovery.include_stderr_path {
4918 if let Some(path) = recovery.stderr_path.as_deref() {
4919 if !paths.contains(&path) {
4920 paths.push(path);
4921 }
4922 }
4923 }
4924
4925 if paths.is_empty() {
4926 return "full output unavailable".to_string();
4927 }
4928
4929 let reads = paths
4930 .into_iter()
4931 .map(|path| format!("read {}", quote_path(path)))
4932 .collect::<Vec<_>>()
4933 .join(" and ");
4934 if recovery.disk_truncated_prefix_bytes > 0 {
4935 format!("retained output: {reads}")
4936 } else {
4937 format!("full output: {reads}")
4938 }
4939}
4940
4941fn strip_plain_truncation_marker_lines(input: &str) -> String {
4942 input
4943 .lines()
4944 .filter(|line| !is_plain_truncation_marker(line.trim()))
4945 .collect::<Vec<_>>()
4946 .join("\n")
4947}
4948
4949fn strip_recovery_marker_lines(input: &str) -> String {
4950 input
4951 .lines()
4952 .filter(|line| !is_recovery_marker(line.trim()))
4953 .collect::<Vec<_>>()
4954 .join("\n")
4955}
4956
4957fn is_plain_truncation_marker(line: &str) -> bool {
4958 let Some(rest) = line.strip_prefix("...<truncated ") else {
4959 return false;
4960 };
4961 let Some(bytes) = rest.strip_suffix(" bytes>...") else {
4962 return false;
4963 };
4964 !bytes.is_empty() && bytes.chars().all(|ch| ch.is_ascii_digit())
4965}
4966
4967fn is_recovery_marker(line: &str) -> bool {
4968 line.starts_with('[')
4969 && line.ends_with(']')
4970 && (line.contains("full output: read ")
4971 || line.contains("retained output: read ")
4972 || line.contains("see remaining: tail -n +")
4973 || line.contains("use bash_status({taskId:")
4974 || line.contains("full output unavailable"))
4975}
4976
4977fn structured_output_pointer(
4978 total_bytes: u64,
4979 output_path: &str,
4980 truncated_prefix_bytes: u64,
4981 artifact_access: &ArtifactRecoveryAccess,
4982) -> String {
4983 if artifact_access.readable {
4984 return if truncated_prefix_bytes > 0 {
4985 retained_json_output_pointer(total_bytes, output_path, truncated_prefix_bytes)
4986 } else {
4987 json_output_pointer(total_bytes, output_path)
4988 };
4989 }
4990
4991 let kb = total_bytes.div_ceil(1024);
4992 let hint = bash_status_recovery_hint(artifact_access);
4993 if truncated_prefix_bytes > 0 {
4994 format!(
4995 "[JSON output {kb} KB; truncated {truncated_prefix_bytes} bytes from saved output prefix; retained output: {hint}]"
4996 )
4997 } else {
4998 format!("[JSON output {kb} KB; full output: {hint}]")
4999 }
5000}
5001
5002fn render_structured_output(
5003 command: &str,
5004 buffer: &BgBuffer,
5005 disk_truncation: DiskTruncation,
5006 artifact_access: ArtifactRecoveryAccess,
5007) -> Option<TerminalOutputCache> {
5008 if !is_gh_structured_command(command) {
5009 return None;
5010 }
5011
5012 let output_path = buffer
5013 .output_path()
5014 .map(|path| path.display().to_string())?;
5015 let stdout_bytes = buffer.stream_len(StreamKind::Stdout);
5016 if stdout_bytes == 0 {
5017 return None;
5018 }
5019
5020 if stdout_bytes > STRUCTURED_OUTPUT_CAP_BYTES as u64 {
5021 if !stream_starts_like_json(buffer, StreamKind::Stdout) {
5022 return None;
5023 }
5024 let output_preview = structured_output_pointer(
5025 stdout_bytes,
5026 &output_path,
5027 disk_truncation.total_prefix_bytes(),
5028 &artifact_access,
5029 );
5030 return Some(TerminalOutputCache {
5031 output_preview,
5032 output_truncated: true,
5033 kind: TerminalOutputKind::Structured,
5034 output_path: Some(output_path),
5035 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
5036 artifact_access,
5037 recovery: None,
5038 });
5039 }
5040
5041 let stdout = buffer.read_stream_bounded(StreamKind::Stdout, STRUCTURED_OUTPUT_CAP_BYTES);
5042 if stdout.truncated || !is_structured_body(&stdout.text) {
5043 return None;
5044 }
5045
5046 Some(TerminalOutputCache {
5047 output_preview: stdout.text,
5048 output_truncated: false,
5049 kind: TerminalOutputKind::Structured,
5050 output_path: Some(output_path),
5051 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
5052 artifact_access,
5053 recovery: None,
5054 })
5055}
5056
5057fn render_raw_passthrough(
5058 buffer: &BgBuffer,
5059 disk_truncation: DiskTruncation,
5060 artifact_access: ArtifactRecoveryAccess,
5061) -> TerminalOutputCache {
5062 let raw = buffer.read_combined_head_tail(
5063 RAW_PASSTHROUGH_CAP_BYTES,
5064 RAW_PASSTHROUGH_HEAD_BYTES,
5065 RAW_PASSTHROUGH_TAIL_BYTES,
5066 );
5067 let output_path = buffer.output_path().map(|path| path.display().to_string());
5068 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
5069 if !raw.truncated && disk_truncation.total_prefix_bytes() == 0 {
5070 return TerminalOutputCache {
5071 output_preview: raw.text,
5072 output_truncated: false,
5073 kind: TerminalOutputKind::Raw,
5074 output_path,
5075 stderr_path,
5076 artifact_access,
5077 recovery: None,
5078 };
5079 }
5080
5081 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
5082 let mut recovery = RecoveryContext {
5083 dropped_by_class: BTreeMap::new(),
5084 had_inner_drop: false,
5085 offset_hint_eligible: false,
5086 offset_start_line: None,
5087 byte_truncated: raw.truncated,
5088 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
5089 output_path: output_path.clone(),
5090 stderr_path: stderr_path.clone(),
5091 include_stderr_path,
5092 artifact_access: artifact_access.clone(),
5093 };
5094 let (output_preview, output_truncated) =
5095 render_raw_body_with_recovery_marker(&raw.text, &mut recovery);
5096 TerminalOutputCache {
5097 output_preview,
5098 output_truncated,
5099 kind: TerminalOutputKind::Raw,
5100 output_path,
5101 stderr_path,
5102 artifact_access,
5103 recovery: Some(recovery),
5104 }
5105}
5106
5107fn completion_preview_for_cache(
5108 cache: &TerminalOutputCache,
5109 exit_code: Option<i32>,
5110) -> (String, bool) {
5111 let exit_ok = exit_code == Some(0);
5114 let threshold = completion_preview_threshold(exit_ok);
5115 if cache.kind == TerminalOutputKind::Structured && cache.output_preview.len() > threshold {
5116 if let Some(path) = cache.output_path.as_deref() {
5117 return (
5118 structured_output_pointer(
5119 cache.output_preview.len() as u64,
5120 path,
5121 0,
5122 &cache.artifact_access,
5123 ),
5124 true,
5125 );
5126 }
5127 return (cache.output_preview.clone(), cache.output_truncated);
5128 }
5129
5130 if let Some(recovery) = cache.recovery.as_ref() {
5131 if cache.output_preview.len() <= threshold {
5132 return (cache.output_preview.clone(), cache.output_truncated);
5133 }
5134 let body = strip_recovery_marker_lines(&cache.output_preview);
5135 let mut completion_recovery = recovery.clone();
5136 completion_recovery.byte_truncated = true;
5137 if let Some(marker) = recovery_marker(&completion_recovery) {
5138 let capped = cap_completion_output_with_marker(&body, &marker, exit_ok);
5139 return (capped.text, true);
5140 }
5141 }
5142
5143 let capped = cap_completion_output(&cache.output_preview, exit_ok);
5144 (capped.text, cache.output_truncated || capped.truncated)
5145}
5146
5147fn is_gh_structured_command(command: &str) -> bool {
5148 let Some(normalized) = crate::compress::plain_command_for_structured_output(command) else {
5149 return false;
5150 };
5151 let tokens = shell_words_for_flags(&normalized);
5152 let Some(head) = tokens.first() else {
5153 return false;
5154 };
5155 let head_name = Path::new(head)
5156 .file_name()
5157 .and_then(|name| name.to_str())
5158 .unwrap_or(head);
5159 if !(head_name == "gh" || head_name.eq_ignore_ascii_case("gh.exe")) {
5160 return false;
5161 }
5162 tokens.iter().any(|token| {
5163 matches!(token.as_str(), "--json" | "--jq" | "--template")
5164 || token.starts_with("--json=")
5165 || token.starts_with("--jq=")
5166 || token.starts_with("--template=")
5167 })
5168}
5169
5170fn shell_words_for_flags(command: &str) -> Vec<String> {
5171 let mut words = Vec::new();
5172 let mut current = String::new();
5173 let mut in_single = false;
5174 let mut in_double = false;
5175 let mut escaped = false;
5176
5177 for ch in command.chars() {
5178 if escaped {
5179 current.push(ch);
5180 escaped = false;
5181 continue;
5182 }
5183 if ch == '\\' && !in_single {
5184 escaped = true;
5185 continue;
5186 }
5187 if ch == '\'' && !in_double {
5188 in_single = !in_single;
5189 continue;
5190 }
5191 if ch == '"' && !in_single {
5192 in_double = !in_double;
5193 continue;
5194 }
5195 if ch.is_whitespace() && !in_single && !in_double {
5196 if !current.is_empty() {
5197 words.push(std::mem::take(&mut current));
5198 }
5199 continue;
5200 }
5201 if matches!(ch, ';' | '&' | '|') && !in_single && !in_double {
5202 if !current.is_empty() {
5203 words.push(std::mem::take(&mut current));
5204 }
5205 continue;
5206 }
5207 current.push(ch);
5208 }
5209 if !current.is_empty() {
5210 words.push(current);
5211 }
5212 words
5213}
5214
5215fn is_structured_body(body: &str) -> bool {
5216 let trimmed = body.trim();
5217 if trimmed.is_empty() {
5218 return false;
5219 }
5220 if serde_json::from_str::<serde_json::Value>(trimmed).is_ok() {
5221 return true;
5222 }
5223
5224 let mut saw_line = false;
5225 for line in trimmed
5226 .lines()
5227 .map(str::trim)
5228 .filter(|line| !line.is_empty())
5229 {
5230 saw_line = true;
5231 if serde_json::from_str::<serde_json::Value>(line).is_err() {
5232 return false;
5233 }
5234 }
5235 saw_line
5236}
5237
5238fn stream_starts_like_json(buffer: &BgBuffer, stream: StreamKind) -> bool {
5239 buffer
5240 .read_stream_bounded(stream, 512)
5241 .text
5242 .chars()
5243 .find(|ch| !ch.is_whitespace())
5244 .is_some_and(|ch| matches!(ch, '{' | '[' | '"' | '-' | '0'..='9' | 't' | 'f' | 'n'))
5245}
5246
5247struct CompletionTokenCounts {
5248 original_tokens: Option<u32>,
5249 compressed_tokens: Option<u32>,
5250 original_bytes: Option<i64>,
5251 compressed_bytes: Option<i64>,
5252 tokens_skipped: bool,
5253}
5254
5255impl CompletionTokenCounts {
5256 fn skipped() -> Self {
5257 Self {
5258 original_tokens: None,
5259 compressed_tokens: None,
5260 original_bytes: None,
5261 compressed_bytes: None,
5262 tokens_skipped: true,
5263 }
5264 }
5265}
5266
5267fn completion_status_text(status: &BgTaskStatus, exit_code: Option<i32>) -> String {
5268 match status {
5269 BgTaskStatus::TimedOut => "timed out".to_string(),
5270 BgTaskStatus::Killed => "killed".to_string(),
5271 _ => exit_code
5272 .map(|code| format!("exit {code}"))
5273 .unwrap_or_else(|| format!("{status:?}").to_lowercase()),
5274 }
5275}
5276
5277fn token_count_u32(text: &str) -> u32 {
5278 aft_tokenizer::count_tokens(text)
5279 .try_into()
5280 .unwrap_or(u32::MAX)
5281}
5282
5283impl Default for BgTaskRegistry {
5284 fn default() -> Self {
5285 Self::new(Arc::new(Mutex::new(None)))
5286 }
5287}
5288
5289fn modified_within(path: &Path, grace: Duration) -> bool {
5290 fs::metadata(path)
5291 .and_then(|metadata| metadata.modified())
5292 .ok()
5293 .and_then(|modified| SystemTime::now().duration_since(modified).ok())
5294 .map(|age| age < grace)
5295 .unwrap_or(false)
5296}
5297
5298fn canonicalized_path(path: &Path) -> PathBuf {
5299 fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf())
5300}
5301
5302fn started_instant_from_unix_millis(started_at: u64) -> Instant {
5303 let now_ms = SystemTime::now()
5304 .duration_since(UNIX_EPOCH)
5305 .ok()
5306 .map(|duration| duration.as_millis() as u64)
5307 .unwrap_or(started_at);
5308 let elapsed_ms = now_ms.saturating_sub(started_at);
5309 Instant::now()
5310 .checked_sub(Duration::from_millis(elapsed_ms))
5311 .unwrap_or_else(Instant::now)
5312}
5313
5314fn gc_quarantine(storage_dir: &Path) {
5315 let quarantine_root = storage_dir.join("bash-tasks-quarantine");
5316 let Ok(session_dirs) = fs::read_dir(&quarantine_root) else {
5317 return;
5318 };
5319 for session_entry in session_dirs.flatten() {
5320 let session_quarantine_dir = session_entry.path();
5321 if !session_quarantine_dir.is_dir() {
5322 continue;
5323 }
5324 let entries = match fs::read_dir(&session_quarantine_dir) {
5325 Ok(entries) => entries,
5326 Err(error) => {
5327 crate::slog_warn!(
5328 "failed to read background task quarantine dir {}: {error}",
5329 session_quarantine_dir.display()
5330 );
5331 continue;
5332 }
5333 };
5334 for entry in entries.flatten() {
5335 let path = entry.path();
5336 if modified_within(&path, QUARANTINE_GC_GRACE) {
5337 continue;
5338 }
5339 let result = if path.is_dir() {
5340 fs::remove_dir_all(&path)
5341 } else {
5342 fs::remove_file(&path)
5343 };
5344 match result {
5345 Ok(()) => log::debug!(
5346 "deleted old background task quarantine entry {}",
5347 path.display()
5348 ),
5349 Err(error) => crate::slog_warn!(
5350 "failed to delete old background task quarantine entry {}: {error}",
5351 path.display()
5352 ),
5353 }
5354 }
5355 let _ = fs::remove_dir(&session_quarantine_dir);
5356 }
5357 let _ = fs::remove_dir(&quarantine_root);
5358}
5359
5360fn read_for_token_count_from_disk(
5361 metadata: &PersistedTask,
5362 paths: &TaskPaths,
5363 max_bytes_per_stream: usize,
5364) -> TokenCountInput {
5365 if metadata.mode == BgMode::Pty {
5366 return TokenCountInput::Skipped;
5367 }
5368 let stdout = read_file_tail_capped(paths, TaskArtifact::Stdout, max_bytes_per_stream);
5375 let stderr = read_file_tail_capped(paths, TaskArtifact::Stderr, max_bytes_per_stream);
5376 match (stdout, stderr) {
5377 (Ok(stdout), Ok(stderr)) => TokenCountInput::Text(combine_streams(
5378 String::from_utf8_lossy(&stdout).as_ref(),
5379 String::from_utf8_lossy(&stderr).as_ref(),
5380 )),
5381 (Ok(stdout), Err(_)) => TokenCountInput::Text(combine_streams(
5382 String::from_utf8_lossy(&stdout).as_ref(),
5383 "",
5384 )),
5385 (Err(_), Ok(stderr)) => TokenCountInput::Text(combine_streams(
5386 "",
5387 String::from_utf8_lossy(&stderr).as_ref(),
5388 )),
5389 (Err(_), Err(_)) => TokenCountInput::Skipped,
5390 }
5391}
5392
5393fn read_file_tail_capped(
5394 paths: &TaskPaths,
5395 artifact: TaskArtifact,
5396 max_bytes: usize,
5397) -> std::io::Result<Vec<u8>> {
5398 let mut file = open_task_artifact(paths, artifact)?;
5399 file.tail(max_bytes).map(|(bytes, _)| bytes)
5400}
5401
5402fn task_bundle_is_absent(storage_dir: &Path, session_id: &str, task_id: &str) -> bool {
5403 let session_dir = session_tasks_dir(storage_dir, session_id);
5404 !session_dir.join(task_id).exists() && !session_dir.join(format!("{task_id}.json")).exists()
5405}
5406
5407fn terminal_db_row_snapshot(row: BashTaskRow, metadata: PersistedTask) -> BgTaskSnapshot {
5408 let existing_path = |path: Option<String>| {
5409 path.filter(|path| {
5410 fs::metadata(path)
5411 .map(|metadata| metadata.is_file())
5412 .unwrap_or(false)
5413 })
5414 };
5415 let duration_ms = metadata.duration_ms.or_else(|| {
5416 metadata
5417 .finished_at
5418 .map(|finished_at| finished_at.saturating_sub(metadata.started_at))
5419 });
5420 BgTaskSnapshot {
5421 info: BgTaskInfo {
5422 task_id: metadata.task_id,
5423 status: metadata.status,
5424 command: metadata.command,
5425 mode: metadata.mode.clone(),
5426 started_at: metadata.started_at,
5427 duration_ms,
5428 status_reason: metadata.status_reason,
5429 },
5430 exit_code: metadata.exit_code,
5431 child_pid: metadata.child_pid,
5432 workdir: metadata.workdir.display().to_string(),
5433 output_preview: String::new(),
5434 output_truncated: false,
5435 output_path: existing_path(row.stdout_path),
5436 stderr_path: existing_path(row.stderr_path),
5437 pty_rows: (metadata.mode == BgMode::Pty).then_some(metadata.pty_rows.unwrap_or(24)),
5438 pty_cols: (metadata.mode == BgMode::Pty).then_some(metadata.pty_cols.unwrap_or(80)),
5439 pty_screen: None,
5440 scanner_report: metadata.scanner_report,
5441 sandbox_native: metadata.sandbox_native,
5442 sandbox_unavailable: false,
5443 }
5444}
5445
5446impl BgTask {
5447 fn snapshot(&self, preview_bytes: usize) -> BgTaskSnapshot {
5448 let state = self
5449 .state
5450 .lock()
5451 .unwrap_or_else(|poison| poison.into_inner());
5452 self.snapshot_locked(&state, preview_bytes)
5453 }
5454
5455 fn snapshot_locked(&self, state: &BgTaskState, preview_bytes: usize) -> BgTaskSnapshot {
5456 let metadata = &state.metadata;
5457 let duration_ms = metadata.duration_ms.or_else(|| {
5458 metadata
5459 .status
5460 .is_terminal()
5461 .then(|| self.started.elapsed().as_millis() as u64)
5462 });
5463 let (output_preview, output_truncated) = if metadata.mode == BgMode::Pty {
5464 (String::new(), false)
5465 } else if metadata.status.is_terminal() {
5466 state
5467 .terminal_output_cache
5468 .as_ref()
5469 .map(|cache| (cache.output_preview.clone(), cache.output_truncated))
5470 .unwrap_or_else(|| (String::new(), false))
5471 } else if preview_bytes == 0 {
5472 (String::new(), false)
5473 } else {
5474 state.buffer.read_tail(preview_bytes)
5475 };
5476 BgTaskSnapshot {
5477 info: BgTaskInfo {
5478 task_id: self.task_id.clone(),
5479 status: metadata.status.clone(),
5480 command: metadata.command.clone(),
5481 mode: metadata.mode.clone(),
5482 started_at: metadata.started_at,
5483 duration_ms,
5484 status_reason: metadata.status_reason.clone(),
5485 },
5486 exit_code: metadata.exit_code,
5487 child_pid: metadata.child_pid,
5488 workdir: metadata.workdir.display().to_string(),
5489 output_preview,
5490 output_truncated,
5491 output_path: state
5492 .buffer
5493 .output_path()
5494 .map(|path| path.display().to_string()),
5495 stderr_path: state
5496 .buffer
5497 .stderr_path()
5498 .map(|path| path.display().to_string()),
5499 pty_rows: (metadata.mode == BgMode::Pty).then_some(metadata.pty_rows.unwrap_or(24)),
5500 pty_cols: (metadata.mode == BgMode::Pty).then_some(metadata.pty_cols.unwrap_or(80)),
5501 pty_screen: None,
5502 scanner_report: metadata.scanner_report.clone(),
5503 sandbox_native: metadata.sandbox_native,
5504 sandbox_unavailable: metadata.sandbox_native
5505 && open_task_artifact(&self.paths, TaskArtifact::SandboxUnavailable)
5506 .and_then(|mut file| file.read_all())
5507 .is_ok_and(|bytes| bytes == b"sandbox_unavailable"),
5508 }
5509 }
5510
5511 pub(crate) fn is_running(&self) -> bool {
5512 self.state
5513 .lock()
5514 .map(|state| {
5515 state.metadata.status == BgTaskStatus::Running
5516 || (state.metadata.mode == BgMode::Pty
5517 && state.metadata.status == BgTaskStatus::Killing)
5518 })
5519 .unwrap_or(false)
5520 }
5521
5522 fn is_terminal(&self) -> bool {
5523 self.state
5524 .lock()
5525 .map(|state| state.metadata.status.is_terminal())
5526 .unwrap_or(false)
5527 }
5528
5529 fn mark_terminal_now(&self) {
5530 if let Ok(mut terminal_at) = self.terminal_at.lock() {
5531 if terminal_at.is_none() {
5532 *terminal_at = Some(Instant::now());
5533 }
5534 }
5535 }
5536
5537 fn set_completion_delivered(
5538 &self,
5539 delivered: bool,
5540 registry: &BgTaskRegistry,
5541 ) -> Result<(), String> {
5542 let mut state = self
5543 .state
5544 .lock()
5545 .map_err(|_| "background task lock poisoned".to_string())?;
5546 let updated = registry
5547 .update_task_metadata(&self.paths, |metadata| {
5548 metadata.completion_delivered = delivered;
5549 })
5550 .map_err(|e| format!("failed to update completion delivery: {e}"))?;
5551 state.metadata = updated;
5552 Ok(())
5553 }
5554}
5555
5556#[cfg(unix)]
5577fn reap_piped_child(child_slot: &mut Option<Child>) {
5578 if let Some(mut child) = child_slot.take() {
5579 if matches!(child.try_wait(), Ok(None)) {
5580 let _ = child.wait();
5581 }
5582 }
5583}
5584
5585#[cfg(windows)]
5590fn reap_piped_child(child_slot: &mut Option<Child>) {
5591 *child_slot = None;
5592}
5593
5594fn terminal_metadata_from_marker(
5595 mut metadata: PersistedTask,
5596 marker: ExitMarker,
5597 reason: Option<String>,
5598) -> PersistedTask {
5599 match marker {
5600 ExitMarker::Code(code) => {
5601 let status = if code == 0 {
5602 BgTaskStatus::Completed
5603 } else {
5604 BgTaskStatus::Failed
5605 };
5606 metadata.mark_terminal(status, Some(code), reason);
5607 }
5608 ExitMarker::Killed => metadata.mark_terminal(
5609 BgTaskStatus::Killed,
5610 terminal_exit_code_for_status(&BgTaskStatus::Killed),
5611 reason,
5612 ),
5613 }
5614 metadata
5615}
5616
5617fn terminal_exit_code_for_status(status: &BgTaskStatus) -> Option<i32> {
5618 match status {
5619 BgTaskStatus::TimedOut => Some(124),
5620 BgTaskStatus::Killed => Some(137),
5621 _ => None,
5622 }
5623}
5624
5625fn attach_sandbox_metadata(metadata: &mut PersistedTask, spawn_plan: &SpawnPlan) {
5626 metadata.sandbox_native = spawn_plan.is_native_launcher();
5627 metadata.sandbox_temp_dir = spawn_plan.temp_dir().map(Path::to_path_buf);
5628}
5629
5630#[cfg(unix)]
5631pub(crate) fn resolve_posix_shell() -> PathBuf {
5632 static POSIX_SHELL: OnceLock<PathBuf> = OnceLock::new();
5633 POSIX_SHELL
5634 .get_or_init(|| {
5635 std::env::var_os("BASH")
5636 .filter(|value| !value.is_empty())
5637 .map(PathBuf::from)
5638 .filter(|path| path.exists())
5639 .or_else(|| which::which("bash").ok())
5640 .or_else(|| which::which("zsh").ok())
5641 .unwrap_or_else(|| PathBuf::from("/bin/sh"))
5642 })
5643 .clone()
5644}
5645
5646#[cfg(windows)]
5647fn detached_shell_command_for(
5648 shell: crate::windows_shell::WindowsShell,
5649 command: &str,
5650 exit_path: &Path,
5651 paths: &TaskPaths,
5652 creation_flags: u32,
5653) -> Result<Command, String> {
5654 use crate::windows_shell::WindowsShell;
5655 let wrapper_body = shell.wrapper_script_bytes(command, exit_path);
5668 let wrapper_ext = match shell {
5669 WindowsShell::Pwsh | WindowsShell::Powershell => "ps1",
5670 WindowsShell::Cmd => "bat",
5671 WindowsShell::Posix(_) => "sh",
5675 };
5676 let wrapper_path = paths.dir.join(format!(
5677 "{}.{}",
5678 paths
5679 .json
5680 .file_stem()
5681 .and_then(|s| s.to_str())
5682 .unwrap_or("wrapper"),
5683 wrapper_ext
5684 ));
5685 fs::write(&wrapper_path, wrapper_body)
5686 .map_err(|e| format!("failed to write background bash wrapper script: {e}"))?;
5687
5688 let mut cmd = Command::new(shell.binary().as_ref());
5689 match shell {
5690 WindowsShell::Pwsh | WindowsShell::Powershell => {
5691 cmd.args([
5694 "-NoLogo",
5695 "-NoProfile",
5696 "-NonInteractive",
5697 "-ExecutionPolicy",
5698 "Bypass",
5699 "-File",
5700 ]);
5701 cmd.arg(&wrapper_path);
5702 }
5703 WindowsShell::Cmd => {
5704 cmd.args(["/D", "/C"]);
5711 cmd.arg(&wrapper_path);
5712 }
5713 WindowsShell::Posix(_) => {
5714 cmd.arg(&wrapper_path);
5719 }
5720 }
5721
5722 cmd.creation_flags(creation_flags);
5726 Ok(cmd)
5727}
5728
5729fn spawn_detached_child(
5745 spawn_plan: &SpawnPlan,
5746 command: &str,
5747 shell: super::BashShell,
5748 shell_path: &Path,
5749 paths: &TaskPaths,
5750 workdir: &Path,
5751 env: &HashMap<String, String>,
5752 io_handles: &mut TaskIoHandles,
5753 capture_pipeline_status: bool,
5754) -> Result<std::process::Child, String> {
5755 #[cfg(windows)]
5756 let _ = capture_pipeline_status;
5757 #[cfg(not(windows))]
5758 let _ = (command, shell);
5759 #[cfg(not(windows))]
5760 {
5761 use std::os::fd::AsRawFd;
5762
5763 let stdout = io_handles
5764 .clone_file(TaskArtifact::Stdout)
5765 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
5766 let stderr = io_handles
5767 .clone_file(TaskArtifact::Stderr)
5768 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
5769 let prepared = spawn_plan
5770 .prepared_task()
5771 .ok_or_else(|| "background task payload was not prepared".to_string())?;
5772 let payload = prepared.invocation()?;
5773 let exit = io_handles
5774 .inheritable_file(TaskArtifact::Exit)
5775 .map_err(|e| format!("failed to inherit exit marker handle: {e}"))?;
5776 let failure = io_handles
5777 .inheritable_file(TaskArtifact::SandboxUnavailable)
5778 .map_err(|e| format!("failed to inherit sandbox failure marker handle: {e}"))?;
5779 let pipeline_status = capture_pipeline_status
5780 .then(|| io_handles.inheritable_file(TaskArtifact::PipelineStatus))
5781 .transpose()
5782 .map_err(|e| format!("failed to inherit pipeline status handle: {e}"))?;
5783 let shell_path = spawn_plan.host_shell_path().unwrap_or(shell_path);
5784 let pipeline_shell = super::process::pipeline_shell_kind(shell_path).unwrap_or("");
5785 let pipeline_status_fd = if capture_pipeline_status {
5786 crate::sandbox_spawn::CHILD_PIPE_STATUS_FD.to_string()
5787 } else {
5788 String::new()
5789 };
5790 let args = vec![
5791 OsString::from("-c"),
5792 payload.wrapper_text.clone(),
5793 OsString::from("aft-payload-wrapper"),
5794 shell_path.as_os_str().to_os_string(),
5795 payload.command_text.clone(),
5796 OsString::from(crate::sandbox_spawn::CHILD_EXIT_FD.to_string()),
5797 OsString::from(pipeline_status_fd),
5798 OsString::from(pipeline_shell),
5799 ];
5800 let (mut child_command, profile_handle) = crate::sandbox_spawn::detached_command_for_plan(
5801 spawn_plan,
5802 std::ffi::OsStr::new("/bin/sh"),
5803 &args,
5804 &paths.json,
5805 crate::sandbox_spawn::CHILD_EXIT_FD,
5806 crate::sandbox_spawn::CHILD_FAILURE_FD,
5807 )?;
5808 crate::sandbox_spawn::apply_marker_fd_allowlist(
5809 &mut child_command,
5810 exit.as_raw_fd(),
5811 failure.as_raw_fd(),
5812 pipeline_status.as_ref().map(|file| file.as_raw_fd()),
5813 )?;
5814 child_command
5815 .current_dir(workdir)
5816 .stdin(Stdio::null())
5817 .stdout(Stdio::from(stdout))
5818 .stderr(Stdio::from(stderr));
5819 crate::agent_child_env::apply_to_command(&mut child_command, env);
5820 crate::sandbox_spawn::apply_sandbox_environment(spawn_plan, &mut child_command, env);
5821 let child = child_command
5822 .spawn()
5823 .map_err(|e| format!("failed to spawn background bash command: {e}"));
5824 drop((payload, exit, failure, pipeline_status, profile_handle));
5825 child
5826 }
5827 #[cfg(windows)]
5828 {
5829 let _ = shell_path;
5830 use crate::windows_shell::shell_candidates;
5831 match spawn_plan {
5832 SpawnPlan::Unsandboxed | SpawnPlan::Host { .. } => {}
5833 SpawnPlan::Refused { code, .. } => return Err((*code).to_string()),
5834 SpawnPlan::Launcher { .. } => return Err("sandbox_unavailable".to_string()),
5835 }
5836 let candidates: Vec<crate::windows_shell::WindowsShell> = if shell.is_powershell() {
5847 vec![crate::windows_shell::WindowsShell::Pwsh]
5848 } else {
5849 shell_candidates()
5850 };
5851 const FLAG_CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200;
5864 const FLAG_CREATE_BREAKAWAY_FROM_JOB: u32 = 0x0100_0000;
5865 const FLAG_CREATE_NO_WINDOW: u32 = 0x0800_0000;
5866 let with_breakaway =
5867 FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP | FLAG_CREATE_BREAKAWAY_FROM_JOB;
5868 let without_breakaway = FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP;
5869 let mut last_error: Option<String> = None;
5870 for (idx, shell) in candidates.iter().enumerate() {
5871 for &flags in &[with_breakaway, without_breakaway] {
5875 let stdout = io_handles
5878 .clone_file(TaskArtifact::Stdout)
5879 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
5880 let stderr = io_handles
5881 .clone_file(TaskArtifact::Stderr)
5882 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
5883 let mut cmd =
5884 detached_shell_command_for(shell.clone(), command, &paths.exit, paths, flags)?;
5885 cmd.current_dir(workdir)
5886 .stdin(Stdio::null())
5887 .stdout(Stdio::from(stdout))
5888 .stderr(Stdio::from(stderr));
5889 crate::agent_child_env::apply_to_command(&mut cmd, env);
5890 match cmd.spawn() {
5891 Ok(child) => {
5892 if idx > 0 {
5893 crate::slog_warn!("background bash spawn fell back to {} after {} earlier candidate(s) failed; \
5894 the cached PATH probe disagreed with runtime spawn — likely PATH \
5895 inheritance, antivirus / AppLocker / Defender ASR, or sandbox policy.",
5896 shell.binary(),
5897 idx);
5898 }
5899 if flags == without_breakaway {
5900 crate::slog_warn!(
5901 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected \
5902 (likely a restrictive Job Object — CI sandbox or MDM policy). \
5903 Spawned without breakaway; the bg task will be torn down if the \
5904 AFT process group is killed."
5905 );
5906 }
5907 return Ok(child);
5908 }
5909 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
5910 crate::slog_warn!("background bash spawn: {} returned NotFound at runtime — trying next candidate",
5911 shell.binary());
5912 last_error = Some(format!("{}: {e}", shell.binary()));
5913 break;
5916 }
5917 Err(e) if flags == with_breakaway && e.raw_os_error() == Some(5) => {
5918 crate::slog_warn!(
5920 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected with \
5921 Access Denied — retrying {} without breakaway",
5922 shell.binary()
5923 );
5924 last_error = Some(format!("{}: {e}", shell.binary()));
5925 continue;
5926 }
5927 Err(e) => {
5928 return Err(format!(
5929 "failed to spawn background bash command via {}: {e}",
5930 shell.binary()
5931 ));
5932 }
5933 }
5934 }
5935 }
5936 Err(format!(
5937 "failed to spawn background bash command: no Windows shell could be spawned. \
5938 Last error: {}. PATH-probed candidates: {:?}",
5939 last_error.unwrap_or_else(|| "no candidates were attempted".to_string()),
5940 candidates.iter().map(|s| s.binary()).collect::<Vec<_>>()
5941 ))
5942 }
5943}
5944
5945#[cfg(test)]
5946fn random_slug() -> String {
5947 let mut bytes = [0u8; 8];
5955 getrandom::fill(&mut bytes).unwrap_or_else(|_| {
5957 let t = SystemTime::now()
5959 .duration_since(UNIX_EPOCH)
5960 .map(|d| d.as_nanos() as u64)
5961 .unwrap_or(0);
5962 let p = u64::from(std::process::id());
5963 bytes.copy_from_slice(&(t ^ p.rotate_left(32)).to_le_bytes());
5964 });
5965 let hex: String = bytes.iter().map(|b| format!("{b:02x}")).collect();
5967 format!("bash-{hex}")
5968}
5969
5970#[cfg(test)]
5971mod tests {
5972 use std::collections::HashMap;
5973 use std::fs;
5974 use std::io::Write;
5975 #[cfg(unix)]
5976 use std::os::unix::fs::PermissionsExt;
5977 use std::sync::atomic::{AtomicBool, AtomicUsize};
5978 use std::sync::{Arc, Mutex};
5979 use std::time::{Duration, Instant, SystemTime};
5980
5981 use super::*;
5982 use crate::bash_background::persistence::{read_task, task_paths, write_task};
5983
5984 #[cfg(unix)]
5985 const QUICK_SUCCESS_COMMAND: &str = "true";
5986 #[cfg(windows)]
5987 const QUICK_SUCCESS_COMMAND: &str = "cmd /c exit 0";
5988
5989 #[cfg(unix)]
5990 const LONG_RUNNING_COMMAND: &str = "sleep 5";
5991
5992 #[cfg(unix)]
5993 #[test]
5994 fn launcher_plans_disable_pipeline_status_capture() {
5995 let launcher = SpawnPlan::launcher_for_test(
5996 crate::sandbox_profile::SandboxProfile {
5997 v: crate::sandbox_profile::SANDBOX_PROFILE_VERSION,
5998 writable_roots: Vec::new(),
5999 write_deny: Vec::new(),
6000 write_deny_nested: Vec::new(),
6001 read_allow: Vec::new(),
6002 read_deny: Vec::new(),
6003 socket_deny: Vec::new(),
6004 cache_roots: Vec::new(),
6005 temp_dir: PathBuf::from("/tmp/aft-test-sandbox"),
6006 },
6007 PathBuf::from("/bin/true"),
6008 );
6009 assert!(!should_capture_pipeline_status(
6010 &launcher,
6011 true,
6012 Path::new("/bin/bash")
6013 ));
6014 assert!(should_capture_pipeline_status(
6015 &SpawnPlan::Unsandboxed,
6016 true,
6017 Path::new("/bin/bash")
6018 ));
6019 }
6020
6021 #[cfg(windows)]
6022 const LONG_RUNNING_COMMAND: &str = "cmd /c timeout /t 5 /nobreak > nul";
6023
6024 #[test]
6025 fn bash_memory_estimate_is_zero_when_empty_and_nonzero_for_completion_cache() {
6026 let registry = BgTaskRegistry::default();
6027 assert_eq!(registry.estimated_memory().estimated_bytes, Some(0));
6028 registry
6029 .inner
6030 .completions
6031 .lock()
6032 .unwrap()
6033 .push_back(BgCompletion {
6034 task_id: "bash-memory".to_string(),
6035 session_id: "session-memory".to_string(),
6036 status: BgTaskStatus::Completed,
6037 exit_code: Some(0),
6038 command: "printf memory".to_string(),
6039 output_preview: "resident completion output".to_string(),
6040 output_truncated: false,
6041 original_tokens: None,
6042 compressed_tokens: None,
6043 tokens_skipped: false,
6044 status_reason: None,
6045 });
6046 let estimate = registry.estimated_memory();
6047 assert!(estimate.estimated_bytes.unwrap() > 0);
6048 assert_eq!(estimate.counts["completion_caches"], 1);
6049 assert_eq!(estimate.counts["sessions"], 1);
6050 }
6051
6052 #[test]
6053 fn gh_structured_detection_rejects_piped_commands() {
6054 assert!(is_gh_structured_command(
6055 "gh issue list --json number,title"
6056 ));
6057 assert!(is_gh_structured_command(
6058 "cd repo && gh issue list --json number,title"
6059 ));
6060
6061 assert!(!is_gh_structured_command(
6062 "gh issue list --json number,title | jq '.[]'"
6063 ));
6064 assert!(!is_gh_structured_command(
6065 "gh issue list --json number,title |"
6066 ));
6067 }
6068
6069 fn insert_terminal_piped_task(
6070 registry: &BgTaskRegistry,
6071 dir: &tempfile::TempDir,
6072 command: &str,
6073 stdout: &str,
6074 stderr: &str,
6075 compressed: bool,
6076 ) -> (String, Arc<BgTask>) {
6077 let task_id = random_slug();
6078 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
6079 fs::create_dir_all(&paths.dir).unwrap();
6080 fs::write(&paths.stdout, stdout).unwrap();
6081 fs::write(&paths.stderr, stderr).unwrap();
6082 let mut metadata = PersistedTask::starting(
6083 task_id.clone(),
6084 "session".to_string(),
6085 command.to_string(),
6086 dir.path().to_path_buf(),
6087 Some(dir.path().to_path_buf()),
6088 Some(30_000),
6089 true,
6090 compressed,
6091 );
6092 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
6093 write_task(&paths.json, &metadata).unwrap();
6094 registry
6095 .insert_rehydrated_task(metadata, paths, true, None)
6096 .expect("insert terminal task");
6097 let task = registry.task_for_session(&task_id, "session").unwrap();
6098 (task_id, task)
6099 }
6100
6101 #[test]
6102 fn bash_zero_preview_running_status_skips_output_read_while_explicit_preview_reads() {
6103 let registry = BgTaskRegistry::default();
6104 let dir = tempfile::tempdir().unwrap();
6105 let task_id = random_slug();
6106 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
6107 fs::create_dir_all(&paths.dir).unwrap();
6108 fs::write(&paths.stdout, "live output\n").unwrap();
6109 fs::write(&paths.stderr, "").unwrap();
6110 let stdout_path = paths.stdout.clone();
6111 let mut metadata = PersistedTask::starting(
6112 task_id.clone(),
6113 "session".to_string(),
6114 "sleep 60".to_string(),
6115 dir.path().to_path_buf(),
6116 Some(dir.path().to_path_buf()),
6117 Some(30_000),
6118 true,
6119 false,
6120 );
6121 metadata.status = BgTaskStatus::Running;
6122 write_task(&paths.json, &metadata).unwrap();
6123 registry
6124 .insert_rehydrated_task(metadata, paths, false, None)
6125 .expect("insert running task");
6126
6127 crate::bash_background::buffer::reset_tail_read_count(&stdout_path);
6128 for _ in 0..5 {
6129 let snapshot = registry
6130 .status(&task_id, "session", Some(dir.path()), Some(dir.path()), 0)
6131 .expect("running snapshot");
6132 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
6133 assert!(snapshot.output_preview.is_empty());
6134 }
6135 assert_eq!(
6136 crate::bash_background::buffer::tail_read_count(&stdout_path),
6137 0
6138 );
6139
6140 let snapshot = registry
6141 .status(
6142 &task_id,
6143 "session",
6144 Some(dir.path()),
6145 Some(dir.path()),
6146 RUNNING_OUTPUT_PREVIEW_BYTES,
6147 )
6148 .expect("explicit running snapshot");
6149 assert_eq!(snapshot.output_preview, "live output\n");
6150 assert_eq!(
6151 crate::bash_background::buffer::tail_read_count(&stdout_path),
6152 1
6153 );
6154 }
6155
6156 #[test]
6157 fn artifact_read_capability_requires_exact_canonical_path_and_session() {
6158 let registry = BgTaskRegistry::default();
6159 let dir = tempfile::tempdir().unwrap();
6160 let (_task_id, task) = insert_terminal_piped_task(
6161 ®istry,
6162 &dir,
6163 "printf output",
6164 "stdout\n",
6165 "stderr\n",
6166 true,
6167 );
6168 fs::write(&task.paths.exit, "0\n").unwrap();
6169
6170 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stdout));
6171 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stderr));
6172 assert!(registry.is_session_owned_artifact_path("session", &task.paths.exit));
6173 assert!(!registry.is_session_owned_artifact_path("different-session", &task.paths.stdout));
6174 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.json));
6175
6176 let unregistered = task.paths.dir.join("unregistered-output");
6177 fs::write(&unregistered, "not a task artifact\n").unwrap();
6178 assert!(!registry.is_session_owned_artifact_path("session", &unregistered));
6179 }
6180
6181 #[cfg(unix)]
6182 #[test]
6183 fn artifact_directory_symlink_does_not_create_a_prefix_exception() {
6184 let registry = BgTaskRegistry::default();
6185 let dir = tempfile::tempdir().unwrap();
6186 let project = dir.path().join("project");
6187 fs::create_dir_all(&project).unwrap();
6188 let (_task_id, task) =
6189 insert_terminal_piped_task(®istry, &dir, "printf output", "stdout\n", "", true);
6190 let link = project.join("task-artifacts");
6191 std::os::unix::fs::symlink(&task.paths.dir, &link).unwrap();
6192 let unregistered = task.paths.dir.join("unregistered-output");
6193 fs::write(&unregistered, "not registered\n").unwrap();
6194
6195 assert!(!registry.is_session_owned_artifact_path("session", &link));
6196 assert!(
6197 !registry.is_session_owned_artifact_path("session", &link.join("unregistered-output"))
6198 );
6199 assert!(registry.is_session_owned_artifact_path(
6200 "session",
6201 &link.join(task.paths.stdout.file_name().unwrap())
6202 ));
6203
6204 let outside = dir.path().join("outside-secret");
6205 fs::write(&outside, "must stay private\n").unwrap();
6206 fs::remove_file(&task.paths.stdout).unwrap();
6207 std::os::unix::fs::symlink(&outside, &task.paths.stdout).unwrap();
6208 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.stdout));
6209 }
6210
6211 #[test]
6212 fn recovery_footer_uses_bash_status_when_artifact_is_not_registered() {
6213 let registry = BgTaskRegistry::default();
6214 let dir = tempfile::tempdir().unwrap();
6215 let task_id = "bash-1111111111111111";
6216 let paths = task_paths(dir.path(), "session", task_id).unwrap();
6217 fs::create_dir_all(&paths.dir).unwrap();
6218 fs::write(
6219 &paths.stdout,
6220 format!("{}tail\n", "output-line\n".repeat(2_000)),
6221 )
6222 .unwrap();
6223 fs::write(&paths.stderr, "").unwrap();
6224 let mut metadata = PersistedTask::starting(
6225 task_id.to_string(),
6226 "session".to_string(),
6227 "printf output".to_string(),
6228 dir.path().to_path_buf(),
6229 Some(dir.path().to_path_buf()),
6230 Some(30_000),
6231 true,
6232 true,
6233 );
6234 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
6235 write_task(&paths.json, &metadata).unwrap();
6236
6237 let cache = registry
6238 .render_terminal_output_from_paths(&metadata, &paths)
6239 .expect("terminal render");
6240
6241 assert!(cache
6242 .output_preview
6243 .contains("use bash_status({taskId: \"bash-1111111111111111\"})"));
6244 assert!(!cache.output_preview.contains("full output: read "));
6245 }
6246
6247 fn insert_terminal_pty_task(
6248 registry: &BgTaskRegistry,
6249 dir: &tempfile::TempDir,
6250 pty_output: &str,
6251 ) -> (String, Arc<BgTask>) {
6252 let task_id = random_slug();
6253 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
6254 fs::create_dir_all(&paths.dir).unwrap();
6255 fs::write(&paths.pty, pty_output).unwrap();
6256 let mut metadata = PersistedTask::starting(
6257 task_id.clone(),
6258 "session".to_string(),
6259 "python".to_string(),
6260 dir.path().to_path_buf(),
6261 Some(dir.path().to_path_buf()),
6262 Some(30_000),
6263 true,
6264 true,
6265 );
6266 metadata.mode = BgMode::Pty;
6267 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
6268 write_task(&paths.json, &metadata).unwrap();
6269 registry
6270 .insert_rehydrated_task(metadata, paths, true, None)
6271 .expect("insert terminal pty task");
6272 let task = registry.task_for_session(&task_id, "session").unwrap();
6273 (task_id, task)
6274 }
6275
6276 #[cfg(unix)]
6277 fn wait_for_terminal_snapshot(
6278 registry: &BgTaskRegistry,
6279 task_id: &str,
6280 session_id: &str,
6281 project: &Path,
6282 storage: &Path,
6283 ) -> BgTaskSnapshot {
6284 let started = Instant::now();
6285 loop {
6286 let snapshot = registry
6287 .status(task_id, session_id, Some(project), Some(storage), 4096)
6288 .expect("spawned task should be visible to status");
6289 if snapshot.info.status.is_terminal() {
6290 return snapshot;
6291 }
6292 assert!(
6293 started.elapsed() < Duration::from_secs(10),
6294 "timed out waiting for task {task_id} to finish; last status={:?}",
6295 snapshot.info.status
6296 );
6297 std::thread::sleep(Duration::from_millis(50));
6298 }
6299 }
6300
6301 fn write_running_project_task(storage: &Path, project: &Path, session: &str, task_id: &str) {
6302 let paths = task_paths(storage, session, task_id).unwrap();
6303 let mut metadata = PersistedTask::starting(
6304 task_id.to_string(),
6305 session.to_string(),
6306 "sleep 60".to_string(),
6307 project.to_path_buf(),
6308 Some(project.to_path_buf()),
6309 Some(30_000),
6310 true,
6311 true,
6312 );
6313 metadata.status = BgTaskStatus::Running;
6314 metadata.child_pid = Some(std::process::id());
6320 write_task(&paths.json, &metadata).unwrap();
6321 fs::write(&paths.stdout, "still running\n").unwrap();
6322 fs::write(&paths.stderr, "").unwrap();
6323 }
6324
6325 #[test]
6326 fn status_replay_filters_same_session_by_project_root() {
6327 let project_a = tempfile::tempdir().unwrap();
6328 let project_b = tempfile::tempdir().unwrap();
6329 let storage = tempfile::tempdir().unwrap();
6330 let session = "shared-session";
6331 let task_id = "bash-2222222222222222";
6332 write_running_project_task(storage.path(), project_a.path(), session, task_id);
6333
6334 let actor_b = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6335 assert!(actor_b
6336 .status(
6337 task_id,
6338 session,
6339 Some(project_b.path()),
6340 Some(storage.path()),
6341 1024,
6342 )
6343 .is_none());
6344 assert!(actor_b.task_for_session(task_id, session).is_none());
6345
6346 let actor_a = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6347 let snapshot = actor_a
6348 .status(
6349 task_id,
6350 session,
6351 Some(project_a.path()),
6352 Some(storage.path()),
6353 1024,
6354 )
6355 .expect("owning project should replay its task");
6356 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
6357 }
6358
6359 #[cfg(unix)]
6360 #[test]
6361 fn multiline_pipeline_stdout_persists_all_lines_after_terminal_status() {
6362 let cases = [
6363 (
6364 "long-first",
6365 "sleep 0.5; printf 'one\\n' | cat\nprintf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
6366 vec!["one", "1", "three"],
6367 ),
6368 (
6369 "short-first",
6370 "printf 'one\\n' | cat\nsleep 0.2; printf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
6371 vec!["one", "1", "three"],
6372 ),
6373 (
6374 "failing-middle",
6375 "sleep 0.2; printf 'one\\n' | cat\nfalse; printf 'after-false\\n' | cat\nprintf 'three\\n' | cat",
6376 vec!["one", "after-false", "three"],
6377 ),
6378 ];
6379
6380 for (name, command, expected_lines) in cases {
6381 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6382 let dir = tempfile::tempdir().unwrap();
6383 let session_id = format!("session-{name}");
6384 let task_id = registry
6385 .spawn(
6386 SpawnPlan::Unsandboxed,
6387 command,
6388 session_id.clone(),
6389 dir.path().to_path_buf(),
6390 HashMap::new(),
6391 Some(Duration::from_secs(30)),
6392 dir.path().to_path_buf(),
6393 10,
6394 true,
6395 true,
6396 Some(dir.path().to_path_buf()),
6397 )
6398 .unwrap();
6399
6400 let snapshot = wait_for_terminal_snapshot(
6401 ®istry,
6402 &task_id,
6403 &session_id,
6404 dir.path(),
6405 dir.path(),
6406 );
6407 assert_eq!(
6408 snapshot.info.status,
6409 BgTaskStatus::Completed,
6410 "{name}: task should complete; snapshot={snapshot:?}"
6411 );
6412 assert_eq!(
6413 snapshot.exit_code,
6414 Some(0),
6415 "{name}: script should use the final command's exit code"
6416 );
6417
6418 let stdout = String::from_utf8(
6419 registry
6420 .read_artifact(&task_id, &session_id, TaskArtifact::Stdout)
6421 .expect("read validated stdout artifact"),
6422 )
6423 .expect("stdout is UTF-8");
6424 let lines: Vec<&str> = stdout.lines().collect();
6425 assert_eq!(
6426 lines, expected_lines,
6427 "{name}: raw stdout artifact must include every newline-separated command's output"
6428 );
6429 }
6430 }
6431
6432 #[test]
6433 fn recognizes_all_recovery_marker_forms() {
6434 assert!(is_recovery_marker(
6435 "[truncated output; full output: read \"/tmp/out\"]"
6436 ));
6437 assert!(is_recovery_marker(
6438 "[omitted output; see remaining: tail -n +42 \"/tmp/out\"]"
6439 ));
6440 assert!(is_recovery_marker(
6441 "[truncated output; full output unavailable]"
6442 ));
6443 assert!(is_recovery_marker(
6444 r#"[truncated 123 bytes from saved output prefix; retained output: read "/tmp/out"]"#
6445 ));
6446 }
6447
6448 #[test]
6449 fn recovery_marker_reports_disk_prefix_truncation_as_retained_output() {
6450 let recovery = RecoveryContext {
6451 dropped_by_class: BTreeMap::new(),
6452 had_inner_drop: false,
6453 offset_hint_eligible: false,
6454 offset_start_line: None,
6455 byte_truncated: false,
6456 disk_truncated_prefix_bytes: 4096,
6457 output_path: Some("/tmp/stdout".to_string()),
6458 stderr_path: None,
6459 include_stderr_path: false,
6460 artifact_access: ArtifactRecoveryAccess {
6461 task_id: "bash-test".to_string(),
6462 readable: true,
6463 },
6464 };
6465
6466 let marker = recovery_marker(&recovery).expect("disk truncation must emit marker");
6467
6468 assert!(marker.contains("truncated 4096 bytes from saved output prefix"));
6469 assert!(marker.contains(r#"retained output: read "/tmp/stdout""#));
6470 assert!(!marker.contains("full output: read"));
6471 }
6472
6473 #[test]
6474 fn killed_exit_marker_sets_nonzero_sentinel_exit_code() {
6475 let metadata = PersistedTask::starting(
6476 "task".to_string(),
6477 "session".to_string(),
6478 "cargo test".to_string(),
6479 PathBuf::from("/tmp"),
6480 None,
6481 None,
6482 true,
6483 true,
6484 );
6485
6486 let terminal = terminal_metadata_from_marker(metadata, ExitMarker::Killed, None);
6487
6488 assert_eq!(terminal.status, BgTaskStatus::Killed);
6489 assert_eq!(terminal.exit_code, Some(137));
6490 }
6491
6492 #[test]
6493 fn terminal_status_polls_use_cached_render_once_and_off_lock() {
6494 let registry = BgTaskRegistry::default();
6495 let dir = tempfile::tempdir().unwrap();
6496 let (_task_id, task) = insert_terminal_piped_task(
6497 ®istry,
6498 &dir,
6499 "custom-tool --verbose",
6500 &"stdout line\n".repeat(200_000),
6501 "",
6502 true,
6503 );
6504 let calls = Arc::new(AtomicUsize::new(0));
6505 let saw_unlocked_state = Arc::new(AtomicBool::new(false));
6506 let task_holder = Arc::new(Mutex::new(Some(Arc::clone(&task))));
6507 let calls_for_closure = Arc::clone(&calls);
6508 let unlocked_for_closure = Arc::clone(&saw_unlocked_state);
6509 let task_for_closure = Arc::clone(&task_holder);
6510 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
6511 calls_for_closure.fetch_add(1, Ordering::SeqCst);
6512 if let Some(task) = task_for_closure.lock().unwrap().as_ref() {
6513 if task.state.try_lock().is_ok() {
6514 unlocked_for_closure.store(true, Ordering::SeqCst);
6515 }
6516 }
6517 CompressionResult::new(format!("compressed {} bytes", output.len()))
6518 });
6519
6520 let first = registry
6521 .status(
6522 &task.task_id,
6523 "session",
6524 None,
6525 Some(dir.path()),
6526 RUNNING_OUTPUT_PREVIEW_BYTES,
6527 )
6528 .unwrap();
6529 let second = registry
6530 .status(
6531 &task.task_id,
6532 "session",
6533 None,
6534 Some(dir.path()),
6535 RUNNING_OUTPUT_PREVIEW_BYTES,
6536 )
6537 .unwrap();
6538 let listed = registry.list(RUNNING_OUTPUT_PREVIEW_BYTES);
6539
6540 assert_eq!(
6541 calls.load(Ordering::SeqCst),
6542 1,
6543 "terminal render must be cached"
6544 );
6545 assert!(
6546 saw_unlocked_state.load(Ordering::SeqCst),
6547 "compressor must run after releasing the task state lock"
6548 );
6549 assert!(first.output_preview.starts_with("compressed "));
6550 assert_eq!(second.output_preview, first.output_preview);
6551 assert_eq!(listed[0].output_preview, first.output_preview);
6552 }
6553
6554 #[test]
6555 fn completion_preview_success_keeps_tail_only() {
6556 let registry = BgTaskRegistry::default();
6561 let dir = tempfile::tempdir().unwrap();
6562 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
6563 let (_task_id, task) =
6564 insert_terminal_piped_task(®istry, &dir, "cat big.log", &output, "", false);
6565
6566 registry.post_terminal_transition(&task, true).unwrap();
6567 let completions = registry.drain_completions_for_session(Some("session"));
6568 assert_eq!(completions.len(), 1);
6569 let preview = &completions[0].output_preview;
6570 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
6571 assert!(!preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
6572 assert!(completions[0].output_truncated);
6573 }
6574
6575 #[test]
6576 fn completion_preview_failure_keeps_head_and_tail() {
6577 let registry = BgTaskRegistry::default();
6580 let dir = tempfile::tempdir().unwrap();
6581 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
6582 let task_id = random_slug();
6583 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
6584 fs::create_dir_all(&paths.dir).unwrap();
6585 fs::write(&paths.stdout, &output).unwrap();
6586 fs::write(&paths.stderr, "").unwrap();
6587 let mut metadata = PersistedTask::starting(
6588 task_id.clone(),
6589 "session".to_string(),
6590 "cat big.log".to_string(),
6591 dir.path().to_path_buf(),
6592 Some(dir.path().to_path_buf()),
6593 Some(30_000),
6594 true,
6595 false,
6596 );
6597 metadata.mark_terminal(BgTaskStatus::Failed, Some(1), None);
6598 write_task(&paths.json, &metadata).unwrap();
6599 registry
6600 .insert_rehydrated_task(metadata, paths, true, None)
6601 .expect("insert terminal task");
6602 let task = registry.task_for_session(&task_id, "session").unwrap();
6603
6604 registry.post_terminal_transition(&task, true).unwrap();
6605 let completions = registry.drain_completions_for_session(Some("session"));
6606 assert_eq!(completions.len(), 1);
6607 let preview = &completions[0].output_preview;
6608 assert!(preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
6609 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
6610 }
6611
6612 #[test]
6613 fn has_completions_for_session_matches_pending_delivery() {
6614 let registry = BgTaskRegistry::default();
6615 assert!(!registry.has_completions_for_session(Some("session")));
6616 assert!(!registry.has_completions_for_session(None));
6617
6618 let dir = tempfile::tempdir().unwrap();
6619 let (_task_id, task) =
6620 insert_terminal_piped_task(®istry, &dir, QUICK_SUCCESS_COMMAND, "done\n", "", false);
6621 registry.post_terminal_transition(&task, true).unwrap();
6622
6623 assert!(registry.has_completions_for_session(Some("session")));
6624 assert!(registry.has_completions_for_session(None));
6625 assert!(!registry.has_completions_for_session(Some("other-session")));
6626
6627 let completions = registry.drain_completions_for_session(Some("session"));
6628 assert_eq!(completions.len(), 1);
6629 assert_eq!(completions[0].task_id, task.task_id);
6630 }
6631
6632 #[test]
6633 fn structured_gh_json_survives_intact_and_ignores_stderr() {
6634 let registry = BgTaskRegistry::default();
6635 let dir = tempfile::tempdir().unwrap();
6636 let calls = Arc::new(AtomicUsize::new(0));
6637 let calls_for_closure = Arc::clone(&calls);
6638 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
6639 calls_for_closure.fetch_add(1, Ordering::SeqCst);
6640 CompressionResult::new(output)
6641 });
6642 let (task_id, _task) = insert_terminal_piped_task(
6643 ®istry,
6644 &dir,
6645 "gh pr view 123 --json body",
6646 "{\"body\":\"hello\"}",
6647 "warning: stderr must not join json",
6648 true,
6649 );
6650
6651 let snapshot = registry
6652 .status(
6653 &task_id,
6654 "session",
6655 None,
6656 Some(dir.path()),
6657 RUNNING_OUTPUT_PREVIEW_BYTES,
6658 )
6659 .unwrap();
6660
6661 assert_eq!(snapshot.output_preview, "{\"body\":\"hello\"}");
6662 assert!(!snapshot.output_preview.contains("warning"));
6663 assert!(!snapshot.output_truncated);
6664 assert_eq!(
6665 calls.load(Ordering::SeqCst),
6666 0,
6667 "structured JSON bypasses compression"
6668 );
6669 }
6670
6671 #[test]
6672 fn registry_emits_single_recovery_marker_for_class_drops() {
6673 let registry = BgTaskRegistry::default();
6674 let dir = tempfile::tempdir().unwrap();
6675 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
6676 let mut dropped = BTreeMap::new();
6677 dropped.insert(DropClass::Error, 18);
6678 dropped.insert(DropClass::Warning, 6);
6679 CompressionResult::with_class_drops("kept diagnostic", dropped)
6680 });
6681 let (task_id, task) =
6682 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
6683
6684 let snapshot = registry
6685 .status(
6686 &task_id,
6687 "session",
6688 None,
6689 Some(dir.path()),
6690 RUNNING_OUTPUT_PREVIEW_BYTES,
6691 )
6692 .unwrap();
6693
6694 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
6695 assert!(snapshot.output_preview.contains("+18 more errors"));
6696 assert!(snapshot.output_preview.contains("+6 more warnings"));
6697 assert!(snapshot
6698 .output_preview
6699 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
6700 assert!(!snapshot.output_preview.contains("tail -n +"));
6701 assert!(snapshot.output_truncated);
6702 }
6703
6704 #[test]
6705 fn registry_marker_reports_semantic_and_byte_drops_once() {
6706 let registry = BgTaskRegistry::default();
6707 let dir = tempfile::tempdir().unwrap();
6708 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
6709 let mut dropped = BTreeMap::new();
6710 dropped.insert(DropClass::Error, 1);
6711 CompressionResult::with_class_drops(
6712 format!("HEAD-SIGNAL\n{}TAIL-SIGNAL", "middle\n".repeat(8_000)),
6713 dropped,
6714 )
6715 });
6716 let (task_id, _task) =
6717 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
6718
6719 let snapshot = registry
6720 .status(
6721 &task_id,
6722 "session",
6723 None,
6724 Some(dir.path()),
6725 RUNNING_OUTPUT_PREVIEW_BYTES,
6726 )
6727 .unwrap();
6728
6729 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
6730 assert!(snapshot.output_preview.contains("+1 more error"));
6731 assert!(snapshot.output_preview.contains("truncated output"));
6732 assert!(snapshot.output_preview.contains("HEAD-SIGNAL"));
6733 assert!(snapshot.output_preview.contains("TAIL-SIGNAL"));
6734 assert!(!snapshot.output_preview.contains("...<truncated"));
6735 assert!(snapshot.output_truncated);
6736 }
6737
6738 #[test]
6739 fn cargo_stderr_class_drops_name_both_capture_paths() {
6740 let registry = BgTaskRegistry::default();
6741 let dir = tempfile::tempdir().unwrap();
6742 let filter_registry = crate::compress::toml_filter::FilterRegistry::default();
6743 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
6744 crate::compress::compress_with_registry_exit_code(
6745 command,
6746 &output,
6747 exit_code,
6748 &filter_registry,
6749 )
6750 });
6751 let stderr = (0..22)
6752 .map(|index| {
6753 format!(
6754 "error: cargo failure {index}\n --> src/lib.rs:{}:1\n |\n{} | boom\n",
6755 index + 1,
6756 index + 1
6757 )
6758 })
6759 .collect::<Vec<_>>()
6760 .join("\n");
6761 let (task_id, task) = insert_terminal_piped_task(
6762 ®istry,
6763 &dir,
6764 "cargo check",
6765 "Finished dev [unoptimized] target(s) in 0.01s\n",
6766 &stderr,
6767 true,
6768 );
6769
6770 let snapshot = registry
6771 .status(
6772 &task_id,
6773 "session",
6774 None,
6775 Some(dir.path()),
6776 RUNNING_OUTPUT_PREVIEW_BYTES,
6777 )
6778 .unwrap();
6779
6780 assert!(snapshot.output_preview.contains("+2 more errors"));
6781 assert!(snapshot
6782 .output_preview
6783 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
6784 assert!(snapshot
6785 .output_preview
6786 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
6787 assert!(!snapshot.output_preview.contains("tail -n +"));
6788 }
6789
6790 #[test]
6791 fn over_ceiling_structured_json_uses_pointer_not_partial_json() {
6792 let registry = BgTaskRegistry::default();
6793 let dir = tempfile::tempdir().unwrap();
6794 let body = format!("{{\"body\":\"{}\"}}", "x".repeat(60 * 1024));
6795 let (task_id, task) = insert_terminal_piped_task(
6796 ®istry,
6797 &dir,
6798 "cd /repo && gh pr view 123 --json body",
6799 &body,
6800 "",
6801 true,
6802 );
6803
6804 let snapshot = registry
6805 .status(
6806 &task_id,
6807 "session",
6808 None,
6809 Some(dir.path()),
6810 RUNNING_OUTPUT_PREVIEW_BYTES,
6811 )
6812 .unwrap();
6813
6814 assert!(snapshot.output_preview.starts_with("[JSON output "));
6815 assert!(snapshot
6816 .output_preview
6817 .contains(&task.paths.stdout.display().to_string()));
6818 assert!(!snapshot.output_preview.contains(&"x".repeat(1024)));
6819 assert!(snapshot.output_truncated);
6820 }
6821
6822 #[test]
6823 fn toml_strip_tail_cap_uses_full_output_hint_not_offset_hint() {
6824 let registry = BgTaskRegistry::default();
6825 let dir = tempfile::tempdir().unwrap();
6826 let filter_registry = crate::compress::toml_filter::build_registry(
6827 crate::compress::builtin_filters::ALL,
6828 None,
6829 None,
6830 );
6831 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
6832 crate::compress::compress_with_registry_exit_code(
6833 command,
6834 &output,
6835 exit_code,
6836 &filter_registry,
6837 )
6838 });
6839 let stdout = format!(
6840 "make[1]: Entering directory `/tmp`\n{}",
6841 (0..100)
6842 .map(|index| format!("compile line {index}"))
6843 .collect::<Vec<_>>()
6844 .join("\n")
6845 );
6846 let (task_id, task) =
6847 insert_terminal_piped_task(®istry, &dir, "make all", &stdout, "", true);
6848
6849 let snapshot = registry
6850 .status(
6851 &task_id,
6852 "session",
6853 None,
6854 Some(dir.path()),
6855 RUNNING_OUTPUT_PREVIEW_BYTES,
6856 )
6857 .unwrap();
6858
6859 assert!(snapshot.output_preview.contains("compile line 99"));
6860 assert!(snapshot.output_preview.contains(&format!(
6861 "full output: read \"{}\"",
6862 task.paths.stdout.display()
6863 )));
6864 assert!(!snapshot
6865 .output_preview
6866 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
6867 assert!(!snapshot.output_preview.contains("tail -n +"));
6868 }
6869
6870 #[test]
6871 fn compressed_false_raw_passthrough_uses_wider_head_tail_cap() {
6872 let registry = BgTaskRegistry::default();
6873 let dir = tempfile::tempdir().unwrap();
6874 let output = format!("RAW-HEAD\n{}RAW-TAIL\n", "raw-middle\n".repeat(8_000));
6875 let (task_id, task) =
6876 insert_terminal_piped_task(®istry, &dir, "cat raw.log", &output, "RAW-ERR\n", false);
6877
6878 let snapshot = registry
6879 .status(
6880 &task_id,
6881 "session",
6882 None,
6883 Some(dir.path()),
6884 RUNNING_OUTPUT_PREVIEW_BYTES,
6885 )
6886 .unwrap();
6887
6888 assert!(snapshot.output_preview.contains("RAW-HEAD"));
6889 assert!(snapshot.output_preview.contains("RAW-TAIL"));
6890 assert!(snapshot.output_preview.contains("truncated output"));
6891 assert!(snapshot
6892 .output_preview
6893 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
6894 assert!(snapshot
6895 .output_preview
6896 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
6897 assert!(!snapshot.output_preview.contains("tail -n +"));
6898 assert!(snapshot.output_preview.len() > 16 * 1024);
6899 assert!(snapshot.output_truncated);
6900 }
6901
6902 #[test]
6903 fn pty_terminal_snapshot_bypasses_line_compression() {
6904 let registry = BgTaskRegistry::default();
6905 let dir = tempfile::tempdir().unwrap();
6906 let calls = Arc::new(AtomicUsize::new(0));
6907 let calls_for_closure = Arc::clone(&calls);
6908 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
6909 calls_for_closure.fetch_add(1, Ordering::SeqCst);
6910 CompressionResult::new(output)
6911 });
6912 let (task_id, _task) = insert_terminal_pty_task(®istry, &dir, "raw\u{1b}[31m pty bytes");
6913
6914 let snapshot = registry
6915 .status(
6916 &task_id,
6917 "session",
6918 None,
6919 Some(dir.path()),
6920 RUNNING_OUTPUT_PREVIEW_BYTES,
6921 )
6922 .unwrap();
6923
6924 assert_eq!(snapshot.info.mode, BgMode::Pty);
6925 assert_eq!(snapshot.output_preview, "");
6926 assert_eq!(calls.load(Ordering::SeqCst), 0);
6927 }
6928
6929 #[test]
6930 fn pty_dimensions_are_persisted_and_returned_in_snapshot() {
6931 let registry = BgTaskRegistry::default();
6932 let dir = tempfile::tempdir().unwrap();
6933 let task_id = registry
6934 .spawn_pty(
6935 SpawnPlan::Unsandboxed,
6936 QUICK_SUCCESS_COMMAND,
6937 "session".to_string(),
6938 dir.path().to_path_buf(),
6939 HashMap::new(),
6940 Some(Duration::from_secs(30)),
6941 dir.path().to_path_buf(),
6942 10,
6943 true,
6944 false,
6945 Some(dir.path().to_path_buf()),
6946 50,
6947 120,
6948 )
6949 .unwrap();
6950
6951 let resolved =
6952 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
6953 let metadata = read_task_at(&resolved).unwrap();
6954 assert_eq!(
6955 metadata.schema_version,
6956 crate::bash_background::persistence::SCHEMA_VERSION
6957 );
6958 assert_eq!(metadata.mode, BgMode::Pty);
6959 assert_eq!(metadata.pty_rows, Some(50));
6960 assert_eq!(metadata.pty_cols, Some(120));
6961
6962 let snapshot = registry
6963 .status(&task_id, "session", None, Some(dir.path()), 1024)
6964 .unwrap();
6965 assert_eq!(snapshot.pty_rows, Some(50));
6966 assert_eq!(snapshot.pty_cols, Some(120));
6967 }
6968
6969 fn spawn_dead_child() -> std::process::Child {
6974 #[cfg(unix)]
6975 let mut cmd = std::process::Command::new("true");
6976 #[cfg(windows)]
6977 let mut cmd = {
6978 let mut c = std::process::Command::new("cmd");
6979 c.args(["/c", "exit", "0"]);
6980 c
6981 };
6982 cmd.stdin(std::process::Stdio::null());
6983 cmd.stdout(std::process::Stdio::null());
6984 cmd.stderr(std::process::Stdio::null());
6985 let mut child = cmd.spawn().expect("spawn replacement child for reap test");
6986 let started = Instant::now();
6995 loop {
6996 match child.try_wait() {
6997 Ok(Some(_)) => break,
6998 Ok(None) => {
6999 if started.elapsed() > Duration::from_secs(5) {
7000 panic!("dead-child stand-in did not exit within 5s");
7001 }
7002 std::thread::sleep(Duration::from_millis(10));
7003 }
7004 Err(error) => panic!("dead-child try_wait failed: {error}"),
7005 }
7006 }
7007 child
7008 }
7009
7010 #[test]
7011 fn ack_marks_delivered_even_when_completion_was_already_consumed_locally() {
7012 let registry = BgTaskRegistry::default();
7013 let dir = tempfile::tempdir().unwrap();
7014 let task_id = registry
7015 .spawn(
7016 SpawnPlan::Unsandboxed,
7017 LONG_RUNNING_COMMAND,
7018 "session".to_string(),
7019 dir.path().to_path_buf(),
7020 HashMap::new(),
7021 Some(Duration::from_secs(30)),
7022 dir.path().to_path_buf(),
7023 10,
7024 true,
7025 false,
7026 Some(dir.path().to_path_buf()),
7027 )
7028 .unwrap();
7029 registry
7030 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
7031 .unwrap();
7032 assert_eq!(
7033 registry
7034 .drain_completions_for_session(Some("session"))
7035 .len(),
7036 1
7037 );
7038
7039 registry.inner.completions.lock().unwrap().clear();
7042
7043 assert_eq!(
7044 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
7045 vec![task_id.clone()]
7046 );
7047 assert!(registry
7048 .drain_completions_for_session(Some("session"))
7049 .is_empty());
7050
7051 let resolved =
7052 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
7053 let metadata = read_task_at(&resolved).unwrap();
7054 assert!(metadata.completion_delivered);
7055
7056 let replayed = BgTaskRegistry::default();
7057 replayed
7058 .replay_session_inner(dir.path(), "session", None)
7059 .unwrap();
7060 assert!(replayed
7061 .drain_completions_for_session(Some("session"))
7062 .is_empty());
7063 }
7064
7065 #[test]
7066 fn reclaimed_root_kills_running_task_and_persists_reason() {
7067 let registry = BgTaskRegistry::default();
7068 let root = tempfile::tempdir().unwrap();
7069 let storage = tempfile::tempdir().unwrap();
7070 let task_id = registry
7071 .spawn(
7072 SpawnPlan::Unsandboxed,
7073 LONG_RUNNING_COMMAND,
7074 "session".to_string(),
7075 root.path().to_path_buf(),
7076 HashMap::new(),
7077 Some(Duration::from_secs(30)),
7078 storage.path().to_path_buf(),
7079 10,
7080 true,
7081 false,
7082 Some(root.path().to_path_buf()),
7083 )
7084 .unwrap();
7085 let pid = registry
7086 .status(
7087 &task_id,
7088 "session",
7089 Some(root.path()),
7090 Some(storage.path()),
7091 0,
7092 )
7093 .unwrap()
7094 .child_pid
7095 .unwrap();
7096 assert!(is_process_alive(pid));
7097
7098 assert_eq!(registry.kill_running_tasks_for_root(root.path()), 1);
7099 let deadline = Instant::now() + Duration::from_secs(5);
7100 while is_process_alive(pid) {
7101 assert!(
7102 Instant::now() < deadline,
7103 "reclaimed task process survived kill"
7104 );
7105 std::thread::sleep(Duration::from_millis(20));
7106 }
7107
7108 let snapshot = registry
7109 .status(
7110 &task_id,
7111 "session",
7112 Some(root.path()),
7113 Some(storage.path()),
7114 0,
7115 )
7116 .unwrap();
7117 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
7118 assert_eq!(
7119 snapshot.info.status_reason.as_deref(),
7120 Some(ROOT_RECLAIMED_REASON)
7121 );
7122 let persisted = read_task(
7123 ®istry
7124 .task_json_path(&task_id, "session")
7125 .expect("reclaimed task metadata path"),
7126 )
7127 .expect("persisted reclaimed task");
7128 assert_eq!(
7129 persisted.status_reason.as_deref(),
7130 Some(ROOT_RECLAIMED_REASON)
7131 );
7132 let completion = registry
7133 .drain_completions_for_session(Some("session"))
7134 .pop()
7135 .expect("reclaimed task completion");
7136 assert_eq!(
7137 completion.status_reason.as_deref(),
7138 Some(ROOT_RECLAIMED_REASON)
7139 );
7140 registry.detach();
7141 }
7142
7143 #[test]
7144 fn reclaimed_root_kills_pty_task_and_preserves_reason() {
7145 let registry = BgTaskRegistry::default();
7146 let root = tempfile::tempdir().unwrap();
7147 let storage = tempfile::tempdir().unwrap();
7148 let command = if cfg!(windows) {
7149 "Start-Sleep -Seconds 30"
7150 } else {
7151 "sleep 30"
7152 };
7153 let task_id = registry
7154 .spawn_pty(
7155 SpawnPlan::Unsandboxed,
7156 command,
7157 "session".to_string(),
7158 root.path().to_path_buf(),
7159 HashMap::new(),
7160 Some(Duration::from_secs(60)),
7161 storage.path().to_path_buf(),
7162 10,
7163 true,
7164 false,
7165 Some(root.path().to_path_buf()),
7166 24,
7167 80,
7168 )
7169 .unwrap();
7170 let pid = registry
7171 .status(
7172 &task_id,
7173 "session",
7174 Some(root.path()),
7175 Some(storage.path()),
7176 0,
7177 )
7178 .unwrap()
7179 .child_pid
7180 .unwrap();
7181 assert!(is_process_alive(pid));
7182
7183 assert_eq!(registry.kill_running_tasks_for_root(root.path()), 1);
7184 let deadline = Instant::now() + Duration::from_secs(10);
7185 loop {
7186 let snapshot = registry
7187 .status(
7188 &task_id,
7189 "session",
7190 Some(root.path()),
7191 Some(storage.path()),
7192 0,
7193 )
7194 .unwrap();
7195 if snapshot.info.status.is_terminal() {
7196 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
7197 assert_eq!(
7198 snapshot.info.status_reason.as_deref(),
7199 Some(ROOT_RECLAIMED_REASON)
7200 );
7201 break;
7202 }
7203 assert!(
7204 Instant::now() < deadline,
7205 "reclaimed PTY task did not terminate"
7206 );
7207 std::thread::sleep(Duration::from_millis(20));
7208 }
7209 assert!(!is_process_alive(pid));
7210 let completion = loop {
7215 if let Some(completion) = registry
7216 .drain_completions_for_session(Some("session"))
7217 .pop()
7218 {
7219 break completion;
7220 }
7221 assert!(
7222 Instant::now() < deadline,
7223 "reclaimed PTY completion never arrived"
7224 );
7225 std::thread::sleep(Duration::from_millis(20));
7226 };
7227 assert_eq!(
7228 completion.status_reason.as_deref(),
7229 Some(ROOT_RECLAIMED_REASON)
7230 );
7231 registry.detach();
7232 }
7233
7234 #[test]
7235 fn register_watch_rejects_unknown_task() {
7236 let registry = BgTaskRegistry::default();
7237
7238 let result = registry.register_watch(
7239 "missing-task".to_string(),
7240 WatchPattern::Substring("READY".into()),
7241 true,
7242 );
7243
7244 assert_eq!(result, Err("task_not_found"));
7245 }
7246
7247 #[test]
7248 fn register_watch_on_terminal_task_scans_existing_output() {
7249 let frames = Arc::new(Mutex::new(Vec::new()));
7250 let captured = Arc::clone(&frames);
7251 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
7252 captured.lock().unwrap().push(frame);
7253 })
7254 as Box<dyn Fn(PushFrame) + Send + Sync>);
7255 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
7256 let dir = tempfile::tempdir().unwrap();
7257 let task_id = registry
7258 .spawn(
7259 SpawnPlan::Unsandboxed,
7260 LONG_RUNNING_COMMAND,
7261 "session".to_string(),
7262 dir.path().to_path_buf(),
7263 HashMap::new(),
7264 Some(Duration::from_secs(30)),
7265 dir.path().to_path_buf(),
7266 10,
7267 true,
7268 false,
7269 Some(dir.path().to_path_buf()),
7270 )
7271 .unwrap();
7272 registry
7273 .inner
7274 .shutdown
7275 .store(true, std::sync::atomic::Ordering::SeqCst);
7276 let task = registry.task_for_session(&task_id, "session").unwrap();
7277 std::fs::write(&task.paths.stdout, "READY\n").unwrap();
7278 registry
7279 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
7280 .unwrap();
7281 frames.lock().unwrap().clear();
7282 registry.inner.completions.lock().unwrap().clear();
7283
7284 registry
7285 .register_watch(
7286 task_id.clone(),
7287 WatchPattern::Substring("READY".into()),
7288 true,
7289 )
7290 .unwrap();
7291
7292 let frames = frames.lock().unwrap();
7293 let frame = frames
7294 .iter()
7295 .find_map(|frame| match frame {
7296 PushFrame::BashPatternMatch(frame) => Some(frame),
7297 _ => None,
7298 })
7299 .expect("terminal watch registration should emit pattern frame");
7300 assert_eq!(frame.reason, "pattern_match");
7301 assert_eq!(frame.task_id, task_id);
7302 assert_eq!(frame.session_id, "session");
7303 assert_eq!(frame.match_text, "READY");
7304 assert_eq!(frame.match_offset, 0);
7305 assert_eq!(registry.active_watch_count(&frame.task_id), 0);
7306 let metadata = read_task(&task.paths.json).unwrap();
7307 assert!(metadata.completion_delivered);
7308 }
7309
7310 #[test]
7311 fn cleanup_finished_removes_terminal_tasks_older_than_threshold() {
7312 let registry = BgTaskRegistry::default();
7313 let dir = tempfile::tempdir().unwrap();
7314 let task_id = registry
7315 .spawn(
7316 SpawnPlan::Unsandboxed,
7317 QUICK_SUCCESS_COMMAND,
7318 "session".to_string(),
7319 dir.path().to_path_buf(),
7320 HashMap::new(),
7321 Some(Duration::from_secs(30)),
7322 dir.path().to_path_buf(),
7323 10,
7324 true,
7325 false,
7326 Some(dir.path().to_path_buf()),
7327 )
7328 .unwrap();
7329 registry
7330 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
7331 .unwrap();
7332 let completions = registry.drain_completions_for_session(Some("session"));
7333 assert_eq!(completions.len(), 1);
7334 assert_eq!(
7335 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
7336 vec![task_id.clone()]
7337 );
7338
7339 registry.cleanup_finished(Duration::ZERO);
7340
7341 assert!(registry.inner.tasks.lock().unwrap().is_empty());
7342 }
7343
7344 #[test]
7345 fn cleanup_finished_retains_undelivered_terminals() {
7346 let registry = BgTaskRegistry::default();
7347 let dir = tempfile::tempdir().unwrap();
7348 let task_id = registry
7349 .spawn(
7350 SpawnPlan::Unsandboxed,
7351 QUICK_SUCCESS_COMMAND,
7352 "session".to_string(),
7353 dir.path().to_path_buf(),
7354 HashMap::new(),
7355 Some(Duration::from_secs(30)),
7356 dir.path().to_path_buf(),
7357 10,
7358 true,
7359 false,
7360 Some(dir.path().to_path_buf()),
7361 )
7362 .unwrap();
7363 registry
7364 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
7365 .unwrap();
7366
7367 registry.cleanup_finished(Duration::ZERO);
7368
7369 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
7370 }
7371
7372 #[test]
7380 fn reap_child_marks_failed_when_child_exits_without_exit_marker() {
7381 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7382 let dir = tempfile::tempdir().unwrap();
7383 let task_id = registry
7384 .spawn(
7385 SpawnPlan::Unsandboxed,
7386 QUICK_SUCCESS_COMMAND,
7387 "session".to_string(),
7388 dir.path().to_path_buf(),
7389 HashMap::new(),
7390 Some(Duration::from_secs(30)),
7391 dir.path().to_path_buf(),
7392 10,
7393 true,
7394 false,
7395 Some(dir.path().to_path_buf()),
7396 )
7397 .unwrap();
7398
7399 let task = registry.task_for_session(&task_id, "session").unwrap();
7400
7401 let started = Instant::now();
7406 loop {
7407 let exited = {
7408 let mut state = task.state.lock().unwrap();
7409 match &mut state.runtime {
7410 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
7411 _ => true,
7412 }
7413 };
7414 if exited {
7415 break;
7416 }
7417 assert!(
7418 started.elapsed() < Duration::from_secs(5),
7419 "child should exit quickly"
7420 );
7421 std::thread::sleep(Duration::from_millis(20));
7422 }
7423
7424 registry
7432 .inner
7433 .shutdown
7434 .store(true, std::sync::atomic::Ordering::SeqCst);
7435 std::thread::sleep(Duration::from_millis(550));
7439
7440 let _ = std::fs::remove_file(&task.paths.exit);
7443
7444 {
7459 let mut state = task.state.lock().unwrap();
7460 state.metadata.status = BgTaskStatus::Running;
7461 state.metadata.status_reason = None;
7462 state.metadata.exit_code = None;
7463 state.metadata.finished_at = None;
7464 state.metadata.duration_ms = None;
7465 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
7468 .expect("persist reset Running metadata for reap_child test");
7469 if matches!(state.runtime, TaskRuntime::Piped(None)) {
7473 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
7474 }
7475 }
7476 *task.terminal_at.lock().unwrap() = None;
7479
7480 assert!(
7483 task.is_running(),
7484 "precondition: metadata.status == Running"
7485 );
7486 assert!(
7487 !task.paths.exit.exists(),
7488 "precondition: exit marker absent"
7489 );
7490
7491 registry.reap_child(&task);
7496
7497 {
7498 let state = task.state.lock().unwrap();
7499 assert_eq!(
7500 state.metadata.status,
7501 BgTaskStatus::Running,
7502 "first reap must leave status Running while waiting one pass for marker"
7503 );
7504 assert_eq!(
7505 state.metadata.status_reason, None,
7506 "first reap must not record a failure reason"
7507 );
7508 assert!(
7509 matches!(state.runtime, TaskRuntime::Piped(None)),
7510 "child handle must be released after first reap"
7511 );
7512 assert!(
7513 state.detached,
7514 "task must be marked detached after first reap"
7515 );
7516 }
7517
7518 registry.reap_child(&task);
7522
7523 let state = task.state.lock().unwrap();
7524 assert!(
7525 state.metadata.status.is_terminal(),
7526 "second reap must transition to terminal when PID dead and no marker. Got status={:?}",
7527 state.metadata.status
7528 );
7529 assert_eq!(
7530 state.metadata.status,
7531 BgTaskStatus::Failed,
7532 "must specifically be Failed (not Killed): status={:?}",
7533 state.metadata.status
7534 );
7535 assert_eq!(
7536 state.metadata.status_reason.as_deref(),
7537 Some("process exited without exit marker"),
7538 "reason must match replay path's wording: {:?}",
7539 state.metadata.status_reason
7540 );
7541 assert!(
7542 matches!(state.runtime, TaskRuntime::Piped(None)),
7543 "child handle must stay released after second reap"
7544 );
7545 assert!(
7546 state.detached,
7547 "task must remain detached after second reap"
7548 );
7549 }
7550
7551 #[test]
7556 fn reap_child_preserves_running_when_exit_marker_exists() {
7557 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7558 let dir = tempfile::tempdir().unwrap();
7559 let task_id = registry
7560 .spawn(
7561 SpawnPlan::Unsandboxed,
7562 QUICK_SUCCESS_COMMAND,
7563 "session".to_string(),
7564 dir.path().to_path_buf(),
7565 HashMap::new(),
7566 Some(Duration::from_secs(30)),
7567 dir.path().to_path_buf(),
7568 10,
7569 true,
7570 false,
7571 Some(dir.path().to_path_buf()),
7572 )
7573 .unwrap();
7574
7575 let task = registry.task_for_session(&task_id, "session").unwrap();
7576
7577 let started = Instant::now();
7580 loop {
7581 let exited = {
7582 let mut state = task.state.lock().unwrap();
7583 match &mut state.runtime {
7584 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
7585 _ => true,
7586 }
7587 };
7588 if exited && task.paths.exit.exists() {
7589 break;
7590 }
7591 assert!(
7592 started.elapsed() < Duration::from_secs(5),
7593 "child should exit and write marker quickly"
7594 );
7595 std::thread::sleep(Duration::from_millis(20));
7596 }
7597
7598 registry
7604 .inner
7605 .shutdown
7606 .store(true, std::sync::atomic::Ordering::SeqCst);
7607 std::thread::sleep(Duration::from_millis(550));
7608
7609 {
7615 let mut state = task.state.lock().unwrap();
7616 state.metadata.status = BgTaskStatus::Running;
7617 state.metadata.status_reason = None;
7618 if matches!(state.runtime, TaskRuntime::Piped(None)) {
7619 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
7620 }
7621 }
7622 *task.terminal_at.lock().unwrap() = None;
7623 if !task.paths.exit.exists() {
7626 std::fs::write(&task.paths.exit, "0").expect("write replacement exit marker");
7627 }
7628
7629 registry.reap_child(&task);
7633
7634 let state = task.state.lock().unwrap();
7635 assert!(
7636 matches!(state.runtime, TaskRuntime::Piped(None)),
7637 "child handle still released even when marker exists"
7638 );
7639 assert!(
7640 state.detached,
7641 "task still marked detached even when marker exists"
7642 );
7643 assert_eq!(
7648 state.metadata.status,
7649 BgTaskStatus::Running,
7650 "reap_child must defer to poll_task when marker exists"
7651 );
7652 }
7653
7654 #[cfg(unix)]
7658 fn pid_stat(pid: u32) -> Option<String> {
7659 let output = std::process::Command::new("ps")
7660 .args(["-o", "stat=", "-p", &pid.to_string()])
7661 .output()
7662 .ok()?;
7663 if !output.status.success() {
7664 return None;
7665 }
7666 let stat = String::from_utf8_lossy(&output.stdout).trim().to_string();
7667 if stat.is_empty() {
7668 None
7669 } else {
7670 Some(stat)
7671 }
7672 }
7673
7674 #[cfg(unix)]
7676 fn is_zombie(pid: u32) -> bool {
7677 pid_stat(pid).is_some_and(|stat| stat.starts_with('Z'))
7678 }
7679
7680 #[cfg(unix)]
7686 fn spawn_unreaped_zombie() -> std::process::Child {
7687 let child = std::process::Command::new("true")
7688 .stdin(std::process::Stdio::null())
7689 .stdout(std::process::Stdio::null())
7690 .stderr(std::process::Stdio::null())
7691 .spawn()
7692 .expect("spawn zombie stand-in");
7693 let pid = child.id();
7694 let started = Instant::now();
7695 while !is_zombie(pid) {
7696 assert!(
7697 started.elapsed() < Duration::from_secs(5),
7698 "stand-in child should become a zombie within 5s"
7699 );
7700 std::thread::sleep(Duration::from_millis(10));
7701 }
7702 child
7704 }
7705
7706 #[cfg(unix)]
7716 #[test]
7717 fn finalize_from_marker_reaps_child_no_zombie() {
7718 use std::sync::atomic::Ordering;
7719
7720 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7721 let dir = tempfile::tempdir().unwrap();
7722 let task_id = registry
7723 .spawn(
7724 SpawnPlan::Unsandboxed,
7725 QUICK_SUCCESS_COMMAND,
7726 "session".to_string(),
7727 dir.path().to_path_buf(),
7728 HashMap::new(),
7729 Some(Duration::from_secs(30)),
7730 dir.path().to_path_buf(),
7731 10,
7732 true,
7733 false,
7734 Some(dir.path().to_path_buf()),
7735 )
7736 .unwrap();
7737
7738 registry.inner.shutdown.store(true, Ordering::SeqCst);
7742 std::thread::sleep(Duration::from_millis(550));
7743
7744 let task = registry.task_for_session(&task_id, "session").unwrap();
7745
7746 let started = Instant::now();
7750 while !task.paths.exit.exists() {
7751 assert!(
7752 started.elapsed() < Duration::from_secs(5),
7753 "exit marker should land quickly for `true`"
7754 );
7755 std::thread::sleep(Duration::from_millis(20));
7756 }
7757
7758 let zombie_pid;
7764 {
7765 let mut state = task.state.lock().unwrap();
7766 state.metadata.status = BgTaskStatus::Running;
7767 state.metadata.status_reason = None;
7768 state.metadata.exit_code = None;
7769 state.metadata.finished_at = None;
7770 state.metadata.duration_ms = None;
7771 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
7772 .expect("persist reset Running metadata");
7773 let zombie = spawn_unreaped_zombie();
7774 zombie_pid = zombie.id();
7775 state.runtime = TaskRuntime::Piped(Some(zombie));
7776 }
7777 *task.terminal_at.lock().unwrap() = None;
7778
7779 assert!(
7781 is_zombie(zombie_pid),
7782 "precondition: stand-in child {zombie_pid} must be a zombie before finalize"
7783 );
7784
7785 registry.poll_task(&task).unwrap();
7788
7789 {
7790 let state = task.state.lock().unwrap();
7791 assert!(
7792 matches!(state.runtime, TaskRuntime::Piped(None)),
7793 "child handle must be released after marker finalize"
7794 );
7795 assert!(
7796 state.metadata.status.is_terminal(),
7797 "task must be terminal after marker finalize: {:?}",
7798 state.metadata.status
7799 );
7800 }
7801
7802 assert!(
7805 !is_zombie(zombie_pid),
7806 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
7807 after the exit-marker terminal transition"
7808 );
7809 }
7810
7811 #[cfg(unix)]
7815 #[test]
7816 fn kill_with_existing_marker_reaps_child_no_zombie() {
7817 use std::sync::atomic::Ordering;
7818
7819 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7820 let dir = tempfile::tempdir().unwrap();
7821 let task_id = registry
7822 .spawn(
7823 SpawnPlan::Unsandboxed,
7824 QUICK_SUCCESS_COMMAND,
7825 "session".to_string(),
7826 dir.path().to_path_buf(),
7827 HashMap::new(),
7828 Some(Duration::from_secs(30)),
7829 dir.path().to_path_buf(),
7830 10,
7831 true,
7832 false,
7833 Some(dir.path().to_path_buf()),
7834 )
7835 .unwrap();
7836
7837 registry.inner.shutdown.store(true, Ordering::SeqCst);
7838 std::thread::sleep(Duration::from_millis(550));
7839
7840 let task = registry.task_for_session(&task_id, "session").unwrap();
7841
7842 let started = Instant::now();
7843 while !task.paths.exit.exists() {
7844 assert!(
7845 started.elapsed() < Duration::from_secs(5),
7846 "exit marker should land quickly for `true`"
7847 );
7848 std::thread::sleep(Duration::from_millis(20));
7849 }
7850
7851 let zombie_pid;
7852 {
7853 let mut state = task.state.lock().unwrap();
7854 state.metadata.status = BgTaskStatus::Running;
7855 state.metadata.status_reason = None;
7856 state.metadata.exit_code = None;
7857 state.metadata.finished_at = None;
7858 state.metadata.duration_ms = None;
7859 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
7860 .expect("persist reset Running metadata");
7861 let zombie = spawn_unreaped_zombie();
7862 zombie_pid = zombie.id();
7863 state.runtime = TaskRuntime::Piped(Some(zombie));
7864 }
7865 *task.terminal_at.lock().unwrap() = None;
7866
7867 assert!(
7868 is_zombie(zombie_pid),
7869 "precondition: stand-in child {zombie_pid} must be a zombie before kill"
7870 );
7871
7872 registry
7874 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
7875 .expect("kill should succeed");
7876
7877 {
7878 let state = task.state.lock().unwrap();
7879 assert!(
7880 matches!(state.runtime, TaskRuntime::Piped(None)),
7881 "child handle must be released after marker-aware kill"
7882 );
7883 assert!(state.metadata.status.is_terminal());
7884 }
7885
7886 assert!(
7887 !is_zombie(zombie_pid),
7888 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
7889 after a marker-aware kill"
7890 );
7891 }
7892
7893 #[test]
7894 fn cleanup_finished_keeps_running_tasks() {
7895 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7896 let dir = tempfile::tempdir().unwrap();
7897 let task_id = registry
7898 .spawn(
7899 SpawnPlan::Unsandboxed,
7900 LONG_RUNNING_COMMAND,
7901 "session".to_string(),
7902 dir.path().to_path_buf(),
7903 HashMap::new(),
7904 Some(Duration::from_secs(30)),
7905 dir.path().to_path_buf(),
7906 10,
7907 true,
7908 false,
7909 Some(dir.path().to_path_buf()),
7910 )
7911 .unwrap();
7912
7913 registry.cleanup_finished(Duration::ZERO);
7914
7915 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
7916 let _ = registry.kill(&task_id, "session");
7917 }
7918
7919 #[cfg(unix)]
7920 #[test]
7921 fn rehydrating_sandboxed_task_never_respawns_persisted_command() {
7922 let project = tempfile::tempdir().unwrap();
7923 let storage = tempfile::tempdir().unwrap();
7924 let sandbox_temp = storage.path().join("sandbox-temp");
7925 fs::create_dir(&sandbox_temp).unwrap();
7926 let launcher_script = project.path().join("sandbox-launch");
7927 let launcher = PathBuf::from("/bin/sh");
7928 fs::write(
7929 &launcher_script,
7930 "while [ \"$#\" -gt 0 ]; do\n if [ \"$1\" = -- ]; then\n shift\n exec \"$@\"\n fi\n shift\ndone\nexit 78\n",
7931 )
7932 .unwrap();
7933 let mut permissions = fs::metadata(&launcher_script).unwrap().permissions();
7934 permissions.set_mode(0o700);
7935 fs::set_permissions(&launcher_script, permissions).unwrap();
7936
7937 let profile = crate::sandbox_profile::SandboxProfile::build(
7938 vec![project.path().to_path_buf()],
7939 Vec::new(),
7940 Vec::new(),
7941 Vec::new(),
7942 Vec::new(),
7943 Vec::new(),
7944 Vec::new(),
7945 sandbox_temp,
7946 )
7947 .unwrap();
7948 let plan = SpawnPlan::launcher_for_test(profile, launcher);
7949 let spawn_marker = project.path().join("spawn-count");
7950 let stop_marker = project.path().join("stop-command");
7951 let quote =
7952 |path: &Path| format!("'{}'", path.display().to_string().replace('\'', "'\\''"));
7953 let command = format!(
7954 "printf 'spawn\\n' >> {}; while [ ! -e {} ]; do sleep 0.05; done",
7955 quote(&spawn_marker),
7956 quote(&stop_marker)
7957 );
7958
7959 let original = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7960 let task_id = original
7961 .spawn(
7962 plan,
7963 &command,
7964 "sandbox-rehydrate".to_string(),
7965 project.path().to_path_buf(),
7966 HashMap::new(),
7967 Some(Duration::from_secs(30)),
7968 storage.path().to_path_buf(),
7969 10,
7970 true,
7971 false,
7972 Some(project.path().to_path_buf()),
7973 )
7974 .unwrap();
7975 let started = Instant::now();
7976 while !spawn_marker.exists() {
7977 assert!(
7978 started.elapsed() < Duration::from_secs(20),
7979 "original sandboxed task did not start"
7980 );
7981 std::thread::sleep(Duration::from_millis(10));
7982 }
7983 original.detach();
7984
7985 let restarted = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7986 restarted
7987 .replay_session(storage.path(), "sandbox-rehydrate")
7988 .unwrap();
7989 let replayed = restarted
7990 .status(
7991 &task_id,
7992 "sandbox-rehydrate",
7993 Some(project.path()),
7994 Some(storage.path()),
7995 4096,
7996 )
7997 .expect("rehydrated sandbox task");
7998 assert_eq!(replayed.info.status, BgTaskStatus::Running);
7999 assert!(replayed.sandbox_native);
8000
8001 std::thread::sleep(Duration::from_millis(650));
8002 assert_eq!(
8003 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
8004 1,
8005 "registry replay must observe the persisted process without spawning its command"
8006 );
8007
8008 fs::write(&stop_marker, "stop").unwrap();
8009 let terminal = wait_for_terminal_snapshot(
8010 &restarted,
8011 &task_id,
8012 "sandbox-rehydrate",
8013 project.path(),
8014 storage.path(),
8015 );
8016 assert_eq!(terminal.info.status, BgTaskStatus::Completed);
8017 assert_eq!(
8018 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
8019 1
8020 );
8021 restarted.detach();
8022 }
8023
8024 #[cfg(windows)]
8025 fn wait_for_file(path: &Path) -> String {
8026 let started = Instant::now();
8033 loop {
8034 if let Ok(content) = fs::read_to_string(path) {
8035 if !content.trim().is_empty() {
8036 return content;
8037 }
8038 }
8039 assert!(
8040 started.elapsed() < Duration::from_secs(30),
8041 "timed out waiting for non-empty {}",
8042 path.display()
8043 );
8044 std::thread::sleep(Duration::from_millis(100));
8045 }
8046 }
8047
8048 #[cfg(windows)]
8049 fn spawn_windows_registry_command(
8050 command: &str,
8051 ) -> (BgTaskRegistry, tempfile::TempDir, String) {
8052 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
8053 let dir = tempfile::tempdir().unwrap();
8054 let task_id = registry
8055 .spawn(
8056 SpawnPlan::Unsandboxed,
8057 command,
8058 "session".to_string(),
8059 dir.path().to_path_buf(),
8060 HashMap::new(),
8061 Some(Duration::from_secs(30)),
8062 dir.path().to_path_buf(),
8063 10,
8064 false,
8065 false,
8066 Some(dir.path().to_path_buf()),
8067 )
8068 .unwrap();
8069 (registry, dir, task_id)
8070 }
8071
8072 #[cfg(windows)]
8073 #[test]
8074 fn windows_spawn_writes_exit_marker_for_zero_exit() {
8075 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 0");
8076 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
8077
8078 let content = wait_for_file(&exit_path);
8079
8080 assert_eq!(content.trim(), "0");
8081 }
8082
8083 #[cfg(windows)]
8084 #[test]
8085 fn windows_spawn_writes_exit_marker_for_nonzero_exit() {
8086 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 42");
8087 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
8088
8089 let content = wait_for_file(&exit_path);
8090
8091 assert_eq!(content.trim(), "42");
8092 }
8093
8094 #[cfg(windows)]
8095 #[test]
8096 fn windows_spawn_captures_stdout_to_disk() {
8097 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c echo hello");
8098 let task = registry.task_for_session(&task_id, "session").unwrap();
8099 let stdout_path = task.paths.stdout.clone();
8100 let exit_path = task.paths.exit.clone();
8101
8102 let _ = wait_for_file(&exit_path);
8103 let stdout = fs::read_to_string(stdout_path).expect("read stdout");
8104
8105 assert!(stdout.contains("hello"), "stdout was {stdout:?}");
8106 }
8107
8108 #[cfg(windows)]
8109 #[test]
8110 fn windows_spawn_uses_pwsh_when_available() {
8111 let candidates = crate::windows_shell::shell_candidates_with(
8115 |binary| match binary {
8116 "pwsh.exe" => Some(std::path::PathBuf::from(r"C:\pwsh\pwsh.exe")),
8117 "powershell.exe" => Some(std::path::PathBuf::from(r"C:\ps\powershell.exe")),
8118 _ => None,
8119 },
8120 || None,
8121 );
8122 let shell = candidates.first().expect("at least one candidate").clone();
8123 assert_eq!(shell, crate::windows_shell::WindowsShell::Pwsh);
8124 assert_eq!(shell.binary().as_ref(), "pwsh.exe");
8125 }
8126
8127 #[cfg(windows)]
8130 #[test]
8131 fn windows_shell_cmd_wrapper_writes_marker_via_temp_rename() {
8132 let exit_path = Path::new(r"C:\Temp\bash-test.exit");
8133 let script =
8134 crate::windows_shell::WindowsShell::Cmd.wrapper_script("cmd /c exit 42", exit_path);
8135
8136 assert!(
8137 script.contains("set CODE=%ERRORLEVEL%"),
8138 "wrapper must capture the child exit code: {script}"
8139 );
8140 assert!(
8141 script.contains("exit /B %CODE%"),
8142 "wrapper must propagate the child exit code: {script}"
8143 );
8144 assert!(
8149 script.contains("bash-test.exit"),
8150 "wrapper must target the exit marker path: {script}"
8151 );
8152 assert!(
8153 script.contains("move /Y"),
8154 "wrapper must write the marker atomically via temp-file + rename: {script}"
8155 );
8156 }
8157
8158 #[cfg(windows)]
8164 #[test]
8165 fn windows_shell_cmd_bg_command_uses_minimal_cmd_flags() {
8166 use crate::windows_shell::WindowsShell;
8167 let cmd = WindowsShell::Cmd.bg_command("echo wrapped");
8168 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
8169 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
8170 assert_eq!(
8171 args_strs,
8172 vec!["/D", "/S", "/C", "echo wrapped"],
8173 "Cmd::bg_command must prepend /D /S /C"
8174 );
8175 }
8176
8177 #[cfg(windows)]
8180 #[test]
8181 fn windows_shell_pwsh_bg_command_uses_standard_args() {
8182 use crate::windows_shell::WindowsShell;
8183 let cmd = WindowsShell::Pwsh.bg_command("Get-Date");
8184 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
8185 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
8186 assert!(
8187 args_strs.contains(&"-Command"),
8188 "Pwsh::bg_command must use -Command: {args_strs:?}"
8189 );
8190 assert!(
8191 args_strs.contains(&"Get-Date"),
8192 "Pwsh::bg_command must include the user command body"
8193 );
8194 }
8195
8196 fn registry_with_db_and_frames(
8197 storage: &Path,
8198 ) -> (
8199 BgTaskRegistry,
8200 Arc<Mutex<Connection>>,
8201 Arc<Mutex<Vec<PushFrame>>>,
8202 ) {
8203 let frames = Arc::new(Mutex::new(Vec::new()));
8204 let captured = Arc::clone(&frames);
8205 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
8206 captured.lock().unwrap().push(frame);
8207 })
8208 as Box<dyn Fn(PushFrame) + Send + Sync>);
8209 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
8210 registry.set_harness(Harness::Opencode);
8211 let conn = crate::db::open(&storage.join("aft.db")).expect("open test DB");
8212 let shared = Arc::new(Mutex::new(conn));
8213 registry.set_db_pool(shared.clone());
8214 (registry, shared, frames)
8215 }
8216
8217 fn pattern_match_frames(frames: &Mutex<Vec<PushFrame>>) -> Vec<BashPatternMatchFrame> {
8218 frames
8219 .lock()
8220 .unwrap()
8221 .iter()
8222 .filter_map(|frame| match frame {
8223 PushFrame::BashPatternMatch(frame) => Some(frame.clone()),
8224 _ => None,
8225 })
8226 .collect()
8227 }
8228
8229 fn install_delivered_terminal_with_pending_watch(
8230 registry: &BgTaskRegistry,
8231 db: &Arc<Mutex<Connection>>,
8232 storage: &Path,
8233 task_id: &str,
8234 ) -> TaskPaths {
8235 let paths = task_paths(storage, "session", task_id).unwrap();
8236 let mut metadata = PersistedTask::starting(
8237 task_id.to_string(),
8238 "session".to_string(),
8239 "false".to_string(),
8240 storage.to_path_buf(),
8241 Some(storage.to_path_buf()),
8242 None,
8243 true,
8244 true,
8245 );
8246 metadata.mark_terminal(BgTaskStatus::Failed, Some(1), None);
8247 metadata.completion_delivered = true;
8248 write_task(&paths.json, &metadata).unwrap();
8249 {
8250 let conn = db.lock().unwrap();
8251 crate::db::bash_tasks::upsert_bash_task(
8252 &conn,
8253 &metadata.to_bash_task_row("opencode", &paths).unwrap(),
8254 )
8255 .unwrap();
8256 crate::db::bash_watches::upsert_bash_pattern_watch(
8257 &conn,
8258 &BashPatternWatchRow {
8259 harness: "opencode".into(),
8260 session_id: "session".into(),
8261 task_id: task_id.into(),
8262 watch_id: "watch-00000001".into(),
8263 pattern_kind: "substring".into(),
8264 pattern: "(fail)".into(),
8265 once: true,
8266 created_at: 1,
8267 stdout_offset: 756_243,
8268 stderr_offset: 0,
8269 pty_offset: 0,
8270 scanning: false,
8271 pending_match: true,
8272 match_text: Some("(fail)".into()),
8273 match_offset: Some(756_237),
8274 match_context: Some("release output ... (fail)".into()),
8275 },
8276 )
8277 .unwrap();
8278 }
8279 registry
8280 .insert_rehydrated_task(metadata, paths.clone(), true, None)
8281 .unwrap();
8282 paths
8283 }
8284
8285 #[cfg(unix)]
8286 #[test]
8287 fn gc_refuses_to_delete_or_quarantine_a_recorded_live_process() {
8288 let dir = tempfile::tempdir().unwrap();
8289 let storage = dir.path();
8290 let (registry, db, _frames) = registry_with_db_and_frames(storage);
8291 let task_id = "bash-0000000000000198";
8292 let paths = task_paths(storage, "session", task_id).unwrap();
8293 let mut running = PersistedTask::starting(
8294 task_id.to_string(),
8295 "session".to_string(),
8296 "live-process-canary".to_string(),
8297 storage.to_path_buf(),
8298 Some(storage.to_path_buf()),
8299 None,
8300 true,
8301 false,
8302 );
8303 running.mark_running(std::process::id(), std::process::id() as i32);
8307 write_task(&paths.json, &running).unwrap();
8308 fs::write(&paths.stdout, b"").unwrap();
8309 fs::write(&paths.stderr, b"").unwrap();
8310 {
8311 let conn = db.lock().unwrap();
8312 crate::db::bash_tasks::upsert_bash_task(
8313 &conn,
8314 &running.to_bash_task_row("opencode", &paths).unwrap(),
8315 )
8316 .unwrap();
8317 }
8318 let running_json = fs::read(&paths.json).unwrap();
8319 let mut terminal: PersistedTask = serde_json::from_slice(&running_json).unwrap();
8320 terminal.mark_terminal(BgTaskStatus::Completed, Some(0), None);
8321 terminal.completion_delivered = true;
8322 write_task_at(
8323 &resolve_task_layout(&paths.session_dir, task_id).unwrap(),
8324 &terminal,
8325 )
8326 .unwrap();
8327 let old = SystemTime::now()
8328 .checked_sub(Duration::from_secs(25 * 60 * 60))
8329 .unwrap();
8330 filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
8331
8332 assert_eq!(registry.maybe_gc_persisted(storage).unwrap(), 0);
8333 assert!(paths.io_dir.exists(), "GC deleted a live task bundle");
8334
8335 fs::write(&paths.json, b"{corrupt").unwrap();
8336 filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
8337 assert_eq!(registry.maybe_gc_persisted(storage).unwrap(), 0);
8338 assert!(
8339 paths.io_dir.exists(),
8340 "GC quarantined a live task with unreadable metadata"
8341 );
8342
8343 fs::write(&paths.json, running_json).unwrap();
8344 }
8345
8346 #[test]
8347 fn pattern_watch_survives_registry_teardown_and_rehydrate() {
8348 let dir = tempfile::tempdir().unwrap();
8349 let storage = dir.path();
8350 let (registry, _db, frames) = registry_with_db_and_frames(storage);
8351 let task_id = registry
8352 .spawn(
8353 SpawnPlan::Unsandboxed,
8354 LONG_RUNNING_COMMAND,
8355 "session".to_string(),
8356 storage.to_path_buf(),
8357 HashMap::new(),
8358 Some(Duration::from_secs(30)),
8359 storage.to_path_buf(),
8360 10,
8361 true,
8362 false,
8363 Some(storage.to_path_buf()),
8364 )
8365 .unwrap();
8366 registry
8367 .register_watch(
8368 task_id.clone(),
8369 WatchPattern::Substring("READY".into()),
8370 true,
8371 )
8372 .unwrap();
8373 let task = registry.task_for_session(&task_id, "session").unwrap();
8374 registry.clear_task_watch_state(&task_id);
8376 assert_eq!(registry.active_watch_count(&task_id), 0);
8377
8378 std::fs::OpenOptions::new()
8379 .append(true)
8380 .open(&task.paths.stdout)
8381 .unwrap()
8382 .write_all(b"READY\n")
8383 .unwrap();
8384 frames.lock().unwrap().clear();
8385
8386 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
8387 registry
8389 .inner
8390 .shutdown
8391 .store(true, std::sync::atomic::Ordering::SeqCst);
8392 replayed
8393 .replay_session_inner(storage, "session", None)
8394 .unwrap();
8395
8396 let matches = pattern_match_frames(&replay_frames);
8397 assert!(
8398 matches.iter().any(|frame| {
8399 frame.task_id == task_id
8400 && frame.reason == "pattern_match"
8401 && frame.match_text == "READY"
8402 }),
8403 "rehydrate should deliver gap match: {matches:?}"
8404 );
8405 }
8406
8407 #[test]
8408 fn pattern_watch_gap_match_between_teardown_and_rehydrate_delivers_once() {
8409 let dir = tempfile::tempdir().unwrap();
8410 let storage = dir.path();
8411 let (registry, _db, frames) = registry_with_db_and_frames(storage);
8412 let task_id = registry
8413 .spawn(
8414 SpawnPlan::Unsandboxed,
8415 LONG_RUNNING_COMMAND,
8416 "session".to_string(),
8417 storage.to_path_buf(),
8418 HashMap::new(),
8419 Some(Duration::from_secs(30)),
8420 storage.to_path_buf(),
8421 10,
8422 true,
8423 false,
8424 Some(storage.to_path_buf()),
8425 )
8426 .unwrap();
8427 registry
8428 .register_watch(
8429 task_id.clone(),
8430 WatchPattern::Substring("GAP-HIT".into()),
8431 true,
8432 )
8433 .unwrap();
8434 let task = registry.task_for_session(&task_id, "session").unwrap();
8435 let cursor_before = registry.watch_stream_cursors(&task_id).0;
8436 registry.clear_task_watch_state(&task_id);
8437
8438 std::fs::OpenOptions::new()
8440 .append(true)
8441 .open(&task.paths.stdout)
8442 .unwrap()
8443 .write_all(b"prefix GAP-HIT suffix\n")
8444 .unwrap();
8445 frames.lock().unwrap().clear();
8446
8447 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
8448 registry
8449 .inner
8450 .shutdown
8451 .store(true, std::sync::atomic::Ordering::SeqCst);
8452 replayed
8453 .replay_session_inner(storage, "session", None)
8454 .unwrap();
8455
8456 let matches: Vec<_> = pattern_match_frames(&replay_frames)
8457 .into_iter()
8458 .filter(|frame| frame.task_id == task_id && frame.match_text.contains("GAP-HIT"))
8459 .collect();
8460 assert_eq!(
8461 matches.len(),
8462 1,
8463 "gap match must deliver exactly once: {matches:?}"
8464 );
8465 assert!(
8466 matches[0].match_offset >= cursor_before,
8467 "match offset should be at/after the persisted cursor ({cursor_before}), got {}",
8468 matches[0].match_offset
8469 );
8470 }
8471
8472 #[test]
8473 fn pattern_watch_acked_match_does_not_redeliver_after_restart() {
8474 let dir = tempfile::tempdir().unwrap();
8475 let storage = dir.path();
8476 let (registry, db, frames) = registry_with_db_and_frames(storage);
8477 let task_id = registry
8478 .spawn(
8479 SpawnPlan::Unsandboxed,
8480 LONG_RUNNING_COMMAND,
8481 "session".to_string(),
8482 storage.to_path_buf(),
8483 HashMap::new(),
8484 Some(Duration::from_secs(30)),
8485 storage.to_path_buf(),
8486 10,
8487 true,
8488 false,
8489 Some(storage.to_path_buf()),
8490 )
8491 .unwrap();
8492 registry
8493 .register_watch(
8494 task_id.clone(),
8495 WatchPattern::Substring("READY".into()),
8496 true,
8497 )
8498 .unwrap();
8499 let task = registry.task_for_session(&task_id, "session").unwrap();
8500 std::fs::OpenOptions::new()
8501 .append(true)
8502 .open(&task.paths.stdout)
8503 .unwrap()
8504 .write_all(b"READY\n")
8505 .unwrap();
8506 registry.scan_task_watch_output(&task);
8507 let delivered = pattern_match_frames(&frames);
8508 assert!(
8509 delivered
8510 .iter()
8511 .any(|frame| frame.task_id == task_id && frame.match_text == "READY"),
8512 "live path should deliver match: {delivered:?}"
8513 );
8514 assert!(registry
8516 .ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id))
8517 .contains(&task_id));
8518 {
8519 let conn = db.lock().unwrap();
8520 let rows = crate::db::bash_watches::list_bash_pattern_watches_for_task(
8521 &conn, "opencode", "session", &task_id,
8522 )
8523 .unwrap();
8524 assert!(
8525 rows.is_empty(),
8526 "acked once-watch rows must be deleted: {rows:?}"
8527 );
8528 }
8529
8530 frames.lock().unwrap().clear();
8531 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
8532 registry
8533 .inner
8534 .shutdown
8535 .store(true, std::sync::atomic::Ordering::SeqCst);
8536 replayed
8537 .replay_session_inner(storage, "session", None)
8538 .unwrap();
8539 let matches = pattern_match_frames(&replay_frames)
8540 .into_iter()
8541 .filter(|frame| frame.task_id == task_id)
8542 .collect::<Vec<_>>();
8543 assert!(
8544 matches.is_empty(),
8545 "acked match must not re-deliver after restart: {matches:?}"
8546 );
8547 }
8548
8549 #[test]
8550 fn pending_watch_match_redelivers_after_terminal_cleanup_removes_task_bundle() {
8551 let dir = tempfile::tempdir().unwrap();
8552 let storage = dir.path();
8553 let (registry, db, frames) = registry_with_db_and_frames(storage);
8554 let task_id = "bash-aaaaaaaaaaaaaaa1";
8555 let paths = install_delivered_terminal_with_pending_watch(®istry, &db, storage, task_id);
8556 frames.lock().unwrap().clear();
8557
8558 registry.cleanup_finished(Duration::ZERO);
8559 assert!(registry.task(task_id).is_none());
8560 assert!(
8561 !paths.json.exists(),
8562 "cleanup must remove the task metadata bundle"
8563 );
8564
8565 let _ = registry.drain_completions_for_session(Some("session"));
8566 let matches = pattern_match_frames(&frames);
8567 assert!(
8568 matches.iter().any(|frame| {
8569 frame.task_id == task_id
8570 && frame.watch_id == "watch-00000001"
8571 && frame.match_text == "(fail)"
8572 }),
8573 "durable pending match must redeliver without an in-memory task: {matches:?}"
8574 );
8575 assert!(registry
8576 .ack_completions_for_session(Some("session"), &[task_id.to_string()])
8577 .contains(&task_id.to_string()));
8578 let rows = crate::db::bash_watches::list_bash_pattern_watches_for_task(
8579 &db.lock().unwrap(),
8580 "opencode",
8581 "session",
8582 task_id,
8583 )
8584 .unwrap();
8585 assert!(rows.is_empty(), "ack must end durable redelivery");
8586 }
8587
8588 #[test]
8589 fn status_uses_intact_terminal_db_row_after_task_bundle_cleanup() {
8590 let dir = tempfile::tempdir().unwrap();
8591 let storage = dir.path();
8592 let (registry, db, _frames) = registry_with_db_and_frames(storage);
8593 let task_id = "bash-aaaaaaaaaaaaaaa2";
8594 let paths = install_delivered_terminal_with_pending_watch(®istry, &db, storage, task_id);
8595
8596 registry.cleanup_finished(Duration::ZERO);
8597 assert!(registry.task(task_id).is_none());
8598 assert!(
8599 !paths.json.exists(),
8600 "cleanup must remove the task metadata bundle"
8601 );
8602 assert!(
8603 crate::db::bash_tasks::get_bash_task(
8604 &db.lock().unwrap(),
8605 "opencode",
8606 "session",
8607 task_id
8608 )
8609 .unwrap()
8610 .is_some(),
8611 "cleanup must retain the terminal database row"
8612 );
8613
8614 let snapshot = registry
8615 .status(
8616 task_id,
8617 "session",
8618 Some(storage),
8619 Some(storage),
8620 RUNNING_OUTPUT_PREVIEW_BYTES,
8621 )
8622 .expect("intact terminal row must remain visible after artifact cleanup");
8623 assert_eq!(snapshot.info.status, BgTaskStatus::Failed);
8624 assert_eq!(snapshot.exit_code, Some(1));
8625 assert!(snapshot.info.duration_ms.is_some());
8626 }
8627
8628 #[test]
8629 fn pattern_watch_rows_become_pending_tombstones_when_task_is_gc_deleted() {
8630 let dir = tempfile::tempdir().unwrap();
8631 let storage = dir.path();
8632 let (registry, db, _frames) = registry_with_db_and_frames(storage);
8633 let task_id = "bash-aaaaaaaaaaaaaaaa";
8634 let paths = task_paths(storage, "session", task_id).unwrap();
8635 let mut metadata = PersistedTask::starting(
8636 task_id.to_string(),
8637 "session".to_string(),
8638 "true".to_string(),
8639 storage.to_path_buf(),
8640 Some(storage.to_path_buf()),
8641 None,
8642 true,
8643 true,
8644 );
8645 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
8646 metadata.completion_delivered = true;
8647 write_task(&paths.json, &metadata).unwrap();
8648 {
8649 let conn = db.lock().unwrap();
8650 crate::db::bash_tasks::upsert_bash_task(
8651 &conn,
8652 &metadata.to_bash_task_row("opencode", &paths).unwrap(),
8653 )
8654 .unwrap();
8655 crate::db::bash_watches::upsert_bash_pattern_watch(
8656 &conn,
8657 &BashPatternWatchRow {
8658 harness: "opencode".into(),
8659 session_id: "session".into(),
8660 task_id: task_id.into(),
8661 watch_id: "watch-00000001".into(),
8662 pattern_kind: "substring".into(),
8663 pattern: "x".into(),
8664 once: true,
8665 created_at: 1,
8666 stdout_offset: 0,
8667 stderr_offset: 0,
8668 pty_offset: 0,
8669 scanning: true,
8670 pending_match: false,
8671 match_text: None,
8672 match_offset: None,
8673 match_context: None,
8674 },
8675 )
8676 .unwrap();
8677 }
8678 let old = SystemTime::now()
8679 .checked_sub(Duration::from_secs(25 * 60 * 60))
8680 .unwrap();
8681 filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
8682
8683 let deleted = registry.maybe_gc_persisted(storage).unwrap();
8684 assert!(
8685 deleted >= 1,
8686 "expected GC to delete the terminal task bundle"
8687 );
8688 let conn = db.lock().unwrap();
8689 let watches = crate::db::bash_watches::list_bash_pattern_watches_for_task(
8690 &conn, "opencode", "session", task_id,
8691 )
8692 .unwrap();
8693 assert_eq!(watches.len(), 1, "watch tombstone must remain until ack");
8694 assert!(!watches[0].scanning);
8695 assert!(watches[0].pending_match);
8696 assert_eq!(
8697 watches[0].match_text.as_deref(),
8698 Some(WATCH_TARGET_ERASED_TEXT)
8699 );
8700 }
8701}