1use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
2#[cfg(not(windows))]
3use std::ffi::OsString;
4use std::fs;
5use std::io::Write;
6use std::path::{Path, PathBuf};
7#[cfg(windows)]
8use std::process::Command;
9use std::process::{Child, Stdio};
10use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
11#[cfg(unix)]
12use std::sync::OnceLock;
13use std::sync::{Arc, Mutex, RwLock};
14use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
15
16use rusqlite::Connection;
17use serde::Serialize;
18
19use crate::bash_permissions::PermissionAsk;
20use crate::compress::caps::DropClass;
21#[cfg(unix)]
22use crate::compress::single_top_level_pipeline;
23use crate::compress::CompressionResult;
24use crate::context::SharedProgressSender;
25use crate::db::compression_events::CompressionAggregateCache;
26use crate::harness::Harness;
27use crate::protocol::{BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, PushFrame};
28use crate::sandbox_spawn::SpawnPlan;
29
30#[cfg(windows)]
31use std::os::windows::process::CommandExt;
32
33use super::buffer::{combine_streams, BgBuffer, DiskTruncation, StreamKind, TokenCountInput};
34use super::output::{
35 cap_completion_output, cap_completion_output_with_marker, cap_final_output,
36 cap_final_output_with_marker, completion_preview_threshold, json_output_pointer, quote_path,
37 retained_json_output_pointer, COMPRESS_INPUT_CAP_BYTES, COMPRESS_INPUT_HEAD_BYTES,
38 COMPRESS_INPUT_TAIL_BYTES, FINAL_OUTPUT_CAP_BYTES, RAW_PASSTHROUGH_CAP_BYTES,
39 RAW_PASSTHROUGH_HEAD_BYTES, RAW_PASSTHROUGH_TAIL_BYTES, RUNNING_OUTPUT_PREVIEW_BYTES,
40 STRUCTURED_OUTPUT_CAP_BYTES,
41};
42use super::persistence::{
43 allocate_task_layout, delete_resolved_task, delete_task_bundle, discover_task_ids,
44 open_task_artifact, quarantine_invalid_entry, quarantine_task_layout, read_exit_marker,
45 read_task_at, resolve_task_layout, session_tasks_dir, uninitialized_layout_is_recent,
46 unix_millis, update_task_at, validate_task_id, write_kill_marker_if_absent, write_task_at,
47 BgMode, ExitMarker, PersistedTask, TaskArtifact, TaskIoHandles, TaskPaths,
48};
49#[cfg(unix)]
50use super::process::terminate_pgid;
51#[cfg(windows)]
52use super::process::terminate_pid;
53use super::process::{is_process_alive, is_recorded_process_alive};
54use super::pty_process::spawn_pty_for_command;
55use super::pty_runtime::PtyRuntime;
56use super::watches::{PatternMatch, WatchPattern, WatchRegistry};
57use super::{BgTaskInfo, BgTaskStatus};
58use crate::db::bash_watches::BashPatternWatchRow;
59const DEFAULT_BG_TIMEOUT: Duration = Duration::from_secs(30 * 60);
62const PERSISTED_GC_GRACE: Duration = Duration::from_secs(24 * 60 * 60);
63const QUARANTINE_GC_GRACE: Duration = Duration::from_secs(30 * 24 * 60 * 60);
64
65const TOKENIZE_CAP_BYTES_PER_STREAM: usize = 128 * 1024;
66pub const ROOT_RECLAIMED_REASON: &str = "root_reclaimed";
67
68#[derive(Debug, Clone, Serialize)]
69pub struct BgCompletion {
70 pub task_id: String,
71 #[serde(skip_serializing)]
74 pub session_id: String,
75 pub status: BgTaskStatus,
76 pub exit_code: Option<i32>,
77 pub command: String,
78 #[serde(default, skip_serializing_if = "String::is_empty")]
84 pub output_preview: String,
85 #[serde(default, skip_serializing_if = "is_false")]
90 pub output_truncated: bool,
91 #[serde(default, skip_serializing_if = "Option::is_none")]
94 pub original_tokens: Option<u32>,
95 #[serde(default, skip_serializing_if = "Option::is_none")]
98 pub compressed_tokens: Option<u32>,
99 #[serde(default, skip_serializing_if = "is_false")]
101 pub tokens_skipped: bool,
102 #[serde(default, skip_serializing_if = "Option::is_none")]
103 pub status_reason: Option<String>,
104}
105
106fn is_false(v: &bool) -> bool {
107 !*v
108}
109
110#[derive(Debug, Clone, Serialize)]
111pub struct BgTaskSnapshot {
112 #[serde(flatten)]
113 pub info: BgTaskInfo,
114 pub exit_code: Option<i32>,
115 pub child_pid: Option<u32>,
116 pub workdir: String,
117 pub output_preview: String,
118 pub output_truncated: bool,
119 pub output_path: Option<String>,
120 pub stderr_path: Option<String>,
121 #[serde(skip_serializing_if = "Option::is_none")]
122 pub pty_rows: Option<u16>,
123 #[serde(skip_serializing_if = "Option::is_none")]
124 pub pty_cols: Option<u16>,
125 #[serde(skip_serializing_if = "Option::is_none")]
126 pub pty_screen: Option<String>,
127 #[serde(default, skip_serializing_if = "Vec::is_empty")]
128 pub scanner_report: Vec<PermissionAsk>,
129 #[serde(default, skip_serializing_if = "is_false")]
130 pub sandbox_native: bool,
131 #[serde(default, skip_serializing_if = "is_false")]
132 pub sandbox_unavailable: bool,
133}
134
135#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
136pub struct BgTaskHealthCounts {
137 pub running: usize,
138 pub pending_completions: usize,
139}
140
141#[derive(Debug, Clone, Copy, PartialEq, Eq)]
142enum TerminalOutputKind {
143 Compressed,
144 Raw,
145 Structured,
146}
147
148#[derive(Debug, Clone, PartialEq, Eq)]
149struct TerminalOutputCache {
150 output_preview: String,
151 output_truncated: bool,
152 kind: TerminalOutputKind,
153 output_path: Option<String>,
154 stderr_path: Option<String>,
155 artifact_access: ArtifactRecoveryAccess,
156 recovery: Option<RecoveryContext>,
157}
158
159#[derive(Debug, Clone, PartialEq, Eq)]
160struct ArtifactRecoveryAccess {
161 task_id: String,
162 readable: bool,
163}
164
165#[derive(Debug, Clone, PartialEq, Eq)]
166struct RecoveryContext {
167 dropped_by_class: BTreeMap<DropClass, usize>,
168 had_inner_drop: bool,
169 offset_hint_eligible: bool,
170 offset_start_line: Option<usize>,
171 byte_truncated: bool,
172 disk_truncated_prefix_bytes: u64,
173 output_path: Option<String>,
174 stderr_path: Option<String>,
175 include_stderr_path: bool,
176 artifact_access: ArtifactRecoveryAccess,
177}
178
179fn optional_string_bytes(value: Option<&String>) -> u64 {
180 value
181 .map(|value| crate::memory::usize_to_u64(value.len()))
182 .unwrap_or(0)
183}
184
185fn terminal_output_cache_estimated_bytes(cache: &TerminalOutputCache) -> u64 {
186 let recovery_bytes = cache
187 .recovery
188 .as_ref()
189 .map(|recovery| {
190 crate::memory::usize_to_u64(recovery.dropped_by_class.len())
191 .saturating_mul(
192 (std::mem::size_of::<DropClass>() + std::mem::size_of::<usize>()) as u64,
193 )
194 .saturating_add(optional_string_bytes(recovery.output_path.as_ref()))
195 .saturating_add(optional_string_bytes(recovery.stderr_path.as_ref()))
196 .saturating_add(crate::memory::usize_to_u64(
197 recovery.artifact_access.task_id.len(),
198 ))
199 })
200 .unwrap_or(0);
201 (std::mem::size_of::<TerminalOutputCache>() as u64)
202 .saturating_add(crate::memory::usize_to_u64(cache.output_preview.len()))
203 .saturating_add(optional_string_bytes(cache.output_path.as_ref()))
204 .saturating_add(optional_string_bytes(cache.stderr_path.as_ref()))
205 .saturating_add(crate::memory::usize_to_u64(
206 cache.artifact_access.task_id.len(),
207 ))
208 .saturating_add(recovery_bytes)
209}
210
211fn completion_estimated_bytes(completion: &BgCompletion) -> u64 {
212 (std::mem::size_of::<BgCompletion>() as u64)
213 .saturating_add(crate::memory::usize_to_u64(completion.task_id.len()))
214 .saturating_add(crate::memory::usize_to_u64(completion.session_id.len()))
215 .saturating_add(crate::memory::usize_to_u64(completion.command.len()))
216 .saturating_add(crate::memory::usize_to_u64(completion.output_preview.len()))
217}
218
219impl RecoveryContext {
220 fn has_visible_drop(&self) -> bool {
221 self.byte_truncated
222 || self.disk_truncated_prefix_bytes > 0
223 || self.had_inner_drop
224 || !self.dropped_by_class.is_empty()
225 }
226}
227
228#[derive(Clone)]
229pub struct BgTaskRegistry {
230 pub(crate) inner: Arc<RegistryInner>,
231}
232
233pub(crate) struct RegistryInner {
234 pub(crate) tasks: Mutex<HashMap<String, Arc<BgTask>>>,
235 pub(crate) completions: Mutex<VecDeque<BgCompletion>>,
236 pub(crate) progress_sender: SharedProgressSender,
237 watchdog_started: AtomicBool,
238 pub(crate) shutdown: AtomicBool,
239 pub(crate) long_running_reminder_enabled: AtomicBool,
240 pub(crate) long_running_reminder_interval_ms: AtomicU64,
241 persisted_gc_started: AtomicBool,
242 #[cfg(test)]
243 persisted_gc_runs: AtomicU64,
244 pub(crate) compressor:
250 Mutex<Option<Box<dyn Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync>>>,
251 pub(crate) db_pool: RwLock<Option<Arc<Mutex<Connection>>>>,
252 pub(crate) db_harness: RwLock<Option<String>>,
253 pub(crate) compression_aggregates: Arc<CompressionAggregateCache>,
254 pub(crate) wake_tx: crossbeam_channel::Sender<()>,
255 pub(crate) wake_rx: crossbeam_channel::Receiver<()>,
256 pub(crate) watch_registry: Mutex<WatchRegistry>,
257 wait_detach_sessions: Mutex<HashSet<String>>,
258 active_wait_sessions: Mutex<HashMap<String, usize>>,
259 wait_registered_tasks: Mutex<HashMap<String, HashSet<String>>>,
260}
261
262pub(crate) struct BgTask {
263 pub(crate) task_id: String,
264 pub(crate) session_id: String,
265 delivery_session_id: String,
266 pub(crate) paths: TaskPaths,
267 artifact_root: PathBuf,
268 pub(crate) started: Instant,
269 pub(crate) last_reminder_at: Mutex<Option<Instant>>,
270 pub(crate) terminal_at: Mutex<Option<Instant>>,
271 pub(crate) state: Mutex<BgTaskState>,
272}
273
274pub(crate) enum TaskRuntime {
275 Piped(Option<Child>),
276 Pty(Option<PtyRuntime>),
277}
278
279pub(crate) struct BgTaskState {
280 pub(crate) metadata: PersistedTask,
281 pub(crate) runtime: TaskRuntime,
282 pub(crate) io_handles: Option<TaskIoHandles>,
285 pub(crate) detached: bool,
286 pub(crate) child_exit_observed: bool,
297 pub(crate) buffer: BgBuffer,
298 terminal_output_cache: Option<TerminalOutputCache>,
299 pub(crate) pending_terminal_override: Option<BgTaskStatus>,
301}
302
303fn completion_matches_session(completion: &BgCompletion, session_id: Option<&str>) -> bool {
304 session_id
305 .map(|session_id| completion.session_id == session_id)
306 .unwrap_or(true)
307}
308
309impl BgTaskRegistry {
310 pub fn new(progress_sender: SharedProgressSender) -> Self {
311 let (wake_tx, wake_rx) = crossbeam_channel::bounded(1);
312 Self {
313 inner: Arc::new(RegistryInner {
314 tasks: Mutex::new(HashMap::new()),
315 completions: Mutex::new(VecDeque::new()),
316 progress_sender,
317 watchdog_started: AtomicBool::new(false),
318 shutdown: AtomicBool::new(false),
319 long_running_reminder_enabled: AtomicBool::new(true),
320 long_running_reminder_interval_ms: AtomicU64::new(600_000),
321 persisted_gc_started: AtomicBool::new(false),
322 #[cfg(test)]
323 persisted_gc_runs: AtomicU64::new(0),
324 compressor: Mutex::new(None),
325 db_pool: RwLock::new(None),
326 db_harness: RwLock::new(None),
327 compression_aggregates: Arc::new(CompressionAggregateCache::default()),
328 wake_tx,
329 wake_rx,
330 watch_registry: Mutex::new(WatchRegistry::default()),
331 wait_detach_sessions: Mutex::new(HashSet::new()),
332 active_wait_sessions: Mutex::new(HashMap::new()),
333 wait_registered_tasks: Mutex::new(HashMap::new()),
334 }),
335 }
336 }
337
338 pub fn is_session_owned_artifact_path(&self, session_id: &str, path: &Path) -> bool {
346 let Ok(requested) = fs::canonicalize(path) else {
347 return false;
348 };
349 let Ok(tasks) = self.inner.tasks.lock() else {
350 return false;
351 };
352
353 tasks.values().any(|task| {
354 task.session_id == session_id
355 && [
356 &task.paths.stdout,
357 &task.paths.stderr,
358 &task.paths.exit,
359 &task.paths.pty,
360 ]
361 .into_iter()
362 .filter_map(|known| known.file_name())
363 .any(|name| task.artifact_root.join(name) == requested)
364 })
365 }
366
367 pub fn read_artifact_path(
368 &self,
369 session_id: &str,
370 path: &Path,
371 ) -> Option<Result<Vec<u8>, String>> {
372 let requested = fs::canonicalize(path).ok()?;
373 let tasks = self.inner.tasks.lock().ok()?;
374 let (task, artifact) = tasks.values().find_map(|task| {
375 if task.session_id != session_id {
376 return None;
377 }
378 TaskArtifact::ALL.into_iter().find_map(|artifact| {
379 let expected = task
380 .paths
381 .artifact_path(artifact)
382 .file_name()
383 .map(|name| task.artifact_root.join(name));
384 (expected.as_deref() == Some(requested.as_path()))
385 .then(|| (Arc::clone(task), artifact))
386 })
387 })?;
388 drop(tasks);
389 Some(self.read_artifact(&task.task_id, session_id, artifact))
390 }
391
392 pub fn read_artifact_range(
393 &self,
394 task_id: &str,
395 session_id: &str,
396 artifact: TaskArtifact,
397 offset: u64,
398 ) -> Result<(Vec<u8>, u64), String> {
399 validate_task_id(task_id).map_err(|error| error.to_string())?;
400 let task = self
401 .task_for_session(task_id, session_id)
402 .ok_or_else(|| "task_not_found".to_string())?;
403 let mut file = open_task_artifact(&task.paths, artifact)
404 .map_err(|error| format!("artifact_refused: {error}"))?;
405 let len = file
406 .len()
407 .map_err(|error| format!("artifact_refused: {error}"))?;
408 let start = offset.min(len);
409 let bytes = file
410 .read_range(start, len.saturating_sub(start))
411 .map_err(|error| format!("artifact_refused: {error}"))?;
412 Ok((bytes, len))
413 }
414
415 pub fn read_artifact(
416 &self,
417 task_id: &str,
418 session_id: &str,
419 artifact: TaskArtifact,
420 ) -> Result<Vec<u8>, String> {
421 validate_task_id(task_id).map_err(|error| error.to_string())?;
422 let task = self
423 .task_for_session(task_id, session_id)
424 .ok_or_else(|| "task_not_found".to_string())?;
425 let mut file = open_task_artifact(&task.paths, artifact)
426 .map_err(|error| format!("artifact_refused: {error}"))?;
427 file.read_all()
428 .map_err(|error| format!("artifact_refused: {error}"))
429 }
430
431 pub fn set_harness(&self, harness: Harness) {
432 if let Ok(mut slot) = self.inner.db_harness.write() {
433 *slot = Some(harness.storage_segment());
434 }
435 }
436
437 pub fn set_db_pool(&self, conn: Arc<Mutex<Connection>>) {
438 if let Ok(mut slot) = self.inner.db_pool.write() {
439 *slot = Some(conn);
440 }
441 self.inner.compression_aggregates.clear();
442 }
443
444 pub fn clear_db_pool(&self) {
445 if let Ok(mut slot) = self.inner.db_pool.write() {
446 *slot = None;
447 }
448 self.inner.compression_aggregates.clear();
449 }
450
451 pub(crate) fn compression_aggregate_cache(&self) -> Arc<CompressionAggregateCache> {
452 Arc::clone(&self.inner.compression_aggregates)
453 }
454
455 pub fn register_foreground_task(&self, session_id: &str, task_id: &str) {
456 if let Ok(mut tasks) = self.inner.wait_registered_tasks.lock() {
457 tasks
458 .entry(session_id.to_string())
459 .or_default()
460 .insert(task_id.to_string());
461 }
462 }
463
464 pub fn begin_wait_mode_session(&self, session_id: &str, task_id: &str) {
465 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
466 *active.entry(session_id.to_string()).or_insert(0) += 1;
467 }
468 self.register_foreground_task(session_id, task_id);
469 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
470 detach.remove(session_id);
471 }
472 }
473
474 pub fn unregister_foreground_task(&self, session_id: &str, task_id: &str) {
475 if let Ok(mut tasks) = self.inner.wait_registered_tasks.lock() {
476 if let Some(session_tasks) = tasks.get_mut(session_id) {
477 session_tasks.remove(task_id);
478 if session_tasks.is_empty() {
479 tasks.remove(session_id);
480 }
481 }
482 }
483 }
484
485 pub fn end_wait_mode_session(&self, session_id: &str, task_id: &str) {
486 let no_active_wait = if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
487 match active.get_mut(session_id) {
488 Some(count) if *count > 1 => *count -= 1,
489 Some(_) => {
490 active.remove(session_id);
491 }
492 None => {}
493 }
494 !active.contains_key(session_id)
495 } else {
496 false
497 };
498 self.unregister_foreground_task(session_id, task_id);
499 if no_active_wait {
500 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
501 detach.remove(session_id);
502 }
503 }
504 }
505
506 pub fn abort_inflight(&self, session_id: &str) -> Result<usize, String> {
510 let task_ids = self
511 .inner
512 .wait_registered_tasks
513 .lock()
514 .map(|mut tasks| tasks.remove(session_id).unwrap_or_default())
515 .map_err(|_| "wait registration lock poisoned".to_string())?;
516 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
517 active.remove(session_id);
518 }
519 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
520 detach.remove(session_id);
521 }
522
523 let mut killed = 0;
524 for task_id in task_ids {
525 let Some(task) = self.task_for_session(&task_id, session_id) else {
526 continue;
527 };
528 let is_terminal = task
529 .state
530 .lock()
531 .map(|state| state.metadata.status.is_terminal())
532 .map_err(|_| "background task lock poisoned".to_string())?;
533 if is_terminal {
534 continue;
535 }
536 let snapshot = self.kill_with_status_reason(
537 &task_id,
538 session_id,
539 BgTaskStatus::Killed,
540 Some("call_aborted".to_string()),
541 )?;
542 if snapshot.info.status == BgTaskStatus::Killed
543 && snapshot.info.status_reason.as_deref() == Some("call_aborted")
544 {
545 killed += 1;
546 }
547 }
548 Ok(killed)
549 }
550
551 pub fn signal_wait_mode_detach(&self, session_id: &str) -> bool {
552 let is_waiting = self
553 .inner
554 .active_wait_sessions
555 .lock()
556 .map(|active| active.get(session_id).copied().unwrap_or(0) > 0)
557 .unwrap_or(false);
558 if !is_waiting {
559 return false;
560 }
561 self.inner
562 .wait_detach_sessions
563 .lock()
564 .map(|mut detach| detach.insert(session_id.to_string()))
565 .unwrap_or(false)
566 }
567
568 pub fn active_wait_session_count(&self) -> usize {
571 self.inner
572 .active_wait_sessions
573 .lock()
574 .map(|active| active.len())
575 .unwrap_or(0)
576 }
577
578 pub fn take_wait_mode_detach(&self, session_id: &str) -> bool {
579 self.inner
580 .wait_detach_sessions
581 .lock()
582 .map(|mut detach| detach.remove(session_id))
583 .unwrap_or(false)
584 }
585
586 pub fn set_compressor<F>(&self, compressor: F)
591 where
592 F: Fn(&str, String) -> CompressionResult + Send + Sync + 'static,
593 {
594 self.set_compressor_with_exit_code(move |command, output, _exit_code| {
595 compressor(command, output)
596 });
597 }
598
599 pub fn set_compressor_with_exit_code<F>(&self, compressor: F)
600 where
601 F: Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync + 'static,
602 {
603 if let Ok(mut slot) = self.inner.compressor.lock() {
604 *slot = Some(Box::new(compressor));
605 }
606 }
607
608 pub(crate) fn compress_output(
611 &self,
612 command: &str,
613 output: String,
614 exit_code: Option<i32>,
615 ) -> CompressionResult {
616 let Ok(slot) = self.inner.compressor.lock() else {
617 return CompressionResult::new(output);
618 };
619 match slot.as_ref() {
620 Some(compressor) => compressor(command, output, exit_code),
621 None => CompressionResult::new(output),
622 }
623 }
624
625 fn ensure_terminal_output_cache(&self, task: &Arc<BgTask>) -> Option<TerminalOutputCache> {
626 let (metadata, buffer) = {
627 let state = task.state.lock().ok()?;
628 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
629 return None;
630 }
631 if let Some(cache) = state.terminal_output_cache.clone() {
632 return Some(cache);
633 }
634 (state.metadata.clone(), state.buffer.clone())
635 };
636
637 let mut cap_buffer = buffer.clone();
638 let disk_truncation = cap_buffer.enforce_terminal_cap();
639 let cache =
640 self.render_terminal_output(&metadata, &cap_buffer, disk_truncation, Some(&task.paths));
641 let mut state = task.state.lock().ok()?;
642 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
643 return None;
644 }
645 if let Some(existing) = state.terminal_output_cache.clone() {
646 return Some(existing);
647 }
648 state.terminal_output_cache = Some(cache.clone());
649 Some(cache)
650 }
651
652 fn render_terminal_output(
653 &self,
654 metadata: &PersistedTask,
655 buffer: &BgBuffer,
656 disk_truncation: DiskTruncation,
657 paths: Option<&TaskPaths>,
658 ) -> TerminalOutputCache {
659 let output_readable = buffer
660 .output_path()
661 .is_some_and(|path| self.is_session_owned_artifact_path(&metadata.session_id, &path));
662 let stderr_readable = buffer
663 .stderr_path()
664 .map(|path| self.is_session_owned_artifact_path(&metadata.session_id, path))
665 .unwrap_or(true);
666 let artifact_access = ArtifactRecoveryAccess {
667 task_id: metadata.task_id.clone(),
668 readable: output_readable && stderr_readable,
669 };
670
671 if metadata.mode == BgMode::Pty {
672 return TerminalOutputCache {
673 output_preview: String::new(),
674 output_truncated: false,
675 kind: TerminalOutputKind::Raw,
676 output_path: buffer.output_path().map(|path| path.display().to_string()),
677 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
678 artifact_access,
679 recovery: None,
680 };
681 }
682
683 let mut rendered = if let Some(structured) = render_structured_output(
684 &metadata.command,
685 buffer,
686 disk_truncation,
687 artifact_access.clone(),
688 ) {
689 structured
690 } else if !metadata.compressed {
691 render_raw_passthrough(buffer, disk_truncation, artifact_access)
692 } else {
693 let raw = buffer.read_combined_head_tail(
694 COMPRESS_INPUT_CAP_BYTES,
695 COMPRESS_INPUT_HEAD_BYTES,
696 COMPRESS_INPUT_TAIL_BYTES,
697 );
698 let compressed = self.compress_output(&metadata.command, raw.text, metadata.exit_code);
699 render_compressed_with_recovery(
700 buffer,
701 compressed,
702 raw.truncated,
703 disk_truncation,
704 artifact_access,
705 )
706 };
707 normalize_piped_display_output(&mut rendered.output_preview);
708 append_pipeline_warning(&mut rendered, metadata, paths);
709 rendered
710 }
711
712 fn snapshot_with_terminal_cache(
713 &self,
714 task: &Arc<BgTask>,
715 preview_bytes: usize,
716 ) -> BgTaskSnapshot {
717 let mut snapshot = task.snapshot(preview_bytes);
718 self.maybe_compress_snapshot(task, &mut snapshot);
719 snapshot
720 }
721
722 fn post_terminal_transition(&self, task: &Arc<BgTask>, emit_frame: bool) -> Result<(), String> {
723 let (metadata, buffer) = {
724 let state = task
725 .state
726 .lock()
727 .map_err(|_| "background task lock poisoned".to_string())?;
728 if !state.metadata.status.is_terminal() {
729 return Ok(());
730 }
731 (state.metadata.clone(), state.buffer.clone())
732 };
733
734 let cache = self.ensure_terminal_output_cache(task);
735 self.enqueue_completion_from_parts(
736 &metadata,
737 Some(&buffer),
738 None,
739 emit_frame,
740 cache.as_ref(),
741 );
742 self.retarget_pending_completion(&metadata.task_id, &task.delivery_session_id);
743 Ok(())
744 }
745
746 fn persist_task(&self, paths: &TaskPaths, metadata: &PersistedTask) -> std::io::Result<()> {
747 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
748 write_task_at(&task, metadata)?;
749 self.dual_write_task(paths, metadata);
750 Ok(())
751 }
752
753 fn update_task_metadata<F>(
754 &self,
755 paths: &TaskPaths,
756 update: F,
757 ) -> std::io::Result<PersistedTask>
758 where
759 F: FnOnce(&mut PersistedTask),
760 {
761 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
762 let metadata = update_task_at(&task, update)?;
763 self.dual_write_task(paths, &metadata);
764 Ok(metadata)
765 }
766
767 fn dual_write_task(&self, paths: &TaskPaths, metadata: &PersistedTask) {
768 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
769 let Some(pool) = pool else {
770 return;
771 };
772 let harness = self
773 .inner
774 .db_harness
775 .read()
776 .ok()
777 .and_then(|slot| slot.clone());
778 let Some(harness) = harness else {
779 crate::slog_warn!(
780 "dual-write bash_task to DB skipped for {}: harness not configured",
781 metadata.task_id
782 );
783 return;
784 };
785 let row = match metadata.to_bash_task_row(&harness, paths) {
786 Ok(row) => row,
787 Err(error) => {
788 crate::slog_warn!(
789 "dual-write bash_task to DB failed for {}: {}",
790 metadata.task_id,
791 error
792 );
793 return;
794 }
795 };
796 let conn = match pool.lock() {
797 Ok(conn) => conn,
798 Err(_) => {
799 crate::slog_warn!(
800 "dual-write bash_task to DB failed for {}: db mutex poisoned",
801 metadata.task_id
802 );
803 return;
804 }
805 };
806 if let Err(error) = crate::db::bash_tasks::upsert_bash_task(&conn, &row) {
807 crate::slog_warn!(
808 "dual-write bash_task to DB failed for {}: {}",
809 metadata.task_id,
810 error
811 );
812 }
813 }
814
815 fn delete_gc_task_from_db(&self, metadata: &PersistedTask) {
816 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
817 let Some(pool) = pool else {
818 return;
819 };
820 let harness = self
821 .inner
822 .db_harness
823 .read()
824 .ok()
825 .and_then(|slot| slot.clone());
826 let Some(harness) = harness else {
827 crate::slog_warn!(
828 "GC bash_task DB delete skipped for {}: harness not configured",
829 metadata.task_id
830 );
831 return;
832 };
833 let conn = match pool.lock() {
834 Ok(conn) => conn,
835 Err(_) => {
836 crate::slog_warn!(
837 "GC bash_task DB delete failed for {}: db mutex poisoned",
838 metadata.task_id
839 );
840 return;
841 }
842 };
843 if let Err(error) = crate::db::bash_tasks::delete_delivered_terminal_bash_task(
844 &conn,
845 &harness,
846 &metadata.session_id,
847 &metadata.task_id,
848 ) {
849 crate::slog_warn!(
850 "GC bash_task DB delete failed for {}: {}",
851 metadata.task_id,
852 error
853 );
854 }
855 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
857 &conn,
858 &harness,
859 &metadata.session_id,
860 &metadata.task_id,
861 ) {
862 crate::slog_warn!(
863 "GC bash_pattern_watches DB delete failed for {}: {}",
864 metadata.task_id,
865 error
866 );
867 }
868 }
869
870 fn persisted_task_process_is_alive(metadata: &PersistedTask) -> bool {
871 let child_pid = metadata.child_pid;
872 let group_leader = metadata.pgid.and_then(|pid| u32::try_from(pid).ok());
873 child_pid
874 .into_iter()
875 .chain(group_leader)
876 .any(|pid| is_recorded_process_alive(pid, metadata.started_at))
877 }
878
879 fn db_has_live_process_for_task(&self, task_id: &str) -> bool {
880 let Some((harness, pool)) = self.db_harness_and_pool() else {
881 return false;
882 };
883 let Ok(conn) = pool.lock() else {
884 return false;
885 };
886 crate::db::bash_tasks::list_bash_tasks_by_id(&conn, &harness, task_id)
887 .map(|rows| {
888 rows.into_iter().any(|row| {
889 let started_at = u64::try_from(row.started_at).unwrap_or_default();
890 row.pid
891 .and_then(|pid| u32::try_from(pid).ok())
892 .into_iter()
893 .chain(row.pgid.and_then(|pid| u32::try_from(pid).ok()))
894 .any(|pid| is_recorded_process_alive(pid, started_at))
895 })
896 })
897 .unwrap_or(false)
898 }
899
900 fn db_harness_and_pool(&self) -> Option<(String, Arc<Mutex<Connection>>)> {
901 let pool = self
902 .inner
903 .db_pool
904 .read()
905 .ok()
906 .and_then(|slot| slot.clone())?;
907 let harness = self
908 .inner
909 .db_harness
910 .read()
911 .ok()
912 .and_then(|slot| slot.clone())?;
913 Some((harness, pool))
914 }
915
916 fn persist_watch_registration(
917 &self,
918 session_id: &str,
919 task_id: &str,
920 watch_id: &str,
921 pattern: &WatchPattern,
922 once: bool,
923 stdout_offset: u64,
924 stderr_offset: u64,
925 pty_offset: u64,
926 ) {
927 let Some((harness, pool)) = self.db_harness_and_pool() else {
928 return;
929 };
930 let Ok(conn) = pool.lock() else {
931 return;
932 };
933 let row = BashPatternWatchRow {
934 harness,
935 session_id: session_id.to_string(),
936 task_id: task_id.to_string(),
937 watch_id: watch_id.to_string(),
938 pattern_kind: pattern.kind_name().to_string(),
939 pattern: pattern.pattern_text().to_string(),
940 once,
941 created_at: unix_millis() as i64,
942 stdout_offset: stdout_offset as i64,
943 stderr_offset: stderr_offset as i64,
944 pty_offset: pty_offset as i64,
945 scanning: true,
946 pending_match: false,
947 match_text: None,
948 match_offset: None,
949 match_context: None,
950 };
951 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
952 crate::slog_warn!(
953 "persist bash_pattern_watch failed for {task_id}/{watch_id}: {error}"
954 );
955 }
956 }
957
958 fn delete_persisted_watch(&self, session_id: &str, task_id: &str, watch_id: &str) {
959 let Some((harness, pool)) = self.db_harness_and_pool() else {
960 return;
961 };
962 let Ok(conn) = pool.lock() else {
963 return;
964 };
965 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watch(
966 &conn, &harness, session_id, task_id, watch_id,
967 ) {
968 crate::slog_warn!("delete bash_pattern_watch failed for {task_id}/{watch_id}: {error}");
969 }
970 }
971
972 fn delete_persisted_watches_for_task(&self, session_id: &str, task_id: &str) {
973 let Some((harness, pool)) = self.db_harness_and_pool() else {
974 return;
975 };
976 let Ok(conn) = pool.lock() else {
977 return;
978 };
979 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
980 &conn, &harness, session_id, task_id,
981 ) {
982 crate::slog_warn!("delete bash_pattern_watches for {task_id} failed: {error}");
983 }
984 }
985
986 fn persist_watch_match(
987 &self,
988 session_id: &str,
989 task_id: &str,
990 pattern_match: &PatternMatch,
991 stdout_offset: u64,
992 stderr_offset: u64,
993 pty_offset: u64,
994 ) {
995 let Some((harness, pool)) = self.db_harness_and_pool() else {
996 return;
997 };
998 let Ok(conn) = pool.lock() else {
999 return;
1000 };
1001 let Ok(Some(mut row)) = crate::db::bash_watches::get_bash_pattern_watch(
1002 &conn,
1003 &harness,
1004 session_id,
1005 task_id,
1006 &pattern_match.watch_id,
1007 ) else {
1008 return;
1009 };
1010 row.stdout_offset = stdout_offset as i64;
1011 row.stderr_offset = stderr_offset as i64;
1012 row.pty_offset = pty_offset as i64;
1013 row.pending_match = true;
1014 row.match_text = Some(pattern_match.match_text.clone());
1015 row.match_offset = Some(pattern_match.match_offset as i64);
1016 row.match_context = Some(pattern_match.context.clone());
1017 if pattern_match.once {
1018 row.scanning = false;
1021 }
1022 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
1023 crate::slog_warn!(
1024 "persist bash_pattern_watch match failed for {}/{}: {error}",
1025 task_id,
1026 pattern_match.watch_id
1027 );
1028 }
1029 }
1030
1031 fn persist_task_watch_cursors(
1032 &self,
1033 session_id: &str,
1034 task_id: &str,
1035 stdout_offset: u64,
1036 stderr_offset: u64,
1037 pty_offset: u64,
1038 ) {
1039 let Some((harness, pool)) = self.db_harness_and_pool() else {
1040 return;
1041 };
1042 let Ok(conn) = pool.lock() else {
1043 return;
1044 };
1045 if let Err(error) = crate::db::bash_watches::update_watch_offsets_for_task(
1046 &conn,
1047 &harness,
1048 session_id,
1049 task_id,
1050 stdout_offset as i64,
1051 stderr_offset as i64,
1052 pty_offset as i64,
1053 ) {
1054 crate::slog_warn!("persist bash_pattern_watch cursors failed for {task_id}: {error}");
1055 }
1056 }
1057
1058 fn watch_stream_cursors(&self, task_id: &str) -> (u64, u64, u64) {
1059 let Ok(registry) = self.inner.watch_registry.lock() else {
1060 return (0, 0, 0);
1061 };
1062 let stdout = registry
1063 .file_cursor(&format!("{task_id}:stdout"))
1064 .unwrap_or(0);
1065 let stderr = registry
1066 .file_cursor(&format!("{task_id}:stderr"))
1067 .unwrap_or(0);
1068 let pty = registry.file_cursor(&format!("{task_id}:pty")).unwrap_or(0);
1069 (stdout, stderr, pty)
1070 }
1071
1072 fn ack_persisted_watches_for_task(&self, session_id: &str, task_id: &str, task_terminal: bool) {
1075 let Some((harness, pool)) = self.db_harness_and_pool() else {
1076 return;
1077 };
1078 let Ok(conn) = pool.lock() else {
1079 return;
1080 };
1081 if task_terminal {
1082 let _ = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
1083 &conn, &harness, session_id, task_id,
1084 );
1085 return;
1086 }
1087 let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
1088 &conn, &harness, session_id, task_id,
1089 ) else {
1090 return;
1091 };
1092 for mut row in rows {
1093 if row.once && (!row.scanning || row.pending_match) {
1094 let _ = crate::db::bash_watches::delete_bash_pattern_watch(
1095 &conn,
1096 &harness,
1097 session_id,
1098 task_id,
1099 &row.watch_id,
1100 );
1101 continue;
1102 }
1103 if row.pending_match {
1104 row.pending_match = false;
1105 row.match_text = None;
1106 row.match_offset = None;
1107 row.match_context = None;
1108 let _ = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row);
1109 }
1110 }
1111 }
1112
1113 pub fn record_scanner_report(
1114 &self,
1115 task_id: &str,
1116 session_id: &str,
1117 scanner_report: Vec<PermissionAsk>,
1118 ) -> Result<(), String> {
1119 if scanner_report.is_empty() {
1120 return Ok(());
1121 }
1122 let task = self.task_for_session(task_id, session_id).ok_or_else(|| {
1123 "background task not found while recording scanner report".to_string()
1124 })?;
1125 let metadata = {
1126 let mut state = task
1127 .state
1128 .lock()
1129 .map_err(|_| "background task lock poisoned".to_string())?;
1130 state.metadata.scanner_report = scanner_report;
1131 state.metadata.clone()
1132 };
1133 self.persist_task(&task.paths, &metadata)
1134 .map_err(|error| format!("failed to persist scanner report: {error}"))
1135 }
1136
1137 pub fn configure_long_running_reminders(&self, enabled: bool, interval_ms: u64) {
1138 self.inner
1139 .long_running_reminder_enabled
1140 .store(enabled, Ordering::SeqCst);
1141 self.inner
1142 .long_running_reminder_interval_ms
1143 .store(interval_ms, Ordering::SeqCst);
1144 }
1145
1146 #[cfg(unix)]
1147 #[allow(clippy::too_many_arguments)]
1148 pub fn spawn(
1149 &self,
1150 spawn_plan: SpawnPlan,
1151 command: &str,
1152 session_id: String,
1153 workdir: PathBuf,
1154 env: HashMap<String, String>,
1155 timeout: Option<Duration>,
1156 storage_dir: PathBuf,
1157 max_running: usize,
1158 notify_on_completion: bool,
1159 compressed: bool,
1160 project_root: Option<PathBuf>,
1161 ) -> Result<String, String> {
1162 self.start_watchdog();
1163
1164 let running = self.running_count();
1165 if running >= max_running {
1166 #[cfg(unix)]
1167 if let Some(prepared) = spawn_plan.prepared_task() {
1168 let _ = delete_resolved_task(&prepared.resolved_task());
1169 }
1170 return Err(format!(
1171 "background bash task limit exceeded: {running} running (max {max_running})"
1172 ));
1173 }
1174
1175 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1176 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1177 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1178 (spawn_plan.clone(), prepared.resolved_task())
1179 } else {
1180 let task = allocate_task_layout(&storage_dir, &session_id)
1181 .map_err(|error| format!("failed to create background task layout: {error}"))?;
1182 let shell = resolve_posix_shell();
1183 let root = project_root.as_deref().unwrap_or(&workdir);
1184 let environment =
1185 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1186 let prepared = match crate::sandbox_spawn::prepare_task_payload(
1187 &task,
1188 command.as_bytes(),
1189 root,
1190 &workdir,
1191 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1192 &shell,
1193 &environment,
1194 ) {
1195 Ok(prepared) => prepared,
1196 Err(error) => {
1197 let _ = delete_resolved_task(&task);
1198 return Err(error);
1199 }
1200 };
1201 let task = prepared.resolved_task();
1202 (spawn_plan.with_prepared_task(prepared), task)
1203 };
1204 let task_id = task_layout.paths.task_id.clone();
1205 let paths = task_layout.paths.clone();
1206
1207 if self.task(&task_id).is_some() {
1208 let _ = delete_resolved_task(&task_layout);
1209 return Err("background task id collided with a live task".to_string());
1210 }
1211
1212 let mut metadata = PersistedTask::starting(
1213 task_id.clone(),
1214 session_id.clone(),
1215 command.to_string(),
1216 workdir.clone(),
1217 project_root,
1218 timeout_ms,
1219 notify_on_completion,
1220 compressed,
1221 );
1222 #[cfg(unix)]
1226 let capture_pipeline_status = {
1227 let shell = spawn_plan
1228 .host_shell_path()
1229 .map(Path::to_path_buf)
1230 .unwrap_or_else(resolve_posix_shell);
1231 let pipeline = single_top_level_pipeline(command);
1232 let capture = should_capture_pipeline_status(&spawn_plan, pipeline.is_some(), &shell);
1233 if capture {
1234 metadata.pipeline_segments = pipeline
1235 .as_ref()
1236 .map(|pipeline| {
1237 pipeline
1238 .segments
1239 .iter()
1240 .map(|segment| segment.label.clone())
1241 .collect()
1242 })
1243 .unwrap_or_default();
1244 }
1245 capture
1246 };
1247 #[cfg(windows)]
1248 let capture_pipeline_status = false;
1249 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1250 if let Err(error) = write_task_at(&task_layout, &metadata) {
1251 let _ = delete_resolved_task(&task_layout);
1252 return Err(format!(
1253 "failed to persist background task metadata: {error}"
1254 ));
1255 }
1256 self.dual_write_task(&paths, &metadata);
1257
1258 let mut io_handles =
1259 TaskIoHandles::create(&task_layout, BgMode::Pipes, capture_pipeline_status)
1260 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
1261 let child = match spawn_detached_child(
1262 &spawn_plan,
1263 command,
1264 &paths,
1265 &workdir,
1266 &env,
1267 &mut io_handles,
1268 capture_pipeline_status,
1269 ) {
1270 Ok(child) => child,
1271 Err(error) => {
1272 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
1273 let _ = delete_task_bundle(&paths);
1274 return Err(error);
1275 }
1276 };
1277
1278 let child_pid = child.id();
1279 metadata.mark_running(child_pid, child_pid as i32);
1280 self.persist_task(&paths, &metadata)
1281 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1282
1283 let task = Arc::new(BgTask {
1284 task_id: task_id.clone(),
1285 delivery_session_id: session_id.clone(),
1286 session_id,
1287 paths: paths.clone(),
1288 artifact_root: canonical_artifact_root(&paths),
1289 started: Instant::now(),
1290 last_reminder_at: Mutex::new(None),
1291 terminal_at: Mutex::new(None),
1292 state: Mutex::new(BgTaskState {
1293 metadata,
1294 runtime: TaskRuntime::Piped(Some(child)),
1295 io_handles: Some(io_handles),
1296 detached: false,
1297 child_exit_observed: false,
1298 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
1299 terminal_output_cache: None,
1300 pending_terminal_override: None,
1301 }),
1302 });
1303
1304 self.inner
1305 .tasks
1306 .lock()
1307 .map_err(|_| "background task registry lock poisoned".to_string())?
1308 .insert(task_id.clone(), task);
1309
1310 Ok(task_id)
1311 }
1312
1313 #[allow(clippy::too_many_arguments)]
1314 pub fn spawn_pty(
1315 &self,
1316 spawn_plan: SpawnPlan,
1317 command: &str,
1318 session_id: String,
1319 workdir: PathBuf,
1320 env: HashMap<String, String>,
1321 timeout: Option<Duration>,
1322 storage_dir: PathBuf,
1323 max_running: usize,
1324 notify_on_completion: bool,
1325 compressed: bool,
1326 project_root: Option<PathBuf>,
1327 rows: u16,
1328 cols: u16,
1329 ) -> Result<String, String> {
1330 self.start_watchdog();
1331
1332 let running = self.running_count();
1333 if running >= max_running {
1334 #[cfg(unix)]
1335 if let Some(prepared) = spawn_plan.prepared_task() {
1336 let _ = delete_resolved_task(&prepared.resolved_task());
1337 }
1338 return Err(format!(
1339 "background bash task limit exceeded: {running} running (max {max_running})"
1340 ));
1341 }
1342
1343 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1344 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1345 #[cfg(unix)]
1346 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1347 (spawn_plan.clone(), prepared.resolved_task())
1348 } else {
1349 let task = allocate_task_layout(&storage_dir, &session_id)
1350 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1351 let shell = super::resolved_shell_path(true);
1352 let root = project_root.as_deref().unwrap_or(&workdir);
1353 let environment =
1354 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1355 let prepared = match crate::sandbox_spawn::prepare_task_payload(
1356 &task,
1357 command.as_bytes(),
1358 root,
1359 &workdir,
1360 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1361 &shell,
1362 &environment,
1363 ) {
1364 Ok(prepared) => prepared,
1365 Err(error) => {
1366 let _ = delete_resolved_task(&task);
1367 return Err(error);
1368 }
1369 };
1370 let task = prepared.resolved_task();
1371 (spawn_plan.with_prepared_task(prepared), task)
1372 };
1373 #[cfg(windows)]
1374 let task_layout = allocate_task_layout(&storage_dir, &session_id)
1375 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1376 let task_id = task_layout.paths.task_id.clone();
1377 let paths = task_layout.paths.clone();
1378
1379 let mut metadata = PersistedTask::starting(
1380 task_id.clone(),
1381 session_id.clone(),
1382 command.to_string(),
1383 workdir.clone(),
1384 project_root,
1385 timeout_ms,
1386 notify_on_completion,
1387 compressed,
1388 );
1389 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1390 metadata.mode = BgMode::Pty;
1391 metadata.pty_rows = Some(rows);
1392 metadata.pty_cols = Some(cols);
1393 if let Err(error) = write_task_at(&task_layout, &metadata) {
1394 let _ = delete_resolved_task(&task_layout);
1395 return Err(format!(
1396 "failed to persist background task metadata: {error}"
1397 ));
1398 }
1399 self.dual_write_task(&paths, &metadata);
1400 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pty, false)
1401 .map_err(|error| format!("failed to pre-open PTY output handles: {error}"))?;
1402
1403 let runtime = match spawn_pty_for_command(
1404 &spawn_plan,
1405 &task_id,
1406 &session_id,
1407 command,
1408 &paths,
1409 &workdir,
1410 &env,
1411 rows,
1412 cols,
1413 self.inner.wake_tx.clone(),
1414 &mut io_handles,
1415 ) {
1416 Ok(runtime) => runtime,
1417 Err(error) => {
1418 crate::slog_warn!(
1419 "failed to spawn PTY background bash task {task_id}; deleting partial bundle: {error}"
1420 );
1421 let _ = delete_task_bundle(&paths);
1422 return Err(error);
1423 }
1424 };
1425
1426 if let Some(child_pid) = runtime.child_pid {
1427 metadata.mark_running(child_pid, child_pid as i32);
1428 } else {
1429 metadata.status = BgTaskStatus::Running;
1430 metadata.pgid = None;
1431 }
1432 self.persist_task(&paths, &metadata)
1433 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1434
1435 let task = Arc::new(BgTask {
1436 task_id: task_id.clone(),
1437 delivery_session_id: session_id.clone(),
1438 session_id,
1439 paths: paths.clone(),
1440 artifact_root: canonical_artifact_root(&paths),
1441 started: Instant::now(),
1442 last_reminder_at: Mutex::new(None),
1443 terminal_at: Mutex::new(None),
1444 state: Mutex::new(BgTaskState {
1445 metadata,
1446 runtime: TaskRuntime::Pty(Some(runtime)),
1447 io_handles: Some(io_handles),
1448 detached: false,
1449 child_exit_observed: false,
1450 buffer: BgBuffer::registered(&paths, BgMode::Pty),
1451 terminal_output_cache: None,
1452 pending_terminal_override: None,
1453 }),
1454 });
1455
1456 self.inner
1457 .tasks
1458 .lock()
1459 .map_err(|_| "background task registry lock poisoned".to_string())?
1460 .insert(task_id.clone(), task);
1461
1462 Ok(task_id)
1463 }
1464
1465 #[cfg(windows)]
1466 #[allow(clippy::too_many_arguments)]
1467 pub fn spawn(
1468 &self,
1469 spawn_plan: SpawnPlan,
1470 command: &str,
1471 session_id: String,
1472 workdir: PathBuf,
1473 env: HashMap<String, String>,
1474 timeout: Option<Duration>,
1475 storage_dir: PathBuf,
1476 max_running: usize,
1477 notify_on_completion: bool,
1478 compressed: bool,
1479 project_root: Option<PathBuf>,
1480 ) -> Result<String, String> {
1481 self.start_watchdog();
1482
1483 let running = self.running_count();
1484 if running >= max_running {
1485 #[cfg(unix)]
1486 if let Some(prepared) = spawn_plan.prepared_task() {
1487 let _ = delete_resolved_task(&prepared.resolved_task());
1488 }
1489 return Err(format!(
1490 "background bash task limit exceeded: {running} running (max {max_running})"
1491 ));
1492 }
1493
1494 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1495 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1496 let task_layout = allocate_task_layout(&storage_dir, &session_id)
1497 .map_err(|error| format!("failed to create background task layout: {error}"))?;
1498 let task_id = task_layout.paths.task_id.clone();
1499 let paths = task_layout.paths.clone();
1500
1501 let mut metadata = PersistedTask::starting(
1502 task_id.clone(),
1503 session_id.clone(),
1504 command.to_string(),
1505 workdir.clone(),
1506 project_root,
1507 timeout_ms,
1508 notify_on_completion,
1509 compressed,
1510 );
1511 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1512 if let Err(error) = write_task_at(&task_layout, &metadata) {
1513 let _ = delete_resolved_task(&task_layout);
1514 return Err(format!(
1515 "failed to persist background task metadata: {error}"
1516 ));
1517 }
1518 self.dual_write_task(&paths, &metadata);
1519 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pipes, false)
1520 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
1521
1522 let child = match spawn_detached_child(
1523 &spawn_plan,
1524 command,
1525 &paths,
1526 &workdir,
1527 &env,
1528 &mut io_handles,
1529 false,
1530 ) {
1531 Ok(child) => child,
1532 Err(error) => {
1533 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
1534 let _ = delete_task_bundle(&paths);
1535 return Err(error);
1536 }
1537 };
1538
1539 let child_pid = child.id();
1540 metadata.status = BgTaskStatus::Running;
1541 metadata.child_pid = Some(child_pid);
1542 metadata.pgid = None;
1543 self.persist_task(&paths, &metadata)
1544 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1545
1546 let task = Arc::new(BgTask {
1547 task_id: task_id.clone(),
1548 delivery_session_id: session_id.clone(),
1549 session_id,
1550 paths: paths.clone(),
1551 artifact_root: canonical_artifact_root(&paths),
1552 started: Instant::now(),
1553 last_reminder_at: Mutex::new(None),
1554 terminal_at: Mutex::new(None),
1555 state: Mutex::new(BgTaskState {
1556 metadata,
1557 runtime: TaskRuntime::Piped(Some(child)),
1558 io_handles: Some(io_handles),
1559 detached: false,
1560 child_exit_observed: false,
1561 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
1562 terminal_output_cache: None,
1563 pending_terminal_override: None,
1564 }),
1565 });
1566
1567 self.inner
1568 .tasks
1569 .lock()
1570 .map_err(|_| "background task registry lock poisoned".to_string())?
1571 .insert(task_id.clone(), task);
1572
1573 Ok(task_id)
1574 }
1575
1576 pub fn write_pty(
1577 &self,
1578 task_id: &str,
1579 session_id: &str,
1580 input: &[u8],
1581 ) -> Result<usize, String> {
1582 let task = self
1583 .task_for_session(task_id, session_id)
1584 .ok_or_else(|| "task_not_found".to_string())?;
1585
1586 let writer = {
1587 let state = task
1588 .state
1589 .lock()
1590 .map_err(|_| "background task lock poisoned".to_string())?;
1591 if state.metadata.mode != BgMode::Pty {
1592 return Err("task_not_pty".to_string());
1593 }
1594 if state.metadata.status.is_terminal() {
1595 return Err("task_exited".to_string());
1596 }
1597 match &state.runtime {
1598 TaskRuntime::Pty(Some(runtime)) => Arc::clone(&runtime.writer),
1599 TaskRuntime::Pty(None) => return Err("task_exited".to_string()),
1600 TaskRuntime::Piped(_) => return Err("task_not_pty".to_string()),
1601 }
1602 };
1603
1604 let mut writer = writer
1605 .lock()
1606 .map_err(|_| "PTY writer lock poisoned".to_string())?;
1607 writer
1608 .write_all(input)
1609 .map_err(|error| format!("failed to write to PTY: {error}"))?;
1610 writer
1611 .flush()
1612 .map_err(|error| format!("failed to flush PTY writer: {error}"))?;
1613 Ok(input.len())
1614 }
1615
1616 pub fn replay_session(&self, storage_dir: &Path, session_id: &str) -> Result<(), String> {
1617 self.replay_session_inner(storage_dir, session_id, None)
1618 }
1619
1620 pub fn replay_session_for_project(
1621 &self,
1622 storage_dir: &Path,
1623 session_id: &str,
1624 project_root: &Path,
1625 ) -> Result<(), String> {
1626 self.replay_session_inner(storage_dir, session_id, Some(project_root))
1627 }
1628
1629 fn replay_session_inner(
1630 &self,
1631 storage_dir: &Path,
1632 session_id: &str,
1633 project_root: Option<&Path>,
1634 ) -> Result<(), String> {
1635 self.start_watchdog();
1636 if !self.inner.persisted_gc_started.swap(true, Ordering::SeqCst) {
1637 if let Err(error) = self.maybe_gc_persisted(storage_dir) {
1638 crate::slog_warn!("failed to GC persisted background bash tasks: {error}");
1639 }
1640 }
1641
1642 let canonical_project = project_root.map(canonicalized_path);
1643 let tasks = match self.replay_session_from_db(session_id, project_root) {
1655 Some(Ok(tasks)) if !tasks.is_empty() => tasks,
1656 Some(Ok(_)) => {
1657 let disk_tasks = self.replay_session_from_disk(storage_dir, session_id)?;
1658 if !disk_tasks.is_empty() {
1659 crate::slog_info!(
1660 "bash task replay: 0 in DB for session {}, {} from disk fallback",
1661 session_id,
1662 disk_tasks.len()
1663 );
1664 }
1665 disk_tasks
1666 }
1667 Some(Err(error)) => {
1668 crate::slog_warn!(
1669 "bash task replay DB lookup failed for session {}; falling back to disk: {}",
1670 session_id,
1671 error
1672 );
1673 self.replay_session_from_disk(storage_dir, session_id)?
1674 }
1675 None => {
1676 self.replay_session_from_disk(storage_dir, session_id)?
1678 }
1679 };
1680
1681 for mut metadata in tasks {
1682 if project_root.is_none() && metadata.session_id != session_id {
1683 continue;
1684 }
1685 if let Some(canonical_project) = canonical_project.as_deref() {
1686 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
1687 if metadata_project.as_deref() != Some(canonical_project) {
1688 continue;
1689 }
1690 }
1691
1692 if validate_task_id(&metadata.task_id).is_err() {
1693 crate::slog_warn!(
1694 "ignoring persisted background task with invalid id {:?}",
1695 metadata.task_id
1696 );
1697 continue;
1698 }
1699 if self.task(&metadata.task_id).is_some() {
1703 continue;
1704 }
1705 let session_dir = session_tasks_dir(storage_dir, &metadata.session_id);
1706 let resolved = match resolve_task_layout(&session_dir, &metadata.task_id) {
1707 Ok(task) => task,
1708 Err(error) => {
1709 if Self::persisted_task_process_is_alive(&metadata) {
1710 crate::slog_warn!(
1711 "refusing to quarantine unresolved live background task {}: {error}",
1712 metadata.task_id
1713 );
1714 continue;
1715 }
1716 crate::slog_warn!(
1717 "quarantining unresolved background task {}: {error}",
1718 metadata.task_id
1719 );
1720 let _ = quarantine_task_layout(
1721 storage_dir,
1722 &session_dir,
1723 &metadata.task_id,
1724 "invalid",
1725 );
1726 continue;
1727 }
1728 };
1729 match read_task_at(&resolved) {
1730 Ok(disk)
1731 if disk.task_id == metadata.task_id
1732 && disk.session_id == metadata.session_id => {}
1733 Ok(_) | Err(_) => {
1734 if Self::persisted_task_process_is_alive(&metadata) {
1735 crate::slog_warn!(
1736 "refusing to quarantine mismatched live background task {}",
1737 metadata.task_id
1738 );
1739 continue;
1740 }
1741 let _ = quarantine_task_layout(
1742 storage_dir,
1743 &session_dir,
1744 &metadata.task_id,
1745 "mismatch",
1746 );
1747 continue;
1748 }
1749 }
1750 let paths = resolved.paths;
1751 let replay_task_id = metadata.task_id.clone();
1752 let delivery_session_id = (metadata.session_id != session_id).then_some(session_id);
1753 match metadata.status {
1754 BgTaskStatus::Starting => {
1755 let completion_was_delivered = metadata.completion_delivered;
1756 metadata.mark_terminal(
1757 BgTaskStatus::Failed,
1758 None,
1759 Some("spawn aborted".to_string()),
1760 );
1761 metadata.completion_delivered |= completion_was_delivered;
1762 let _ = self.persist_task(&paths, &metadata);
1763 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1764 self.insert_rehydrated_task(metadata, paths, true, delivery_session_id)?;
1765 }
1766 BgTaskStatus::Running | BgTaskStatus::Killing => {
1767 if metadata.mode == BgMode::Pty {
1768 if let Ok(Some(marker)) = read_exit_marker(&paths) {
1769 let completion_was_delivered = metadata.completion_delivered;
1770 metadata = terminal_metadata_from_marker(metadata, marker, None);
1771 metadata.completion_delivered |= completion_was_delivered;
1772 let _ = self.persist_task(&paths, &metadata);
1773 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1774 self.insert_rehydrated_task(
1775 metadata,
1776 paths,
1777 true,
1778 delivery_session_id,
1779 )?;
1780 } else if metadata.status.is_terminal() {
1781 self.insert_rehydrated_task(
1782 metadata,
1783 paths,
1784 true,
1785 delivery_session_id,
1786 )?;
1787 } else {
1788 let completion_was_delivered = metadata.completion_delivered;
1789 metadata.mark_terminal(
1790 BgTaskStatus::Killed,
1791 None,
1792 Some("pty_lost_on_bridge_restart".to_string()),
1793 );
1794 metadata.completion_delivered |= completion_was_delivered;
1795 let _ = self.persist_task(&paths, &metadata);
1796 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1797 self.insert_rehydrated_task(
1798 metadata,
1799 paths,
1800 true,
1801 delivery_session_id,
1802 )?;
1803 }
1804 } else if let Ok(Some(marker)) = read_exit_marker(&paths) {
1805 let reason = (metadata.status == BgTaskStatus::Killing).then(|| {
1806 "recovered from inconsistent killing state on replay".to_string()
1807 });
1808 if reason.is_some() {
1809 crate::slog_warn!("background task {} had killing state with exit marker; preferring marker",
1810 metadata.task_id);
1811 }
1812 let completion_was_delivered = metadata.completion_delivered;
1813 metadata = terminal_metadata_from_marker(metadata, marker, reason);
1814 metadata.completion_delivered |= completion_was_delivered;
1815 let _ = self.persist_task(&paths, &metadata);
1816 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1817 self.insert_rehydrated_task(metadata, paths, true, delivery_session_id)?;
1818 } else if metadata.status == BgTaskStatus::Killing {
1819 let _ = write_kill_marker_if_absent(&paths);
1820 let completion_was_delivered = metadata.completion_delivered;
1821 metadata.mark_terminal(
1822 BgTaskStatus::Killed,
1823 None,
1824 Some("recovered from inconsistent killing state on replay".to_string()),
1825 );
1826 metadata.completion_delivered |= completion_was_delivered;
1827 let _ = self.persist_task(&paths, &metadata);
1828 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1829 self.insert_rehydrated_task(metadata, paths, true, delivery_session_id)?;
1830 } else if Self::persisted_task_process_is_alive(&metadata) {
1831 self.insert_rehydrated_task(metadata, paths, true, delivery_session_id)?;
1832 } else {
1833 let completion_was_delivered = metadata.completion_delivered;
1834 metadata.mark_terminal(
1835 BgTaskStatus::FateUnknown,
1836 None,
1837 Some(restart_fate_unknown_reason(&metadata, &paths)),
1838 );
1839 metadata.completion_delivered |= completion_was_delivered;
1840 let _ = self.persist_task(&paths, &metadata);
1841 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1842 self.insert_rehydrated_task(metadata, paths, true, delivery_session_id)?;
1843 }
1844 }
1845 _ if metadata.status.is_terminal() => {
1846 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1852 self.insert_rehydrated_task(metadata, paths, true, delivery_session_id)?;
1853 }
1854 _ => {}
1855 }
1856 self.retarget_pending_completion(&replay_task_id, session_id);
1857 }
1858
1859 Ok(())
1860 }
1861
1862 fn replay_session_from_db(
1863 &self,
1864 session_id: &str,
1865 project_root: Option<&Path>,
1866 ) -> Option<Result<Vec<PersistedTask>, String>> {
1867 let pool = self
1868 .inner
1869 .db_pool
1870 .read()
1871 .ok()
1872 .and_then(|slot| slot.clone())?;
1873 let harness = self
1874 .inner
1875 .db_harness
1876 .read()
1877 .ok()
1878 .and_then(|slot| slot.clone())?;
1879 let conn = match pool.lock() {
1880 Ok(conn) => conn,
1881 Err(_) => return Some(Err("db mutex poisoned".to_string())),
1882 };
1883 let rows = if let Some(project_root) = project_root {
1884 let project_key = crate::path_identity::project_scope_key(project_root);
1885 crate::db::bash_tasks::list_replayable_bash_tasks_for_project(
1886 &conn,
1887 &harness,
1888 &project_key,
1889 )
1890 } else {
1891 crate::db::bash_tasks::list_bash_tasks_for_session(&conn, &harness, session_id)
1892 };
1893 Some(
1894 rows.map(|rows| rows.into_iter().map(PersistedTask::from).collect())
1895 .map_err(|error| error.to_string()),
1896 )
1897 }
1898
1899 fn replay_session_from_disk(
1900 &self,
1901 storage_dir: &Path,
1902 session_id: &str,
1903 ) -> Result<Vec<PersistedTask>, String> {
1904 let dir = session_tasks_dir(storage_dir, session_id);
1905 if !dir.exists() {
1906 return Ok(Vec::new());
1907 }
1908
1909 let (task_ids, invalid_entries) = discover_task_ids(&dir)
1910 .map_err(|error| format!("failed to discover background task layouts: {error}"))?;
1911 for entry in invalid_entries {
1912 if let Err(error) = quarantine_invalid_entry(storage_dir, &dir, &entry) {
1913 crate::slog_warn!(
1914 "failed to quarantine invalid background task entry {:?}: {error}",
1915 entry
1916 );
1917 }
1918 }
1919
1920 let mut tasks = Vec::new();
1921 for task_id in task_ids {
1922 let task = match resolve_task_layout(&dir, &task_id) {
1923 Ok(task) => task,
1924 Err(error)
1925 if error.kind() == std::io::ErrorKind::NotFound
1926 && uninitialized_layout_is_recent(
1927 &dir,
1928 &task_id,
1929 Duration::from_secs(5 * 60),
1930 )
1931 .unwrap_or(false) =>
1932 {
1933 continue;
1934 }
1935 Err(error) => {
1936 if self.db_has_live_process_for_task(&task_id) {
1937 crate::slog_warn!(
1938 "refusing to quarantine unresolved live background task {task_id} during replay: {error}"
1939 );
1940 continue;
1941 }
1942 crate::slog_warn!(
1943 "quarantining unresolved background task {task_id} during replay: {error}"
1944 );
1945 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
1946 continue;
1947 }
1948 };
1949 match read_task_at(&task) {
1950 Ok(metadata) if metadata.session_id == session_id => tasks.push(metadata),
1951 Ok(_) => {
1952 crate::slog_warn!(
1953 "quarantining background task {task_id} with mismatched session metadata"
1954 );
1955 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "mismatch");
1956 }
1957 Err(error) => {
1958 if self.db_has_live_process_for_task(&task_id) {
1959 crate::slog_warn!(
1960 "refusing to quarantine unreadable live background task {task_id} during replay: {error}"
1961 );
1962 continue;
1963 }
1964 crate::slog_warn!(
1965 "quarantining invalid background task metadata {task_id} during replay: {error}"
1966 );
1967 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
1968 }
1969 }
1970 }
1971 Ok(tasks)
1972 }
1973
1974 pub fn register_watch(
1975 &self,
1976 task_id: String,
1977 pattern: WatchPattern,
1978 once: bool,
1979 ) -> Result<String, &'static str> {
1980 let task = self.task(&task_id).ok_or("task_not_found")?;
1981 validate_task_id(&task_id).map_err(|_| "invalid_task_id")?;
1982 let (mode, terminal_at_registration) = task
1983 .state
1984 .lock()
1985 .map(|state| {
1986 (
1987 state.metadata.mode.clone(),
1988 state.metadata.status.is_terminal(),
1989 )
1990 })
1991 .map_err(|_| "background_task_lock_poisoned")?;
1992 let mut stdout = (mode == BgMode::Pipes)
1993 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
1994 .transpose()
1995 .map_err(|_| "artifact_refused")?;
1996 let mut stderr = (mode == BgMode::Pipes)
1997 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
1998 .transpose()
1999 .map_err(|_| "artifact_refused")?;
2000 let mut pty = (mode == BgMode::Pty)
2001 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
2002 .transpose()
2003 .map_err(|_| "artifact_refused")?;
2004
2005 let mut terminal_matches = Vec::new();
2006 let scanned_terminal = terminal_at_registration;
2007 let watch_id = {
2008 let mut registry = self
2009 .inner
2010 .watch_registry
2011 .lock()
2012 .map_err(|_| "watch_registry_poisoned")?;
2013 let watch_id = registry.register(task_id.clone(), pattern.clone(), once)?;
2014 match &mode {
2015 BgMode::Pipes => {
2016 let stdout_key = format!("{task_id}:stdout");
2017 let stderr_key = format!("{task_id}:stderr");
2018 if terminal_at_registration {
2019 registry.set_file_cursor(&stdout_key, 0);
2020 registry.set_file_cursor(&stderr_key, 0);
2021 terminal_matches.extend(registry.scan_file_new_bytes(
2022 &stdout_key,
2023 &task_id,
2024 stdout.as_mut().expect("pipe stdout opened"),
2025 ));
2026 terminal_matches.extend(registry.scan_file_new_bytes(
2027 &stderr_key,
2028 &task_id,
2029 stderr.as_mut().expect("pipe stderr opened"),
2030 ));
2031 } else {
2032 registry.prime_file_cursor(
2033 &stdout_key,
2034 stdout.as_ref().expect("pipe stdout opened"),
2035 );
2036 registry.prime_file_cursor(
2037 &stderr_key,
2038 stderr.as_ref().expect("pipe stderr opened"),
2039 );
2040 }
2041 }
2042 BgMode::Pty => {
2043 let pty_key = format!("{task_id}:pty");
2044 if terminal_at_registration {
2045 registry.set_file_cursor(&pty_key, 0);
2046 terminal_matches.extend(registry.scan_file_new_bytes(
2047 &pty_key,
2048 &task_id,
2049 pty.as_mut().expect("PTY artifact opened"),
2050 ));
2051 } else {
2052 registry.prime_file_cursor(
2053 &pty_key,
2054 pty.as_ref().expect("PTY artifact opened"),
2055 );
2056 }
2057 }
2058 }
2059 watch_id
2060 };
2061
2062 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
2063 self.persist_watch_registration(
2064 &task.session_id,
2065 &task_id,
2066 &watch_id,
2067 &pattern,
2068 once,
2069 stdout_offset,
2070 stderr_offset,
2071 pty_offset,
2072 );
2073
2074 if task.is_terminal() {
2075 if !scanned_terminal {
2076 terminal_matches = {
2077 let mut registry = self
2078 .inner
2079 .watch_registry
2080 .lock()
2081 .map_err(|_| "watch_registry_poisoned")?;
2082 match &mode {
2083 BgMode::Pipes => {
2084 let stdout_key = format!("{task_id}:stdout");
2085 let stderr_key = format!("{task_id}:stderr");
2086 registry.set_file_cursor(&stdout_key, 0);
2087 registry.set_file_cursor(&stderr_key, 0);
2088 let mut matches = registry.scan_file_new_bytes(
2089 &stdout_key,
2090 &task_id,
2091 stdout.as_mut().expect("pipe stdout opened"),
2092 );
2093 matches.extend(registry.scan_file_new_bytes(
2094 &stderr_key,
2095 &task_id,
2096 stderr.as_mut().expect("pipe stderr opened"),
2097 ));
2098 matches
2099 }
2100 BgMode::Pty => {
2101 let pty_key = format!("{task_id}:pty");
2102 registry.set_file_cursor(&pty_key, 0);
2103 registry.scan_file_new_bytes(
2104 &pty_key,
2105 &task_id,
2106 pty.as_mut().expect("PTY artifact opened"),
2107 )
2108 }
2109 }
2110 };
2111 }
2112
2113 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
2114 let (watch_controlled, watch_matched) = self.task_watch_state(&task_id);
2115 if terminal_matches.is_empty() && (!watch_controlled || watch_matched) {
2116 if watch_matched {
2117 let _ = task.set_completion_delivered(true, self);
2118 self.clear_task_watch_state(&task_id);
2119 self.delete_persisted_watches_for_task(&task.session_id, &task_id);
2121 }
2122 return Ok(watch_id);
2123 }
2124
2125 let completion = self
2126 .remove_pending_completion(&task_id)
2127 .or_else(|| self.completion_snapshot_for_task(&task));
2128 if terminal_matches.is_empty() {
2129 if let Some(completion) = completion.as_ref() {
2130 self.emit_bash_watch_exit(completion);
2131 }
2132 } else {
2133 for pattern_match in &terminal_matches {
2134 self.persist_watch_match(
2135 &task.session_id,
2136 &task_id,
2137 pattern_match,
2138 stdout_offset,
2139 stderr_offset,
2140 pty_offset,
2141 );
2142 self.emit_bash_pattern_match(&task.delivery_session_id, pattern_match.clone());
2143 }
2144 }
2145 let _ = task.set_completion_delivered(true, self);
2146 self.clear_task_watch_state(&task_id);
2147 self.delete_persisted_watches_for_task(&task.session_id, &task_id);
2149 }
2150
2151 Ok(watch_id)
2152 }
2153
2154 pub fn unregister_watch(&self, task_id: &str, watch_id: &str) {
2155 let session_id = self.task(task_id).map(|task| task.session_id.clone());
2156 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2157 registry.unregister(task_id, watch_id);
2158 }
2159 if let Some(session_id) = session_id {
2160 self.delete_persisted_watch(&session_id, task_id, watch_id);
2161 }
2162 }
2163
2164 pub fn active_watch_count(&self, task_id: &str) -> usize {
2165 self.inner
2166 .watch_registry
2167 .lock()
2168 .map(|registry| registry.active_count(task_id))
2169 .unwrap_or(0)
2170 }
2171
2172 fn task_watch_state(&self, task_id: &str) -> (bool, bool) {
2173 self.inner
2174 .watch_registry
2175 .lock()
2176 .map(|registry| {
2177 (
2178 registry.has_controlled_task(task_id),
2179 registry.has_matched_task(task_id),
2180 )
2181 })
2182 .unwrap_or((false, false))
2183 }
2184
2185 fn task_has_watch_control(&self, task_id: &str) -> bool {
2186 self.inner
2187 .watch_registry
2188 .lock()
2189 .map(|registry| registry.has_controlled_task(task_id))
2190 .unwrap_or(false)
2191 }
2192
2193 fn clear_task_watch_state(&self, task_id: &str) {
2194 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2195 registry.clear_task(task_id);
2196 }
2197 }
2198
2199 pub(crate) fn scan_task_watch_output(&self, task: &Arc<BgTask>) {
2200 let mode = match task.state.lock() {
2201 Ok(state) => state.metadata.mode.clone(),
2202 Err(_) => return,
2203 };
2204 let mut stdout = (mode == BgMode::Pipes)
2205 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
2206 .transpose()
2207 .ok()
2208 .flatten();
2209 let mut stderr = (mode == BgMode::Pipes)
2210 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
2211 .transpose()
2212 .ok()
2213 .flatten();
2214 let mut pty = (mode == BgMode::Pty)
2215 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
2216 .transpose()
2217 .ok()
2218 .flatten();
2219 let mut matches = Vec::new();
2220 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2221 match mode {
2222 BgMode::Pipes => {
2223 let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) else {
2224 return;
2225 };
2226 let stdout_key = format!("{}:stdout", task.task_id);
2227 let stderr_key = format!("{}:stderr", task.task_id);
2228 matches.extend(registry.scan_file_new_bytes(
2229 &stdout_key,
2230 &task.task_id,
2231 stdout,
2232 ));
2233 matches.extend(registry.scan_file_new_bytes(
2234 &stderr_key,
2235 &task.task_id,
2236 stderr,
2237 ));
2238 }
2239 BgMode::Pty => {
2240 let Some(pty) = pty.as_mut() else {
2241 return;
2242 };
2243 let pty_key = format!("{}:pty", task.task_id);
2244 matches.extend(registry.scan_file_new_bytes(&pty_key, &task.task_id, pty));
2245 }
2246 }
2247 }
2248 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
2249 if matches.is_empty() {
2250 if self.task_has_watch_control(&task.task_id) {
2253 self.persist_task_watch_cursors(
2254 &task.session_id,
2255 &task.task_id,
2256 stdout_offset,
2257 stderr_offset,
2258 pty_offset,
2259 );
2260 }
2261 return;
2262 }
2263 for pattern_match in matches {
2264 self.persist_watch_match(
2265 &task.session_id,
2266 &task.task_id,
2267 &pattern_match,
2268 stdout_offset,
2269 stderr_offset,
2270 pty_offset,
2271 );
2272 self.emit_bash_pattern_match(&task.delivery_session_id, pattern_match);
2273 }
2274 self.persist_task_watch_cursors(
2275 &task.session_id,
2276 &task.task_id,
2277 stdout_offset,
2278 stderr_offset,
2279 pty_offset,
2280 );
2281 }
2282
2283 pub fn status(
2284 &self,
2285 task_id: &str,
2286 session_id: &str,
2287 project_root: Option<&Path>,
2288 storage_dir: Option<&Path>,
2289 preview_bytes: usize,
2290 ) -> Option<BgTaskSnapshot> {
2291 validate_task_id(task_id).ok()?;
2292 let mut task = self.task_for_session(task_id, session_id);
2293 if task.is_none() {
2294 if let Some(storage_dir) = storage_dir {
2295 let _ = if let Some(project_root) = project_root {
2296 self.replay_session_for_project(storage_dir, session_id, project_root)
2297 } else {
2298 self.replay_session(storage_dir, session_id)
2299 };
2300 task = self.task_for_session(task_id, session_id);
2301 }
2302 }
2303 let Some(task) = task else {
2304 return self.status_relaxed(
2305 task_id,
2306 session_id,
2307 project_root?,
2308 storage_dir?,
2309 preview_bytes,
2310 );
2311 };
2312 let _ = self.poll_task(&task);
2313 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
2314 }
2315
2316 fn status_relaxed_task(
2317 &self,
2318 task_id: &str,
2319 project_root: &Path,
2320 storage_dir: &Path,
2321 ) -> Option<Arc<BgTask>> {
2322 validate_task_id(task_id).ok()?;
2323 let canonical_project = canonicalized_path(project_root);
2324 match self.lookup_relaxed_task_from_db(task_id, project_root) {
2325 Some(Ok(Some(metadata))) => {
2326 if let Some(task) = self.task(task_id) {
2327 let matches_project = task
2328 .state
2329 .lock()
2330 .map(|state| {
2331 state
2332 .metadata
2333 .project_root
2334 .as_deref()
2335 .map(canonicalized_path)
2336 .as_deref()
2337 == Some(canonical_project.as_path())
2338 })
2339 .unwrap_or(false);
2340 return matches_project.then_some(task);
2341 }
2342 let resolved = resolve_task_layout(
2343 &session_tasks_dir(storage_dir, &metadata.session_id),
2344 &metadata.task_id,
2345 )
2346 .ok()?;
2347 let disk = read_task_at(&resolved).ok()?;
2348 if disk.task_id != metadata.task_id || disk.session_id != metadata.session_id {
2349 return None;
2350 }
2351 if self
2352 .insert_rehydrated_task(metadata, resolved.paths, true, None)
2353 .is_err()
2354 {
2355 return None;
2356 }
2357 return self.task(task_id);
2358 }
2359 Some(Ok(None)) => {
2360 crate::slog_info!(
2361 "bash task relaxed DB miss for {}; falling back to disk",
2362 task_id
2363 );
2364 }
2365 Some(Err(error)) => {
2366 crate::slog_warn!(
2367 "bash task relaxed DB lookup failed for {}; falling back to disk: {}",
2368 task_id,
2369 error
2370 );
2371 }
2372 None => {
2373 crate::slog_info!(
2374 "bash task relaxed DB unavailable for {}; falling back to disk",
2375 task_id
2376 );
2377 }
2378 }
2379 let root = storage_dir.join("bash-tasks");
2380 let entries = fs::read_dir(&root).ok()?;
2381 for entry in entries.flatten() {
2382 let dir = entry.path();
2383 if !dir.is_dir() {
2384 continue;
2385 }
2386 let resolved = match resolve_task_layout(&dir, task_id) {
2387 Ok(task) => task,
2388 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
2389 Err(error) => {
2390 if self.db_has_live_process_for_task(task_id) {
2391 crate::slog_warn!(
2392 "refusing to quarantine unresolved live background task {task_id} during relaxed lookup: {error}"
2393 );
2394 continue;
2395 }
2396 crate::slog_warn!(
2397 "quarantining unresolved background task {task_id} during relaxed lookup: {error}"
2398 );
2399 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
2400 continue;
2401 }
2402 };
2403 let metadata = match read_task_at(&resolved) {
2404 Ok(metadata) => metadata,
2405 Err(error) => {
2406 if self.db_has_live_process_for_task(task_id) {
2407 crate::slog_warn!(
2408 "refusing to quarantine unreadable live background task {task_id} during relaxed lookup: {error}"
2409 );
2410 continue;
2411 }
2412 crate::slog_warn!(
2413 "quarantining invalid background task metadata {task_id} during relaxed lookup: {error}"
2414 );
2415 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
2416 continue;
2417 }
2418 };
2419 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
2420 if metadata_project.as_deref() != Some(canonical_project.as_path()) {
2421 continue;
2422 }
2423 if let Some(task) = self.task(task_id) {
2424 let matches_project = task
2425 .state
2426 .lock()
2427 .map(|state| {
2428 state
2429 .metadata
2430 .project_root
2431 .as_deref()
2432 .map(canonicalized_path)
2433 .as_deref()
2434 == Some(canonical_project.as_path())
2435 })
2436 .unwrap_or(false);
2437 return matches_project.then_some(task);
2438 }
2439 if self
2440 .insert_rehydrated_task(metadata, resolved.paths, true, None)
2441 .is_err()
2442 {
2443 return None;
2444 }
2445 return self.task(task_id);
2446 }
2447 None
2448 }
2449
2450 fn lookup_relaxed_task_from_db(
2451 &self,
2452 task_id: &str,
2453 project_root: &Path,
2454 ) -> Option<Result<Option<PersistedTask>, String>> {
2455 let pool = self
2456 .inner
2457 .db_pool
2458 .read()
2459 .ok()
2460 .and_then(|slot| slot.clone())?;
2461 let harness = self
2462 .inner
2463 .db_harness
2464 .read()
2465 .ok()
2466 .and_then(|slot| slot.clone())?;
2467 let conn = match pool.lock() {
2468 Ok(conn) => conn,
2469 Err(_) => return Some(Err("db mutex poisoned".to_string())),
2470 };
2471 let project_key = crate::path_identity::project_scope_key(project_root);
2472 Some(
2473 crate::db::bash_tasks::find_bash_task_for_project(
2474 &conn,
2475 &harness,
2476 &project_key,
2477 task_id,
2478 )
2479 .map(|row| row.map(PersistedTask::from))
2480 .map_err(|error| error.to_string()),
2481 )
2482 }
2483
2484 pub(super) fn status_relaxed(
2485 &self,
2486 task_id: &str,
2487 _session_id: &str,
2488 project_root: &Path,
2489 storage_dir: &Path,
2490 preview_bytes: usize,
2491 ) -> Option<BgTaskSnapshot> {
2492 let task = self.status_relaxed_task(task_id, project_root, storage_dir)?;
2493 let _ = self.poll_task(&task);
2494 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
2495 }
2496
2497 pub fn kill_relaxed(
2498 &self,
2499 task_id: &str,
2500 project_root: &Path,
2501 storage_dir: &Path,
2502 ) -> Result<BgTaskSnapshot, String> {
2503 let task = self
2504 .status_relaxed_task(task_id, project_root, storage_dir)
2505 .ok_or_else(|| format!("background task not found: {task_id}"))?;
2506 self.kill_with_status(task_id, &task.session_id, BgTaskStatus::Killed)
2507 }
2508
2509 pub fn maybe_gc_persisted(&self, storage_dir: &Path) -> Result<usize, String> {
2510 #[cfg(test)]
2511 self.inner.persisted_gc_runs.fetch_add(1, Ordering::SeqCst);
2512
2513 let mut deleted = 0usize;
2514
2515 let root = storage_dir.join("bash-tasks");
2516 if root.exists() {
2517 let session_dirs = fs::read_dir(&root).map_err(|e| {
2518 format!(
2519 "failed to read background task root {}: {e}",
2520 root.display()
2521 )
2522 })?;
2523 for session_entry in session_dirs.flatten() {
2524 let session_dir = session_entry.path();
2525 if !session_dir.is_dir() {
2526 continue;
2527 }
2528 let (task_ids, invalid_entries) = match discover_task_ids(&session_dir) {
2529 Ok(discovery) => discovery,
2530 Err(error) => {
2531 crate::slog_warn!(
2532 "failed to discover background task session {}: {error}",
2533 session_dir.display()
2534 );
2535 continue;
2536 }
2537 };
2538 for entry in invalid_entries {
2539 let _ = quarantine_invalid_entry(storage_dir, &session_dir, &entry);
2540 }
2541 for task_id in task_ids {
2542 let resolved = match resolve_task_layout(&session_dir, &task_id) {
2543 Ok(task) => task,
2544 Err(error)
2545 if error.kind() == std::io::ErrorKind::NotFound
2546 && uninitialized_layout_is_recent(
2547 &session_dir,
2548 &task_id,
2549 Duration::from_secs(5 * 60),
2550 )
2551 .unwrap_or(false) =>
2552 {
2553 continue;
2554 }
2555 Err(error) => {
2556 if self.db_has_live_process_for_task(&task_id) {
2557 crate::slog_warn!(
2558 "refusing to quarantine unresolved live background task {task_id} during GC: {error}"
2559 );
2560 continue;
2561 }
2562 crate::slog_warn!(
2563 "quarantining unresolved background task {task_id}: {error}"
2564 );
2565 quarantine_task_layout(storage_dir, &session_dir, &task_id, "invalid")
2566 .map_err(|error| error.to_string())?;
2567 continue;
2568 }
2569 };
2570 if modified_within(&resolved.paths.json, PERSISTED_GC_GRACE) {
2571 continue;
2572 }
2573 let metadata = match read_task_at(&resolved) {
2574 Ok(metadata) => metadata,
2575 Err(error) => {
2576 if self.db_has_live_process_for_task(&task_id) {
2577 crate::slog_warn!(
2578 "refusing to quarantine unreadable live background task {task_id} during GC: {error}"
2579 );
2580 continue;
2581 }
2582 crate::slog_warn!(
2583 "quarantining corrupt background task metadata {task_id}: {error}"
2584 );
2585 quarantine_task_layout(storage_dir, &session_dir, &task_id, "corrupt")
2586 .map_err(|error| error.to_string())?;
2587 continue;
2588 }
2589 };
2590 if !(metadata.status.is_terminal() && metadata.completion_delivered) {
2591 continue;
2592 }
2593 if Self::persisted_task_process_is_alive(&metadata)
2594 || self.db_has_live_process_for_task(&task_id)
2595 {
2596 crate::slog_warn!(
2597 "refusing to delete terminal background task bundle {task_id}: recorded process is still alive"
2598 );
2599 continue;
2600 }
2601 match delete_task_bundle(&resolved.paths) {
2602 Ok(()) => {
2603 self.delete_gc_task_from_db(&metadata);
2604 deleted += 1;
2605 log::debug!(
2606 "deleted persisted background task bundle {}",
2607 metadata.task_id
2608 );
2609 }
2610 Err(error) => {
2611 crate::slog_warn!(
2612 "failed to delete background task bundle {}: {error}",
2613 metadata.task_id
2614 );
2615 }
2616 }
2617 }
2618 }
2619 }
2620 gc_quarantine(storage_dir);
2621 Ok(deleted)
2622 }
2623
2624 pub fn list(&self, preview_bytes: usize) -> Vec<BgTaskSnapshot> {
2625 let tasks = self
2626 .inner
2627 .tasks
2628 .lock()
2629 .map(|tasks| tasks.values().cloned().collect::<Vec<_>>())
2630 .unwrap_or_default();
2631 tasks
2632 .into_iter()
2633 .map(|task| {
2634 let _ = self.poll_task(&task);
2635 self.snapshot_with_terminal_cache(&task, preview_bytes)
2636 })
2637 .collect()
2638 }
2639
2640 fn maybe_compress_snapshot(&self, task: &Arc<BgTask>, snapshot: &mut BgTaskSnapshot) {
2646 if !snapshot.info.status.is_terminal() || snapshot.info.mode == BgMode::Pty {
2647 return;
2648 }
2649 if let Some(cache) = self.ensure_terminal_output_cache(task) {
2650 snapshot.output_preview = cache.output_preview;
2651 snapshot.output_truncated = cache.output_truncated;
2652 }
2653 }
2654
2655 pub fn kill(&self, task_id: &str, session_id: &str) -> Result<BgTaskSnapshot, String> {
2656 self.kill_with_status(task_id, session_id, BgTaskStatus::Killed)
2657 }
2658
2659 pub fn kill_running_tasks_for_root(&self, project_root: &Path) -> usize {
2669 let canonical_root = canonicalized_path(project_root);
2670 let targets = self
2671 .inner
2672 .tasks
2673 .lock()
2674 .map(|tasks| {
2675 tasks
2676 .values()
2677 .filter_map(|task| {
2678 let state = task.state.lock().ok()?;
2679 let status = &state.metadata.status;
2680 let running = matches!(status, BgTaskStatus::Running)
2681 || (state.metadata.mode == BgMode::Pty
2682 && matches!(status, BgTaskStatus::Killing));
2683 if !running {
2684 return None;
2685 }
2686 let task_root = state
2687 .metadata
2688 .project_root
2689 .as_deref()
2690 .unwrap_or(&state.metadata.workdir);
2691 (canonicalized_path(task_root) == canonical_root)
2692 .then(|| (task.task_id.clone(), task.session_id.clone()))
2693 })
2694 .collect::<Vec<_>>()
2695 })
2696 .unwrap_or_default();
2697
2698 let mut killed = 0;
2699 for (task_id, session_id) in targets {
2700 match self.kill_with_status_reason(
2701 &task_id,
2702 &session_id,
2703 BgTaskStatus::Killed,
2704 Some(ROOT_RECLAIMED_REASON.to_string()),
2705 ) {
2706 Ok(_) => killed += 1,
2707 Err(error) => crate::slog_warn!(
2708 "failed to terminate background task {task_id} for reclaimed root {}: {error}",
2709 project_root.display()
2710 ),
2711 }
2712 }
2713 killed
2714 }
2715
2716 pub fn promote(&self, task_id: &str, session_id: &str) -> Result<bool, String> {
2717 let task = self
2718 .task_for_session(task_id, session_id)
2719 .ok_or_else(|| format!("background task not found: {task_id}"))?;
2720 let terminal_after_promote = {
2721 let mut state = task
2722 .state
2723 .lock()
2724 .map_err(|_| "background task lock poisoned".to_string())?;
2725 let updated = self
2726 .update_task_metadata(&task.paths, |metadata| {
2727 metadata.notify_on_completion = true;
2728 metadata.completion_delivered = false;
2729 })
2730 .map_err(|e| format!("failed to promote background task: {e}"))?;
2731 state.metadata = updated;
2732 state.metadata.status.is_terminal()
2733 };
2734 if terminal_after_promote {
2735 self.post_terminal_transition(&task, true)?;
2736 }
2737 Ok(true)
2738 }
2739
2740 pub(crate) fn kill_for_timeout(&self, task_id: &str, session_id: &str) -> Result<(), String> {
2741 self.kill_with_status(task_id, session_id, BgTaskStatus::TimedOut)
2742 .map(|_| ())
2743 }
2744
2745 pub fn cleanup_finished(&self, older_than: Duration) {
2746 let cutoff = Instant::now().checked_sub(older_than);
2747 let removable_paths: Vec<(String, TaskPaths)> =
2748 if let Ok(mut tasks) = self.inner.tasks.lock() {
2749 let removable = tasks
2750 .iter()
2751 .filter_map(|(task_id, task)| {
2752 let delivered_terminal = task
2753 .state
2754 .lock()
2755 .map(|state| {
2756 state.metadata.status.is_terminal()
2757 && state.metadata.completion_delivered
2758 })
2759 .unwrap_or(false);
2760 if !delivered_terminal {
2761 return None;
2762 }
2763
2764 let terminal_at = task.terminal_at.lock().ok().and_then(|at| *at);
2765 let expired = match (terminal_at, cutoff) {
2766 (Some(terminal_at), Some(cutoff)) => terminal_at <= cutoff,
2767 (Some(_), None) => true,
2768 (None, _) => false,
2769 };
2770 expired.then(|| task_id.clone())
2771 })
2772 .collect::<Vec<_>>();
2773
2774 removable
2775 .into_iter()
2776 .filter_map(|task_id| {
2777 tasks
2778 .remove(&task_id)
2779 .map(|task| (task_id, task.paths.clone()))
2780 })
2781 .collect()
2782 } else {
2783 Vec::new()
2784 };
2785
2786 for (task_id, paths) in removable_paths {
2787 match delete_task_bundle(&paths) {
2788 Ok(()) => log::debug!("deleted persisted background task bundle {task_id}"),
2789 Err(error) => crate::slog_warn!(
2790 "failed to delete persisted background task bundle {task_id}: {error}"
2791 ),
2792 }
2793 }
2794 }
2795
2796 pub fn drain_completions(&self) -> Vec<BgCompletion> {
2797 self.drain_completions_for_session(None)
2798 }
2799
2800 pub fn drain_completions_for_session(&self, session_id: Option<&str>) -> Vec<BgCompletion> {
2801 let completions = match self.inner.completions.lock() {
2802 Ok(completions) => completions,
2803 Err(_) => return Vec::new(),
2804 };
2805
2806 completions
2807 .iter()
2808 .filter(|completion| completion_matches_session(completion, session_id))
2809 .cloned()
2810 .collect()
2811 }
2812
2813 pub fn has_completions_for_session(&self, session_id: Option<&str>) -> bool {
2814 match self.inner.completions.lock() {
2815 Ok(completions) => completions
2816 .iter()
2817 .any(|completion| completion_matches_session(completion, session_id)),
2818 Err(_) => true,
2822 }
2823 }
2824
2825 pub fn ack_completions_for_session(
2826 &self,
2827 session_id: Option<&str>,
2828 task_ids: &[String],
2829 ) -> Vec<String> {
2830 if task_ids.is_empty() {
2831 return Vec::new();
2832 }
2833 let requested_task_ids = task_ids.iter().map(String::as_str).collect::<HashSet<_>>();
2834 let mut completion_sessions = HashMap::new();
2835 if let Ok(mut completions) = self.inner.completions.lock() {
2836 completions.retain(|completion| {
2837 let session_matches = session_id
2838 .map(|session_id| completion.session_id == session_id)
2839 .unwrap_or(true);
2840 if session_matches && requested_task_ids.contains(completion.task_id.as_str()) {
2841 completion_sessions
2842 .insert(completion.task_id.clone(), completion.session_id.clone());
2843 false
2844 } else {
2845 true
2846 }
2847 });
2848 }
2849
2850 let mut delivered = Vec::new();
2851 for task_id in task_ids {
2852 let task = if let Some(session_id) = session_id {
2853 self.task_for_session(task_id, session_id)
2854 .or_else(|| {
2855 self.task(task_id)
2856 .filter(|task| task.delivery_session_id == session_id)
2857 })
2858 .or_else(|| {
2859 completion_sessions
2860 .contains_key(task_id)
2861 .then(|| self.task(task_id))
2862 .flatten()
2863 })
2864 } else if let Some(completion_session_id) = completion_sessions.get(task_id) {
2865 self.task_for_session(task_id, completion_session_id)
2866 .or_else(|| self.task(task_id))
2867 } else {
2868 self.task(task_id)
2869 };
2870 if let Some(task) = task {
2871 let terminal = task
2872 .state
2873 .lock()
2874 .map(|state| state.metadata.status.is_terminal())
2875 .unwrap_or(false);
2876 self.ack_persisted_watches_for_task(&task.session_id, task_id, terminal);
2880 if terminal {
2881 self.clear_task_watch_state(task_id);
2882 if task.set_completion_delivered(true, self).is_ok() {
2883 delivered.push(task_id.clone());
2884 }
2885 } else {
2886 self.sync_memory_watches_after_ack(task_id);
2889 delivered.push(task_id.clone());
2890 }
2891 } else if let Some(session_id) = session_id {
2892 self.ack_persisted_watches_for_task(session_id, task_id, true);
2894 delivered.push(task_id.clone());
2895 }
2896 }
2897
2898 delivered
2899 }
2900
2901 fn sync_memory_watches_after_ack(&self, task_id: &str) {
2902 let Some((harness, pool)) = self.db_harness_and_pool() else {
2903 return;
2904 };
2905 let session_id = match self.task(task_id) {
2906 Some(task) => task.session_id.clone(),
2907 None => return,
2908 };
2909 let Ok(conn) = pool.lock() else {
2910 return;
2911 };
2912 let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
2913 &conn,
2914 &harness,
2915 &session_id,
2916 task_id,
2917 ) else {
2918 return;
2919 };
2920 let remaining: HashSet<String> = rows.into_iter().map(|row| row.watch_id).collect();
2921 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2922 registry.retain_watch_ids(task_id, &remaining);
2923 }
2924 }
2925
2926 pub fn pending_completions_for_session(&self, session_id: &str) -> Vec<BgCompletion> {
2927 self.inner
2928 .completions
2929 .lock()
2930 .map(|completions| {
2931 completions
2932 .iter()
2933 .filter(|completion| completion.session_id == session_id)
2934 .cloned()
2935 .collect()
2936 })
2937 .unwrap_or_default()
2938 }
2939
2940 fn remove_pending_completion(&self, task_id: &str) -> Option<BgCompletion> {
2941 let mut completions = self.inner.completions.lock().ok()?;
2942 let idx = completions
2943 .iter()
2944 .position(|completion| completion.task_id == task_id)?;
2945 completions.remove(idx)
2946 }
2947
2948 fn retarget_pending_completion(&self, task_id: &str, session_id: &str) {
2949 if let Ok(mut completions) = self.inner.completions.lock() {
2950 if let Some(completion) = completions
2951 .iter_mut()
2952 .find(|completion| completion.task_id == task_id)
2953 {
2954 completion.session_id = session_id.to_string();
2955 }
2956 }
2957 }
2958
2959 fn completion_snapshot_for_task(&self, task: &Arc<BgTask>) -> Option<BgCompletion> {
2960 let snapshot = self.snapshot_with_terminal_cache(task, RUNNING_OUTPUT_PREVIEW_BYTES);
2961 if !snapshot.info.status.is_terminal() {
2962 return None;
2963 }
2964 let (output_preview, output_truncated) = if snapshot.info.mode == BgMode::Pty {
2965 (String::new(), false)
2966 } else {
2967 self.ensure_terminal_output_cache(task)
2968 .map(|cache| completion_preview_for_cache(&cache, snapshot.exit_code))
2969 .unwrap_or_else(|| (String::new(), false))
2970 };
2971 Some(BgCompletion {
2972 task_id: snapshot.info.task_id,
2973 session_id: task.delivery_session_id.clone(),
2974 status: snapshot.info.status,
2975 exit_code: snapshot.exit_code,
2976 command: snapshot.info.command,
2977 output_preview,
2978 output_truncated,
2979 original_tokens: None,
2980 compressed_tokens: None,
2981 tokens_skipped: false,
2982 status_reason: snapshot.info.status_reason,
2983 })
2984 }
2985
2986 pub fn detach(&self) {
2987 self.inner.shutdown.store(true, Ordering::SeqCst);
2988 if let Ok(mut tasks) = self.inner.tasks.lock() {
2989 for task in tasks.values() {
2990 if let Ok(mut state) = task.state.lock() {
2991 match &mut state.runtime {
2992 TaskRuntime::Piped(child) => *child = None,
2993 TaskRuntime::Pty(runtime) => *runtime = None,
2994 }
2995 state.detached = true;
2996 }
2997 }
2998 tasks.clear();
2999 }
3000 }
3001
3002 pub fn shutdown(&self) {
3003 let tasks = self
3004 .inner
3005 .tasks
3006 .lock()
3007 .map(|tasks| {
3008 tasks
3009 .values()
3010 .map(|task| (task.task_id.clone(), task.session_id.clone()))
3011 .collect::<Vec<_>>()
3012 })
3013 .unwrap_or_default();
3014 for (task_id, session_id) in tasks {
3015 let _ = self.kill(&task_id, &session_id);
3016 }
3017 }
3018
3019 pub(crate) fn poll_task(&self, task: &Arc<BgTask>) -> Result<(), String> {
3020 if let Ok(state) = task.state.lock() {
3021 if let TaskRuntime::Pty(Some(pty)) = &state.runtime {
3022 if !pty.exit_observed.load(Ordering::SeqCst) {
3030 return Ok(());
3031 }
3032 }
3033 }
3034 let marker = match read_exit_marker(&task.paths) {
3035 Ok(Some(marker)) => marker,
3036 Ok(None) => return Ok(()),
3037 Err(error) => return Err(format!("failed to read exit marker: {error}")),
3038 };
3039 self.finalize_from_marker(task, marker, None)
3040 }
3041
3042 pub(crate) fn reap_child(&self, task: &Arc<BgTask>) {
3043 let mut needs_completion = false;
3044 {
3045 let Ok(mut state) = task.state.lock() else {
3046 return;
3047 };
3048 match &mut state.runtime {
3049 TaskRuntime::Piped(child_slot) => {
3050 if let Some(child) = child_slot.as_mut() {
3051 if let Ok(Some(status)) = child.try_wait() {
3052 *child_slot = None;
3053 state.detached = true;
3054 state.child_exit_observed = true;
3055 if let Some(handles) = state.io_handles.as_mut() {
3056 if handles.artifact_len(TaskArtifact::Exit).unwrap_or(1) == 0 {
3057 let marker = status
3058 .code()
3059 .map(|code| code.to_string())
3060 .unwrap_or_else(|| "1".to_string());
3061 let _ = handles.write(TaskArtifact::Exit, marker.as_bytes());
3062 }
3063 }
3064 }
3065 } else if state.detached {
3066 let child_known_dead = state.child_exit_observed
3067 || state
3068 .metadata
3069 .child_pid
3070 .is_some_and(|pid| !is_process_alive(pid));
3071 if child_known_dead {
3072 needs_completion =
3073 self.fail_without_exit_marker_if_needed(task, &mut state);
3074 }
3075 }
3076 }
3077 TaskRuntime::Pty(Some(pty)) => {
3078 if pty.exit_observed.load(Ordering::SeqCst) {
3079 drop(state);
3080 let _ = self.poll_task(task);
3081 return;
3082 }
3083 }
3084 TaskRuntime::Pty(None) => {}
3085 }
3086 }
3087 if needs_completion {
3088 let _ = self.post_terminal_transition(task, true);
3089 }
3090 }
3091
3092 fn fail_without_exit_marker_if_needed(
3093 &self,
3094 task: &Arc<BgTask>,
3095 state: &mut BgTaskState,
3096 ) -> bool {
3097 if state.metadata.status.is_terminal() {
3098 return false;
3099 }
3100 if matches!(read_exit_marker(&task.paths), Ok(Some(_))) {
3101 return false;
3102 }
3103 let watch_controlled = self.task_has_watch_control(&task.task_id);
3104 let child_exit_observed = state.child_exit_observed;
3105 let updated = self.update_task_metadata(&task.paths, |metadata| {
3106 let (status, reason) = if child_exit_observed {
3107 (
3108 BgTaskStatus::Failed,
3109 "process exited without exit marker".to_string(),
3110 )
3111 } else {
3112 (
3113 BgTaskStatus::FateUnknown,
3114 restart_fate_unknown_reason(metadata, &task.paths),
3115 )
3116 };
3117 metadata.mark_terminal(status, None, Some(reason));
3118 if watch_controlled {
3119 metadata.completion_delivered = true;
3120 }
3121 });
3122 if let Ok(metadata) = updated {
3123 state.pending_terminal_override = None;
3124 state.metadata = metadata;
3125 task.mark_terminal_now();
3126 return true;
3127 }
3128 false
3129 }
3130
3131 pub(crate) fn running_tasks(&self) -> Vec<Arc<BgTask>> {
3132 self.inner
3133 .tasks
3134 .lock()
3135 .map(|tasks| {
3136 tasks
3137 .values()
3138 .filter(|task| task.is_running())
3139 .cloned()
3140 .collect()
3141 })
3142 .unwrap_or_default()
3143 }
3144
3145 fn insert_rehydrated_task(
3146 &self,
3147 metadata: PersistedTask,
3148 paths: TaskPaths,
3149 detached: bool,
3150 delivery_session_id: Option<&str>,
3151 ) -> Result<(), String> {
3152 let task_id = metadata.task_id.clone();
3153 let session_id = metadata.session_id.clone();
3154 let started = started_instant_from_unix_millis(metadata.started_at);
3155 let suppress_replayed_running_reminder = metadata.status == BgTaskStatus::Running;
3156 let mode = metadata.mode.clone();
3157 let task = Arc::new(BgTask {
3158 task_id: task_id.clone(),
3159 delivery_session_id: delivery_session_id.unwrap_or(&session_id).to_string(),
3160 session_id,
3161 paths: paths.clone(),
3162 artifact_root: canonical_artifact_root(&paths),
3163 started,
3164 last_reminder_at: Mutex::new(suppress_replayed_running_reminder.then(Instant::now)),
3165 terminal_at: Mutex::new(metadata.status.is_terminal().then(Instant::now)),
3166 state: Mutex::new(BgTaskState {
3167 metadata,
3168 runtime: if mode == BgMode::Pty {
3169 TaskRuntime::Pty(None)
3170 } else {
3171 TaskRuntime::Piped(None)
3172 },
3173 io_handles: None,
3174 detached,
3175 child_exit_observed: false,
3182 buffer: BgBuffer::registered(&paths, mode.clone()),
3183 terminal_output_cache: None,
3184 pending_terminal_override: None,
3185 }),
3186 });
3187 self.inner
3188 .tasks
3189 .lock()
3190 .map_err(|_| "background task registry lock poisoned".to_string())?
3191 .insert(task_id.clone(), Arc::clone(&task));
3192 self.rearm_persisted_watches(&task);
3196 Ok(())
3197 }
3198
3199 fn rearm_persisted_watches(&self, task: &Arc<BgTask>) {
3200 let Some((harness, pool)) = self.db_harness_and_pool() else {
3201 return;
3202 };
3203 let rows = {
3204 let Ok(conn) = pool.lock() else {
3205 return;
3206 };
3207 match crate::db::bash_watches::list_bash_pattern_watches_for_task(
3208 &conn,
3209 &harness,
3210 &task.session_id,
3211 &task.task_id,
3212 ) {
3213 Ok(rows) if !rows.is_empty() => rows,
3214 _ => return,
3215 }
3216 };
3217
3218 let mode = match task.state.lock() {
3219 Ok(state) => state.metadata.mode.clone(),
3220 Err(_) => return,
3221 };
3222 let terminal = task
3223 .state
3224 .lock()
3225 .map(|state| state.metadata.status.is_terminal())
3226 .unwrap_or(false);
3227 let completion_delivered = task
3228 .state
3229 .lock()
3230 .map(|state| state.metadata.completion_delivered)
3231 .unwrap_or(true);
3232
3233 let mut stdout = (mode == BgMode::Pipes)
3234 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
3235 .transpose()
3236 .ok()
3237 .flatten();
3238 let mut stderr = (mode == BgMode::Pipes)
3239 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
3240 .transpose()
3241 .ok()
3242 .flatten();
3243 let mut pty = (mode == BgMode::Pty)
3244 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
3245 .transpose()
3246 .ok()
3247 .flatten();
3248
3249 let mut pending_to_emit = Vec::new();
3250 let mut gap_matches = Vec::new();
3251 {
3252 let Ok(mut registry) = self.inner.watch_registry.lock() else {
3253 return;
3254 };
3255 let stdout_key = format!("{}:stdout", task.task_id);
3256 let stderr_key = format!("{}:stderr", task.task_id);
3257 let pty_key = format!("{}:pty", task.task_id);
3258
3259 let first = &rows[0];
3261 match mode {
3262 BgMode::Pipes => {
3263 registry.set_file_cursor(&stdout_key, first.stdout_offset.max(0) as u64);
3264 registry.set_file_cursor(&stderr_key, first.stderr_offset.max(0) as u64);
3265 }
3266 BgMode::Pty => {
3267 registry.set_file_cursor(&pty_key, first.pty_offset.max(0) as u64);
3268 }
3269 }
3270
3271 for row in &rows {
3272 let Ok(pattern) = WatchPattern::from_persisted(&row.pattern_kind, &row.pattern)
3273 else {
3274 crate::slog_warn!(
3275 "skipping unreadable persisted watch {}/{}",
3276 row.task_id,
3277 row.watch_id
3278 );
3279 continue;
3280 };
3281 if let Err(error) = registry.restore(
3282 row.watch_id.clone(),
3283 row.task_id.clone(),
3284 pattern,
3285 row.once,
3286 row.scanning,
3287 ) {
3288 crate::slog_warn!(
3289 "failed to restore watch {}/{}: {error}",
3290 row.task_id,
3291 row.watch_id
3292 );
3293 continue;
3294 }
3295 if row.pending_match {
3296 if let (Some(match_text), Some(match_offset), Some(context)) = (
3297 row.match_text.clone(),
3298 row.match_offset,
3299 row.match_context.clone(),
3300 ) {
3301 pending_to_emit.push(PatternMatch {
3302 watch_id: row.watch_id.clone(),
3303 task_id: row.task_id.clone(),
3304 match_text,
3305 match_offset: match_offset.max(0) as u64,
3306 context,
3307 once: row.once,
3308 });
3309 }
3310 }
3311 }
3312
3313 let should_gap_scan =
3317 rows.iter().any(|row| row.scanning) && !pending_to_emit.iter().any(|m| m.once);
3318 if should_gap_scan {
3319 match mode {
3320 BgMode::Pipes => {
3321 if let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) {
3322 gap_matches.extend(registry.scan_file_new_bytes(
3323 &stdout_key,
3324 &task.task_id,
3325 stdout,
3326 ));
3327 gap_matches.extend(registry.scan_file_new_bytes(
3328 &stderr_key,
3329 &task.task_id,
3330 stderr,
3331 ));
3332 }
3333 }
3334 BgMode::Pty => {
3335 if let Some(pty) = pty.as_mut() {
3336 gap_matches.extend(registry.scan_file_new_bytes(
3337 &pty_key,
3338 &task.task_id,
3339 pty,
3340 ));
3341 }
3342 }
3343 }
3344 }
3345 }
3346
3347 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
3348 for pattern_match in &gap_matches {
3349 self.persist_watch_match(
3350 &task.session_id,
3351 &task.task_id,
3352 pattern_match,
3353 stdout_offset,
3354 stderr_offset,
3355 pty_offset,
3356 );
3357 }
3358 if !gap_matches.is_empty() || rows.iter().any(|row| row.scanning) {
3359 self.persist_task_watch_cursors(
3360 &task.session_id,
3361 &task.task_id,
3362 stdout_offset,
3363 stderr_offset,
3364 pty_offset,
3365 );
3366 }
3367
3368 let emitted_pending = !pending_to_emit.is_empty();
3370 let to_emit = if emitted_pending {
3371 pending_to_emit
3372 } else {
3373 gap_matches
3374 };
3375 for pattern_match in to_emit {
3376 self.emit_bash_pattern_match(&task.delivery_session_id, pattern_match);
3377 }
3378
3379 if !terminal {
3380 return;
3381 }
3382
3383 let _ = self.remove_pending_completion(&task.task_id);
3386 let (watch_controlled, watch_matched) = self.task_watch_state(&task.task_id);
3387 if !watch_controlled {
3388 return;
3389 }
3390 if watch_matched {
3391 return;
3393 }
3394 if completion_delivered {
3395 self.clear_task_watch_state(&task.task_id);
3397 self.delete_persisted_watches_for_task(&task.session_id, &task.task_id);
3398 return;
3399 }
3400 if let Some(completion) = self.completion_snapshot_for_task(task) {
3401 self.emit_bash_watch_exit(&completion);
3402 }
3403 self.clear_task_watch_state(&task.task_id);
3405 }
3406
3407 fn kill_with_status(
3408 &self,
3409 task_id: &str,
3410 session_id: &str,
3411 terminal_status: BgTaskStatus,
3412 ) -> Result<BgTaskSnapshot, String> {
3413 self.kill_with_status_reason(task_id, session_id, terminal_status, None)
3414 }
3415
3416 fn kill_with_status_reason(
3417 &self,
3418 task_id: &str,
3419 session_id: &str,
3420 terminal_status: BgTaskStatus,
3421 reason: Option<String>,
3422 ) -> Result<BgTaskSnapshot, String> {
3423 let task = self
3424 .task_for_session(task_id, session_id)
3425 .ok_or_else(|| format!("background task not found: {task_id}"))?;
3426 let mut terminalized = false;
3427
3428 {
3429 let mut state = task
3430 .state
3431 .lock()
3432 .map_err(|_| "background task lock poisoned".to_string())?;
3433 if state.metadata.status.is_terminal() {
3434 state.pending_terminal_override = None;
3435 } else if let Ok(Some(marker)) = read_exit_marker(&task.paths) {
3436 state.metadata =
3437 terminal_metadata_from_marker(state.metadata.clone(), marker, reason.clone());
3438 if self.task_has_watch_control(&task.task_id) {
3439 state.metadata.completion_delivered = true;
3440 }
3441 state.pending_terminal_override = None;
3442 task.mark_terminal_now();
3443 match &mut state.runtime {
3444 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
3451 TaskRuntime::Pty(runtime) => *runtime = None,
3452 }
3453 state.detached = true;
3454 self.persist_task(&task.paths, &state.metadata)
3455 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
3456 terminalized = true;
3457 } else {
3458 let was_already_killing = state.metadata.status == BgTaskStatus::Killing;
3459 if !was_already_killing {
3460 state.metadata.status = BgTaskStatus::Killing;
3461 }
3462 if reason.is_some() {
3463 state.metadata.status_reason = reason.clone();
3464 }
3465 if !was_already_killing || reason.is_some() {
3466 self.persist_task(&task.paths, &state.metadata)
3467 .map_err(|e| format!("failed to persist killing state: {e}"))?;
3468 }
3469
3470 #[cfg(unix)]
3471 let pgid = state.metadata.pgid;
3472 #[cfg(windows)]
3473 let child_pid = state.metadata.child_pid;
3474 if !was_already_killing
3475 && state.metadata.mode == BgMode::Pty
3476 && terminal_status == BgTaskStatus::TimedOut
3477 {
3478 state.pending_terminal_override = Some(BgTaskStatus::TimedOut);
3479 }
3480
3481 #[cfg(windows)]
3482 let mut pty_forced_terminal_status: Option<BgTaskStatus> = None;
3483
3484 match &mut state.runtime {
3485 TaskRuntime::Piped(child_slot) => {
3486 #[cfg(unix)]
3487 if let Some(pgid) = pgid {
3488 terminate_pgid(pgid, child_slot.as_mut());
3489 }
3490 #[cfg(windows)]
3491 if let Some(child) = child_slot.as_mut() {
3492 super::process::terminate_process(child);
3493 } else if let Some(pid) = child_pid {
3494 terminate_pid(pid);
3495 }
3496 if let Some(child) = child_slot.as_mut() {
3497 let _ = child.wait();
3498 }
3499 *child_slot = None;
3500 state.detached = true;
3501
3502 if let Some(handles) = state.io_handles.as_mut() {
3503 handles.write(TaskArtifact::Exit, b"killed").map_err(|e| {
3504 format!("failed to write retained kill marker: {e}")
3505 })?;
3506 } else {
3507 write_kill_marker_if_absent(&task.paths)
3508 .map_err(|e| format!("failed to write kill marker: {e}"))?;
3509 }
3510
3511 let exit_code = terminal_exit_code_for_status(&terminal_status);
3512 state
3513 .metadata
3514 .mark_terminal(terminal_status, exit_code, reason.clone());
3515 if self.task_has_watch_control(&task.task_id) {
3516 state.metadata.completion_delivered = true;
3517 }
3518 state.pending_terminal_override = None;
3519 task.mark_terminal_now();
3520 self.persist_task(&task.paths, &state.metadata)
3521 .map_err(|e| format!("failed to persist killed state: {e}"))?;
3522 terminalized = true;
3523 }
3524 TaskRuntime::Pty(Some(pty)) => {
3525 pty.was_killed.store(true, Ordering::SeqCst);
3526 if let Err(error) = pty.killer.kill() {
3527 crate::slog_warn!(
3528 "[pty-kill] {task_id} ChildKiller::kill failed: {error}"
3529 );
3530 }
3531 if let Some(pid) = pty.child_pid {
3532 #[cfg(unix)]
3533 terminate_pgid(pid as i32, None);
3534 #[cfg(windows)]
3535 terminate_pid(pid);
3536 }
3537 drop(pty.master.take());
3538
3539 #[cfg(windows)]
3540 {
3541 let default_status = if terminal_status == BgTaskStatus::TimedOut {
3542 BgTaskStatus::TimedOut
3543 } else {
3544 BgTaskStatus::Killed
3545 };
3546 pty_forced_terminal_status = Some(
3547 state
3548 .pending_terminal_override
3549 .take()
3550 .unwrap_or(default_status),
3551 );
3552 }
3553 }
3554 TaskRuntime::Pty(None) => {}
3555 }
3556
3557 #[cfg(windows)]
3558 if let Some(target_status) = pty_forced_terminal_status {
3559 if !task.paths.exit.exists() {
3560 write_kill_marker_if_absent(&task.paths)
3561 .map_err(|e| format!("failed to write kill marker: {e}"))?;
3562 }
3563
3564 let exit_code = terminal_exit_code_for_status(&target_status);
3565 state
3566 .metadata
3567 .mark_terminal(target_status, exit_code, reason.clone());
3568 if self.task_has_watch_control(&task.task_id) {
3569 state.metadata.completion_delivered = true;
3570 }
3571 state.pending_terminal_override = None;
3572 task.mark_terminal_now();
3573 if let TaskRuntime::Pty(runtime) = &mut state.runtime {
3574 *runtime = None;
3575 }
3576 state.detached = true;
3577 self.persist_task(&task.paths, &state.metadata)
3578 .map_err(|e| format!("failed to persist killed PTY state: {e}"))?;
3579 terminalized = true;
3580 }
3581 }
3582 }
3583
3584 if terminalized {
3585 self.post_terminal_transition(&task, true)?;
3586 }
3587 Ok(self.snapshot_with_terminal_cache(&task, RUNNING_OUTPUT_PREVIEW_BYTES))
3588 }
3589
3590 fn finalize_from_marker(
3591 &self,
3592 task: &Arc<BgTask>,
3593 marker: ExitMarker,
3594 reason: Option<String>,
3595 ) -> Result<(), String> {
3596 let watch_controlled = self.task_has_watch_control(&task.task_id);
3597 let mut pty_reader_done = None;
3598 {
3599 let mut state = task
3600 .state
3601 .lock()
3602 .map_err(|_| "background task lock poisoned".to_string())?;
3603 if state.metadata.status.is_terminal() {
3604 state.pending_terminal_override = None;
3605 return Ok(());
3606 }
3607
3608 let pending_override = state.pending_terminal_override.take();
3609 let is_pty = state.metadata.mode == BgMode::Pty;
3610 let reason = reason.or_else(|| state.metadata.status_reason.clone());
3611 let updated = self
3612 .update_task_metadata(&task.paths, |metadata| {
3613 let mut new_metadata = if is_pty && marker == ExitMarker::Killed {
3614 let mut metadata = metadata.clone();
3615 let target_status = pending_override.unwrap_or(BgTaskStatus::Killed);
3616 let exit_code = terminal_exit_code_for_status(&target_status);
3617 metadata.mark_terminal(target_status, exit_code, reason.clone());
3618 metadata
3619 } else {
3620 terminal_metadata_from_marker(metadata.clone(), marker, reason.clone())
3621 };
3622 if watch_controlled {
3623 new_metadata.completion_delivered = true;
3624 }
3625 *metadata = new_metadata;
3626 })
3627 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
3628 state.metadata = updated;
3629 task.mark_terminal_now();
3630 match &mut state.runtime {
3631 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
3636 TaskRuntime::Pty(runtime) => {
3637 pty_reader_done = runtime
3638 .as_ref()
3639 .map(|runtime| Arc::clone(&runtime.reader_done));
3640 *runtime = None;
3641 }
3642 }
3643 state.detached = true;
3644 }
3645
3646 if let Some(reader_done) = pty_reader_done {
3647 let deadline = Instant::now() + Duration::from_millis(200);
3648 while !reader_done.load(Ordering::SeqCst) && Instant::now() < deadline {
3649 std::thread::sleep(Duration::from_millis(10));
3650 }
3651 }
3652
3653 self.scan_task_watch_output(task);
3656
3657 self.post_terminal_transition(task, true)
3658 }
3659
3660 fn enqueue_completion_if_needed(
3661 &self,
3662 metadata: &PersistedTask,
3663 paths: Option<&TaskPaths>,
3664 emit_frame: bool,
3665 ) {
3666 if metadata.status.is_terminal() && !metadata.completion_delivered {
3667 let cache =
3668 paths.and_then(|paths| self.render_terminal_output_from_paths(metadata, paths));
3669 self.enqueue_completion_from_parts(metadata, None, paths, emit_frame, cache.as_ref());
3670 }
3671 }
3672
3673 fn render_terminal_output_from_paths(
3674 &self,
3675 metadata: &PersistedTask,
3676 paths: &TaskPaths,
3677 ) -> Option<TerminalOutputCache> {
3678 if metadata.mode == BgMode::Pty {
3679 return None;
3680 }
3681 let mut buffer = BgBuffer::registered(paths, BgMode::Pipes);
3682 let disk_truncation = buffer.enforce_terminal_cap();
3683 Some(self.render_terminal_output(metadata, &buffer, disk_truncation, Some(paths)))
3684 }
3685
3686 fn enqueue_completion_from_parts(
3687 &self,
3688 metadata: &PersistedTask,
3689 buffer: Option<&BgBuffer>,
3690 paths: Option<&TaskPaths>,
3691 emit_frame: bool,
3692 terminal_render: Option<&TerminalOutputCache>,
3693 ) {
3694 if !metadata.status.is_terminal() {
3705 return;
3706 }
3707
3708 let owned_buffer = if buffer.is_none() && metadata.mode != BgMode::Pty {
3709 paths.map(|paths| BgBuffer::registered(paths, BgMode::Pipes))
3710 } else {
3711 None
3712 };
3713 let render_buffer = buffer.or(owned_buffer.as_ref());
3714 let owned_render = if terminal_render.is_none() {
3715 render_buffer.map(|buffer| {
3716 let mut capped_buffer = buffer.clone();
3717 let disk_truncation = capped_buffer.enforce_terminal_cap();
3718 self.render_terminal_output(metadata, &capped_buffer, disk_truncation, paths)
3719 })
3720 } else {
3721 None
3722 };
3723 let render = terminal_render.or(owned_render.as_ref());
3724
3725 let (mut output_preview, output_truncated) = render
3729 .map(|cache| completion_preview_for_cache(cache, metadata.exit_code))
3730 .unwrap_or_else(|| (String::new(), false));
3731 if metadata.status == BgTaskStatus::FateUnknown {
3732 if let Some(reason) = metadata.status_reason.as_deref() {
3733 output_preview = if output_preview.is_empty() {
3734 reason.to_string()
3735 } else {
3736 format!("{reason}\n{output_preview}")
3737 };
3738 }
3739 }
3740
3741 let token_counts = self.completion_token_counts(
3742 metadata,
3743 buffer,
3744 paths,
3745 render.map(|render| render.output_preview.as_str()),
3746 );
3747 let completion = BgCompletion {
3748 task_id: metadata.task_id.clone(),
3749 session_id: metadata.session_id.clone(),
3750 status: metadata.status.clone(),
3751 exit_code: metadata.exit_code,
3752 command: metadata.command.clone(),
3753 output_preview,
3754 output_truncated,
3755 original_tokens: token_counts.original_tokens,
3756 compressed_tokens: token_counts.compressed_tokens,
3757 tokens_skipped: token_counts.tokens_skipped,
3758 status_reason: metadata.status_reason.clone(),
3759 };
3760
3761 self.record_compression_event_if_applicable(metadata, &token_counts);
3772
3773 let (watch_controlled, watch_matched) = self.task_watch_state(&metadata.task_id);
3774 if watch_controlled {
3775 if emit_frame && !watch_matched {
3776 self.emit_bash_watch_exit(&completion);
3777 } else if watch_matched {
3778 if let Some(task) = self.task(&metadata.task_id) {
3783 let _ = task.set_completion_delivered(true, self);
3784 }
3785 }
3786 self.clear_task_watch_state(&metadata.task_id);
3788 return;
3789 }
3790
3791 if metadata.completion_delivered {
3801 return;
3802 }
3803
3804 let pushed = if let Ok(mut completions) = self.inner.completions.lock() {
3807 if completions
3808 .iter()
3809 .any(|existing| existing.task_id == metadata.task_id)
3810 {
3811 false
3812 } else {
3813 completions.push_back(completion.clone());
3814 true
3815 }
3816 } else {
3817 false
3818 };
3819
3820 if pushed && emit_frame {
3821 self.emit_bash_completed(completion);
3822 }
3823 }
3824
3825 fn record_compression_event_if_applicable(
3826 &self,
3827 metadata: &PersistedTask,
3828 token_counts: &CompletionTokenCounts,
3829 ) {
3830 if metadata.mode == BgMode::Pty {
3831 return;
3832 }
3833
3834 let (original_tokens, compressed_tokens, original_bytes, compressed_bytes) = match (
3835 token_counts.original_tokens,
3836 token_counts.compressed_tokens,
3837 token_counts.original_bytes,
3838 token_counts.compressed_bytes,
3839 ) {
3840 (
3841 Some(original_tokens),
3842 Some(compressed_tokens),
3843 Some(original_bytes),
3844 Some(compressed_bytes),
3845 ) => (
3846 original_tokens,
3847 compressed_tokens,
3848 original_bytes,
3849 compressed_bytes,
3850 ),
3851 _ => {
3852 crate::slog_warn!(
3853 "compression event skipped for {}: token counts unavailable (likely spill file missing or unreadable)",
3854 metadata.task_id
3855 );
3856 return;
3857 }
3858 };
3859
3860 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
3861 let Some(pool) = pool else {
3862 crate::slog_warn!(
3863 "compression event skipped for {}: db_pool not initialized — was configure run?",
3864 metadata.task_id
3865 );
3866 return;
3867 };
3868 let harness = self
3869 .inner
3870 .db_harness
3871 .read()
3872 .ok()
3873 .and_then(|slot| slot.clone());
3874 let Some(harness) = harness else {
3875 crate::slog_warn!(
3876 "compression event insert skipped for {}: harness not configured",
3877 metadata.task_id
3878 );
3879 return;
3880 };
3881
3882 let project_root = metadata
3883 .project_root
3884 .as_deref()
3885 .unwrap_or(&metadata.workdir);
3886 let project_key = crate::path_identity::project_scope_key(project_root);
3887 let row = crate::db::compression_events::CompressionEventRow {
3888 harness: &harness,
3889 session_id: Some(&metadata.session_id),
3890 project_key: &project_key,
3891 tool: "bash",
3892 task_id: Some(&metadata.task_id),
3893 command: Some(&metadata.command),
3894 compressor: if metadata.compressed {
3895 "registry"
3896 } else {
3897 "none"
3898 },
3899 original_bytes,
3900 compressed_bytes,
3901 original_tokens,
3902 compressed_tokens,
3903 created_at: unix_millis() as i64,
3904 };
3905
3906 let conn = match pool.lock() {
3907 Ok(conn) => conn,
3908 Err(_) => {
3909 crate::slog_warn!(
3910 "compression event insert failed for {}: db mutex poisoned",
3911 metadata.task_id
3912 );
3913 return;
3914 }
3915 };
3916 match crate::db::compression_events::insert_compression_event(&conn, &row) {
3917 Ok(Some(row_id)) => {
3918 self.inner
3922 .compression_aggregates
3923 .record_successful_insert(&conn, &row, row_id);
3924 crate::slog_debug!(
3928 "compression event recorded for {} (project={}, session={}, {} → {} tokens)",
3929 metadata.task_id,
3930 project_key,
3931 metadata.session_id,
3932 original_tokens,
3933 compressed_tokens
3934 );
3935 }
3936 Ok(None) => {
3937 crate::slog_debug!(
3938 "duplicate compression event ignored for {} (project={}, session={})",
3939 metadata.task_id,
3940 project_key,
3941 metadata.session_id
3942 );
3943 }
3944 Err(error) => {
3945 crate::slog_warn!(
3946 "compression event insert failed for {}: {}",
3947 metadata.task_id,
3948 error
3949 );
3950 }
3951 }
3952 }
3953
3954 fn emit_bash_pattern_match(&self, session_id: &str, pattern_match: PatternMatch) {
3955 let Ok(progress_sender) = self
3956 .inner
3957 .progress_sender
3958 .lock()
3959 .map(|sender| sender.clone())
3960 else {
3961 return;
3962 };
3963 if let Some(sender) = progress_sender.as_ref() {
3964 sender(PushFrame::BashPatternMatch(BashPatternMatchFrame::new(
3965 pattern_match.task_id,
3966 session_id.to_string(),
3967 pattern_match.watch_id,
3968 pattern_match.match_text,
3969 pattern_match.match_offset,
3970 pattern_match.context,
3971 pattern_match.once,
3972 )));
3973 }
3974 }
3975
3976 fn emit_bash_watch_exit(&self, completion: &BgCompletion) {
3977 let Ok(progress_sender) = self
3978 .inner
3979 .progress_sender
3980 .lock()
3981 .map(|sender| sender.clone())
3982 else {
3983 return;
3984 };
3985 let Some(sender) = progress_sender.as_ref() else {
3986 return;
3987 };
3988 let status = completion_status_text(&completion.status, completion.exit_code);
3989 let preview = completion.output_preview.trim_end();
3990 let context = if preview.is_empty() {
3991 format!("task {} exited ({status})", completion.task_id)
3992 } else {
3993 format!(
3994 "task {} exited ({status})
3995{preview}",
3996 completion.task_id
3997 )
3998 };
3999 sender(PushFrame::BashPatternMatch(
4000 BashPatternMatchFrame::task_exit(
4001 completion.task_id.clone(),
4002 completion.session_id.clone(),
4003 format!("exited ({status})"),
4004 context,
4005 ),
4006 ));
4007 }
4008
4009 fn emit_bash_completed(&self, completion: BgCompletion) {
4010 let Ok(progress_sender) = self
4011 .inner
4012 .progress_sender
4013 .lock()
4014 .map(|sender| sender.clone())
4015 else {
4016 return;
4017 };
4018 let Some(sender) = progress_sender.as_ref() else {
4019 return;
4020 };
4021 let mut frame = BashCompletedFrame::new(
4029 completion.task_id,
4030 completion.session_id,
4031 completion.status,
4032 completion.exit_code,
4033 completion.command,
4034 completion.output_preview,
4035 completion.output_truncated,
4036 completion.original_tokens,
4037 completion.compressed_tokens,
4038 completion.tokens_skipped,
4039 );
4040 frame.status_reason = completion.status_reason;
4041 sender(PushFrame::BashCompleted(frame));
4042 }
4043
4044 fn completion_token_counts(
4045 &self,
4046 metadata: &PersistedTask,
4047 buffer: Option<&BgBuffer>,
4048 paths: Option<&TaskPaths>,
4049 rendered_output: Option<&str>,
4050 ) -> CompletionTokenCounts {
4051 if metadata.mode == BgMode::Pty {
4052 return CompletionTokenCounts::skipped();
4053 }
4054
4055 let raw = match buffer {
4056 Some(buffer) => buffer.read_for_token_count(TOKENIZE_CAP_BYTES_PER_STREAM),
4057 None => paths
4058 .map(|paths| {
4059 read_for_token_count_from_disk(metadata, paths, TOKENIZE_CAP_BYTES_PER_STREAM)
4060 })
4061 .unwrap_or(TokenCountInput::Skipped),
4062 };
4063
4064 let TokenCountInput::Text(raw_output) = raw else {
4065 return CompletionTokenCounts::skipped();
4066 };
4067
4068 let original_tokens = token_count_u32(&raw_output);
4069 let original_bytes = raw_output.len() as i64;
4070 let compressed_output = rendered_output.unwrap_or(&raw_output);
4071 let compressed_tokens = token_count_u32(compressed_output);
4072 let compressed_bytes = compressed_output.len() as i64;
4073 CompletionTokenCounts {
4074 original_tokens: Some(original_tokens),
4075 compressed_tokens: Some(compressed_tokens),
4076 original_bytes: Some(original_bytes),
4077 compressed_bytes: Some(compressed_bytes),
4078 tokens_skipped: false,
4079 }
4080 }
4081
4082 pub(crate) fn maybe_emit_long_running_reminder(&self, task: &Arc<BgTask>) {
4083 if !self
4084 .inner
4085 .long_running_reminder_enabled
4086 .load(Ordering::SeqCst)
4087 {
4088 return;
4089 }
4090 let interval_ms = self
4091 .inner
4092 .long_running_reminder_interval_ms
4093 .load(Ordering::SeqCst);
4094 if interval_ms == 0 {
4095 return;
4096 }
4097 let interval = Duration::from_millis(interval_ms);
4098 let now = Instant::now();
4099 let Ok(mut last_reminder_at) = task.last_reminder_at.lock() else {
4100 return;
4101 };
4102 let since = last_reminder_at.unwrap_or(task.started);
4103 if now.duration_since(since) < interval {
4104 return;
4105 }
4106 let command = task
4107 .state
4108 .lock()
4109 .map(|state| state.metadata.command.clone())
4110 .unwrap_or_default();
4111 *last_reminder_at = Some(now);
4112 self.emit_bash_long_running(BashLongRunningFrame::new(
4113 task.task_id.clone(),
4114 task.session_id.clone(),
4115 command,
4116 task.started.elapsed().as_millis() as u64,
4117 ));
4118 }
4119
4120 fn emit_bash_long_running(&self, frame: BashLongRunningFrame) {
4121 let Ok(progress_sender) = self
4122 .inner
4123 .progress_sender
4124 .lock()
4125 .map(|sender| sender.clone())
4126 else {
4127 return;
4128 };
4129 if let Some(sender) = progress_sender.as_ref() {
4130 sender(PushFrame::BashLongRunning(frame));
4131 }
4132 }
4133
4134 fn task(&self, task_id: &str) -> Option<Arc<BgTask>> {
4135 validate_task_id(task_id).ok()?;
4136 self.inner
4137 .tasks
4138 .lock()
4139 .ok()
4140 .and_then(|tasks| tasks.get(task_id).cloned())
4141 }
4142
4143 fn task_for_session(&self, task_id: &str, session_id: &str) -> Option<Arc<BgTask>> {
4144 self.task(task_id)
4145 .filter(|task| task.session_id == session_id)
4146 }
4147
4148 pub fn try_health_counts(&self) -> Option<BgTaskHealthCounts> {
4149 let running = self
4150 .inner
4151 .tasks
4152 .try_lock()
4153 .ok()
4154 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())?;
4155 let pending_completions = self.inner.completions.try_lock().ok().map(|q| q.len())?;
4156 Some(BgTaskHealthCounts {
4157 running,
4158 pending_completions,
4159 })
4160 }
4161
4162 pub fn estimated_memory(&self) -> crate::memory::MemoryEstimate {
4166 let tasks = match self.inner.tasks.try_lock() {
4167 Ok(tasks) => tasks.values().cloned().collect::<Vec<_>>(),
4168 Err(_) => return crate::memory::MemoryEstimate::busy(),
4169 };
4170 let mut bytes = 0u64;
4171 let mut terminal_output_caches = 0usize;
4172 let mut sessions = HashSet::new();
4173 for task in &tasks {
4174 sessions.insert(task.session_id.clone());
4175 let state = match task.state.try_lock() {
4176 Ok(state) => state,
4177 Err(_) => return crate::memory::MemoryEstimate::busy(),
4178 };
4179 if let Some(cache) = state.terminal_output_cache.as_ref() {
4180 terminal_output_caches = terminal_output_caches.saturating_add(1);
4181 bytes = bytes.saturating_add(terminal_output_cache_estimated_bytes(cache));
4182 }
4183 }
4184 let completion_count = match self.inner.completions.try_lock() {
4185 Ok(completions) => {
4186 for completion in completions.iter() {
4187 sessions.insert(completion.session_id.clone());
4188 bytes = bytes.saturating_add(completion_estimated_bytes(completion));
4189 }
4190 completions.len()
4191 }
4192 Err(_) => return crate::memory::MemoryEstimate::busy(),
4193 };
4194
4195 crate::memory::MemoryEstimate::estimated(bytes)
4196 .count("tasks", tasks.len())
4197 .count("sessions", sessions.len())
4198 .count("terminal_output_caches", terminal_output_caches)
4199 .count("completion_caches", completion_count)
4200 .count_u64("output_ring_bytes", 0)
4201 }
4202
4203 fn running_count(&self) -> usize {
4204 self.inner
4205 .tasks
4206 .lock()
4207 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())
4208 .unwrap_or(0)
4209 }
4210
4211 fn start_watchdog(&self) {
4212 if !self.inner.watchdog_started.swap(true, Ordering::SeqCst) {
4213 super::watchdog::start(self.clone());
4214 }
4215 }
4216
4217 #[cfg(test)]
4218 pub fn task_json_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
4219 self.task_for_session(task_id, session_id)
4220 .map(|task| task.paths.json.clone())
4221 }
4222
4223 #[cfg(test)]
4224 pub fn task_exit_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
4225 self.task_for_session(task_id, session_id)
4226 .map(|task| task.paths.exit.clone())
4227 }
4228}
4229
4230#[cfg(unix)]
4231fn should_capture_pipeline_status(
4232 spawn_plan: &SpawnPlan,
4233 has_pipeline: bool,
4234 shell: &Path,
4235) -> bool {
4236 if spawn_plan.is_native_launcher() {
4237 return false;
4240 }
4241 has_pipeline && super::process::pipeline_shell_kind(shell).is_some()
4242}
4243
4244fn canonical_artifact_root(paths: &TaskPaths) -> PathBuf {
4245 fs::canonicalize(&paths.io_dir).unwrap_or_else(|_| paths.io_dir.clone())
4246}
4247
4248fn restart_fate_unknown_reason(metadata: &PersistedTask, paths: &TaskPaths) -> String {
4249 let output = match metadata.mode {
4250 BgMode::Pipes => &paths.stdout,
4251 BgMode::Pty => &paths.pty,
4252 };
4253 format!(
4254 "task {}: daemon restarted, process fate unknown, last output at {}",
4255 metadata.task_id,
4256 output.display()
4257 )
4258}
4259
4260fn append_pipeline_warning(
4265 cache: &mut TerminalOutputCache,
4266 metadata: &PersistedTask,
4267 paths: Option<&TaskPaths>,
4268) {
4269 if metadata.exit_code != Some(0) {
4270 return;
4271 }
4272 let Some(paths) = paths else {
4273 return;
4274 };
4275 if metadata.pipeline_segments.len() < 2 {
4276 return;
4277 }
4278 let Ok(mut status_file) = open_task_artifact(paths, TaskArtifact::PipelineStatus) else {
4279 return;
4280 };
4281 let Ok(status_bytes) = status_file.read_all() else {
4282 return;
4283 };
4284 let Some(statuses) = String::from_utf8_lossy(&status_bytes)
4285 .lines()
4286 .map(|line| line.trim().parse::<i32>().ok())
4287 .collect::<Option<Vec<_>>>()
4288 else {
4289 return;
4290 };
4291 if statuses.len() != metadata.pipeline_segments.len() {
4292 return;
4293 }
4294 let Some((failing_index, failing_code)) = statuses
4295 .iter()
4296 .enumerate()
4297 .take(statuses.len().saturating_sub(1))
4298 .find(|(_, code)| **code != 0)
4299 .map(|(index, code)| (index, *code))
4300 else {
4301 return;
4302 };
4303 let Some(final_segment) = metadata.pipeline_segments.last() else {
4304 return;
4305 };
4306 let failing_segment = &metadata.pipeline_segments[failing_index];
4307 let footer = format!(
4308 "note: `{}` (segment {} of {}) exited {}; the pipeline's exit code is `{}`'s.",
4309 failing_segment,
4310 failing_index + 1,
4311 metadata.pipeline_segments.len(),
4312 failing_code,
4313 final_segment,
4314 );
4315 if cache.output_preview.trim().is_empty() {
4316 cache.output_preview = footer;
4317 } else {
4318 cache.output_preview = format!("{}\n{}", cache.output_preview.trim_end(), footer,);
4319 }
4320}
4321
4322fn normalize_piped_display_output(text: &mut String) {
4325 if !text.contains('\r') {
4326 return;
4327 }
4328
4329 let mut rendered = String::with_capacity(text.len());
4330 let mut line = Vec::new();
4331 let mut column = 0;
4332 let mut chars = text.chars().peekable();
4333
4334 while let Some(character) = chars.next() {
4335 match character {
4336 '\r' if chars.peek() == Some(&'\n') => {
4337 chars.next();
4338 for character in &line {
4339 rendered.push(*character);
4340 }
4341 rendered.push('\n');
4342 line.clear();
4343 column = 0;
4344 }
4345 '\r' => column = 0,
4346 '\n' => {
4347 for character in &line {
4348 rendered.push(*character);
4349 }
4350 rendered.push('\n');
4351 line.clear();
4352 column = 0;
4353 }
4354 character => {
4355 if column < line.len() {
4356 line[column] = character;
4357 } else {
4358 line.resize(column, ' ');
4359 line.push(character);
4360 }
4361 column += 1;
4362 }
4363 }
4364 }
4365
4366 for character in &line {
4367 rendered.push(*character);
4368 }
4369 *text = rendered;
4370}
4371
4372fn render_compressed_with_recovery(
4373 buffer: &BgBuffer,
4374 mut compressed: CompressionResult,
4375 input_truncated: bool,
4376 disk_truncation: DiskTruncation,
4377 artifact_access: ArtifactRecoveryAccess,
4378) -> TerminalOutputCache {
4379 let had_trailing_newline = compressed.text.ends_with('\n');
4387 let mut text = strip_plain_truncation_marker_lines(&compressed.text)
4388 .trim_end()
4389 .to_string();
4390 if had_trailing_newline && !text.is_empty() {
4391 text.push('\n');
4392 }
4393 compressed.text = text;
4394
4395 let output_path = buffer.output_path().map(|path| path.display().to_string());
4396 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
4397 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
4398 let mut recovery = RecoveryContext {
4399 dropped_by_class: compressed.dropped_by_class,
4400 had_inner_drop: compressed.had_inner_drop,
4401 offset_hint_eligible: compressed.offset_hint_eligible,
4402 offset_start_line: compressed.offset_start_line,
4403 byte_truncated: input_truncated,
4404 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
4405 output_path: output_path.clone(),
4406 stderr_path: stderr_path.clone(),
4407 include_stderr_path,
4408 artifact_access: artifact_access.clone(),
4409 };
4410
4411 let (output_preview, output_truncated) =
4412 render_body_with_recovery_marker(&compressed.text, &mut recovery);
4413 TerminalOutputCache {
4414 output_preview,
4415 output_truncated,
4416 kind: TerminalOutputKind::Compressed,
4417 output_path,
4418 stderr_path,
4419 artifact_access,
4420 recovery: Some(recovery),
4421 }
4422}
4423
4424fn render_body_with_recovery_marker(body: &str, recovery: &mut RecoveryContext) -> (String, bool) {
4425 render_body_with_recovery_marker_at_cap(
4426 body,
4427 recovery,
4428 FINAL_OUTPUT_CAP_BYTES,
4429 cap_final_output,
4430 cap_final_output_with_marker,
4431 )
4432}
4433
4434fn render_raw_body_with_recovery_marker(
4435 body: &str,
4436 recovery: &mut RecoveryContext,
4437) -> (String, bool) {
4438 render_body_with_recovery_marker_at_cap(
4439 body,
4440 recovery,
4441 RAW_PASSTHROUGH_CAP_BYTES,
4442 |input| {
4443 super::output::cap_head_tail(
4444 input,
4445 RAW_PASSTHROUGH_CAP_BYTES,
4446 RAW_PASSTHROUGH_HEAD_BYTES,
4447 RAW_PASSTHROUGH_TAIL_BYTES,
4448 )
4449 },
4450 |input, marker| {
4451 super::output::cap_head_tail_with_marker(
4452 input,
4453 RAW_PASSTHROUGH_CAP_BYTES,
4454 RAW_PASSTHROUGH_HEAD_BYTES,
4455 RAW_PASSTHROUGH_TAIL_BYTES,
4456 marker,
4457 )
4458 },
4459 )
4460}
4461
4462fn render_body_with_recovery_marker_at_cap<F, G>(
4463 body: &str,
4464 recovery: &mut RecoveryContext,
4465 cap_bytes: usize,
4466 cap_plain: F,
4467 cap_with_marker: G,
4468) -> (String, bool)
4469where
4470 F: Fn(&str) -> super::output::CappedText,
4471 G: Fn(&str, &str) -> super::output::CappedText,
4472{
4473 let needs_marker = recovery.has_visible_drop();
4474 if body.len() > cap_bytes {
4475 recovery.byte_truncated = true;
4476 if let Some(marker) = recovery_marker(recovery) {
4477 let capped = cap_with_marker(body, &marker);
4478 return (capped.text, true);
4479 }
4480 let capped = cap_plain(body);
4481 return (capped.text, capped.truncated || needs_marker);
4482 }
4483
4484 if !needs_marker {
4485 return (body.to_string(), false);
4486 }
4487
4488 let Some(marker) = recovery_marker(recovery) else {
4489 return (body.to_string(), true);
4490 };
4491 let with_marker = append_recovery_marker(body, &marker);
4492 if with_marker.len() <= cap_bytes {
4493 return (with_marker, true);
4494 }
4495
4496 recovery.byte_truncated = true;
4497 let marker = recovery_marker(recovery).unwrap_or(marker);
4498 let capped = cap_with_marker(body, &marker);
4499 (capped.text, true)
4500}
4501
4502fn append_recovery_marker(body: &str, marker: &str) -> String {
4503 if body.is_empty() {
4504 return marker.to_string();
4505 }
4506 let mut output = body.trim_end().to_string();
4507 output.push('\n');
4508 output.push_str(marker);
4509 output
4510}
4511
4512fn recovery_marker(recovery: &RecoveryContext) -> Option<String> {
4513 let mut parts = Vec::new();
4514 for (class, count) in &recovery.dropped_by_class {
4515 let label = if *count == 1 {
4516 class.singular()
4517 } else {
4518 class.plural()
4519 };
4520 parts.push(format!("+{count} more {label}"));
4521 }
4522 if recovery.byte_truncated {
4523 parts.push("truncated output".to_string());
4524 }
4525 let disk_truncated_prefix_bytes = recovery.disk_truncated_prefix_bytes;
4526 if disk_truncated_prefix_bytes > 0 {
4527 parts.push(format!(
4528 "truncated {disk_truncated_prefix_bytes} bytes from saved output prefix"
4529 ));
4530 } else if recovery.had_inner_drop && parts.is_empty() {
4531 parts.push("omitted output".to_string());
4532 }
4533
4534 if parts.is_empty() {
4535 return None;
4536 }
4537
4538 let hint = recovery_hint(recovery);
4539 Some(format!("[{}; {hint}]", parts.join(", ")))
4540}
4541
4542fn bash_status_recovery_hint(access: &ArtifactRecoveryAccess) -> String {
4543 let task_id = serde_json::to_string(&access.task_id)
4544 .unwrap_or_else(|_| format!("\"{}\"", access.task_id));
4545 format!("use bash_status({{taskId: {task_id}}})")
4546}
4547
4548fn recovery_hint(recovery: &RecoveryContext) -> String {
4549 if !recovery.artifact_access.readable {
4550 return bash_status_recovery_hint(&recovery.artifact_access);
4551 }
4552
4553 if recovery.offset_hint_eligible
4557 && !recovery.byte_truncated
4558 && recovery.dropped_by_class.is_empty()
4559 && !recovery.include_stderr_path
4560 {
4561 if let (Some(path), Some(line)) =
4562 (recovery.output_path.as_deref(), recovery.offset_start_line)
4563 {
4564 return format!("see remaining: tail -n +{line} {}", quote_path(path));
4565 }
4566 }
4567
4568 let mut paths = Vec::new();
4569 if let Some(path) = recovery.output_path.as_deref() {
4570 paths.push(path);
4571 }
4572 if recovery.include_stderr_path {
4573 if let Some(path) = recovery.stderr_path.as_deref() {
4574 if !paths.contains(&path) {
4575 paths.push(path);
4576 }
4577 }
4578 }
4579
4580 if paths.is_empty() {
4581 return "full output unavailable".to_string();
4582 }
4583
4584 let reads = paths
4585 .into_iter()
4586 .map(|path| format!("read {}", quote_path(path)))
4587 .collect::<Vec<_>>()
4588 .join(" and ");
4589 if recovery.disk_truncated_prefix_bytes > 0 {
4590 format!("retained output: {reads}")
4591 } else {
4592 format!("full output: {reads}")
4593 }
4594}
4595
4596fn strip_plain_truncation_marker_lines(input: &str) -> String {
4597 input
4598 .lines()
4599 .filter(|line| !is_plain_truncation_marker(line.trim()))
4600 .collect::<Vec<_>>()
4601 .join("\n")
4602}
4603
4604fn strip_recovery_marker_lines(input: &str) -> String {
4605 input
4606 .lines()
4607 .filter(|line| !is_recovery_marker(line.trim()))
4608 .collect::<Vec<_>>()
4609 .join("\n")
4610}
4611
4612fn is_plain_truncation_marker(line: &str) -> bool {
4613 let Some(rest) = line.strip_prefix("...<truncated ") else {
4614 return false;
4615 };
4616 let Some(bytes) = rest.strip_suffix(" bytes>...") else {
4617 return false;
4618 };
4619 !bytes.is_empty() && bytes.chars().all(|ch| ch.is_ascii_digit())
4620}
4621
4622fn is_recovery_marker(line: &str) -> bool {
4623 line.starts_with('[')
4624 && line.ends_with(']')
4625 && (line.contains("full output: read ")
4626 || line.contains("retained output: read ")
4627 || line.contains("see remaining: tail -n +")
4628 || line.contains("use bash_status({taskId:")
4629 || line.contains("full output unavailable"))
4630}
4631
4632fn structured_output_pointer(
4633 total_bytes: u64,
4634 output_path: &str,
4635 truncated_prefix_bytes: u64,
4636 artifact_access: &ArtifactRecoveryAccess,
4637) -> String {
4638 if artifact_access.readable {
4639 return if truncated_prefix_bytes > 0 {
4640 retained_json_output_pointer(total_bytes, output_path, truncated_prefix_bytes)
4641 } else {
4642 json_output_pointer(total_bytes, output_path)
4643 };
4644 }
4645
4646 let kb = total_bytes.div_ceil(1024);
4647 let hint = bash_status_recovery_hint(artifact_access);
4648 if truncated_prefix_bytes > 0 {
4649 format!(
4650 "[JSON output {kb} KB; truncated {truncated_prefix_bytes} bytes from saved output prefix; retained output: {hint}]"
4651 )
4652 } else {
4653 format!("[JSON output {kb} KB; full output: {hint}]")
4654 }
4655}
4656
4657fn render_structured_output(
4658 command: &str,
4659 buffer: &BgBuffer,
4660 disk_truncation: DiskTruncation,
4661 artifact_access: ArtifactRecoveryAccess,
4662) -> Option<TerminalOutputCache> {
4663 if !is_gh_structured_command(command) {
4664 return None;
4665 }
4666
4667 let output_path = buffer
4668 .output_path()
4669 .map(|path| path.display().to_string())?;
4670 let stdout_bytes = buffer.stream_len(StreamKind::Stdout);
4671 if stdout_bytes == 0 {
4672 return None;
4673 }
4674
4675 if stdout_bytes > STRUCTURED_OUTPUT_CAP_BYTES as u64 {
4676 if !stream_starts_like_json(buffer, StreamKind::Stdout) {
4677 return None;
4678 }
4679 let output_preview = structured_output_pointer(
4680 stdout_bytes,
4681 &output_path,
4682 disk_truncation.total_prefix_bytes(),
4683 &artifact_access,
4684 );
4685 return Some(TerminalOutputCache {
4686 output_preview,
4687 output_truncated: true,
4688 kind: TerminalOutputKind::Structured,
4689 output_path: Some(output_path),
4690 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
4691 artifact_access,
4692 recovery: None,
4693 });
4694 }
4695
4696 let stdout = buffer.read_stream_bounded(StreamKind::Stdout, STRUCTURED_OUTPUT_CAP_BYTES);
4697 if stdout.truncated || !is_structured_body(&stdout.text) {
4698 return None;
4699 }
4700
4701 Some(TerminalOutputCache {
4702 output_preview: stdout.text,
4703 output_truncated: false,
4704 kind: TerminalOutputKind::Structured,
4705 output_path: Some(output_path),
4706 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
4707 artifact_access,
4708 recovery: None,
4709 })
4710}
4711
4712fn render_raw_passthrough(
4713 buffer: &BgBuffer,
4714 disk_truncation: DiskTruncation,
4715 artifact_access: ArtifactRecoveryAccess,
4716) -> TerminalOutputCache {
4717 let raw = buffer.read_combined_head_tail(
4718 RAW_PASSTHROUGH_CAP_BYTES,
4719 RAW_PASSTHROUGH_HEAD_BYTES,
4720 RAW_PASSTHROUGH_TAIL_BYTES,
4721 );
4722 let output_path = buffer.output_path().map(|path| path.display().to_string());
4723 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
4724 if !raw.truncated && disk_truncation.total_prefix_bytes() == 0 {
4725 return TerminalOutputCache {
4726 output_preview: raw.text,
4727 output_truncated: false,
4728 kind: TerminalOutputKind::Raw,
4729 output_path,
4730 stderr_path,
4731 artifact_access,
4732 recovery: None,
4733 };
4734 }
4735
4736 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
4737 let mut recovery = RecoveryContext {
4738 dropped_by_class: BTreeMap::new(),
4739 had_inner_drop: false,
4740 offset_hint_eligible: false,
4741 offset_start_line: None,
4742 byte_truncated: raw.truncated,
4743 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
4744 output_path: output_path.clone(),
4745 stderr_path: stderr_path.clone(),
4746 include_stderr_path,
4747 artifact_access: artifact_access.clone(),
4748 };
4749 let (output_preview, output_truncated) =
4750 render_raw_body_with_recovery_marker(&raw.text, &mut recovery);
4751 TerminalOutputCache {
4752 output_preview,
4753 output_truncated,
4754 kind: TerminalOutputKind::Raw,
4755 output_path,
4756 stderr_path,
4757 artifact_access,
4758 recovery: Some(recovery),
4759 }
4760}
4761
4762fn completion_preview_for_cache(
4763 cache: &TerminalOutputCache,
4764 exit_code: Option<i32>,
4765) -> (String, bool) {
4766 let exit_ok = exit_code == Some(0);
4769 let threshold = completion_preview_threshold(exit_ok);
4770 if cache.kind == TerminalOutputKind::Structured && cache.output_preview.len() > threshold {
4771 if let Some(path) = cache.output_path.as_deref() {
4772 return (
4773 structured_output_pointer(
4774 cache.output_preview.len() as u64,
4775 path,
4776 0,
4777 &cache.artifact_access,
4778 ),
4779 true,
4780 );
4781 }
4782 return (cache.output_preview.clone(), cache.output_truncated);
4783 }
4784
4785 if let Some(recovery) = cache.recovery.as_ref() {
4786 if cache.output_preview.len() <= threshold {
4787 return (cache.output_preview.clone(), cache.output_truncated);
4788 }
4789 let body = strip_recovery_marker_lines(&cache.output_preview);
4790 let mut completion_recovery = recovery.clone();
4791 completion_recovery.byte_truncated = true;
4792 if let Some(marker) = recovery_marker(&completion_recovery) {
4793 let capped = cap_completion_output_with_marker(&body, &marker, exit_ok);
4794 return (capped.text, true);
4795 }
4796 }
4797
4798 let capped = cap_completion_output(&cache.output_preview, exit_ok);
4799 (capped.text, cache.output_truncated || capped.truncated)
4800}
4801
4802fn is_gh_structured_command(command: &str) -> bool {
4803 let Some(normalized) = crate::compress::plain_command_for_structured_output(command) else {
4804 return false;
4805 };
4806 let tokens = shell_words_for_flags(&normalized);
4807 let Some(head) = tokens.first() else {
4808 return false;
4809 };
4810 let head_name = Path::new(head)
4811 .file_name()
4812 .and_then(|name| name.to_str())
4813 .unwrap_or(head);
4814 if !(head_name == "gh" || head_name.eq_ignore_ascii_case("gh.exe")) {
4815 return false;
4816 }
4817 tokens.iter().any(|token| {
4818 matches!(token.as_str(), "--json" | "--jq" | "--template")
4819 || token.starts_with("--json=")
4820 || token.starts_with("--jq=")
4821 || token.starts_with("--template=")
4822 })
4823}
4824
4825fn shell_words_for_flags(command: &str) -> Vec<String> {
4826 let mut words = Vec::new();
4827 let mut current = String::new();
4828 let mut in_single = false;
4829 let mut in_double = false;
4830 let mut escaped = false;
4831
4832 for ch in command.chars() {
4833 if escaped {
4834 current.push(ch);
4835 escaped = false;
4836 continue;
4837 }
4838 if ch == '\\' && !in_single {
4839 escaped = true;
4840 continue;
4841 }
4842 if ch == '\'' && !in_double {
4843 in_single = !in_single;
4844 continue;
4845 }
4846 if ch == '"' && !in_single {
4847 in_double = !in_double;
4848 continue;
4849 }
4850 if ch.is_whitespace() && !in_single && !in_double {
4851 if !current.is_empty() {
4852 words.push(std::mem::take(&mut current));
4853 }
4854 continue;
4855 }
4856 if matches!(ch, ';' | '&' | '|') && !in_single && !in_double {
4857 if !current.is_empty() {
4858 words.push(std::mem::take(&mut current));
4859 }
4860 continue;
4861 }
4862 current.push(ch);
4863 }
4864 if !current.is_empty() {
4865 words.push(current);
4866 }
4867 words
4868}
4869
4870fn is_structured_body(body: &str) -> bool {
4871 let trimmed = body.trim();
4872 if trimmed.is_empty() {
4873 return false;
4874 }
4875 if serde_json::from_str::<serde_json::Value>(trimmed).is_ok() {
4876 return true;
4877 }
4878
4879 let mut saw_line = false;
4880 for line in trimmed
4881 .lines()
4882 .map(str::trim)
4883 .filter(|line| !line.is_empty())
4884 {
4885 saw_line = true;
4886 if serde_json::from_str::<serde_json::Value>(line).is_err() {
4887 return false;
4888 }
4889 }
4890 saw_line
4891}
4892
4893fn stream_starts_like_json(buffer: &BgBuffer, stream: StreamKind) -> bool {
4894 buffer
4895 .read_stream_bounded(stream, 512)
4896 .text
4897 .chars()
4898 .find(|ch| !ch.is_whitespace())
4899 .is_some_and(|ch| matches!(ch, '{' | '[' | '"' | '-' | '0'..='9' | 't' | 'f' | 'n'))
4900}
4901
4902struct CompletionTokenCounts {
4903 original_tokens: Option<u32>,
4904 compressed_tokens: Option<u32>,
4905 original_bytes: Option<i64>,
4906 compressed_bytes: Option<i64>,
4907 tokens_skipped: bool,
4908}
4909
4910impl CompletionTokenCounts {
4911 fn skipped() -> Self {
4912 Self {
4913 original_tokens: None,
4914 compressed_tokens: None,
4915 original_bytes: None,
4916 compressed_bytes: None,
4917 tokens_skipped: true,
4918 }
4919 }
4920}
4921
4922fn completion_status_text(status: &BgTaskStatus, exit_code: Option<i32>) -> String {
4923 match status {
4924 BgTaskStatus::TimedOut => "timed out".to_string(),
4925 BgTaskStatus::Killed => "killed".to_string(),
4926 _ => exit_code
4927 .map(|code| format!("exit {code}"))
4928 .unwrap_or_else(|| format!("{status:?}").to_lowercase()),
4929 }
4930}
4931
4932fn token_count_u32(text: &str) -> u32 {
4933 aft_tokenizer::count_tokens(text)
4934 .try_into()
4935 .unwrap_or(u32::MAX)
4936}
4937
4938impl Default for BgTaskRegistry {
4939 fn default() -> Self {
4940 Self::new(Arc::new(Mutex::new(None)))
4941 }
4942}
4943
4944fn modified_within(path: &Path, grace: Duration) -> bool {
4945 fs::metadata(path)
4946 .and_then(|metadata| metadata.modified())
4947 .ok()
4948 .and_then(|modified| SystemTime::now().duration_since(modified).ok())
4949 .map(|age| age < grace)
4950 .unwrap_or(false)
4951}
4952
4953fn canonicalized_path(path: &Path) -> PathBuf {
4954 fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf())
4955}
4956
4957fn started_instant_from_unix_millis(started_at: u64) -> Instant {
4958 let now_ms = SystemTime::now()
4959 .duration_since(UNIX_EPOCH)
4960 .ok()
4961 .map(|duration| duration.as_millis() as u64)
4962 .unwrap_or(started_at);
4963 let elapsed_ms = now_ms.saturating_sub(started_at);
4964 Instant::now()
4965 .checked_sub(Duration::from_millis(elapsed_ms))
4966 .unwrap_or_else(Instant::now)
4967}
4968
4969fn gc_quarantine(storage_dir: &Path) {
4970 let quarantine_root = storage_dir.join("bash-tasks-quarantine");
4971 let Ok(session_dirs) = fs::read_dir(&quarantine_root) else {
4972 return;
4973 };
4974 for session_entry in session_dirs.flatten() {
4975 let session_quarantine_dir = session_entry.path();
4976 if !session_quarantine_dir.is_dir() {
4977 continue;
4978 }
4979 let entries = match fs::read_dir(&session_quarantine_dir) {
4980 Ok(entries) => entries,
4981 Err(error) => {
4982 crate::slog_warn!(
4983 "failed to read background task quarantine dir {}: {error}",
4984 session_quarantine_dir.display()
4985 );
4986 continue;
4987 }
4988 };
4989 for entry in entries.flatten() {
4990 let path = entry.path();
4991 if modified_within(&path, QUARANTINE_GC_GRACE) {
4992 continue;
4993 }
4994 let result = if path.is_dir() {
4995 fs::remove_dir_all(&path)
4996 } else {
4997 fs::remove_file(&path)
4998 };
4999 match result {
5000 Ok(()) => log::debug!(
5001 "deleted old background task quarantine entry {}",
5002 path.display()
5003 ),
5004 Err(error) => crate::slog_warn!(
5005 "failed to delete old background task quarantine entry {}: {error}",
5006 path.display()
5007 ),
5008 }
5009 }
5010 let _ = fs::remove_dir(&session_quarantine_dir);
5011 }
5012 let _ = fs::remove_dir(&quarantine_root);
5013}
5014
5015fn read_for_token_count_from_disk(
5016 metadata: &PersistedTask,
5017 paths: &TaskPaths,
5018 max_bytes_per_stream: usize,
5019) -> TokenCountInput {
5020 if metadata.mode == BgMode::Pty {
5021 return TokenCountInput::Skipped;
5022 }
5023 let stdout = read_file_tail_capped(paths, TaskArtifact::Stdout, max_bytes_per_stream);
5030 let stderr = read_file_tail_capped(paths, TaskArtifact::Stderr, max_bytes_per_stream);
5031 match (stdout, stderr) {
5032 (Ok(stdout), Ok(stderr)) => TokenCountInput::Text(combine_streams(
5033 String::from_utf8_lossy(&stdout).as_ref(),
5034 String::from_utf8_lossy(&stderr).as_ref(),
5035 )),
5036 (Ok(stdout), Err(_)) => TokenCountInput::Text(combine_streams(
5037 String::from_utf8_lossy(&stdout).as_ref(),
5038 "",
5039 )),
5040 (Err(_), Ok(stderr)) => TokenCountInput::Text(combine_streams(
5041 "",
5042 String::from_utf8_lossy(&stderr).as_ref(),
5043 )),
5044 (Err(_), Err(_)) => TokenCountInput::Skipped,
5045 }
5046}
5047
5048fn read_file_tail_capped(
5049 paths: &TaskPaths,
5050 artifact: TaskArtifact,
5051 max_bytes: usize,
5052) -> std::io::Result<Vec<u8>> {
5053 let mut file = open_task_artifact(paths, artifact)?;
5054 file.tail(max_bytes).map(|(bytes, _)| bytes)
5055}
5056
5057impl BgTask {
5058 fn snapshot(&self, preview_bytes: usize) -> BgTaskSnapshot {
5059 let state = self
5060 .state
5061 .lock()
5062 .unwrap_or_else(|poison| poison.into_inner());
5063 self.snapshot_locked(&state, preview_bytes)
5064 }
5065
5066 fn snapshot_locked(&self, state: &BgTaskState, preview_bytes: usize) -> BgTaskSnapshot {
5067 let metadata = &state.metadata;
5068 let duration_ms = metadata.duration_ms.or_else(|| {
5069 metadata
5070 .status
5071 .is_terminal()
5072 .then(|| self.started.elapsed().as_millis() as u64)
5073 });
5074 let (output_preview, output_truncated) = if metadata.mode == BgMode::Pty {
5075 (String::new(), false)
5076 } else if metadata.status.is_terminal() {
5077 state
5078 .terminal_output_cache
5079 .as_ref()
5080 .map(|cache| (cache.output_preview.clone(), cache.output_truncated))
5081 .unwrap_or_else(|| (String::new(), false))
5082 } else if preview_bytes == 0 {
5083 (String::new(), false)
5084 } else {
5085 state.buffer.read_tail(preview_bytes)
5086 };
5087 BgTaskSnapshot {
5088 info: BgTaskInfo {
5089 task_id: self.task_id.clone(),
5090 status: metadata.status.clone(),
5091 command: metadata.command.clone(),
5092 mode: metadata.mode.clone(),
5093 started_at: metadata.started_at,
5094 duration_ms,
5095 status_reason: metadata.status_reason.clone(),
5096 },
5097 exit_code: metadata.exit_code,
5098 child_pid: metadata.child_pid,
5099 workdir: metadata.workdir.display().to_string(),
5100 output_preview,
5101 output_truncated,
5102 output_path: state
5103 .buffer
5104 .output_path()
5105 .map(|path| path.display().to_string()),
5106 stderr_path: state
5107 .buffer
5108 .stderr_path()
5109 .map(|path| path.display().to_string()),
5110 pty_rows: (metadata.mode == BgMode::Pty).then_some(metadata.pty_rows.unwrap_or(24)),
5111 pty_cols: (metadata.mode == BgMode::Pty).then_some(metadata.pty_cols.unwrap_or(80)),
5112 pty_screen: None,
5113 scanner_report: metadata.scanner_report.clone(),
5114 sandbox_native: metadata.sandbox_native,
5115 sandbox_unavailable: metadata.sandbox_native
5116 && open_task_artifact(&self.paths, TaskArtifact::SandboxUnavailable)
5117 .and_then(|mut file| file.read_all())
5118 .is_ok_and(|bytes| bytes == b"sandbox_unavailable"),
5119 }
5120 }
5121
5122 pub(crate) fn is_running(&self) -> bool {
5123 self.state
5124 .lock()
5125 .map(|state| {
5126 state.metadata.status == BgTaskStatus::Running
5127 || (state.metadata.mode == BgMode::Pty
5128 && state.metadata.status == BgTaskStatus::Killing)
5129 })
5130 .unwrap_or(false)
5131 }
5132
5133 fn is_terminal(&self) -> bool {
5134 self.state
5135 .lock()
5136 .map(|state| state.metadata.status.is_terminal())
5137 .unwrap_or(false)
5138 }
5139
5140 fn mark_terminal_now(&self) {
5141 if let Ok(mut terminal_at) = self.terminal_at.lock() {
5142 if terminal_at.is_none() {
5143 *terminal_at = Some(Instant::now());
5144 }
5145 }
5146 }
5147
5148 fn set_completion_delivered(
5149 &self,
5150 delivered: bool,
5151 registry: &BgTaskRegistry,
5152 ) -> Result<(), String> {
5153 let mut state = self
5154 .state
5155 .lock()
5156 .map_err(|_| "background task lock poisoned".to_string())?;
5157 let updated = registry
5158 .update_task_metadata(&self.paths, |metadata| {
5159 metadata.completion_delivered = delivered;
5160 })
5161 .map_err(|e| format!("failed to update completion delivery: {e}"))?;
5162 state.metadata = updated;
5163 Ok(())
5164 }
5165}
5166
5167#[cfg(unix)]
5188fn reap_piped_child(child_slot: &mut Option<Child>) {
5189 if let Some(mut child) = child_slot.take() {
5190 if matches!(child.try_wait(), Ok(None)) {
5191 let _ = child.wait();
5192 }
5193 }
5194}
5195
5196#[cfg(windows)]
5201fn reap_piped_child(child_slot: &mut Option<Child>) {
5202 *child_slot = None;
5203}
5204
5205fn terminal_metadata_from_marker(
5206 mut metadata: PersistedTask,
5207 marker: ExitMarker,
5208 reason: Option<String>,
5209) -> PersistedTask {
5210 match marker {
5211 ExitMarker::Code(code) => {
5212 let status = if code == 0 {
5213 BgTaskStatus::Completed
5214 } else {
5215 BgTaskStatus::Failed
5216 };
5217 metadata.mark_terminal(status, Some(code), reason);
5218 }
5219 ExitMarker::Killed => metadata.mark_terminal(
5220 BgTaskStatus::Killed,
5221 terminal_exit_code_for_status(&BgTaskStatus::Killed),
5222 reason,
5223 ),
5224 }
5225 metadata
5226}
5227
5228fn terminal_exit_code_for_status(status: &BgTaskStatus) -> Option<i32> {
5229 match status {
5230 BgTaskStatus::TimedOut => Some(124),
5231 BgTaskStatus::Killed => Some(137),
5232 _ => None,
5233 }
5234}
5235
5236fn attach_sandbox_metadata(metadata: &mut PersistedTask, spawn_plan: &SpawnPlan) {
5237 metadata.sandbox_native = spawn_plan.is_native_launcher();
5238 metadata.sandbox_temp_dir = spawn_plan.temp_dir().map(Path::to_path_buf);
5239}
5240
5241#[cfg(unix)]
5242pub(crate) fn resolve_posix_shell() -> PathBuf {
5243 static POSIX_SHELL: OnceLock<PathBuf> = OnceLock::new();
5244 POSIX_SHELL
5245 .get_or_init(|| {
5246 std::env::var_os("BASH")
5247 .filter(|value| !value.is_empty())
5248 .map(PathBuf::from)
5249 .filter(|path| path.exists())
5250 .or_else(|| which::which("bash").ok())
5251 .or_else(|| which::which("zsh").ok())
5252 .unwrap_or_else(|| PathBuf::from("/bin/sh"))
5253 })
5254 .clone()
5255}
5256
5257#[cfg(windows)]
5258fn detached_shell_command_for(
5259 shell: crate::windows_shell::WindowsShell,
5260 command: &str,
5261 exit_path: &Path,
5262 paths: &TaskPaths,
5263 creation_flags: u32,
5264) -> Result<Command, String> {
5265 use crate::windows_shell::WindowsShell;
5266 let wrapper_body = shell.wrapper_script_bytes(command, exit_path);
5279 let wrapper_ext = match shell {
5280 WindowsShell::Pwsh | WindowsShell::Powershell => "ps1",
5281 WindowsShell::Cmd => "bat",
5282 WindowsShell::Posix(_) => "sh",
5286 };
5287 let wrapper_path = paths.dir.join(format!(
5288 "{}.{}",
5289 paths
5290 .json
5291 .file_stem()
5292 .and_then(|s| s.to_str())
5293 .unwrap_or("wrapper"),
5294 wrapper_ext
5295 ));
5296 fs::write(&wrapper_path, wrapper_body)
5297 .map_err(|e| format!("failed to write background bash wrapper script: {e}"))?;
5298
5299 let mut cmd = Command::new(shell.binary().as_ref());
5300 match shell {
5301 WindowsShell::Pwsh | WindowsShell::Powershell => {
5302 cmd.args([
5305 "-NoLogo",
5306 "-NoProfile",
5307 "-NonInteractive",
5308 "-ExecutionPolicy",
5309 "Bypass",
5310 "-File",
5311 ]);
5312 cmd.arg(&wrapper_path);
5313 }
5314 WindowsShell::Cmd => {
5315 cmd.args(["/D", "/C"]);
5322 cmd.arg(&wrapper_path);
5323 }
5324 WindowsShell::Posix(_) => {
5325 cmd.arg(&wrapper_path);
5330 }
5331 }
5332
5333 cmd.creation_flags(creation_flags);
5337 Ok(cmd)
5338}
5339
5340fn spawn_detached_child(
5356 spawn_plan: &SpawnPlan,
5357 command: &str,
5358 paths: &TaskPaths,
5359 workdir: &Path,
5360 env: &HashMap<String, String>,
5361 io_handles: &mut TaskIoHandles,
5362 capture_pipeline_status: bool,
5363) -> Result<std::process::Child, String> {
5364 #[cfg(windows)]
5365 let _ = capture_pipeline_status;
5366 #[cfg(not(windows))]
5367 let _ = command;
5368 #[cfg(not(windows))]
5369 {
5370 use std::os::fd::AsRawFd;
5371
5372 let stdout = io_handles
5373 .clone_file(TaskArtifact::Stdout)
5374 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
5375 let stderr = io_handles
5376 .clone_file(TaskArtifact::Stderr)
5377 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
5378 let prepared = spawn_plan
5379 .prepared_task()
5380 .ok_or_else(|| "background task payload was not prepared".to_string())?;
5381 let payload = prepared.invocation()?;
5382 let exit = io_handles
5383 .inheritable_file(TaskArtifact::Exit)
5384 .map_err(|e| format!("failed to inherit exit marker handle: {e}"))?;
5385 let failure = io_handles
5386 .inheritable_file(TaskArtifact::SandboxUnavailable)
5387 .map_err(|e| format!("failed to inherit sandbox failure marker handle: {e}"))?;
5388 let pipeline_status = capture_pipeline_status
5389 .then(|| io_handles.inheritable_file(TaskArtifact::PipelineStatus))
5390 .transpose()
5391 .map_err(|e| format!("failed to inherit pipeline status handle: {e}"))?;
5392 let shell = spawn_plan
5393 .host_shell_path()
5394 .map(Path::to_path_buf)
5395 .unwrap_or_else(resolve_posix_shell);
5396 let pipeline_shell = super::process::pipeline_shell_kind(&shell).unwrap_or("");
5397 let pipeline_status_fd = if capture_pipeline_status {
5398 crate::sandbox_spawn::CHILD_PIPE_STATUS_FD.to_string()
5399 } else {
5400 String::new()
5401 };
5402 let args = vec![
5403 OsString::from("-c"),
5404 payload.wrapper_text.clone(),
5405 OsString::from("aft-payload-wrapper"),
5406 shell.as_os_str().to_os_string(),
5407 payload.command_text.clone(),
5408 OsString::from(crate::sandbox_spawn::CHILD_EXIT_FD.to_string()),
5409 OsString::from(pipeline_status_fd),
5410 OsString::from(pipeline_shell),
5411 ];
5412 let (mut child_command, profile_handle) = crate::sandbox_spawn::detached_command_for_plan(
5413 spawn_plan,
5414 std::ffi::OsStr::new("/bin/sh"),
5415 &args,
5416 &paths.json,
5417 crate::sandbox_spawn::CHILD_EXIT_FD,
5418 crate::sandbox_spawn::CHILD_FAILURE_FD,
5419 )?;
5420 crate::sandbox_spawn::apply_marker_fd_allowlist(
5421 &mut child_command,
5422 exit.as_raw_fd(),
5423 failure.as_raw_fd(),
5424 pipeline_status.as_ref().map(|file| file.as_raw_fd()),
5425 )?;
5426 child_command
5427 .current_dir(workdir)
5428 .envs(env)
5429 .stdin(Stdio::null())
5430 .stdout(Stdio::from(stdout))
5431 .stderr(Stdio::from(stderr));
5432 crate::sandbox_spawn::apply_sandbox_environment(spawn_plan, &mut child_command, env);
5433 let child = child_command
5434 .spawn()
5435 .map_err(|e| format!("failed to spawn background bash command: {e}"));
5436 drop((payload, exit, failure, pipeline_status, profile_handle));
5437 child
5438 }
5439 #[cfg(windows)]
5440 {
5441 use crate::windows_shell::shell_candidates;
5442 match spawn_plan {
5443 SpawnPlan::Unsandboxed | SpawnPlan::Host { .. } => {}
5444 SpawnPlan::Refused { code, .. } => return Err((*code).to_string()),
5445 SpawnPlan::Launcher { .. } => return Err("sandbox_unavailable".to_string()),
5446 }
5447 let candidates: Vec<crate::windows_shell::WindowsShell> = shell_candidates();
5458 const FLAG_CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200;
5471 const FLAG_CREATE_BREAKAWAY_FROM_JOB: u32 = 0x0100_0000;
5472 const FLAG_CREATE_NO_WINDOW: u32 = 0x0800_0000;
5473 let with_breakaway =
5474 FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP | FLAG_CREATE_BREAKAWAY_FROM_JOB;
5475 let without_breakaway = FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP;
5476 let mut last_error: Option<String> = None;
5477 for (idx, shell) in candidates.iter().enumerate() {
5478 for &flags in &[with_breakaway, without_breakaway] {
5482 let stdout = io_handles
5485 .clone_file(TaskArtifact::Stdout)
5486 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
5487 let stderr = io_handles
5488 .clone_file(TaskArtifact::Stderr)
5489 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
5490 let mut cmd =
5491 detached_shell_command_for(shell.clone(), command, &paths.exit, paths, flags)?;
5492 cmd.current_dir(workdir)
5493 .envs(env)
5494 .stdin(Stdio::null())
5495 .stdout(Stdio::from(stdout))
5496 .stderr(Stdio::from(stderr));
5497 match cmd.spawn() {
5498 Ok(child) => {
5499 if idx > 0 {
5500 crate::slog_warn!("background bash spawn fell back to {} after {} earlier candidate(s) failed; \
5501 the cached PATH probe disagreed with runtime spawn — likely PATH \
5502 inheritance, antivirus / AppLocker / Defender ASR, or sandbox policy.",
5503 shell.binary(),
5504 idx);
5505 }
5506 if flags == without_breakaway {
5507 crate::slog_warn!(
5508 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected \
5509 (likely a restrictive Job Object — CI sandbox or MDM policy). \
5510 Spawned without breakaway; the bg task will be torn down if the \
5511 AFT process group is killed."
5512 );
5513 }
5514 return Ok(child);
5515 }
5516 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
5517 crate::slog_warn!("background bash spawn: {} returned NotFound at runtime — trying next candidate",
5518 shell.binary());
5519 last_error = Some(format!("{}: {e}", shell.binary()));
5520 break;
5523 }
5524 Err(e) if flags == with_breakaway && e.raw_os_error() == Some(5) => {
5525 crate::slog_warn!(
5527 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected with \
5528 Access Denied — retrying {} without breakaway",
5529 shell.binary()
5530 );
5531 last_error = Some(format!("{}: {e}", shell.binary()));
5532 continue;
5533 }
5534 Err(e) => {
5535 return Err(format!(
5536 "failed to spawn background bash command via {}: {e}",
5537 shell.binary()
5538 ));
5539 }
5540 }
5541 }
5542 }
5543 Err(format!(
5544 "failed to spawn background bash command: no Windows shell could be spawned. \
5545 Last error: {}. PATH-probed candidates: {:?}",
5546 last_error.unwrap_or_else(|| "no candidates were attempted".to_string()),
5547 candidates.iter().map(|s| s.binary()).collect::<Vec<_>>()
5548 ))
5549 }
5550}
5551
5552#[cfg(test)]
5553fn random_slug() -> String {
5554 let mut bytes = [0u8; 8];
5562 getrandom::fill(&mut bytes).unwrap_or_else(|_| {
5564 let t = SystemTime::now()
5566 .duration_since(UNIX_EPOCH)
5567 .map(|d| d.as_nanos() as u64)
5568 .unwrap_or(0);
5569 let p = u64::from(std::process::id());
5570 bytes.copy_from_slice(&(t ^ p.rotate_left(32)).to_le_bytes());
5571 });
5572 let hex: String = bytes.iter().map(|b| format!("{b:02x}")).collect();
5574 format!("bash-{hex}")
5575}
5576
5577#[cfg(test)]
5578mod tests {
5579 use std::collections::HashMap;
5580 use std::fs;
5581 use std::io::Write;
5582 #[cfg(unix)]
5583 use std::os::unix::fs::PermissionsExt;
5584 use std::sync::atomic::{AtomicBool, AtomicUsize};
5585 use std::sync::{Arc, Mutex};
5586 use std::time::{Duration, Instant, SystemTime};
5587
5588 use super::*;
5589 use crate::bash_background::persistence::{read_task, task_paths, write_task};
5590
5591 #[cfg(unix)]
5592 const QUICK_SUCCESS_COMMAND: &str = "true";
5593 #[cfg(windows)]
5594 const QUICK_SUCCESS_COMMAND: &str = "cmd /c exit 0";
5595
5596 #[cfg(unix)]
5597 const LONG_RUNNING_COMMAND: &str = "sleep 5";
5598
5599 #[cfg(unix)]
5600 #[test]
5601 fn launcher_plans_disable_pipeline_status_capture() {
5602 let launcher = SpawnPlan::launcher_for_test(
5603 crate::sandbox_profile::SandboxProfile {
5604 v: crate::sandbox_profile::SANDBOX_PROFILE_VERSION,
5605 writable_roots: Vec::new(),
5606 write_deny: Vec::new(),
5607 write_deny_nested: Vec::new(),
5608 read_allow: Vec::new(),
5609 read_deny: Vec::new(),
5610 socket_deny: Vec::new(),
5611 cache_roots: Vec::new(),
5612 temp_dir: PathBuf::from("/tmp/aft-test-sandbox"),
5613 },
5614 PathBuf::from("/bin/true"),
5615 );
5616 assert!(!should_capture_pipeline_status(
5617 &launcher,
5618 true,
5619 Path::new("/bin/bash")
5620 ));
5621 assert!(should_capture_pipeline_status(
5622 &SpawnPlan::Unsandboxed,
5623 true,
5624 Path::new("/bin/bash")
5625 ));
5626 }
5627
5628 #[cfg(windows)]
5629 const LONG_RUNNING_COMMAND: &str = "cmd /c timeout /t 5 /nobreak > nul";
5630
5631 #[test]
5632 fn bash_memory_estimate_is_zero_when_empty_and_nonzero_for_completion_cache() {
5633 let registry = BgTaskRegistry::default();
5634 assert_eq!(registry.estimated_memory().estimated_bytes, Some(0));
5635 registry
5636 .inner
5637 .completions
5638 .lock()
5639 .unwrap()
5640 .push_back(BgCompletion {
5641 task_id: "bash-memory".to_string(),
5642 session_id: "session-memory".to_string(),
5643 status: BgTaskStatus::Completed,
5644 exit_code: Some(0),
5645 command: "printf memory".to_string(),
5646 output_preview: "resident completion output".to_string(),
5647 output_truncated: false,
5648 original_tokens: None,
5649 compressed_tokens: None,
5650 tokens_skipped: false,
5651 status_reason: None,
5652 });
5653 let estimate = registry.estimated_memory();
5654 assert!(estimate.estimated_bytes.unwrap() > 0);
5655 assert_eq!(estimate.counts["completion_caches"], 1);
5656 assert_eq!(estimate.counts["sessions"], 1);
5657 }
5658
5659 #[test]
5660 fn gh_structured_detection_rejects_piped_commands() {
5661 assert!(is_gh_structured_command(
5662 "gh issue list --json number,title"
5663 ));
5664 assert!(is_gh_structured_command(
5665 "cd repo && gh issue list --json number,title"
5666 ));
5667
5668 assert!(!is_gh_structured_command(
5669 "gh issue list --json number,title | jq '.[]'"
5670 ));
5671 assert!(!is_gh_structured_command(
5672 "gh issue list --json number,title |"
5673 ));
5674 }
5675
5676 fn insert_terminal_piped_task(
5677 registry: &BgTaskRegistry,
5678 dir: &tempfile::TempDir,
5679 command: &str,
5680 stdout: &str,
5681 stderr: &str,
5682 compressed: bool,
5683 ) -> (String, Arc<BgTask>) {
5684 let task_id = random_slug();
5685 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5686 fs::create_dir_all(&paths.dir).unwrap();
5687 fs::write(&paths.stdout, stdout).unwrap();
5688 fs::write(&paths.stderr, stderr).unwrap();
5689 let mut metadata = PersistedTask::starting(
5690 task_id.clone(),
5691 "session".to_string(),
5692 command.to_string(),
5693 dir.path().to_path_buf(),
5694 Some(dir.path().to_path_buf()),
5695 Some(30_000),
5696 true,
5697 compressed,
5698 );
5699 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
5700 write_task(&paths.json, &metadata).unwrap();
5701 registry
5702 .insert_rehydrated_task(metadata, paths, true, None)
5703 .expect("insert terminal task");
5704 let task = registry.task_for_session(&task_id, "session").unwrap();
5705 (task_id, task)
5706 }
5707
5708 #[test]
5709 fn bash_zero_preview_running_status_skips_output_read_while_explicit_preview_reads() {
5710 let registry = BgTaskRegistry::default();
5711 let dir = tempfile::tempdir().unwrap();
5712 let task_id = random_slug();
5713 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5714 fs::create_dir_all(&paths.dir).unwrap();
5715 fs::write(&paths.stdout, "live output\n").unwrap();
5716 fs::write(&paths.stderr, "").unwrap();
5717 let stdout_path = paths.stdout.clone();
5718 let mut metadata = PersistedTask::starting(
5719 task_id.clone(),
5720 "session".to_string(),
5721 "sleep 60".to_string(),
5722 dir.path().to_path_buf(),
5723 Some(dir.path().to_path_buf()),
5724 Some(30_000),
5725 true,
5726 false,
5727 );
5728 metadata.status = BgTaskStatus::Running;
5729 write_task(&paths.json, &metadata).unwrap();
5730 registry
5731 .insert_rehydrated_task(metadata, paths, false, None)
5732 .expect("insert running task");
5733
5734 crate::bash_background::buffer::reset_tail_read_count(&stdout_path);
5735 for _ in 0..5 {
5736 let snapshot = registry
5737 .status(&task_id, "session", Some(dir.path()), Some(dir.path()), 0)
5738 .expect("running snapshot");
5739 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
5740 assert!(snapshot.output_preview.is_empty());
5741 }
5742 assert_eq!(
5743 crate::bash_background::buffer::tail_read_count(&stdout_path),
5744 0
5745 );
5746
5747 let snapshot = registry
5748 .status(
5749 &task_id,
5750 "session",
5751 Some(dir.path()),
5752 Some(dir.path()),
5753 RUNNING_OUTPUT_PREVIEW_BYTES,
5754 )
5755 .expect("explicit running snapshot");
5756 assert_eq!(snapshot.output_preview, "live output\n");
5757 assert_eq!(
5758 crate::bash_background::buffer::tail_read_count(&stdout_path),
5759 1
5760 );
5761 }
5762
5763 #[test]
5764 fn artifact_read_capability_requires_exact_canonical_path_and_session() {
5765 let registry = BgTaskRegistry::default();
5766 let dir = tempfile::tempdir().unwrap();
5767 let (_task_id, task) = insert_terminal_piped_task(
5768 ®istry,
5769 &dir,
5770 "printf output",
5771 "stdout\n",
5772 "stderr\n",
5773 true,
5774 );
5775 fs::write(&task.paths.exit, "0\n").unwrap();
5776
5777 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stdout));
5778 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stderr));
5779 assert!(registry.is_session_owned_artifact_path("session", &task.paths.exit));
5780 assert!(!registry.is_session_owned_artifact_path("different-session", &task.paths.stdout));
5781 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.json));
5782
5783 let unregistered = task.paths.dir.join("unregistered-output");
5784 fs::write(&unregistered, "not a task artifact\n").unwrap();
5785 assert!(!registry.is_session_owned_artifact_path("session", &unregistered));
5786 }
5787
5788 #[cfg(unix)]
5789 #[test]
5790 fn artifact_directory_symlink_does_not_create_a_prefix_exception() {
5791 let registry = BgTaskRegistry::default();
5792 let dir = tempfile::tempdir().unwrap();
5793 let project = dir.path().join("project");
5794 fs::create_dir_all(&project).unwrap();
5795 let (_task_id, task) =
5796 insert_terminal_piped_task(®istry, &dir, "printf output", "stdout\n", "", true);
5797 let link = project.join("task-artifacts");
5798 std::os::unix::fs::symlink(&task.paths.dir, &link).unwrap();
5799 let unregistered = task.paths.dir.join("unregistered-output");
5800 fs::write(&unregistered, "not registered\n").unwrap();
5801
5802 assert!(!registry.is_session_owned_artifact_path("session", &link));
5803 assert!(
5804 !registry.is_session_owned_artifact_path("session", &link.join("unregistered-output"))
5805 );
5806 assert!(registry.is_session_owned_artifact_path(
5807 "session",
5808 &link.join(task.paths.stdout.file_name().unwrap())
5809 ));
5810
5811 let outside = dir.path().join("outside-secret");
5812 fs::write(&outside, "must stay private\n").unwrap();
5813 fs::remove_file(&task.paths.stdout).unwrap();
5814 std::os::unix::fs::symlink(&outside, &task.paths.stdout).unwrap();
5815 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.stdout));
5816 }
5817
5818 #[test]
5819 fn recovery_footer_uses_bash_status_when_artifact_is_not_registered() {
5820 let registry = BgTaskRegistry::default();
5821 let dir = tempfile::tempdir().unwrap();
5822 let task_id = "bash-1111111111111111";
5823 let paths = task_paths(dir.path(), "session", task_id).unwrap();
5824 fs::create_dir_all(&paths.dir).unwrap();
5825 fs::write(
5826 &paths.stdout,
5827 format!("{}tail\n", "output-line\n".repeat(2_000)),
5828 )
5829 .unwrap();
5830 fs::write(&paths.stderr, "").unwrap();
5831 let mut metadata = PersistedTask::starting(
5832 task_id.to_string(),
5833 "session".to_string(),
5834 "printf output".to_string(),
5835 dir.path().to_path_buf(),
5836 Some(dir.path().to_path_buf()),
5837 Some(30_000),
5838 true,
5839 true,
5840 );
5841 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
5842 write_task(&paths.json, &metadata).unwrap();
5843
5844 let cache = registry
5845 .render_terminal_output_from_paths(&metadata, &paths)
5846 .expect("terminal render");
5847
5848 assert!(cache
5849 .output_preview
5850 .contains("use bash_status({taskId: \"bash-1111111111111111\"})"));
5851 assert!(!cache.output_preview.contains("full output: read "));
5852 }
5853
5854 fn insert_terminal_pty_task(
5855 registry: &BgTaskRegistry,
5856 dir: &tempfile::TempDir,
5857 pty_output: &str,
5858 ) -> (String, Arc<BgTask>) {
5859 let task_id = random_slug();
5860 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5861 fs::create_dir_all(&paths.dir).unwrap();
5862 fs::write(&paths.pty, pty_output).unwrap();
5863 let mut metadata = PersistedTask::starting(
5864 task_id.clone(),
5865 "session".to_string(),
5866 "python".to_string(),
5867 dir.path().to_path_buf(),
5868 Some(dir.path().to_path_buf()),
5869 Some(30_000),
5870 true,
5871 true,
5872 );
5873 metadata.mode = BgMode::Pty;
5874 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
5875 write_task(&paths.json, &metadata).unwrap();
5876 registry
5877 .insert_rehydrated_task(metadata, paths, true, None)
5878 .expect("insert terminal pty task");
5879 let task = registry.task_for_session(&task_id, "session").unwrap();
5880 (task_id, task)
5881 }
5882
5883 #[cfg(unix)]
5884 fn wait_for_terminal_snapshot(
5885 registry: &BgTaskRegistry,
5886 task_id: &str,
5887 session_id: &str,
5888 project: &Path,
5889 storage: &Path,
5890 ) -> BgTaskSnapshot {
5891 let started = Instant::now();
5892 loop {
5893 let snapshot = registry
5894 .status(task_id, session_id, Some(project), Some(storage), 4096)
5895 .expect("spawned task should be visible to status");
5896 if snapshot.info.status.is_terminal() {
5897 return snapshot;
5898 }
5899 assert!(
5900 started.elapsed() < Duration::from_secs(10),
5901 "timed out waiting for task {task_id} to finish; last status={:?}",
5902 snapshot.info.status
5903 );
5904 std::thread::sleep(Duration::from_millis(50));
5905 }
5906 }
5907
5908 fn write_running_project_task(storage: &Path, project: &Path, session: &str, task_id: &str) {
5909 let paths = task_paths(storage, session, task_id).unwrap();
5910 let mut metadata = PersistedTask::starting(
5911 task_id.to_string(),
5912 session.to_string(),
5913 "sleep 60".to_string(),
5914 project.to_path_buf(),
5915 Some(project.to_path_buf()),
5916 Some(30_000),
5917 true,
5918 true,
5919 );
5920 metadata.status = BgTaskStatus::Running;
5921 metadata.child_pid = Some(std::process::id());
5927 write_task(&paths.json, &metadata).unwrap();
5928 fs::write(&paths.stdout, "still running\n").unwrap();
5929 fs::write(&paths.stderr, "").unwrap();
5930 }
5931
5932 #[test]
5933 fn status_replay_filters_same_session_by_project_root() {
5934 let project_a = tempfile::tempdir().unwrap();
5935 let project_b = tempfile::tempdir().unwrap();
5936 let storage = tempfile::tempdir().unwrap();
5937 let session = "shared-session";
5938 let task_id = "bash-2222222222222222";
5939 write_running_project_task(storage.path(), project_a.path(), session, task_id);
5940
5941 let actor_b = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5942 assert!(actor_b
5943 .status(
5944 task_id,
5945 session,
5946 Some(project_b.path()),
5947 Some(storage.path()),
5948 1024,
5949 )
5950 .is_none());
5951 assert!(actor_b.task_for_session(task_id, session).is_none());
5952
5953 let actor_a = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5954 let snapshot = actor_a
5955 .status(
5956 task_id,
5957 session,
5958 Some(project_a.path()),
5959 Some(storage.path()),
5960 1024,
5961 )
5962 .expect("owning project should replay its task");
5963 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
5964 }
5965
5966 #[cfg(unix)]
5967 #[test]
5968 fn multiline_pipeline_stdout_persists_all_lines_after_terminal_status() {
5969 let cases = [
5970 (
5971 "long-first",
5972 "sleep 0.5; printf 'one\\n' | cat\nprintf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
5973 vec!["one", "1", "three"],
5974 ),
5975 (
5976 "short-first",
5977 "printf 'one\\n' | cat\nsleep 0.2; printf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
5978 vec!["one", "1", "three"],
5979 ),
5980 (
5981 "failing-middle",
5982 "sleep 0.2; printf 'one\\n' | cat\nfalse; printf 'after-false\\n' | cat\nprintf 'three\\n' | cat",
5983 vec!["one", "after-false", "three"],
5984 ),
5985 ];
5986
5987 for (name, command, expected_lines) in cases {
5988 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5989 let dir = tempfile::tempdir().unwrap();
5990 let session_id = format!("session-{name}");
5991 let task_id = registry
5992 .spawn(
5993 SpawnPlan::Unsandboxed,
5994 command,
5995 session_id.clone(),
5996 dir.path().to_path_buf(),
5997 HashMap::new(),
5998 Some(Duration::from_secs(30)),
5999 dir.path().to_path_buf(),
6000 10,
6001 true,
6002 true,
6003 Some(dir.path().to_path_buf()),
6004 )
6005 .unwrap();
6006
6007 let snapshot = wait_for_terminal_snapshot(
6008 ®istry,
6009 &task_id,
6010 &session_id,
6011 dir.path(),
6012 dir.path(),
6013 );
6014 assert_eq!(
6015 snapshot.info.status,
6016 BgTaskStatus::Completed,
6017 "{name}: task should complete; snapshot={snapshot:?}"
6018 );
6019 assert_eq!(
6020 snapshot.exit_code,
6021 Some(0),
6022 "{name}: script should use the final command's exit code"
6023 );
6024
6025 let stdout = String::from_utf8(
6026 registry
6027 .read_artifact(&task_id, &session_id, TaskArtifact::Stdout)
6028 .expect("read validated stdout artifact"),
6029 )
6030 .expect("stdout is UTF-8");
6031 let lines: Vec<&str> = stdout.lines().collect();
6032 assert_eq!(
6033 lines, expected_lines,
6034 "{name}: raw stdout artifact must include every newline-separated command's output"
6035 );
6036 }
6037 }
6038
6039 #[test]
6040 fn recognizes_all_recovery_marker_forms() {
6041 assert!(is_recovery_marker(
6042 "[truncated output; full output: read \"/tmp/out\"]"
6043 ));
6044 assert!(is_recovery_marker(
6045 "[omitted output; see remaining: tail -n +42 \"/tmp/out\"]"
6046 ));
6047 assert!(is_recovery_marker(
6048 "[truncated output; full output unavailable]"
6049 ));
6050 assert!(is_recovery_marker(
6051 r#"[truncated 123 bytes from saved output prefix; retained output: read "/tmp/out"]"#
6052 ));
6053 }
6054
6055 #[test]
6056 fn recovery_marker_reports_disk_prefix_truncation_as_retained_output() {
6057 let recovery = RecoveryContext {
6058 dropped_by_class: BTreeMap::new(),
6059 had_inner_drop: false,
6060 offset_hint_eligible: false,
6061 offset_start_line: None,
6062 byte_truncated: false,
6063 disk_truncated_prefix_bytes: 4096,
6064 output_path: Some("/tmp/stdout".to_string()),
6065 stderr_path: None,
6066 include_stderr_path: false,
6067 artifact_access: ArtifactRecoveryAccess {
6068 task_id: "bash-test".to_string(),
6069 readable: true,
6070 },
6071 };
6072
6073 let marker = recovery_marker(&recovery).expect("disk truncation must emit marker");
6074
6075 assert!(marker.contains("truncated 4096 bytes from saved output prefix"));
6076 assert!(marker.contains(r#"retained output: read "/tmp/stdout""#));
6077 assert!(!marker.contains("full output: read"));
6078 }
6079
6080 #[test]
6081 fn killed_exit_marker_sets_nonzero_sentinel_exit_code() {
6082 let metadata = PersistedTask::starting(
6083 "task".to_string(),
6084 "session".to_string(),
6085 "cargo test".to_string(),
6086 PathBuf::from("/tmp"),
6087 None,
6088 None,
6089 true,
6090 true,
6091 );
6092
6093 let terminal = terminal_metadata_from_marker(metadata, ExitMarker::Killed, None);
6094
6095 assert_eq!(terminal.status, BgTaskStatus::Killed);
6096 assert_eq!(terminal.exit_code, Some(137));
6097 }
6098
6099 #[test]
6100 fn terminal_status_polls_use_cached_render_once_and_off_lock() {
6101 let registry = BgTaskRegistry::default();
6102 let dir = tempfile::tempdir().unwrap();
6103 let (_task_id, task) = insert_terminal_piped_task(
6104 ®istry,
6105 &dir,
6106 "custom-tool --verbose",
6107 &"stdout line\n".repeat(200_000),
6108 "",
6109 true,
6110 );
6111 let calls = Arc::new(AtomicUsize::new(0));
6112 let saw_unlocked_state = Arc::new(AtomicBool::new(false));
6113 let task_holder = Arc::new(Mutex::new(Some(Arc::clone(&task))));
6114 let calls_for_closure = Arc::clone(&calls);
6115 let unlocked_for_closure = Arc::clone(&saw_unlocked_state);
6116 let task_for_closure = Arc::clone(&task_holder);
6117 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
6118 calls_for_closure.fetch_add(1, Ordering::SeqCst);
6119 if let Some(task) = task_for_closure.lock().unwrap().as_ref() {
6120 if task.state.try_lock().is_ok() {
6121 unlocked_for_closure.store(true, Ordering::SeqCst);
6122 }
6123 }
6124 CompressionResult::new(format!("compressed {} bytes", output.len()))
6125 });
6126
6127 let first = registry
6128 .status(
6129 &task.task_id,
6130 "session",
6131 None,
6132 Some(dir.path()),
6133 RUNNING_OUTPUT_PREVIEW_BYTES,
6134 )
6135 .unwrap();
6136 let second = registry
6137 .status(
6138 &task.task_id,
6139 "session",
6140 None,
6141 Some(dir.path()),
6142 RUNNING_OUTPUT_PREVIEW_BYTES,
6143 )
6144 .unwrap();
6145 let listed = registry.list(RUNNING_OUTPUT_PREVIEW_BYTES);
6146
6147 assert_eq!(
6148 calls.load(Ordering::SeqCst),
6149 1,
6150 "terminal render must be cached"
6151 );
6152 assert!(
6153 saw_unlocked_state.load(Ordering::SeqCst),
6154 "compressor must run after releasing the task state lock"
6155 );
6156 assert!(first.output_preview.starts_with("compressed "));
6157 assert_eq!(second.output_preview, first.output_preview);
6158 assert_eq!(listed[0].output_preview, first.output_preview);
6159 }
6160
6161 #[test]
6162 fn completion_preview_success_keeps_tail_only() {
6163 let registry = BgTaskRegistry::default();
6168 let dir = tempfile::tempdir().unwrap();
6169 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
6170 let (_task_id, task) =
6171 insert_terminal_piped_task(®istry, &dir, "cat big.log", &output, "", false);
6172
6173 registry.post_terminal_transition(&task, true).unwrap();
6174 let completions = registry.drain_completions_for_session(Some("session"));
6175 assert_eq!(completions.len(), 1);
6176 let preview = &completions[0].output_preview;
6177 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
6178 assert!(!preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
6179 assert!(completions[0].output_truncated);
6180 }
6181
6182 #[test]
6183 fn completion_preview_failure_keeps_head_and_tail() {
6184 let registry = BgTaskRegistry::default();
6187 let dir = tempfile::tempdir().unwrap();
6188 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
6189 let task_id = random_slug();
6190 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
6191 fs::create_dir_all(&paths.dir).unwrap();
6192 fs::write(&paths.stdout, &output).unwrap();
6193 fs::write(&paths.stderr, "").unwrap();
6194 let mut metadata = PersistedTask::starting(
6195 task_id.clone(),
6196 "session".to_string(),
6197 "cat big.log".to_string(),
6198 dir.path().to_path_buf(),
6199 Some(dir.path().to_path_buf()),
6200 Some(30_000),
6201 true,
6202 false,
6203 );
6204 metadata.mark_terminal(BgTaskStatus::Failed, Some(1), None);
6205 write_task(&paths.json, &metadata).unwrap();
6206 registry
6207 .insert_rehydrated_task(metadata, paths, true, None)
6208 .expect("insert terminal task");
6209 let task = registry.task_for_session(&task_id, "session").unwrap();
6210
6211 registry.post_terminal_transition(&task, true).unwrap();
6212 let completions = registry.drain_completions_for_session(Some("session"));
6213 assert_eq!(completions.len(), 1);
6214 let preview = &completions[0].output_preview;
6215 assert!(preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
6216 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
6217 }
6218
6219 #[test]
6220 fn has_completions_for_session_matches_pending_delivery() {
6221 let registry = BgTaskRegistry::default();
6222 assert!(!registry.has_completions_for_session(Some("session")));
6223 assert!(!registry.has_completions_for_session(None));
6224
6225 let dir = tempfile::tempdir().unwrap();
6226 let (_task_id, task) =
6227 insert_terminal_piped_task(®istry, &dir, QUICK_SUCCESS_COMMAND, "done\n", "", false);
6228 registry.post_terminal_transition(&task, true).unwrap();
6229
6230 assert!(registry.has_completions_for_session(Some("session")));
6231 assert!(registry.has_completions_for_session(None));
6232 assert!(!registry.has_completions_for_session(Some("other-session")));
6233
6234 let completions = registry.drain_completions_for_session(Some("session"));
6235 assert_eq!(completions.len(), 1);
6236 assert_eq!(completions[0].task_id, task.task_id);
6237 }
6238
6239 #[test]
6240 fn structured_gh_json_survives_intact_and_ignores_stderr() {
6241 let registry = BgTaskRegistry::default();
6242 let dir = tempfile::tempdir().unwrap();
6243 let calls = Arc::new(AtomicUsize::new(0));
6244 let calls_for_closure = Arc::clone(&calls);
6245 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
6246 calls_for_closure.fetch_add(1, Ordering::SeqCst);
6247 CompressionResult::new(output)
6248 });
6249 let (task_id, _task) = insert_terminal_piped_task(
6250 ®istry,
6251 &dir,
6252 "gh pr view 123 --json body",
6253 "{\"body\":\"hello\"}",
6254 "warning: stderr must not join json",
6255 true,
6256 );
6257
6258 let snapshot = registry
6259 .status(
6260 &task_id,
6261 "session",
6262 None,
6263 Some(dir.path()),
6264 RUNNING_OUTPUT_PREVIEW_BYTES,
6265 )
6266 .unwrap();
6267
6268 assert_eq!(snapshot.output_preview, "{\"body\":\"hello\"}");
6269 assert!(!snapshot.output_preview.contains("warning"));
6270 assert!(!snapshot.output_truncated);
6271 assert_eq!(
6272 calls.load(Ordering::SeqCst),
6273 0,
6274 "structured JSON bypasses compression"
6275 );
6276 }
6277
6278 #[test]
6279 fn registry_emits_single_recovery_marker_for_class_drops() {
6280 let registry = BgTaskRegistry::default();
6281 let dir = tempfile::tempdir().unwrap();
6282 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
6283 let mut dropped = BTreeMap::new();
6284 dropped.insert(DropClass::Error, 18);
6285 dropped.insert(DropClass::Warning, 6);
6286 CompressionResult::with_class_drops("kept diagnostic", dropped)
6287 });
6288 let (task_id, task) =
6289 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
6290
6291 let snapshot = registry
6292 .status(
6293 &task_id,
6294 "session",
6295 None,
6296 Some(dir.path()),
6297 RUNNING_OUTPUT_PREVIEW_BYTES,
6298 )
6299 .unwrap();
6300
6301 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
6302 assert!(snapshot.output_preview.contains("+18 more errors"));
6303 assert!(snapshot.output_preview.contains("+6 more warnings"));
6304 assert!(snapshot
6305 .output_preview
6306 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
6307 assert!(!snapshot.output_preview.contains("tail -n +"));
6308 assert!(snapshot.output_truncated);
6309 }
6310
6311 #[test]
6312 fn registry_marker_reports_semantic_and_byte_drops_once() {
6313 let registry = BgTaskRegistry::default();
6314 let dir = tempfile::tempdir().unwrap();
6315 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
6316 let mut dropped = BTreeMap::new();
6317 dropped.insert(DropClass::Error, 1);
6318 CompressionResult::with_class_drops(
6319 format!("HEAD-SIGNAL\n{}TAIL-SIGNAL", "middle\n".repeat(8_000)),
6320 dropped,
6321 )
6322 });
6323 let (task_id, _task) =
6324 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
6325
6326 let snapshot = registry
6327 .status(
6328 &task_id,
6329 "session",
6330 None,
6331 Some(dir.path()),
6332 RUNNING_OUTPUT_PREVIEW_BYTES,
6333 )
6334 .unwrap();
6335
6336 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
6337 assert!(snapshot.output_preview.contains("+1 more error"));
6338 assert!(snapshot.output_preview.contains("truncated output"));
6339 assert!(snapshot.output_preview.contains("HEAD-SIGNAL"));
6340 assert!(snapshot.output_preview.contains("TAIL-SIGNAL"));
6341 assert!(!snapshot.output_preview.contains("...<truncated"));
6342 assert!(snapshot.output_truncated);
6343 }
6344
6345 #[test]
6346 fn cargo_stderr_class_drops_name_both_capture_paths() {
6347 let registry = BgTaskRegistry::default();
6348 let dir = tempfile::tempdir().unwrap();
6349 let filter_registry = crate::compress::toml_filter::FilterRegistry::default();
6350 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
6351 crate::compress::compress_with_registry_exit_code(
6352 command,
6353 &output,
6354 exit_code,
6355 &filter_registry,
6356 )
6357 });
6358 let stderr = (0..22)
6359 .map(|index| {
6360 format!(
6361 "error: cargo failure {index}\n --> src/lib.rs:{}:1\n |\n{} | boom\n",
6362 index + 1,
6363 index + 1
6364 )
6365 })
6366 .collect::<Vec<_>>()
6367 .join("\n");
6368 let (task_id, task) = insert_terminal_piped_task(
6369 ®istry,
6370 &dir,
6371 "cargo check",
6372 "Finished dev [unoptimized] target(s) in 0.01s\n",
6373 &stderr,
6374 true,
6375 );
6376
6377 let snapshot = registry
6378 .status(
6379 &task_id,
6380 "session",
6381 None,
6382 Some(dir.path()),
6383 RUNNING_OUTPUT_PREVIEW_BYTES,
6384 )
6385 .unwrap();
6386
6387 assert!(snapshot.output_preview.contains("+2 more errors"));
6388 assert!(snapshot
6389 .output_preview
6390 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
6391 assert!(snapshot
6392 .output_preview
6393 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
6394 assert!(!snapshot.output_preview.contains("tail -n +"));
6395 }
6396
6397 #[test]
6398 fn over_ceiling_structured_json_uses_pointer_not_partial_json() {
6399 let registry = BgTaskRegistry::default();
6400 let dir = tempfile::tempdir().unwrap();
6401 let body = format!("{{\"body\":\"{}\"}}", "x".repeat(60 * 1024));
6402 let (task_id, task) = insert_terminal_piped_task(
6403 ®istry,
6404 &dir,
6405 "cd /repo && gh pr view 123 --json body",
6406 &body,
6407 "",
6408 true,
6409 );
6410
6411 let snapshot = registry
6412 .status(
6413 &task_id,
6414 "session",
6415 None,
6416 Some(dir.path()),
6417 RUNNING_OUTPUT_PREVIEW_BYTES,
6418 )
6419 .unwrap();
6420
6421 assert!(snapshot.output_preview.starts_with("[JSON output "));
6422 assert!(snapshot
6423 .output_preview
6424 .contains(&task.paths.stdout.display().to_string()));
6425 assert!(!snapshot.output_preview.contains(&"x".repeat(1024)));
6426 assert!(snapshot.output_truncated);
6427 }
6428
6429 #[test]
6430 fn toml_strip_tail_cap_uses_full_output_hint_not_offset_hint() {
6431 let registry = BgTaskRegistry::default();
6432 let dir = tempfile::tempdir().unwrap();
6433 let filter_registry = crate::compress::toml_filter::build_registry(
6434 crate::compress::builtin_filters::ALL,
6435 None,
6436 None,
6437 );
6438 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
6439 crate::compress::compress_with_registry_exit_code(
6440 command,
6441 &output,
6442 exit_code,
6443 &filter_registry,
6444 )
6445 });
6446 let stdout = format!(
6447 "make[1]: Entering directory `/tmp`\n{}",
6448 (0..100)
6449 .map(|index| format!("compile line {index}"))
6450 .collect::<Vec<_>>()
6451 .join("\n")
6452 );
6453 let (task_id, task) =
6454 insert_terminal_piped_task(®istry, &dir, "make all", &stdout, "", true);
6455
6456 let snapshot = registry
6457 .status(
6458 &task_id,
6459 "session",
6460 None,
6461 Some(dir.path()),
6462 RUNNING_OUTPUT_PREVIEW_BYTES,
6463 )
6464 .unwrap();
6465
6466 assert!(snapshot.output_preview.contains("compile line 99"));
6467 assert!(snapshot.output_preview.contains(&format!(
6468 "full output: read \"{}\"",
6469 task.paths.stdout.display()
6470 )));
6471 assert!(!snapshot
6472 .output_preview
6473 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
6474 assert!(!snapshot.output_preview.contains("tail -n +"));
6475 }
6476
6477 #[test]
6478 fn compressed_false_raw_passthrough_uses_wider_head_tail_cap() {
6479 let registry = BgTaskRegistry::default();
6480 let dir = tempfile::tempdir().unwrap();
6481 let output = format!("RAW-HEAD\n{}RAW-TAIL\n", "raw-middle\n".repeat(8_000));
6482 let (task_id, task) =
6483 insert_terminal_piped_task(®istry, &dir, "cat raw.log", &output, "RAW-ERR\n", false);
6484
6485 let snapshot = registry
6486 .status(
6487 &task_id,
6488 "session",
6489 None,
6490 Some(dir.path()),
6491 RUNNING_OUTPUT_PREVIEW_BYTES,
6492 )
6493 .unwrap();
6494
6495 assert!(snapshot.output_preview.contains("RAW-HEAD"));
6496 assert!(snapshot.output_preview.contains("RAW-TAIL"));
6497 assert!(snapshot.output_preview.contains("truncated output"));
6498 assert!(snapshot
6499 .output_preview
6500 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
6501 assert!(snapshot
6502 .output_preview
6503 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
6504 assert!(!snapshot.output_preview.contains("tail -n +"));
6505 assert!(snapshot.output_preview.len() > 16 * 1024);
6506 assert!(snapshot.output_truncated);
6507 }
6508
6509 #[test]
6510 fn pty_terminal_snapshot_bypasses_line_compression() {
6511 let registry = BgTaskRegistry::default();
6512 let dir = tempfile::tempdir().unwrap();
6513 let calls = Arc::new(AtomicUsize::new(0));
6514 let calls_for_closure = Arc::clone(&calls);
6515 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
6516 calls_for_closure.fetch_add(1, Ordering::SeqCst);
6517 CompressionResult::new(output)
6518 });
6519 let (task_id, _task) = insert_terminal_pty_task(®istry, &dir, "raw\u{1b}[31m pty bytes");
6520
6521 let snapshot = registry
6522 .status(
6523 &task_id,
6524 "session",
6525 None,
6526 Some(dir.path()),
6527 RUNNING_OUTPUT_PREVIEW_BYTES,
6528 )
6529 .unwrap();
6530
6531 assert_eq!(snapshot.info.mode, BgMode::Pty);
6532 assert_eq!(snapshot.output_preview, "");
6533 assert_eq!(calls.load(Ordering::SeqCst), 0);
6534 }
6535
6536 #[test]
6537 fn pty_dimensions_are_persisted_and_returned_in_snapshot() {
6538 let registry = BgTaskRegistry::default();
6539 let dir = tempfile::tempdir().unwrap();
6540 let task_id = registry
6541 .spawn_pty(
6542 SpawnPlan::Unsandboxed,
6543 QUICK_SUCCESS_COMMAND,
6544 "session".to_string(),
6545 dir.path().to_path_buf(),
6546 HashMap::new(),
6547 Some(Duration::from_secs(30)),
6548 dir.path().to_path_buf(),
6549 10,
6550 true,
6551 false,
6552 Some(dir.path().to_path_buf()),
6553 50,
6554 120,
6555 )
6556 .unwrap();
6557
6558 let resolved =
6559 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
6560 let metadata = read_task_at(&resolved).unwrap();
6561 assert_eq!(
6562 metadata.schema_version,
6563 crate::bash_background::persistence::SCHEMA_VERSION
6564 );
6565 assert_eq!(metadata.mode, BgMode::Pty);
6566 assert_eq!(metadata.pty_rows, Some(50));
6567 assert_eq!(metadata.pty_cols, Some(120));
6568
6569 let snapshot = registry
6570 .status(&task_id, "session", None, Some(dir.path()), 1024)
6571 .unwrap();
6572 assert_eq!(snapshot.pty_rows, Some(50));
6573 assert_eq!(snapshot.pty_cols, Some(120));
6574 }
6575
6576 fn spawn_dead_child() -> std::process::Child {
6581 #[cfg(unix)]
6582 let mut cmd = std::process::Command::new("true");
6583 #[cfg(windows)]
6584 let mut cmd = {
6585 let mut c = std::process::Command::new("cmd");
6586 c.args(["/c", "exit", "0"]);
6587 c
6588 };
6589 cmd.stdin(std::process::Stdio::null());
6590 cmd.stdout(std::process::Stdio::null());
6591 cmd.stderr(std::process::Stdio::null());
6592 let mut child = cmd.spawn().expect("spawn replacement child for reap test");
6593 let started = Instant::now();
6602 loop {
6603 match child.try_wait() {
6604 Ok(Some(_)) => break,
6605 Ok(None) => {
6606 if started.elapsed() > Duration::from_secs(5) {
6607 panic!("dead-child stand-in did not exit within 5s");
6608 }
6609 std::thread::sleep(Duration::from_millis(10));
6610 }
6611 Err(error) => panic!("dead-child try_wait failed: {error}"),
6612 }
6613 }
6614 child
6615 }
6616
6617 #[test]
6618 fn ack_marks_delivered_even_when_completion_was_already_consumed_locally() {
6619 let registry = BgTaskRegistry::default();
6620 let dir = tempfile::tempdir().unwrap();
6621 let task_id = registry
6622 .spawn(
6623 SpawnPlan::Unsandboxed,
6624 LONG_RUNNING_COMMAND,
6625 "session".to_string(),
6626 dir.path().to_path_buf(),
6627 HashMap::new(),
6628 Some(Duration::from_secs(30)),
6629 dir.path().to_path_buf(),
6630 10,
6631 true,
6632 false,
6633 Some(dir.path().to_path_buf()),
6634 )
6635 .unwrap();
6636 registry
6637 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6638 .unwrap();
6639 assert_eq!(
6640 registry
6641 .drain_completions_for_session(Some("session"))
6642 .len(),
6643 1
6644 );
6645
6646 registry.inner.completions.lock().unwrap().clear();
6649
6650 assert_eq!(
6651 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
6652 vec![task_id.clone()]
6653 );
6654 assert!(registry
6655 .drain_completions_for_session(Some("session"))
6656 .is_empty());
6657
6658 let resolved =
6659 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
6660 let metadata = read_task_at(&resolved).unwrap();
6661 assert!(metadata.completion_delivered);
6662
6663 let replayed = BgTaskRegistry::default();
6664 replayed
6665 .replay_session_inner(dir.path(), "session", None)
6666 .unwrap();
6667 assert!(replayed
6668 .drain_completions_for_session(Some("session"))
6669 .is_empty());
6670 }
6671
6672 #[test]
6673 fn reclaimed_root_kills_running_task_and_persists_reason() {
6674 let registry = BgTaskRegistry::default();
6675 let root = tempfile::tempdir().unwrap();
6676 let storage = tempfile::tempdir().unwrap();
6677 let task_id = registry
6678 .spawn(
6679 SpawnPlan::Unsandboxed,
6680 LONG_RUNNING_COMMAND,
6681 "session".to_string(),
6682 root.path().to_path_buf(),
6683 HashMap::new(),
6684 Some(Duration::from_secs(30)),
6685 storage.path().to_path_buf(),
6686 10,
6687 true,
6688 false,
6689 Some(root.path().to_path_buf()),
6690 )
6691 .unwrap();
6692 let pid = registry
6693 .status(
6694 &task_id,
6695 "session",
6696 Some(root.path()),
6697 Some(storage.path()),
6698 0,
6699 )
6700 .unwrap()
6701 .child_pid
6702 .unwrap();
6703 assert!(is_process_alive(pid));
6704
6705 assert_eq!(registry.kill_running_tasks_for_root(root.path()), 1);
6706 let deadline = Instant::now() + Duration::from_secs(5);
6707 while is_process_alive(pid) {
6708 assert!(
6709 Instant::now() < deadline,
6710 "reclaimed task process survived kill"
6711 );
6712 std::thread::sleep(Duration::from_millis(20));
6713 }
6714
6715 let snapshot = registry
6716 .status(
6717 &task_id,
6718 "session",
6719 Some(root.path()),
6720 Some(storage.path()),
6721 0,
6722 )
6723 .unwrap();
6724 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
6725 assert_eq!(
6726 snapshot.info.status_reason.as_deref(),
6727 Some(ROOT_RECLAIMED_REASON)
6728 );
6729 let persisted = read_task(
6730 ®istry
6731 .task_json_path(&task_id, "session")
6732 .expect("reclaimed task metadata path"),
6733 )
6734 .expect("persisted reclaimed task");
6735 assert_eq!(
6736 persisted.status_reason.as_deref(),
6737 Some(ROOT_RECLAIMED_REASON)
6738 );
6739 let completion = registry
6740 .drain_completions_for_session(Some("session"))
6741 .pop()
6742 .expect("reclaimed task completion");
6743 assert_eq!(
6744 completion.status_reason.as_deref(),
6745 Some(ROOT_RECLAIMED_REASON)
6746 );
6747 registry.detach();
6748 }
6749
6750 #[test]
6751 fn reclaimed_root_kills_pty_task_and_preserves_reason() {
6752 let registry = BgTaskRegistry::default();
6753 let root = tempfile::tempdir().unwrap();
6754 let storage = tempfile::tempdir().unwrap();
6755 let command = if cfg!(windows) {
6756 "Start-Sleep -Seconds 30"
6757 } else {
6758 "sleep 30"
6759 };
6760 let task_id = registry
6761 .spawn_pty(
6762 SpawnPlan::Unsandboxed,
6763 command,
6764 "session".to_string(),
6765 root.path().to_path_buf(),
6766 HashMap::new(),
6767 Some(Duration::from_secs(60)),
6768 storage.path().to_path_buf(),
6769 10,
6770 true,
6771 false,
6772 Some(root.path().to_path_buf()),
6773 24,
6774 80,
6775 )
6776 .unwrap();
6777 let pid = registry
6778 .status(
6779 &task_id,
6780 "session",
6781 Some(root.path()),
6782 Some(storage.path()),
6783 0,
6784 )
6785 .unwrap()
6786 .child_pid
6787 .unwrap();
6788 assert!(is_process_alive(pid));
6789
6790 assert_eq!(registry.kill_running_tasks_for_root(root.path()), 1);
6791 let deadline = Instant::now() + Duration::from_secs(10);
6792 loop {
6793 let snapshot = registry
6794 .status(
6795 &task_id,
6796 "session",
6797 Some(root.path()),
6798 Some(storage.path()),
6799 0,
6800 )
6801 .unwrap();
6802 if snapshot.info.status.is_terminal() {
6803 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
6804 assert_eq!(
6805 snapshot.info.status_reason.as_deref(),
6806 Some(ROOT_RECLAIMED_REASON)
6807 );
6808 break;
6809 }
6810 assert!(
6811 Instant::now() < deadline,
6812 "reclaimed PTY task did not terminate"
6813 );
6814 std::thread::sleep(Duration::from_millis(20));
6815 }
6816 assert!(!is_process_alive(pid));
6817 let completion = loop {
6822 if let Some(completion) = registry
6823 .drain_completions_for_session(Some("session"))
6824 .pop()
6825 {
6826 break completion;
6827 }
6828 assert!(
6829 Instant::now() < deadline,
6830 "reclaimed PTY completion never arrived"
6831 );
6832 std::thread::sleep(Duration::from_millis(20));
6833 };
6834 assert_eq!(
6835 completion.status_reason.as_deref(),
6836 Some(ROOT_RECLAIMED_REASON)
6837 );
6838 registry.detach();
6839 }
6840
6841 #[test]
6842 fn register_watch_rejects_unknown_task() {
6843 let registry = BgTaskRegistry::default();
6844
6845 let result = registry.register_watch(
6846 "missing-task".to_string(),
6847 WatchPattern::Substring("READY".into()),
6848 true,
6849 );
6850
6851 assert_eq!(result, Err("task_not_found"));
6852 }
6853
6854 #[test]
6855 fn register_watch_on_terminal_task_scans_existing_output() {
6856 let frames = Arc::new(Mutex::new(Vec::new()));
6857 let captured = Arc::clone(&frames);
6858 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
6859 captured.lock().unwrap().push(frame);
6860 })
6861 as Box<dyn Fn(PushFrame) + Send + Sync>);
6862 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
6863 let dir = tempfile::tempdir().unwrap();
6864 let task_id = registry
6865 .spawn(
6866 SpawnPlan::Unsandboxed,
6867 LONG_RUNNING_COMMAND,
6868 "session".to_string(),
6869 dir.path().to_path_buf(),
6870 HashMap::new(),
6871 Some(Duration::from_secs(30)),
6872 dir.path().to_path_buf(),
6873 10,
6874 true,
6875 false,
6876 Some(dir.path().to_path_buf()),
6877 )
6878 .unwrap();
6879 registry
6880 .inner
6881 .shutdown
6882 .store(true, std::sync::atomic::Ordering::SeqCst);
6883 let task = registry.task_for_session(&task_id, "session").unwrap();
6884 std::fs::write(&task.paths.stdout, "READY\n").unwrap();
6885 registry
6886 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6887 .unwrap();
6888 frames.lock().unwrap().clear();
6889 registry.inner.completions.lock().unwrap().clear();
6890
6891 registry
6892 .register_watch(
6893 task_id.clone(),
6894 WatchPattern::Substring("READY".into()),
6895 true,
6896 )
6897 .unwrap();
6898
6899 let frames = frames.lock().unwrap();
6900 let frame = frames
6901 .iter()
6902 .find_map(|frame| match frame {
6903 PushFrame::BashPatternMatch(frame) => Some(frame),
6904 _ => None,
6905 })
6906 .expect("terminal watch registration should emit pattern frame");
6907 assert_eq!(frame.reason, "pattern_match");
6908 assert_eq!(frame.task_id, task_id);
6909 assert_eq!(frame.session_id, "session");
6910 assert_eq!(frame.match_text, "READY");
6911 assert_eq!(frame.match_offset, 0);
6912 assert_eq!(registry.active_watch_count(&frame.task_id), 0);
6913 let metadata = read_task(&task.paths.json).unwrap();
6914 assert!(metadata.completion_delivered);
6915 }
6916
6917 #[test]
6918 fn cleanup_finished_removes_terminal_tasks_older_than_threshold() {
6919 let registry = BgTaskRegistry::default();
6920 let dir = tempfile::tempdir().unwrap();
6921 let task_id = registry
6922 .spawn(
6923 SpawnPlan::Unsandboxed,
6924 QUICK_SUCCESS_COMMAND,
6925 "session".to_string(),
6926 dir.path().to_path_buf(),
6927 HashMap::new(),
6928 Some(Duration::from_secs(30)),
6929 dir.path().to_path_buf(),
6930 10,
6931 true,
6932 false,
6933 Some(dir.path().to_path_buf()),
6934 )
6935 .unwrap();
6936 registry
6937 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6938 .unwrap();
6939 let completions = registry.drain_completions_for_session(Some("session"));
6940 assert_eq!(completions.len(), 1);
6941 assert_eq!(
6942 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
6943 vec![task_id.clone()]
6944 );
6945
6946 registry.cleanup_finished(Duration::ZERO);
6947
6948 assert!(registry.inner.tasks.lock().unwrap().is_empty());
6949 }
6950
6951 #[test]
6952 fn cleanup_finished_retains_undelivered_terminals() {
6953 let registry = BgTaskRegistry::default();
6954 let dir = tempfile::tempdir().unwrap();
6955 let task_id = registry
6956 .spawn(
6957 SpawnPlan::Unsandboxed,
6958 QUICK_SUCCESS_COMMAND,
6959 "session".to_string(),
6960 dir.path().to_path_buf(),
6961 HashMap::new(),
6962 Some(Duration::from_secs(30)),
6963 dir.path().to_path_buf(),
6964 10,
6965 true,
6966 false,
6967 Some(dir.path().to_path_buf()),
6968 )
6969 .unwrap();
6970 registry
6971 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6972 .unwrap();
6973
6974 registry.cleanup_finished(Duration::ZERO);
6975
6976 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
6977 }
6978
6979 #[test]
6987 fn reap_child_marks_failed_when_child_exits_without_exit_marker() {
6988 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6989 let dir = tempfile::tempdir().unwrap();
6990 let task_id = registry
6991 .spawn(
6992 SpawnPlan::Unsandboxed,
6993 QUICK_SUCCESS_COMMAND,
6994 "session".to_string(),
6995 dir.path().to_path_buf(),
6996 HashMap::new(),
6997 Some(Duration::from_secs(30)),
6998 dir.path().to_path_buf(),
6999 10,
7000 true,
7001 false,
7002 Some(dir.path().to_path_buf()),
7003 )
7004 .unwrap();
7005
7006 let task = registry.task_for_session(&task_id, "session").unwrap();
7007
7008 let started = Instant::now();
7013 loop {
7014 let exited = {
7015 let mut state = task.state.lock().unwrap();
7016 match &mut state.runtime {
7017 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
7018 _ => true,
7019 }
7020 };
7021 if exited {
7022 break;
7023 }
7024 assert!(
7025 started.elapsed() < Duration::from_secs(5),
7026 "child should exit quickly"
7027 );
7028 std::thread::sleep(Duration::from_millis(20));
7029 }
7030
7031 registry
7039 .inner
7040 .shutdown
7041 .store(true, std::sync::atomic::Ordering::SeqCst);
7042 std::thread::sleep(Duration::from_millis(550));
7046
7047 let _ = std::fs::remove_file(&task.paths.exit);
7050
7051 {
7066 let mut state = task.state.lock().unwrap();
7067 state.metadata.status = BgTaskStatus::Running;
7068 state.metadata.status_reason = None;
7069 state.metadata.exit_code = None;
7070 state.metadata.finished_at = None;
7071 state.metadata.duration_ms = None;
7072 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
7075 .expect("persist reset Running metadata for reap_child test");
7076 if matches!(state.runtime, TaskRuntime::Piped(None)) {
7080 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
7081 }
7082 }
7083 *task.terminal_at.lock().unwrap() = None;
7086
7087 assert!(
7090 task.is_running(),
7091 "precondition: metadata.status == Running"
7092 );
7093 assert!(
7094 !task.paths.exit.exists(),
7095 "precondition: exit marker absent"
7096 );
7097
7098 registry.reap_child(&task);
7103
7104 {
7105 let state = task.state.lock().unwrap();
7106 assert_eq!(
7107 state.metadata.status,
7108 BgTaskStatus::Running,
7109 "first reap must leave status Running while waiting one pass for marker"
7110 );
7111 assert_eq!(
7112 state.metadata.status_reason, None,
7113 "first reap must not record a failure reason"
7114 );
7115 assert!(
7116 matches!(state.runtime, TaskRuntime::Piped(None)),
7117 "child handle must be released after first reap"
7118 );
7119 assert!(
7120 state.detached,
7121 "task must be marked detached after first reap"
7122 );
7123 }
7124
7125 registry.reap_child(&task);
7129
7130 let state = task.state.lock().unwrap();
7131 assert!(
7132 state.metadata.status.is_terminal(),
7133 "second reap must transition to terminal when PID dead and no marker. Got status={:?}",
7134 state.metadata.status
7135 );
7136 assert_eq!(
7137 state.metadata.status,
7138 BgTaskStatus::Failed,
7139 "must specifically be Failed (not Killed): status={:?}",
7140 state.metadata.status
7141 );
7142 assert_eq!(
7143 state.metadata.status_reason.as_deref(),
7144 Some("process exited without exit marker"),
7145 "reason must match replay path's wording: {:?}",
7146 state.metadata.status_reason
7147 );
7148 assert!(
7149 matches!(state.runtime, TaskRuntime::Piped(None)),
7150 "child handle must stay released after second reap"
7151 );
7152 assert!(
7153 state.detached,
7154 "task must remain detached after second reap"
7155 );
7156 }
7157
7158 #[test]
7163 fn reap_child_preserves_running_when_exit_marker_exists() {
7164 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7165 let dir = tempfile::tempdir().unwrap();
7166 let task_id = registry
7167 .spawn(
7168 SpawnPlan::Unsandboxed,
7169 QUICK_SUCCESS_COMMAND,
7170 "session".to_string(),
7171 dir.path().to_path_buf(),
7172 HashMap::new(),
7173 Some(Duration::from_secs(30)),
7174 dir.path().to_path_buf(),
7175 10,
7176 true,
7177 false,
7178 Some(dir.path().to_path_buf()),
7179 )
7180 .unwrap();
7181
7182 let task = registry.task_for_session(&task_id, "session").unwrap();
7183
7184 let started = Instant::now();
7187 loop {
7188 let exited = {
7189 let mut state = task.state.lock().unwrap();
7190 match &mut state.runtime {
7191 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
7192 _ => true,
7193 }
7194 };
7195 if exited && task.paths.exit.exists() {
7196 break;
7197 }
7198 assert!(
7199 started.elapsed() < Duration::from_secs(5),
7200 "child should exit and write marker quickly"
7201 );
7202 std::thread::sleep(Duration::from_millis(20));
7203 }
7204
7205 registry
7211 .inner
7212 .shutdown
7213 .store(true, std::sync::atomic::Ordering::SeqCst);
7214 std::thread::sleep(Duration::from_millis(550));
7215
7216 {
7222 let mut state = task.state.lock().unwrap();
7223 state.metadata.status = BgTaskStatus::Running;
7224 state.metadata.status_reason = None;
7225 if matches!(state.runtime, TaskRuntime::Piped(None)) {
7226 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
7227 }
7228 }
7229 *task.terminal_at.lock().unwrap() = None;
7230 if !task.paths.exit.exists() {
7233 std::fs::write(&task.paths.exit, "0").expect("write replacement exit marker");
7234 }
7235
7236 registry.reap_child(&task);
7240
7241 let state = task.state.lock().unwrap();
7242 assert!(
7243 matches!(state.runtime, TaskRuntime::Piped(None)),
7244 "child handle still released even when marker exists"
7245 );
7246 assert!(
7247 state.detached,
7248 "task still marked detached even when marker exists"
7249 );
7250 assert_eq!(
7255 state.metadata.status,
7256 BgTaskStatus::Running,
7257 "reap_child must defer to poll_task when marker exists"
7258 );
7259 }
7260
7261 #[cfg(unix)]
7265 fn pid_stat(pid: u32) -> Option<String> {
7266 let output = std::process::Command::new("ps")
7267 .args(["-o", "stat=", "-p", &pid.to_string()])
7268 .output()
7269 .ok()?;
7270 if !output.status.success() {
7271 return None;
7272 }
7273 let stat = String::from_utf8_lossy(&output.stdout).trim().to_string();
7274 if stat.is_empty() {
7275 None
7276 } else {
7277 Some(stat)
7278 }
7279 }
7280
7281 #[cfg(unix)]
7283 fn is_zombie(pid: u32) -> bool {
7284 pid_stat(pid).is_some_and(|stat| stat.starts_with('Z'))
7285 }
7286
7287 #[cfg(unix)]
7293 fn spawn_unreaped_zombie() -> std::process::Child {
7294 let child = std::process::Command::new("true")
7295 .stdin(std::process::Stdio::null())
7296 .stdout(std::process::Stdio::null())
7297 .stderr(std::process::Stdio::null())
7298 .spawn()
7299 .expect("spawn zombie stand-in");
7300 let pid = child.id();
7301 let started = Instant::now();
7302 while !is_zombie(pid) {
7303 assert!(
7304 started.elapsed() < Duration::from_secs(5),
7305 "stand-in child should become a zombie within 5s"
7306 );
7307 std::thread::sleep(Duration::from_millis(10));
7308 }
7309 child
7311 }
7312
7313 #[cfg(unix)]
7323 #[test]
7324 fn finalize_from_marker_reaps_child_no_zombie() {
7325 use std::sync::atomic::Ordering;
7326
7327 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7328 let dir = tempfile::tempdir().unwrap();
7329 let task_id = registry
7330 .spawn(
7331 SpawnPlan::Unsandboxed,
7332 QUICK_SUCCESS_COMMAND,
7333 "session".to_string(),
7334 dir.path().to_path_buf(),
7335 HashMap::new(),
7336 Some(Duration::from_secs(30)),
7337 dir.path().to_path_buf(),
7338 10,
7339 true,
7340 false,
7341 Some(dir.path().to_path_buf()),
7342 )
7343 .unwrap();
7344
7345 registry.inner.shutdown.store(true, Ordering::SeqCst);
7349 std::thread::sleep(Duration::from_millis(550));
7350
7351 let task = registry.task_for_session(&task_id, "session").unwrap();
7352
7353 let started = Instant::now();
7357 while !task.paths.exit.exists() {
7358 assert!(
7359 started.elapsed() < Duration::from_secs(5),
7360 "exit marker should land quickly for `true`"
7361 );
7362 std::thread::sleep(Duration::from_millis(20));
7363 }
7364
7365 let zombie_pid;
7371 {
7372 let mut state = task.state.lock().unwrap();
7373 state.metadata.status = BgTaskStatus::Running;
7374 state.metadata.status_reason = None;
7375 state.metadata.exit_code = None;
7376 state.metadata.finished_at = None;
7377 state.metadata.duration_ms = None;
7378 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
7379 .expect("persist reset Running metadata");
7380 let zombie = spawn_unreaped_zombie();
7381 zombie_pid = zombie.id();
7382 state.runtime = TaskRuntime::Piped(Some(zombie));
7383 }
7384 *task.terminal_at.lock().unwrap() = None;
7385
7386 assert!(
7388 is_zombie(zombie_pid),
7389 "precondition: stand-in child {zombie_pid} must be a zombie before finalize"
7390 );
7391
7392 registry.poll_task(&task).unwrap();
7395
7396 {
7397 let state = task.state.lock().unwrap();
7398 assert!(
7399 matches!(state.runtime, TaskRuntime::Piped(None)),
7400 "child handle must be released after marker finalize"
7401 );
7402 assert!(
7403 state.metadata.status.is_terminal(),
7404 "task must be terminal after marker finalize: {:?}",
7405 state.metadata.status
7406 );
7407 }
7408
7409 assert!(
7412 !is_zombie(zombie_pid),
7413 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
7414 after the exit-marker terminal transition"
7415 );
7416 }
7417
7418 #[cfg(unix)]
7422 #[test]
7423 fn kill_with_existing_marker_reaps_child_no_zombie() {
7424 use std::sync::atomic::Ordering;
7425
7426 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7427 let dir = tempfile::tempdir().unwrap();
7428 let task_id = registry
7429 .spawn(
7430 SpawnPlan::Unsandboxed,
7431 QUICK_SUCCESS_COMMAND,
7432 "session".to_string(),
7433 dir.path().to_path_buf(),
7434 HashMap::new(),
7435 Some(Duration::from_secs(30)),
7436 dir.path().to_path_buf(),
7437 10,
7438 true,
7439 false,
7440 Some(dir.path().to_path_buf()),
7441 )
7442 .unwrap();
7443
7444 registry.inner.shutdown.store(true, Ordering::SeqCst);
7445 std::thread::sleep(Duration::from_millis(550));
7446
7447 let task = registry.task_for_session(&task_id, "session").unwrap();
7448
7449 let started = Instant::now();
7450 while !task.paths.exit.exists() {
7451 assert!(
7452 started.elapsed() < Duration::from_secs(5),
7453 "exit marker should land quickly for `true`"
7454 );
7455 std::thread::sleep(Duration::from_millis(20));
7456 }
7457
7458 let zombie_pid;
7459 {
7460 let mut state = task.state.lock().unwrap();
7461 state.metadata.status = BgTaskStatus::Running;
7462 state.metadata.status_reason = None;
7463 state.metadata.exit_code = None;
7464 state.metadata.finished_at = None;
7465 state.metadata.duration_ms = None;
7466 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
7467 .expect("persist reset Running metadata");
7468 let zombie = spawn_unreaped_zombie();
7469 zombie_pid = zombie.id();
7470 state.runtime = TaskRuntime::Piped(Some(zombie));
7471 }
7472 *task.terminal_at.lock().unwrap() = None;
7473
7474 assert!(
7475 is_zombie(zombie_pid),
7476 "precondition: stand-in child {zombie_pid} must be a zombie before kill"
7477 );
7478
7479 registry
7481 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
7482 .expect("kill should succeed");
7483
7484 {
7485 let state = task.state.lock().unwrap();
7486 assert!(
7487 matches!(state.runtime, TaskRuntime::Piped(None)),
7488 "child handle must be released after marker-aware kill"
7489 );
7490 assert!(state.metadata.status.is_terminal());
7491 }
7492
7493 assert!(
7494 !is_zombie(zombie_pid),
7495 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
7496 after a marker-aware kill"
7497 );
7498 }
7499
7500 #[test]
7501 fn cleanup_finished_keeps_running_tasks() {
7502 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7503 let dir = tempfile::tempdir().unwrap();
7504 let task_id = registry
7505 .spawn(
7506 SpawnPlan::Unsandboxed,
7507 LONG_RUNNING_COMMAND,
7508 "session".to_string(),
7509 dir.path().to_path_buf(),
7510 HashMap::new(),
7511 Some(Duration::from_secs(30)),
7512 dir.path().to_path_buf(),
7513 10,
7514 true,
7515 false,
7516 Some(dir.path().to_path_buf()),
7517 )
7518 .unwrap();
7519
7520 registry.cleanup_finished(Duration::ZERO);
7521
7522 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
7523 let _ = registry.kill(&task_id, "session");
7524 }
7525
7526 #[cfg(unix)]
7527 #[test]
7528 fn rehydrating_sandboxed_task_never_respawns_persisted_command() {
7529 let project = tempfile::tempdir().unwrap();
7530 let storage = tempfile::tempdir().unwrap();
7531 let sandbox_temp = storage.path().join("sandbox-temp");
7532 fs::create_dir(&sandbox_temp).unwrap();
7533 let launcher_script = project.path().join("sandbox-launch");
7534 let launcher = PathBuf::from("/bin/sh");
7535 fs::write(
7536 &launcher_script,
7537 "while [ \"$#\" -gt 0 ]; do\n if [ \"$1\" = -- ]; then\n shift\n exec \"$@\"\n fi\n shift\ndone\nexit 78\n",
7538 )
7539 .unwrap();
7540 let mut permissions = fs::metadata(&launcher_script).unwrap().permissions();
7541 permissions.set_mode(0o700);
7542 fs::set_permissions(&launcher_script, permissions).unwrap();
7543
7544 let profile = crate::sandbox_profile::SandboxProfile::build(
7545 vec![project.path().to_path_buf()],
7546 Vec::new(),
7547 Vec::new(),
7548 Vec::new(),
7549 Vec::new(),
7550 Vec::new(),
7551 Vec::new(),
7552 sandbox_temp,
7553 )
7554 .unwrap();
7555 let plan = SpawnPlan::launcher_for_test(profile, launcher);
7556 let spawn_marker = project.path().join("spawn-count");
7557 let stop_marker = project.path().join("stop-command");
7558 let quote =
7559 |path: &Path| format!("'{}'", path.display().to_string().replace('\'', "'\\''"));
7560 let command = format!(
7561 "printf 'spawn\\n' >> {}; while [ ! -e {} ]; do sleep 0.05; done",
7562 quote(&spawn_marker),
7563 quote(&stop_marker)
7564 );
7565
7566 let original = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7567 let task_id = original
7568 .spawn(
7569 plan,
7570 &command,
7571 "sandbox-rehydrate".to_string(),
7572 project.path().to_path_buf(),
7573 HashMap::new(),
7574 Some(Duration::from_secs(30)),
7575 storage.path().to_path_buf(),
7576 10,
7577 true,
7578 false,
7579 Some(project.path().to_path_buf()),
7580 )
7581 .unwrap();
7582 let started = Instant::now();
7583 while !spawn_marker.exists() {
7584 assert!(
7585 started.elapsed() < Duration::from_secs(20),
7586 "original sandboxed task did not start"
7587 );
7588 std::thread::sleep(Duration::from_millis(10));
7589 }
7590 original.detach();
7591
7592 let restarted = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7593 restarted
7594 .replay_session(storage.path(), "sandbox-rehydrate")
7595 .unwrap();
7596 let replayed = restarted
7597 .status(
7598 &task_id,
7599 "sandbox-rehydrate",
7600 Some(project.path()),
7601 Some(storage.path()),
7602 4096,
7603 )
7604 .expect("rehydrated sandbox task");
7605 assert_eq!(replayed.info.status, BgTaskStatus::Running);
7606 assert!(replayed.sandbox_native);
7607
7608 std::thread::sleep(Duration::from_millis(650));
7609 assert_eq!(
7610 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
7611 1,
7612 "registry replay must observe the persisted process without spawning its command"
7613 );
7614
7615 fs::write(&stop_marker, "stop").unwrap();
7616 let terminal = wait_for_terminal_snapshot(
7617 &restarted,
7618 &task_id,
7619 "sandbox-rehydrate",
7620 project.path(),
7621 storage.path(),
7622 );
7623 assert_eq!(terminal.info.status, BgTaskStatus::Completed);
7624 assert_eq!(
7625 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
7626 1
7627 );
7628 restarted.detach();
7629 }
7630
7631 #[cfg(windows)]
7632 fn wait_for_file(path: &Path) -> String {
7633 let started = Instant::now();
7640 loop {
7641 if let Ok(content) = fs::read_to_string(path) {
7642 if !content.trim().is_empty() {
7643 return content;
7644 }
7645 }
7646 assert!(
7647 started.elapsed() < Duration::from_secs(30),
7648 "timed out waiting for non-empty {}",
7649 path.display()
7650 );
7651 std::thread::sleep(Duration::from_millis(100));
7652 }
7653 }
7654
7655 #[cfg(windows)]
7656 fn spawn_windows_registry_command(
7657 command: &str,
7658 ) -> (BgTaskRegistry, tempfile::TempDir, String) {
7659 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7660 let dir = tempfile::tempdir().unwrap();
7661 let task_id = registry
7662 .spawn(
7663 SpawnPlan::Unsandboxed,
7664 command,
7665 "session".to_string(),
7666 dir.path().to_path_buf(),
7667 HashMap::new(),
7668 Some(Duration::from_secs(30)),
7669 dir.path().to_path_buf(),
7670 10,
7671 false,
7672 false,
7673 Some(dir.path().to_path_buf()),
7674 )
7675 .unwrap();
7676 (registry, dir, task_id)
7677 }
7678
7679 #[cfg(windows)]
7680 #[test]
7681 fn windows_spawn_writes_exit_marker_for_zero_exit() {
7682 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 0");
7683 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
7684
7685 let content = wait_for_file(&exit_path);
7686
7687 assert_eq!(content.trim(), "0");
7688 }
7689
7690 #[cfg(windows)]
7691 #[test]
7692 fn windows_spawn_writes_exit_marker_for_nonzero_exit() {
7693 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 42");
7694 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
7695
7696 let content = wait_for_file(&exit_path);
7697
7698 assert_eq!(content.trim(), "42");
7699 }
7700
7701 #[cfg(windows)]
7702 #[test]
7703 fn windows_spawn_captures_stdout_to_disk() {
7704 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c echo hello");
7705 let task = registry.task_for_session(&task_id, "session").unwrap();
7706 let stdout_path = task.paths.stdout.clone();
7707 let exit_path = task.paths.exit.clone();
7708
7709 let _ = wait_for_file(&exit_path);
7710 let stdout = fs::read_to_string(stdout_path).expect("read stdout");
7711
7712 assert!(stdout.contains("hello"), "stdout was {stdout:?}");
7713 }
7714
7715 #[cfg(windows)]
7716 #[test]
7717 fn windows_spawn_uses_pwsh_when_available() {
7718 let candidates = crate::windows_shell::shell_candidates_with(
7722 |binary| match binary {
7723 "pwsh.exe" => Some(std::path::PathBuf::from(r"C:\pwsh\pwsh.exe")),
7724 "powershell.exe" => Some(std::path::PathBuf::from(r"C:\ps\powershell.exe")),
7725 _ => None,
7726 },
7727 || None,
7728 );
7729 let shell = candidates.first().expect("at least one candidate").clone();
7730 assert_eq!(shell, crate::windows_shell::WindowsShell::Pwsh);
7731 assert_eq!(shell.binary().as_ref(), "pwsh.exe");
7732 }
7733
7734 #[cfg(windows)]
7737 #[test]
7738 fn windows_shell_cmd_wrapper_writes_marker_via_temp_rename() {
7739 let exit_path = Path::new(r"C:\Temp\bash-test.exit");
7740 let script =
7741 crate::windows_shell::WindowsShell::Cmd.wrapper_script("cmd /c exit 42", exit_path);
7742
7743 assert!(
7744 script.contains("set CODE=%ERRORLEVEL%"),
7745 "wrapper must capture the child exit code: {script}"
7746 );
7747 assert!(
7748 script.contains("exit /B %CODE%"),
7749 "wrapper must propagate the child exit code: {script}"
7750 );
7751 assert!(
7756 script.contains("bash-test.exit"),
7757 "wrapper must target the exit marker path: {script}"
7758 );
7759 assert!(
7760 script.contains("move /Y"),
7761 "wrapper must write the marker atomically via temp-file + rename: {script}"
7762 );
7763 }
7764
7765 #[cfg(windows)]
7771 #[test]
7772 fn windows_shell_cmd_bg_command_uses_minimal_cmd_flags() {
7773 use crate::windows_shell::WindowsShell;
7774 let cmd = WindowsShell::Cmd.bg_command("echo wrapped");
7775 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
7776 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
7777 assert_eq!(
7778 args_strs,
7779 vec!["/D", "/S", "/C", "echo wrapped"],
7780 "Cmd::bg_command must prepend /D /S /C"
7781 );
7782 }
7783
7784 #[cfg(windows)]
7787 #[test]
7788 fn windows_shell_pwsh_bg_command_uses_standard_args() {
7789 use crate::windows_shell::WindowsShell;
7790 let cmd = WindowsShell::Pwsh.bg_command("Get-Date");
7791 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
7792 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
7793 assert!(
7794 args_strs.contains(&"-Command"),
7795 "Pwsh::bg_command must use -Command: {args_strs:?}"
7796 );
7797 assert!(
7798 args_strs.contains(&"Get-Date"),
7799 "Pwsh::bg_command must include the user command body"
7800 );
7801 }
7802
7803 fn registry_with_db_and_frames(
7804 storage: &Path,
7805 ) -> (
7806 BgTaskRegistry,
7807 Arc<Mutex<Connection>>,
7808 Arc<Mutex<Vec<PushFrame>>>,
7809 ) {
7810 let frames = Arc::new(Mutex::new(Vec::new()));
7811 let captured = Arc::clone(&frames);
7812 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
7813 captured.lock().unwrap().push(frame);
7814 })
7815 as Box<dyn Fn(PushFrame) + Send + Sync>);
7816 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
7817 registry.set_harness(Harness::Opencode);
7818 let conn = crate::db::open(&storage.join("aft.db")).expect("open test DB");
7819 let shared = Arc::new(Mutex::new(conn));
7820 registry.set_db_pool(shared.clone());
7821 (registry, shared, frames)
7822 }
7823
7824 fn pattern_match_frames(frames: &Mutex<Vec<PushFrame>>) -> Vec<BashPatternMatchFrame> {
7825 frames
7826 .lock()
7827 .unwrap()
7828 .iter()
7829 .filter_map(|frame| match frame {
7830 PushFrame::BashPatternMatch(frame) => Some(frame.clone()),
7831 _ => None,
7832 })
7833 .collect()
7834 }
7835
7836 #[cfg(unix)]
7837 #[test]
7838 fn gc_refuses_to_delete_or_quarantine_a_recorded_live_process() {
7839 let dir = tempfile::tempdir().unwrap();
7840 let storage = dir.path();
7841 let (registry, db, _frames) = registry_with_db_and_frames(storage);
7842 let task_id = "bash-0000000000000198";
7843 let paths = task_paths(storage, "session", task_id).unwrap();
7844 let mut running = PersistedTask::starting(
7845 task_id.to_string(),
7846 "session".to_string(),
7847 "live-process-canary".to_string(),
7848 storage.to_path_buf(),
7849 Some(storage.to_path_buf()),
7850 None,
7851 true,
7852 false,
7853 );
7854 running.mark_running(std::process::id(), std::process::id() as i32);
7858 write_task(&paths.json, &running).unwrap();
7859 fs::write(&paths.stdout, b"").unwrap();
7860 fs::write(&paths.stderr, b"").unwrap();
7861 {
7862 let conn = db.lock().unwrap();
7863 crate::db::bash_tasks::upsert_bash_task(
7864 &conn,
7865 &running.to_bash_task_row("opencode", &paths).unwrap(),
7866 )
7867 .unwrap();
7868 }
7869 let running_json = fs::read(&paths.json).unwrap();
7870 let mut terminal: PersistedTask = serde_json::from_slice(&running_json).unwrap();
7871 terminal.mark_terminal(BgTaskStatus::Completed, Some(0), None);
7872 terminal.completion_delivered = true;
7873 write_task_at(
7874 &resolve_task_layout(&paths.session_dir, task_id).unwrap(),
7875 &terminal,
7876 )
7877 .unwrap();
7878 let old = SystemTime::now()
7879 .checked_sub(Duration::from_secs(25 * 60 * 60))
7880 .unwrap();
7881 filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
7882
7883 assert_eq!(registry.maybe_gc_persisted(storage).unwrap(), 0);
7884 assert!(paths.io_dir.exists(), "GC deleted a live task bundle");
7885
7886 fs::write(&paths.json, b"{corrupt").unwrap();
7887 filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
7888 assert_eq!(registry.maybe_gc_persisted(storage).unwrap(), 0);
7889 assert!(
7890 paths.io_dir.exists(),
7891 "GC quarantined a live task with unreadable metadata"
7892 );
7893
7894 fs::write(&paths.json, running_json).unwrap();
7895 }
7896
7897 #[test]
7898 fn pattern_watch_survives_registry_teardown_and_rehydrate() {
7899 let dir = tempfile::tempdir().unwrap();
7900 let storage = dir.path();
7901 let (registry, _db, frames) = registry_with_db_and_frames(storage);
7902 let task_id = registry
7903 .spawn(
7904 SpawnPlan::Unsandboxed,
7905 LONG_RUNNING_COMMAND,
7906 "session".to_string(),
7907 storage.to_path_buf(),
7908 HashMap::new(),
7909 Some(Duration::from_secs(30)),
7910 storage.to_path_buf(),
7911 10,
7912 true,
7913 false,
7914 Some(storage.to_path_buf()),
7915 )
7916 .unwrap();
7917 registry
7918 .register_watch(
7919 task_id.clone(),
7920 WatchPattern::Substring("READY".into()),
7921 true,
7922 )
7923 .unwrap();
7924 let task = registry.task_for_session(&task_id, "session").unwrap();
7925 registry.clear_task_watch_state(&task_id);
7927 assert_eq!(registry.active_watch_count(&task_id), 0);
7928
7929 std::fs::OpenOptions::new()
7930 .append(true)
7931 .open(&task.paths.stdout)
7932 .unwrap()
7933 .write_all(b"READY\n")
7934 .unwrap();
7935 frames.lock().unwrap().clear();
7936
7937 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
7938 registry
7940 .inner
7941 .shutdown
7942 .store(true, std::sync::atomic::Ordering::SeqCst);
7943 replayed
7944 .replay_session_inner(storage, "session", None)
7945 .unwrap();
7946
7947 let matches = pattern_match_frames(&replay_frames);
7948 assert!(
7949 matches.iter().any(|frame| {
7950 frame.task_id == task_id
7951 && frame.reason == "pattern_match"
7952 && frame.match_text == "READY"
7953 }),
7954 "rehydrate should deliver gap match: {matches:?}"
7955 );
7956 }
7957
7958 #[test]
7959 fn pattern_watch_gap_match_between_teardown_and_rehydrate_delivers_once() {
7960 let dir = tempfile::tempdir().unwrap();
7961 let storage = dir.path();
7962 let (registry, _db, frames) = registry_with_db_and_frames(storage);
7963 let task_id = registry
7964 .spawn(
7965 SpawnPlan::Unsandboxed,
7966 LONG_RUNNING_COMMAND,
7967 "session".to_string(),
7968 storage.to_path_buf(),
7969 HashMap::new(),
7970 Some(Duration::from_secs(30)),
7971 storage.to_path_buf(),
7972 10,
7973 true,
7974 false,
7975 Some(storage.to_path_buf()),
7976 )
7977 .unwrap();
7978 registry
7979 .register_watch(
7980 task_id.clone(),
7981 WatchPattern::Substring("GAP-HIT".into()),
7982 true,
7983 )
7984 .unwrap();
7985 let task = registry.task_for_session(&task_id, "session").unwrap();
7986 let cursor_before = registry.watch_stream_cursors(&task_id).0;
7987 registry.clear_task_watch_state(&task_id);
7988
7989 std::fs::OpenOptions::new()
7991 .append(true)
7992 .open(&task.paths.stdout)
7993 .unwrap()
7994 .write_all(b"prefix GAP-HIT suffix\n")
7995 .unwrap();
7996 frames.lock().unwrap().clear();
7997
7998 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
7999 registry
8000 .inner
8001 .shutdown
8002 .store(true, std::sync::atomic::Ordering::SeqCst);
8003 replayed
8004 .replay_session_inner(storage, "session", None)
8005 .unwrap();
8006
8007 let matches: Vec<_> = pattern_match_frames(&replay_frames)
8008 .into_iter()
8009 .filter(|frame| frame.task_id == task_id && frame.match_text.contains("GAP-HIT"))
8010 .collect();
8011 assert_eq!(
8012 matches.len(),
8013 1,
8014 "gap match must deliver exactly once: {matches:?}"
8015 );
8016 assert!(
8017 matches[0].match_offset >= cursor_before,
8018 "match offset should be at/after the persisted cursor ({cursor_before}), got {}",
8019 matches[0].match_offset
8020 );
8021 }
8022
8023 #[test]
8024 fn pattern_watch_acked_match_does_not_redeliver_after_restart() {
8025 let dir = tempfile::tempdir().unwrap();
8026 let storage = dir.path();
8027 let (registry, db, frames) = registry_with_db_and_frames(storage);
8028 let task_id = registry
8029 .spawn(
8030 SpawnPlan::Unsandboxed,
8031 LONG_RUNNING_COMMAND,
8032 "session".to_string(),
8033 storage.to_path_buf(),
8034 HashMap::new(),
8035 Some(Duration::from_secs(30)),
8036 storage.to_path_buf(),
8037 10,
8038 true,
8039 false,
8040 Some(storage.to_path_buf()),
8041 )
8042 .unwrap();
8043 registry
8044 .register_watch(
8045 task_id.clone(),
8046 WatchPattern::Substring("READY".into()),
8047 true,
8048 )
8049 .unwrap();
8050 let task = registry.task_for_session(&task_id, "session").unwrap();
8051 std::fs::OpenOptions::new()
8052 .append(true)
8053 .open(&task.paths.stdout)
8054 .unwrap()
8055 .write_all(b"READY\n")
8056 .unwrap();
8057 registry.scan_task_watch_output(&task);
8058 let delivered = pattern_match_frames(&frames);
8059 assert!(
8060 delivered
8061 .iter()
8062 .any(|frame| frame.task_id == task_id && frame.match_text == "READY"),
8063 "live path should deliver match: {delivered:?}"
8064 );
8065 assert!(registry
8067 .ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id))
8068 .contains(&task_id));
8069 {
8070 let conn = db.lock().unwrap();
8071 let rows = crate::db::bash_watches::list_bash_pattern_watches_for_task(
8072 &conn, "opencode", "session", &task_id,
8073 )
8074 .unwrap();
8075 assert!(
8076 rows.is_empty(),
8077 "acked once-watch rows must be deleted: {rows:?}"
8078 );
8079 }
8080
8081 frames.lock().unwrap().clear();
8082 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
8083 registry
8084 .inner
8085 .shutdown
8086 .store(true, std::sync::atomic::Ordering::SeqCst);
8087 replayed
8088 .replay_session_inner(storage, "session", None)
8089 .unwrap();
8090 let matches = pattern_match_frames(&replay_frames)
8091 .into_iter()
8092 .filter(|frame| frame.task_id == task_id)
8093 .collect::<Vec<_>>();
8094 assert!(
8095 matches.is_empty(),
8096 "acked match must not re-deliver after restart: {matches:?}"
8097 );
8098 }
8099
8100 #[test]
8101 fn pattern_watch_rows_are_removed_when_task_is_gc_deleted() {
8102 let dir = tempfile::tempdir().unwrap();
8103 let storage = dir.path();
8104 let (registry, db, _frames) = registry_with_db_and_frames(storage);
8105 let task_id = "bash-aaaaaaaaaaaaaaaa";
8106 let paths = task_paths(storage, "session", task_id).unwrap();
8107 let mut metadata = PersistedTask::starting(
8108 task_id.to_string(),
8109 "session".to_string(),
8110 "true".to_string(),
8111 storage.to_path_buf(),
8112 Some(storage.to_path_buf()),
8113 None,
8114 true,
8115 true,
8116 );
8117 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
8118 metadata.completion_delivered = true;
8119 write_task(&paths.json, &metadata).unwrap();
8120 {
8121 let conn = db.lock().unwrap();
8122 crate::db::bash_tasks::upsert_bash_task(
8123 &conn,
8124 &metadata.to_bash_task_row("opencode", &paths).unwrap(),
8125 )
8126 .unwrap();
8127 crate::db::bash_watches::upsert_bash_pattern_watch(
8128 &conn,
8129 &BashPatternWatchRow {
8130 harness: "opencode".into(),
8131 session_id: "session".into(),
8132 task_id: task_id.into(),
8133 watch_id: "watch-00000001".into(),
8134 pattern_kind: "substring".into(),
8135 pattern: "x".into(),
8136 once: true,
8137 created_at: 1,
8138 stdout_offset: 0,
8139 stderr_offset: 0,
8140 pty_offset: 0,
8141 scanning: true,
8142 pending_match: false,
8143 match_text: None,
8144 match_offset: None,
8145 match_context: None,
8146 },
8147 )
8148 .unwrap();
8149 }
8150 let old = SystemTime::now()
8151 .checked_sub(Duration::from_secs(25 * 60 * 60))
8152 .unwrap();
8153 filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
8154
8155 let deleted = registry.maybe_gc_persisted(storage).unwrap();
8156 assert!(
8157 deleted >= 1,
8158 "expected GC to delete the terminal task bundle"
8159 );
8160 let conn = db.lock().unwrap();
8161 let watches = crate::db::bash_watches::list_bash_pattern_watches_for_task(
8162 &conn, "opencode", "session", task_id,
8163 )
8164 .unwrap();
8165 assert!(
8166 watches.is_empty(),
8167 "task GC must remove watch rows: {watches:?}"
8168 );
8169 }
8170}