1use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
2#[cfg(not(windows))]
3use std::ffi::OsString;
4use std::fs;
5use std::io::Write;
6use std::path::{Path, PathBuf};
7#[cfg(windows)]
8use std::process::Command;
9use std::process::{Child, Stdio};
10use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
11#[cfg(unix)]
12use std::sync::OnceLock;
13use std::sync::{Arc, Mutex, RwLock};
14use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
15
16use rusqlite::Connection;
17use serde::Serialize;
18
19use crate::bash_permissions::PermissionAsk;
20use crate::compress::caps::DropClass;
21#[cfg(unix)]
22use crate::compress::single_top_level_pipeline;
23use crate::compress::CompressionResult;
24use crate::context::SharedProgressSender;
25use crate::db::compression_events::CompressionAggregateCache;
26use crate::harness::Harness;
27use crate::protocol::{BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, PushFrame};
28use crate::sandbox_spawn::SpawnPlan;
29
30#[cfg(windows)]
31use std::os::windows::process::CommandExt;
32
33use super::buffer::{combine_streams, BgBuffer, DiskTruncation, StreamKind, TokenCountInput};
34use super::output::{
35 cap_completion_output, cap_completion_output_with_marker, cap_final_output,
36 cap_final_output_with_marker, completion_preview_threshold, json_output_pointer, quote_path,
37 retained_json_output_pointer, COMPRESS_INPUT_CAP_BYTES, COMPRESS_INPUT_HEAD_BYTES,
38 COMPRESS_INPUT_TAIL_BYTES, FINAL_OUTPUT_CAP_BYTES, RAW_PASSTHROUGH_CAP_BYTES,
39 RAW_PASSTHROUGH_HEAD_BYTES, RAW_PASSTHROUGH_TAIL_BYTES, RUNNING_OUTPUT_PREVIEW_BYTES,
40 STRUCTURED_OUTPUT_CAP_BYTES,
41};
42use super::persistence::{
43 allocate_task_layout, delete_resolved_task, delete_task_bundle, discover_task_ids,
44 open_task_artifact, quarantine_invalid_entry, quarantine_task_layout, read_exit_marker,
45 read_task_at, resolve_task_layout, session_tasks_dir, uninitialized_layout_is_recent,
46 unix_millis, update_task_at, validate_task_id, write_kill_marker_if_absent, write_task_at,
47 BgMode, ExitMarker, PersistedTask, TaskArtifact, TaskIoHandles, TaskPaths,
48};
49#[cfg(unix)]
50use super::process::terminate_pgid;
51#[cfg(windows)]
52use super::process::terminate_pid;
53use super::process::{is_process_alive, is_recorded_process_alive};
54use super::pty_process::spawn_pty_for_command;
55use super::pty_runtime::PtyRuntime;
56use super::watches::{PatternMatch, WatchPattern, WatchRegistry};
57use super::{BgTaskInfo, BgTaskStatus};
58use crate::db::bash_watches::BashPatternWatchRow;
59const DEFAULT_BG_TIMEOUT: Duration = Duration::from_secs(30 * 60);
62const PERSISTED_GC_GRACE: Duration = Duration::from_secs(24 * 60 * 60);
63const QUARANTINE_GC_GRACE: Duration = Duration::from_secs(30 * 24 * 60 * 60);
64
65const TOKENIZE_CAP_BYTES_PER_STREAM: usize = 128 * 1024;
66pub const ROOT_RECLAIMED_REASON: &str = "root_reclaimed";
67
68#[derive(Debug, Clone, Serialize)]
69pub struct BgCompletion {
70 pub task_id: String,
71 #[serde(skip_serializing)]
74 pub session_id: String,
75 pub status: BgTaskStatus,
76 pub exit_code: Option<i32>,
77 pub command: String,
78 #[serde(default, skip_serializing_if = "String::is_empty")]
84 pub output_preview: String,
85 #[serde(default, skip_serializing_if = "is_false")]
90 pub output_truncated: bool,
91 #[serde(default, skip_serializing_if = "Option::is_none")]
94 pub original_tokens: Option<u32>,
95 #[serde(default, skip_serializing_if = "Option::is_none")]
98 pub compressed_tokens: Option<u32>,
99 #[serde(default, skip_serializing_if = "is_false")]
101 pub tokens_skipped: bool,
102 #[serde(default, skip_serializing_if = "Option::is_none")]
103 pub status_reason: Option<String>,
104}
105
106fn is_false(v: &bool) -> bool {
107 !*v
108}
109
110#[derive(Debug, Clone, Serialize)]
111pub struct BgTaskSnapshot {
112 #[serde(flatten)]
113 pub info: BgTaskInfo,
114 pub exit_code: Option<i32>,
115 pub child_pid: Option<u32>,
116 pub workdir: String,
117 pub output_preview: String,
118 pub output_truncated: bool,
119 pub output_path: Option<String>,
120 pub stderr_path: Option<String>,
121 #[serde(skip_serializing_if = "Option::is_none")]
122 pub pty_rows: Option<u16>,
123 #[serde(skip_serializing_if = "Option::is_none")]
124 pub pty_cols: Option<u16>,
125 #[serde(skip_serializing_if = "Option::is_none")]
126 pub pty_screen: Option<String>,
127 #[serde(default, skip_serializing_if = "Vec::is_empty")]
128 pub scanner_report: Vec<PermissionAsk>,
129 #[serde(default, skip_serializing_if = "is_false")]
130 pub sandbox_native: bool,
131 #[serde(default, skip_serializing_if = "is_false")]
132 pub sandbox_unavailable: bool,
133}
134
135#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
136pub struct BgTaskHealthCounts {
137 pub running: usize,
138 pub pending_completions: usize,
139}
140
141#[derive(Debug, Clone, Copy, PartialEq, Eq)]
142enum TerminalOutputKind {
143 Compressed,
144 Raw,
145 Structured,
146}
147
148#[derive(Debug, Clone, PartialEq, Eq)]
149struct TerminalOutputCache {
150 output_preview: String,
151 output_truncated: bool,
152 kind: TerminalOutputKind,
153 output_path: Option<String>,
154 stderr_path: Option<String>,
155 artifact_access: ArtifactRecoveryAccess,
156 recovery: Option<RecoveryContext>,
157}
158
159#[derive(Debug, Clone, PartialEq, Eq)]
160struct ArtifactRecoveryAccess {
161 task_id: String,
162 readable: bool,
163}
164
165#[derive(Debug, Clone, PartialEq, Eq)]
166struct RecoveryContext {
167 dropped_by_class: BTreeMap<DropClass, usize>,
168 had_inner_drop: bool,
169 offset_hint_eligible: bool,
170 offset_start_line: Option<usize>,
171 byte_truncated: bool,
172 disk_truncated_prefix_bytes: u64,
173 output_path: Option<String>,
174 stderr_path: Option<String>,
175 include_stderr_path: bool,
176 artifact_access: ArtifactRecoveryAccess,
177}
178
179fn optional_string_bytes(value: Option<&String>) -> u64 {
180 value
181 .map(|value| crate::memory::usize_to_u64(value.len()))
182 .unwrap_or(0)
183}
184
185fn terminal_output_cache_estimated_bytes(cache: &TerminalOutputCache) -> u64 {
186 let recovery_bytes = cache
187 .recovery
188 .as_ref()
189 .map(|recovery| {
190 crate::memory::usize_to_u64(recovery.dropped_by_class.len())
191 .saturating_mul(
192 (std::mem::size_of::<DropClass>() + std::mem::size_of::<usize>()) as u64,
193 )
194 .saturating_add(optional_string_bytes(recovery.output_path.as_ref()))
195 .saturating_add(optional_string_bytes(recovery.stderr_path.as_ref()))
196 .saturating_add(crate::memory::usize_to_u64(
197 recovery.artifact_access.task_id.len(),
198 ))
199 })
200 .unwrap_or(0);
201 (std::mem::size_of::<TerminalOutputCache>() as u64)
202 .saturating_add(crate::memory::usize_to_u64(cache.output_preview.len()))
203 .saturating_add(optional_string_bytes(cache.output_path.as_ref()))
204 .saturating_add(optional_string_bytes(cache.stderr_path.as_ref()))
205 .saturating_add(crate::memory::usize_to_u64(
206 cache.artifact_access.task_id.len(),
207 ))
208 .saturating_add(recovery_bytes)
209}
210
211fn completion_estimated_bytes(completion: &BgCompletion) -> u64 {
212 (std::mem::size_of::<BgCompletion>() as u64)
213 .saturating_add(crate::memory::usize_to_u64(completion.task_id.len()))
214 .saturating_add(crate::memory::usize_to_u64(completion.session_id.len()))
215 .saturating_add(crate::memory::usize_to_u64(completion.command.len()))
216 .saturating_add(crate::memory::usize_to_u64(completion.output_preview.len()))
217}
218
219impl RecoveryContext {
220 fn has_visible_drop(&self) -> bool {
221 self.byte_truncated
222 || self.disk_truncated_prefix_bytes > 0
223 || self.had_inner_drop
224 || !self.dropped_by_class.is_empty()
225 }
226}
227
228#[derive(Clone)]
229pub struct BgTaskRegistry {
230 pub(crate) inner: Arc<RegistryInner>,
231}
232
233pub(crate) struct RegistryInner {
234 pub(crate) tasks: Mutex<HashMap<String, Arc<BgTask>>>,
235 pub(crate) completions: Mutex<VecDeque<BgCompletion>>,
236 pub(crate) progress_sender: SharedProgressSender,
237 watchdog_started: AtomicBool,
238 pub(crate) shutdown: AtomicBool,
239 pub(crate) long_running_reminder_enabled: AtomicBool,
240 pub(crate) long_running_reminder_interval_ms: AtomicU64,
241 persisted_gc_started: AtomicBool,
242 #[cfg(test)]
243 persisted_gc_runs: AtomicU64,
244 pub(crate) compressor:
250 Mutex<Option<Box<dyn Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync>>>,
251 pub(crate) db_pool: RwLock<Option<Arc<Mutex<Connection>>>>,
252 pub(crate) db_harness: RwLock<Option<String>>,
253 pub(crate) compression_aggregates: Arc<CompressionAggregateCache>,
254 pub(crate) wake_tx: crossbeam_channel::Sender<()>,
255 pub(crate) wake_rx: crossbeam_channel::Receiver<()>,
256 pub(crate) watch_registry: Mutex<WatchRegistry>,
257 wait_detach_sessions: Mutex<HashSet<String>>,
258 active_wait_sessions: Mutex<HashMap<String, usize>>,
259 wait_registered_tasks: Mutex<HashMap<String, HashSet<String>>>,
260}
261
262pub(crate) struct BgTask {
263 pub(crate) task_id: String,
264 pub(crate) session_id: String,
265 delivery_session_id: String,
266 pub(crate) paths: TaskPaths,
267 artifact_root: PathBuf,
268 pub(crate) started: Instant,
269 pub(crate) last_reminder_at: Mutex<Option<Instant>>,
270 pub(crate) terminal_at: Mutex<Option<Instant>>,
271 pub(crate) state: Mutex<BgTaskState>,
272}
273
274pub(crate) enum TaskRuntime {
275 Piped(Option<Child>),
276 Pty(Option<PtyRuntime>),
277}
278
279pub(crate) struct BgTaskState {
280 pub(crate) metadata: PersistedTask,
281 pub(crate) runtime: TaskRuntime,
282 pub(crate) io_handles: Option<TaskIoHandles>,
285 pub(crate) detached: bool,
286 pub(crate) child_exit_observed: bool,
297 pub(crate) buffer: BgBuffer,
298 terminal_output_cache: Option<TerminalOutputCache>,
299 pub(crate) pending_terminal_override: Option<BgTaskStatus>,
301}
302
303fn completion_matches_session(completion: &BgCompletion, session_id: Option<&str>) -> bool {
304 session_id
305 .map(|session_id| completion.session_id == session_id)
306 .unwrap_or(true)
307}
308
309impl BgTaskRegistry {
310 pub fn new(progress_sender: SharedProgressSender) -> Self {
311 let (wake_tx, wake_rx) = crossbeam_channel::bounded(1);
312 Self {
313 inner: Arc::new(RegistryInner {
314 tasks: Mutex::new(HashMap::new()),
315 completions: Mutex::new(VecDeque::new()),
316 progress_sender,
317 watchdog_started: AtomicBool::new(false),
318 shutdown: AtomicBool::new(false),
319 long_running_reminder_enabled: AtomicBool::new(true),
320 long_running_reminder_interval_ms: AtomicU64::new(600_000),
321 persisted_gc_started: AtomicBool::new(false),
322 #[cfg(test)]
323 persisted_gc_runs: AtomicU64::new(0),
324 compressor: Mutex::new(None),
325 db_pool: RwLock::new(None),
326 db_harness: RwLock::new(None),
327 compression_aggregates: Arc::new(CompressionAggregateCache::default()),
328 wake_tx,
329 wake_rx,
330 watch_registry: Mutex::new(WatchRegistry::default()),
331 wait_detach_sessions: Mutex::new(HashSet::new()),
332 active_wait_sessions: Mutex::new(HashMap::new()),
333 wait_registered_tasks: Mutex::new(HashMap::new()),
334 }),
335 }
336 }
337
338 pub fn is_session_owned_artifact_path(&self, session_id: &str, path: &Path) -> bool {
346 let Ok(requested) = fs::canonicalize(path) else {
347 return false;
348 };
349 let Ok(tasks) = self.inner.tasks.lock() else {
350 return false;
351 };
352
353 tasks.values().any(|task| {
354 task.session_id == session_id
355 && [
356 &task.paths.stdout,
357 &task.paths.stderr,
358 &task.paths.exit,
359 &task.paths.pty,
360 ]
361 .into_iter()
362 .filter_map(|known| known.file_name())
363 .any(|name| task.artifact_root.join(name) == requested)
364 })
365 }
366
367 pub fn read_artifact_path(
368 &self,
369 session_id: &str,
370 path: &Path,
371 ) -> Option<Result<Vec<u8>, String>> {
372 let requested = fs::canonicalize(path).ok()?;
373 let tasks = self.inner.tasks.lock().ok()?;
374 let (task, artifact) = tasks.values().find_map(|task| {
375 if task.session_id != session_id {
376 return None;
377 }
378 TaskArtifact::ALL.into_iter().find_map(|artifact| {
379 let expected = task
380 .paths
381 .artifact_path(artifact)
382 .file_name()
383 .map(|name| task.artifact_root.join(name));
384 (expected.as_deref() == Some(requested.as_path()))
385 .then(|| (Arc::clone(task), artifact))
386 })
387 })?;
388 drop(tasks);
389 Some(self.read_artifact(&task.task_id, session_id, artifact))
390 }
391
392 pub fn read_artifact_range(
393 &self,
394 task_id: &str,
395 session_id: &str,
396 artifact: TaskArtifact,
397 offset: u64,
398 ) -> Result<(Vec<u8>, u64), String> {
399 validate_task_id(task_id).map_err(|error| error.to_string())?;
400 let task = self
401 .task_for_session(task_id, session_id)
402 .ok_or_else(|| "task_not_found".to_string())?;
403 let mut file = open_task_artifact(&task.paths, artifact)
404 .map_err(|error| format!("artifact_refused: {error}"))?;
405 let len = file
406 .len()
407 .map_err(|error| format!("artifact_refused: {error}"))?;
408 let start = offset.min(len);
409 let bytes = file
410 .read_range(start, len.saturating_sub(start))
411 .map_err(|error| format!("artifact_refused: {error}"))?;
412 Ok((bytes, len))
413 }
414
415 pub fn read_artifact(
416 &self,
417 task_id: &str,
418 session_id: &str,
419 artifact: TaskArtifact,
420 ) -> Result<Vec<u8>, String> {
421 validate_task_id(task_id).map_err(|error| error.to_string())?;
422 let task = self
423 .task_for_session(task_id, session_id)
424 .ok_or_else(|| "task_not_found".to_string())?;
425 let mut file = open_task_artifact(&task.paths, artifact)
426 .map_err(|error| format!("artifact_refused: {error}"))?;
427 file.read_all()
428 .map_err(|error| format!("artifact_refused: {error}"))
429 }
430
431 pub fn set_harness(&self, harness: Harness) {
432 if let Ok(mut slot) = self.inner.db_harness.write() {
433 *slot = Some(harness.storage_segment());
434 }
435 }
436
437 pub fn set_db_pool(&self, conn: Arc<Mutex<Connection>>) {
438 if let Ok(mut slot) = self.inner.db_pool.write() {
439 *slot = Some(conn);
440 }
441 self.inner.compression_aggregates.clear();
442 }
443
444 pub fn clear_db_pool(&self) {
445 if let Ok(mut slot) = self.inner.db_pool.write() {
446 *slot = None;
447 }
448 self.inner.compression_aggregates.clear();
449 }
450
451 pub(crate) fn compression_aggregate_cache(&self) -> Arc<CompressionAggregateCache> {
452 Arc::clone(&self.inner.compression_aggregates)
453 }
454
455 pub fn register_foreground_task(&self, session_id: &str, task_id: &str) {
456 if let Ok(mut tasks) = self.inner.wait_registered_tasks.lock() {
457 tasks
458 .entry(session_id.to_string())
459 .or_default()
460 .insert(task_id.to_string());
461 }
462 }
463
464 pub fn begin_wait_mode_session(&self, session_id: &str, task_id: &str) {
465 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
466 *active.entry(session_id.to_string()).or_insert(0) += 1;
467 }
468 self.register_foreground_task(session_id, task_id);
469 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
470 detach.remove(session_id);
471 }
472 }
473
474 pub fn unregister_foreground_task(&self, session_id: &str, task_id: &str) {
475 if let Ok(mut tasks) = self.inner.wait_registered_tasks.lock() {
476 if let Some(session_tasks) = tasks.get_mut(session_id) {
477 session_tasks.remove(task_id);
478 if session_tasks.is_empty() {
479 tasks.remove(session_id);
480 }
481 }
482 }
483 }
484
485 pub fn end_wait_mode_session(&self, session_id: &str, task_id: &str) {
486 let no_active_wait = if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
487 match active.get_mut(session_id) {
488 Some(count) if *count > 1 => *count -= 1,
489 Some(_) => {
490 active.remove(session_id);
491 }
492 None => {}
493 }
494 !active.contains_key(session_id)
495 } else {
496 false
497 };
498 self.unregister_foreground_task(session_id, task_id);
499 if no_active_wait {
500 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
501 detach.remove(session_id);
502 }
503 }
504 }
505
506 pub fn abort_inflight(&self, session_id: &str) -> Result<usize, String> {
510 let task_ids = self
511 .inner
512 .wait_registered_tasks
513 .lock()
514 .map(|mut tasks| tasks.remove(session_id).unwrap_or_default())
515 .map_err(|_| "wait registration lock poisoned".to_string())?;
516 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
517 active.remove(session_id);
518 }
519 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
520 detach.remove(session_id);
521 }
522
523 let mut killed = 0;
524 for task_id in task_ids {
525 let Some(task) = self.task_for_session(&task_id, session_id) else {
526 continue;
527 };
528 let is_terminal = task
529 .state
530 .lock()
531 .map(|state| state.metadata.status.is_terminal())
532 .map_err(|_| "background task lock poisoned".to_string())?;
533 if is_terminal {
534 continue;
535 }
536 let snapshot = self.kill_with_status_reason(
537 &task_id,
538 session_id,
539 BgTaskStatus::Killed,
540 Some("call_aborted".to_string()),
541 )?;
542 if snapshot.info.status == BgTaskStatus::Killed
543 && snapshot.info.status_reason.as_deref() == Some("call_aborted")
544 {
545 killed += 1;
546 }
547 }
548 Ok(killed)
549 }
550
551 pub fn signal_wait_mode_detach(&self, session_id: &str) -> bool {
552 let is_waiting = self
553 .inner
554 .active_wait_sessions
555 .lock()
556 .map(|active| active.get(session_id).copied().unwrap_or(0) > 0)
557 .unwrap_or(false);
558 if !is_waiting {
559 return false;
560 }
561 self.inner
562 .wait_detach_sessions
563 .lock()
564 .map(|mut detach| detach.insert(session_id.to_string()))
565 .unwrap_or(false)
566 }
567
568 pub fn active_wait_session_count(&self) -> usize {
571 self.inner
572 .active_wait_sessions
573 .lock()
574 .map(|active| active.len())
575 .unwrap_or(0)
576 }
577
578 pub fn take_wait_mode_detach(&self, session_id: &str) -> bool {
579 self.inner
580 .wait_detach_sessions
581 .lock()
582 .map(|mut detach| detach.remove(session_id))
583 .unwrap_or(false)
584 }
585
586 pub fn set_compressor<F>(&self, compressor: F)
591 where
592 F: Fn(&str, String) -> CompressionResult + Send + Sync + 'static,
593 {
594 self.set_compressor_with_exit_code(move |command, output, _exit_code| {
595 compressor(command, output)
596 });
597 }
598
599 pub fn set_compressor_with_exit_code<F>(&self, compressor: F)
600 where
601 F: Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync + 'static,
602 {
603 if let Ok(mut slot) = self.inner.compressor.lock() {
604 *slot = Some(Box::new(compressor));
605 }
606 }
607
608 pub(crate) fn compress_output(
611 &self,
612 command: &str,
613 output: String,
614 exit_code: Option<i32>,
615 ) -> CompressionResult {
616 let Ok(slot) = self.inner.compressor.lock() else {
617 return CompressionResult::new(output);
618 };
619 match slot.as_ref() {
620 Some(compressor) => compressor(command, output, exit_code),
621 None => CompressionResult::new(output),
622 }
623 }
624
625 fn ensure_terminal_output_cache(&self, task: &Arc<BgTask>) -> Option<TerminalOutputCache> {
626 let (metadata, buffer) = {
627 let state = task.state.lock().ok()?;
628 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
629 return None;
630 }
631 if let Some(cache) = state.terminal_output_cache.clone() {
632 return Some(cache);
633 }
634 (state.metadata.clone(), state.buffer.clone())
635 };
636
637 let mut cap_buffer = buffer.clone();
638 let disk_truncation = cap_buffer.enforce_terminal_cap();
639 let cache =
640 self.render_terminal_output(&metadata, &cap_buffer, disk_truncation, Some(&task.paths));
641 let mut state = task.state.lock().ok()?;
642 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
643 return None;
644 }
645 if let Some(existing) = state.terminal_output_cache.clone() {
646 return Some(existing);
647 }
648 state.terminal_output_cache = Some(cache.clone());
649 Some(cache)
650 }
651
652 fn render_terminal_output(
653 &self,
654 metadata: &PersistedTask,
655 buffer: &BgBuffer,
656 disk_truncation: DiskTruncation,
657 paths: Option<&TaskPaths>,
658 ) -> TerminalOutputCache {
659 let output_readable = buffer
660 .output_path()
661 .is_some_and(|path| self.is_session_owned_artifact_path(&metadata.session_id, &path));
662 let stderr_readable = buffer
663 .stderr_path()
664 .map(|path| self.is_session_owned_artifact_path(&metadata.session_id, path))
665 .unwrap_or(true);
666 let artifact_access = ArtifactRecoveryAccess {
667 task_id: metadata.task_id.clone(),
668 readable: output_readable && stderr_readable,
669 };
670
671 if metadata.mode == BgMode::Pty {
672 return TerminalOutputCache {
673 output_preview: String::new(),
674 output_truncated: false,
675 kind: TerminalOutputKind::Raw,
676 output_path: buffer.output_path().map(|path| path.display().to_string()),
677 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
678 artifact_access,
679 recovery: None,
680 };
681 }
682
683 let mut rendered = if let Some(structured) = render_structured_output(
684 &metadata.command,
685 buffer,
686 disk_truncation,
687 artifact_access.clone(),
688 ) {
689 structured
690 } else if !metadata.compressed {
691 render_raw_passthrough(buffer, disk_truncation, artifact_access)
692 } else {
693 let raw = buffer.read_combined_head_tail(
694 COMPRESS_INPUT_CAP_BYTES,
695 COMPRESS_INPUT_HEAD_BYTES,
696 COMPRESS_INPUT_TAIL_BYTES,
697 );
698 let compressed = self.compress_output(&metadata.command, raw.text, metadata.exit_code);
699 render_compressed_with_recovery(
700 buffer,
701 compressed,
702 raw.truncated,
703 disk_truncation,
704 artifact_access,
705 )
706 };
707 normalize_piped_display_output(&mut rendered.output_preview);
708 append_pipeline_warning(&mut rendered, metadata, paths);
709 rendered
710 }
711
712 fn snapshot_with_terminal_cache(
713 &self,
714 task: &Arc<BgTask>,
715 preview_bytes: usize,
716 ) -> BgTaskSnapshot {
717 let mut snapshot = task.snapshot(preview_bytes);
718 self.maybe_compress_snapshot(task, &mut snapshot);
719 snapshot
720 }
721
722 fn post_terminal_transition(&self, task: &Arc<BgTask>, emit_frame: bool) -> Result<(), String> {
723 let (metadata, buffer) = {
724 let state = task
725 .state
726 .lock()
727 .map_err(|_| "background task lock poisoned".to_string())?;
728 if !state.metadata.status.is_terminal() {
729 return Ok(());
730 }
731 (state.metadata.clone(), state.buffer.clone())
732 };
733
734 let cache = self.ensure_terminal_output_cache(task);
735 self.enqueue_completion_from_parts(
736 &metadata,
737 Some(&buffer),
738 None,
739 emit_frame,
740 cache.as_ref(),
741 );
742 self.retarget_pending_completion(&metadata.task_id, &task.delivery_session_id);
743 Ok(())
744 }
745
746 fn persist_task(&self, paths: &TaskPaths, metadata: &PersistedTask) -> std::io::Result<()> {
747 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
748 write_task_at(&task, metadata)?;
749 self.dual_write_task(paths, metadata);
750 Ok(())
751 }
752
753 fn update_task_metadata<F>(
754 &self,
755 paths: &TaskPaths,
756 update: F,
757 ) -> std::io::Result<PersistedTask>
758 where
759 F: FnOnce(&mut PersistedTask),
760 {
761 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
762 let metadata = update_task_at(&task, update)?;
763 self.dual_write_task(paths, &metadata);
764 Ok(metadata)
765 }
766
767 fn dual_write_task(&self, paths: &TaskPaths, metadata: &PersistedTask) {
768 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
769 let Some(pool) = pool else {
770 return;
771 };
772 let harness = self
773 .inner
774 .db_harness
775 .read()
776 .ok()
777 .and_then(|slot| slot.clone());
778 let Some(harness) = harness else {
779 crate::slog_warn!(
780 "dual-write bash_task to DB skipped for {}: harness not configured",
781 metadata.task_id
782 );
783 return;
784 };
785 let row = match metadata.to_bash_task_row(&harness, paths) {
786 Ok(row) => row,
787 Err(error) => {
788 crate::slog_warn!(
789 "dual-write bash_task to DB failed for {}: {}",
790 metadata.task_id,
791 error
792 );
793 return;
794 }
795 };
796 let conn = match pool.lock() {
797 Ok(conn) => conn,
798 Err(_) => {
799 crate::slog_warn!(
800 "dual-write bash_task to DB failed for {}: db mutex poisoned",
801 metadata.task_id
802 );
803 return;
804 }
805 };
806 if let Err(error) = crate::db::bash_tasks::upsert_bash_task(&conn, &row) {
807 crate::slog_warn!(
808 "dual-write bash_task to DB failed for {}: {}",
809 metadata.task_id,
810 error
811 );
812 }
813 }
814
815 fn delete_gc_task_from_db(&self, metadata: &PersistedTask) {
816 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
817 let Some(pool) = pool else {
818 return;
819 };
820 let harness = self
821 .inner
822 .db_harness
823 .read()
824 .ok()
825 .and_then(|slot| slot.clone());
826 let Some(harness) = harness else {
827 crate::slog_warn!(
828 "GC bash_task DB delete skipped for {}: harness not configured",
829 metadata.task_id
830 );
831 return;
832 };
833 let conn = match pool.lock() {
834 Ok(conn) => conn,
835 Err(_) => {
836 crate::slog_warn!(
837 "GC bash_task DB delete failed for {}: db mutex poisoned",
838 metadata.task_id
839 );
840 return;
841 }
842 };
843 if let Err(error) = crate::db::bash_tasks::delete_delivered_terminal_bash_task(
844 &conn,
845 &harness,
846 &metadata.session_id,
847 &metadata.task_id,
848 ) {
849 crate::slog_warn!(
850 "GC bash_task DB delete failed for {}: {}",
851 metadata.task_id,
852 error
853 );
854 }
855 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
857 &conn,
858 &harness,
859 &metadata.session_id,
860 &metadata.task_id,
861 ) {
862 crate::slog_warn!(
863 "GC bash_pattern_watches DB delete failed for {}: {}",
864 metadata.task_id,
865 error
866 );
867 }
868 }
869
870 fn persisted_task_process_is_alive(metadata: &PersistedTask) -> bool {
871 let child_pid = metadata.child_pid;
872 let group_leader = metadata.pgid.and_then(|pid| u32::try_from(pid).ok());
873 child_pid
874 .into_iter()
875 .chain(group_leader)
876 .any(|pid| is_recorded_process_alive(pid, metadata.started_at))
877 }
878
879 fn db_has_live_process_for_task(&self, task_id: &str) -> bool {
880 let Some((harness, pool)) = self.db_harness_and_pool() else {
881 return false;
882 };
883 let Ok(conn) = pool.lock() else {
884 return false;
885 };
886 crate::db::bash_tasks::list_bash_tasks_by_id(&conn, &harness, task_id)
887 .map(|rows| {
888 rows.into_iter().any(|row| {
889 let started_at = u64::try_from(row.started_at).unwrap_or_default();
890 row.pid
891 .and_then(|pid| u32::try_from(pid).ok())
892 .into_iter()
893 .chain(row.pgid.and_then(|pid| u32::try_from(pid).ok()))
894 .any(|pid| is_recorded_process_alive(pid, started_at))
895 })
896 })
897 .unwrap_or(false)
898 }
899
900 fn db_harness_and_pool(&self) -> Option<(String, Arc<Mutex<Connection>>)> {
901 let pool = self
902 .inner
903 .db_pool
904 .read()
905 .ok()
906 .and_then(|slot| slot.clone())?;
907 let harness = self
908 .inner
909 .db_harness
910 .read()
911 .ok()
912 .and_then(|slot| slot.clone())?;
913 Some((harness, pool))
914 }
915
916 fn persist_watch_registration(
917 &self,
918 session_id: &str,
919 task_id: &str,
920 watch_id: &str,
921 pattern: &WatchPattern,
922 once: bool,
923 stdout_offset: u64,
924 stderr_offset: u64,
925 pty_offset: u64,
926 ) {
927 let Some((harness, pool)) = self.db_harness_and_pool() else {
928 return;
929 };
930 let Ok(conn) = pool.lock() else {
931 return;
932 };
933 let row = BashPatternWatchRow {
934 harness,
935 session_id: session_id.to_string(),
936 task_id: task_id.to_string(),
937 watch_id: watch_id.to_string(),
938 pattern_kind: pattern.kind_name().to_string(),
939 pattern: pattern.pattern_text().to_string(),
940 once,
941 created_at: unix_millis() as i64,
942 stdout_offset: stdout_offset as i64,
943 stderr_offset: stderr_offset as i64,
944 pty_offset: pty_offset as i64,
945 scanning: true,
946 pending_match: false,
947 match_text: None,
948 match_offset: None,
949 match_context: None,
950 };
951 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
952 crate::slog_warn!(
953 "persist bash_pattern_watch failed for {task_id}/{watch_id}: {error}"
954 );
955 }
956 }
957
958 fn delete_persisted_watch(&self, session_id: &str, task_id: &str, watch_id: &str) {
959 let Some((harness, pool)) = self.db_harness_and_pool() else {
960 return;
961 };
962 let Ok(conn) = pool.lock() else {
963 return;
964 };
965 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watch(
966 &conn, &harness, session_id, task_id, watch_id,
967 ) {
968 crate::slog_warn!("delete bash_pattern_watch failed for {task_id}/{watch_id}: {error}");
969 }
970 }
971
972 fn delete_persisted_watches_for_task(&self, session_id: &str, task_id: &str) {
973 let Some((harness, pool)) = self.db_harness_and_pool() else {
974 return;
975 };
976 let Ok(conn) = pool.lock() else {
977 return;
978 };
979 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
980 &conn, &harness, session_id, task_id,
981 ) {
982 crate::slog_warn!("delete bash_pattern_watches for {task_id} failed: {error}");
983 }
984 }
985
986 fn persist_watch_match(
987 &self,
988 session_id: &str,
989 task_id: &str,
990 pattern_match: &PatternMatch,
991 stdout_offset: u64,
992 stderr_offset: u64,
993 pty_offset: u64,
994 ) {
995 let Some((harness, pool)) = self.db_harness_and_pool() else {
996 return;
997 };
998 let Ok(conn) = pool.lock() else {
999 return;
1000 };
1001 let Ok(Some(mut row)) = crate::db::bash_watches::get_bash_pattern_watch(
1002 &conn,
1003 &harness,
1004 session_id,
1005 task_id,
1006 &pattern_match.watch_id,
1007 ) else {
1008 return;
1009 };
1010 row.stdout_offset = stdout_offset as i64;
1011 row.stderr_offset = stderr_offset as i64;
1012 row.pty_offset = pty_offset as i64;
1013 row.pending_match = true;
1014 row.match_text = Some(pattern_match.match_text.clone());
1015 row.match_offset = Some(pattern_match.match_offset as i64);
1016 row.match_context = Some(pattern_match.context.clone());
1017 if pattern_match.once {
1018 row.scanning = false;
1021 }
1022 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
1023 crate::slog_warn!(
1024 "persist bash_pattern_watch match failed for {}/{}: {error}",
1025 task_id,
1026 pattern_match.watch_id
1027 );
1028 }
1029 }
1030
1031 fn persist_task_watch_cursors(
1032 &self,
1033 session_id: &str,
1034 task_id: &str,
1035 stdout_offset: u64,
1036 stderr_offset: u64,
1037 pty_offset: u64,
1038 ) {
1039 let Some((harness, pool)) = self.db_harness_and_pool() else {
1040 return;
1041 };
1042 let Ok(conn) = pool.lock() else {
1043 return;
1044 };
1045 if let Err(error) = crate::db::bash_watches::update_watch_offsets_for_task(
1046 &conn,
1047 &harness,
1048 session_id,
1049 task_id,
1050 stdout_offset as i64,
1051 stderr_offset as i64,
1052 pty_offset as i64,
1053 ) {
1054 crate::slog_warn!("persist bash_pattern_watch cursors failed for {task_id}: {error}");
1055 }
1056 }
1057
1058 fn watch_stream_cursors(&self, task_id: &str) -> (u64, u64, u64) {
1059 let Ok(registry) = self.inner.watch_registry.lock() else {
1060 return (0, 0, 0);
1061 };
1062 let stdout = registry
1063 .file_cursor(&format!("{task_id}:stdout"))
1064 .unwrap_or(0);
1065 let stderr = registry
1066 .file_cursor(&format!("{task_id}:stderr"))
1067 .unwrap_or(0);
1068 let pty = registry.file_cursor(&format!("{task_id}:pty")).unwrap_or(0);
1069 (stdout, stderr, pty)
1070 }
1071
1072 fn ack_persisted_watches_for_task(&self, session_id: &str, task_id: &str, task_terminal: bool) {
1075 let Some((harness, pool)) = self.db_harness_and_pool() else {
1076 return;
1077 };
1078 let Ok(conn) = pool.lock() else {
1079 return;
1080 };
1081 if task_terminal {
1082 let _ = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
1083 &conn, &harness, session_id, task_id,
1084 );
1085 return;
1086 }
1087 let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
1088 &conn, &harness, session_id, task_id,
1089 ) else {
1090 return;
1091 };
1092 for mut row in rows {
1093 if row.once && (!row.scanning || row.pending_match) {
1094 let _ = crate::db::bash_watches::delete_bash_pattern_watch(
1095 &conn,
1096 &harness,
1097 session_id,
1098 task_id,
1099 &row.watch_id,
1100 );
1101 continue;
1102 }
1103 if row.pending_match {
1104 row.pending_match = false;
1105 row.match_text = None;
1106 row.match_offset = None;
1107 row.match_context = None;
1108 let _ = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row);
1109 }
1110 }
1111 }
1112
1113 pub fn record_scanner_report(
1114 &self,
1115 task_id: &str,
1116 session_id: &str,
1117 scanner_report: Vec<PermissionAsk>,
1118 ) -> Result<(), String> {
1119 if scanner_report.is_empty() {
1120 return Ok(());
1121 }
1122 let task = self.task_for_session(task_id, session_id).ok_or_else(|| {
1123 "background task not found while recording scanner report".to_string()
1124 })?;
1125 let metadata = {
1126 let mut state = task
1127 .state
1128 .lock()
1129 .map_err(|_| "background task lock poisoned".to_string())?;
1130 state.metadata.scanner_report = scanner_report;
1131 state.metadata.clone()
1132 };
1133 self.persist_task(&task.paths, &metadata)
1134 .map_err(|error| format!("failed to persist scanner report: {error}"))
1135 }
1136
1137 pub fn configure_long_running_reminders(&self, enabled: bool, interval_ms: u64) {
1138 self.inner
1139 .long_running_reminder_enabled
1140 .store(enabled, Ordering::SeqCst);
1141 self.inner
1142 .long_running_reminder_interval_ms
1143 .store(interval_ms, Ordering::SeqCst);
1144 }
1145
1146 #[cfg(unix)]
1147 #[allow(clippy::too_many_arguments)]
1148 pub fn spawn(
1149 &self,
1150 spawn_plan: SpawnPlan,
1151 command: &str,
1152 session_id: String,
1153 workdir: PathBuf,
1154 env: HashMap<String, String>,
1155 timeout: Option<Duration>,
1156 storage_dir: PathBuf,
1157 max_running: usize,
1158 notify_on_completion: bool,
1159 compressed: bool,
1160 project_root: Option<PathBuf>,
1161 ) -> Result<String, String> {
1162 self.spawn_with_shell(
1163 spawn_plan,
1164 command,
1165 super::BashShell::Bash,
1166 resolve_posix_shell(),
1167 session_id,
1168 workdir,
1169 env,
1170 timeout,
1171 storage_dir,
1172 max_running,
1173 notify_on_completion,
1174 compressed,
1175 project_root,
1176 )
1177 }
1178
1179 #[cfg(unix)]
1180 #[allow(clippy::too_many_arguments)]
1181 pub fn spawn_with_shell(
1182 &self,
1183 spawn_plan: SpawnPlan,
1184 command: &str,
1185 shell: super::BashShell,
1186 shell_path: PathBuf,
1187 session_id: String,
1188 workdir: PathBuf,
1189 env: HashMap<String, String>,
1190 timeout: Option<Duration>,
1191 storage_dir: PathBuf,
1192 max_running: usize,
1193 notify_on_completion: bool,
1194 compressed: bool,
1195 project_root: Option<PathBuf>,
1196 ) -> Result<String, String> {
1197 self.start_watchdog();
1198
1199 let running = self.running_count();
1200 if running >= max_running {
1201 #[cfg(unix)]
1202 if let Some(prepared) = spawn_plan.prepared_task() {
1203 let _ = delete_resolved_task(&prepared.resolved_task());
1204 }
1205 return Err(format!(
1206 "background bash task limit exceeded: {running} running (max {max_running})"
1207 ));
1208 }
1209
1210 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1211 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1212 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1213 (spawn_plan.clone(), prepared.resolved_task())
1214 } else {
1215 let task = allocate_task_layout(&storage_dir, &session_id)
1216 .map_err(|error| format!("failed to create background task layout: {error}"))?;
1217 let root = project_root.as_deref().unwrap_or(&workdir);
1218 let environment =
1219 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1220 let prepared = match crate::sandbox_spawn::prepare_task_payload(
1221 &task,
1222 command.as_bytes(),
1223 root,
1224 &workdir,
1225 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1226 &shell_path,
1227 &environment,
1228 ) {
1229 Ok(prepared) => prepared,
1230 Err(error) => {
1231 let _ = delete_resolved_task(&task);
1232 return Err(error);
1233 }
1234 };
1235 let task = prepared.resolved_task();
1236 (spawn_plan.with_prepared_task(prepared), task)
1237 };
1238 let task_id = task_layout.paths.task_id.clone();
1239 let paths = task_layout.paths.clone();
1240
1241 if self.task(&task_id).is_some() {
1242 let _ = delete_resolved_task(&task_layout);
1243 return Err("background task id collided with a live task".to_string());
1244 }
1245
1246 let mut metadata = PersistedTask::starting(
1247 task_id.clone(),
1248 session_id.clone(),
1249 command.to_string(),
1250 workdir.clone(),
1251 project_root,
1252 timeout_ms,
1253 notify_on_completion,
1254 compressed,
1255 );
1256 #[cfg(unix)]
1260 let capture_pipeline_status = {
1261 let pipeline = single_top_level_pipeline(command);
1262 let capture = !shell.is_powershell()
1263 && should_capture_pipeline_status(&spawn_plan, pipeline.is_some(), &shell_path);
1264 if capture {
1265 metadata.pipeline_segments = pipeline
1266 .as_ref()
1267 .map(|pipeline| {
1268 pipeline
1269 .segments
1270 .iter()
1271 .map(|segment| segment.label.clone())
1272 .collect()
1273 })
1274 .unwrap_or_default();
1275 }
1276 capture
1277 };
1278 #[cfg(windows)]
1279 let capture_pipeline_status = false;
1280 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1281 if let Err(error) = write_task_at(&task_layout, &metadata) {
1282 let _ = delete_resolved_task(&task_layout);
1283 return Err(format!(
1284 "failed to persist background task metadata: {error}"
1285 ));
1286 }
1287 self.dual_write_task(&paths, &metadata);
1288
1289 let mut io_handles =
1290 TaskIoHandles::create(&task_layout, BgMode::Pipes, capture_pipeline_status)
1291 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
1292 let child = match spawn_detached_child(
1293 &spawn_plan,
1294 command,
1295 shell,
1296 &shell_path,
1297 &paths,
1298 &workdir,
1299 &env,
1300 &mut io_handles,
1301 capture_pipeline_status,
1302 ) {
1303 Ok(child) => child,
1304 Err(error) => {
1305 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
1306 let _ = delete_task_bundle(&paths);
1307 return Err(error);
1308 }
1309 };
1310
1311 let child_pid = child.id();
1312 metadata.mark_running(child_pid, child_pid as i32);
1313 self.persist_task(&paths, &metadata)
1314 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1315
1316 let task = Arc::new(BgTask {
1317 task_id: task_id.clone(),
1318 delivery_session_id: session_id.clone(),
1319 session_id,
1320 paths: paths.clone(),
1321 artifact_root: canonical_artifact_root(&paths),
1322 started: Instant::now(),
1323 last_reminder_at: Mutex::new(None),
1324 terminal_at: Mutex::new(None),
1325 state: Mutex::new(BgTaskState {
1326 metadata,
1327 runtime: TaskRuntime::Piped(Some(child)),
1328 io_handles: Some(io_handles),
1329 detached: false,
1330 child_exit_observed: false,
1331 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
1332 terminal_output_cache: None,
1333 pending_terminal_override: None,
1334 }),
1335 });
1336
1337 self.inner
1338 .tasks
1339 .lock()
1340 .map_err(|_| "background task registry lock poisoned".to_string())?
1341 .insert(task_id.clone(), task);
1342
1343 Ok(task_id)
1344 }
1345
1346 #[allow(clippy::too_many_arguments)]
1347 pub fn spawn_pty(
1348 &self,
1349 spawn_plan: SpawnPlan,
1350 command: &str,
1351 session_id: String,
1352 workdir: PathBuf,
1353 env: HashMap<String, String>,
1354 timeout: Option<Duration>,
1355 storage_dir: PathBuf,
1356 max_running: usize,
1357 notify_on_completion: bool,
1358 compressed: bool,
1359 project_root: Option<PathBuf>,
1360 rows: u16,
1361 cols: u16,
1362 ) -> Result<String, String> {
1363 self.spawn_pty_with_shell(
1364 spawn_plan,
1365 command,
1366 super::BashShell::Bash,
1367 super::resolve_shell_path(true, super::BashShell::Bash)
1368 .expect("POSIX shell must resolve for bash PTY"),
1369 session_id,
1370 workdir,
1371 env,
1372 timeout,
1373 storage_dir,
1374 max_running,
1375 notify_on_completion,
1376 compressed,
1377 project_root,
1378 rows,
1379 cols,
1380 )
1381 }
1382
1383 #[allow(clippy::too_many_arguments)]
1384 pub fn spawn_pty_with_shell(
1385 &self,
1386 spawn_plan: SpawnPlan,
1387 command: &str,
1388 shell: super::BashShell,
1389 shell_path: PathBuf,
1390 session_id: String,
1391 workdir: PathBuf,
1392 env: HashMap<String, String>,
1393 timeout: Option<Duration>,
1394 storage_dir: PathBuf,
1395 max_running: usize,
1396 notify_on_completion: bool,
1397 compressed: bool,
1398 project_root: Option<PathBuf>,
1399 rows: u16,
1400 cols: u16,
1401 ) -> Result<String, String> {
1402 self.start_watchdog();
1403
1404 let running = self.running_count();
1405 if running >= max_running {
1406 #[cfg(unix)]
1407 if let Some(prepared) = spawn_plan.prepared_task() {
1408 let _ = delete_resolved_task(&prepared.resolved_task());
1409 }
1410 return Err(format!(
1411 "background bash task limit exceeded: {running} running (max {max_running})"
1412 ));
1413 }
1414
1415 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1416 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1417 #[cfg(unix)]
1418 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1419 (spawn_plan.clone(), prepared.resolved_task())
1420 } else {
1421 let task = allocate_task_layout(&storage_dir, &session_id)
1422 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1423 let root = project_root.as_deref().unwrap_or(&workdir);
1424 let environment =
1425 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1426 let prepared = match crate::sandbox_spawn::prepare_task_payload(
1427 &task,
1428 command.as_bytes(),
1429 root,
1430 &workdir,
1431 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1432 &shell_path,
1433 &environment,
1434 ) {
1435 Ok(prepared) => prepared,
1436 Err(error) => {
1437 let _ = delete_resolved_task(&task);
1438 return Err(error);
1439 }
1440 };
1441 let task = prepared.resolved_task();
1442 (spawn_plan.with_prepared_task(prepared), task)
1443 };
1444 #[cfg(windows)]
1445 let task_layout = allocate_task_layout(&storage_dir, &session_id)
1446 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1447 let task_id = task_layout.paths.task_id.clone();
1448 let paths = task_layout.paths.clone();
1449
1450 let mut metadata = PersistedTask::starting(
1451 task_id.clone(),
1452 session_id.clone(),
1453 command.to_string(),
1454 workdir.clone(),
1455 project_root,
1456 timeout_ms,
1457 notify_on_completion,
1458 compressed,
1459 );
1460 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1461 metadata.mode = BgMode::Pty;
1462 metadata.pty_rows = Some(rows);
1463 metadata.pty_cols = Some(cols);
1464 if let Err(error) = write_task_at(&task_layout, &metadata) {
1465 let _ = delete_resolved_task(&task_layout);
1466 return Err(format!(
1467 "failed to persist background task metadata: {error}"
1468 ));
1469 }
1470 self.dual_write_task(&paths, &metadata);
1471 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pty, false)
1472 .map_err(|error| format!("failed to pre-open PTY output handles: {error}"))?;
1473
1474 let runtime = match spawn_pty_for_command(
1475 &spawn_plan,
1476 &task_id,
1477 &session_id,
1478 command,
1479 shell,
1480 &shell_path,
1481 &paths,
1482 &workdir,
1483 &env,
1484 rows,
1485 cols,
1486 self.inner.wake_tx.clone(),
1487 &mut io_handles,
1488 ) {
1489 Ok(runtime) => runtime,
1490 Err(error) => {
1491 crate::slog_warn!(
1492 "failed to spawn PTY background bash task {task_id}; deleting partial bundle: {error}"
1493 );
1494 let _ = delete_task_bundle(&paths);
1495 return Err(error);
1496 }
1497 };
1498
1499 if let Some(child_pid) = runtime.child_pid {
1500 metadata.mark_running(child_pid, child_pid as i32);
1501 } else {
1502 metadata.status = BgTaskStatus::Running;
1503 metadata.pgid = None;
1504 }
1505 self.persist_task(&paths, &metadata)
1506 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1507
1508 let task = Arc::new(BgTask {
1509 task_id: task_id.clone(),
1510 delivery_session_id: session_id.clone(),
1511 session_id,
1512 paths: paths.clone(),
1513 artifact_root: canonical_artifact_root(&paths),
1514 started: Instant::now(),
1515 last_reminder_at: Mutex::new(None),
1516 terminal_at: Mutex::new(None),
1517 state: Mutex::new(BgTaskState {
1518 metadata,
1519 runtime: TaskRuntime::Pty(Some(runtime)),
1520 io_handles: Some(io_handles),
1521 detached: false,
1522 child_exit_observed: false,
1523 buffer: BgBuffer::registered(&paths, BgMode::Pty),
1524 terminal_output_cache: None,
1525 pending_terminal_override: None,
1526 }),
1527 });
1528
1529 self.inner
1530 .tasks
1531 .lock()
1532 .map_err(|_| "background task registry lock poisoned".to_string())?
1533 .insert(task_id.clone(), task);
1534
1535 Ok(task_id)
1536 }
1537
1538 #[cfg(windows)]
1539 #[allow(clippy::too_many_arguments)]
1540 pub fn spawn(
1541 &self,
1542 spawn_plan: SpawnPlan,
1543 command: &str,
1544 session_id: String,
1545 workdir: PathBuf,
1546 env: HashMap<String, String>,
1547 timeout: Option<Duration>,
1548 storage_dir: PathBuf,
1549 max_running: usize,
1550 notify_on_completion: bool,
1551 compressed: bool,
1552 project_root: Option<PathBuf>,
1553 ) -> Result<String, String> {
1554 self.spawn_with_shell(
1555 spawn_plan,
1556 command,
1557 super::BashShell::Bash,
1558 PathBuf::from("cmd.exe"),
1559 session_id,
1560 workdir,
1561 env,
1562 timeout,
1563 storage_dir,
1564 max_running,
1565 notify_on_completion,
1566 compressed,
1567 project_root,
1568 )
1569 }
1570
1571 #[cfg(windows)]
1572 #[allow(clippy::too_many_arguments)]
1573 pub fn spawn_with_shell(
1574 &self,
1575 spawn_plan: SpawnPlan,
1576 command: &str,
1577 shell: super::BashShell,
1578 shell_path: PathBuf,
1579 session_id: String,
1580 workdir: PathBuf,
1581 env: HashMap<String, String>,
1582 timeout: Option<Duration>,
1583 storage_dir: PathBuf,
1584 max_running: usize,
1585 notify_on_completion: bool,
1586 compressed: bool,
1587 project_root: Option<PathBuf>,
1588 ) -> Result<String, String> {
1589 self.start_watchdog();
1590
1591 let running = self.running_count();
1592 if running >= max_running {
1593 #[cfg(unix)]
1594 if let Some(prepared) = spawn_plan.prepared_task() {
1595 let _ = delete_resolved_task(&prepared.resolved_task());
1596 }
1597 return Err(format!(
1598 "background bash task limit exceeded: {running} running (max {max_running})"
1599 ));
1600 }
1601
1602 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1603 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1604 let task_layout = allocate_task_layout(&storage_dir, &session_id)
1605 .map_err(|error| format!("failed to create background task layout: {error}"))?;
1606 let task_id = task_layout.paths.task_id.clone();
1607 let paths = task_layout.paths.clone();
1608
1609 let mut metadata = PersistedTask::starting(
1610 task_id.clone(),
1611 session_id.clone(),
1612 command.to_string(),
1613 workdir.clone(),
1614 project_root,
1615 timeout_ms,
1616 notify_on_completion,
1617 compressed,
1618 );
1619 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1620 if let Err(error) = write_task_at(&task_layout, &metadata) {
1621 let _ = delete_resolved_task(&task_layout);
1622 return Err(format!(
1623 "failed to persist background task metadata: {error}"
1624 ));
1625 }
1626 self.dual_write_task(&paths, &metadata);
1627 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pipes, false)
1628 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
1629
1630 let child = match spawn_detached_child(
1631 &spawn_plan,
1632 command,
1633 shell,
1634 &shell_path,
1635 &paths,
1636 &workdir,
1637 &env,
1638 &mut io_handles,
1639 false,
1640 ) {
1641 Ok(child) => child,
1642 Err(error) => {
1643 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
1644 let _ = delete_task_bundle(&paths);
1645 return Err(error);
1646 }
1647 };
1648
1649 let child_pid = child.id();
1650 metadata.status = BgTaskStatus::Running;
1651 metadata.child_pid = Some(child_pid);
1652 metadata.pgid = None;
1653 self.persist_task(&paths, &metadata)
1654 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1655
1656 let task = Arc::new(BgTask {
1657 task_id: task_id.clone(),
1658 delivery_session_id: session_id.clone(),
1659 session_id,
1660 paths: paths.clone(),
1661 artifact_root: canonical_artifact_root(&paths),
1662 started: Instant::now(),
1663 last_reminder_at: Mutex::new(None),
1664 terminal_at: Mutex::new(None),
1665 state: Mutex::new(BgTaskState {
1666 metadata,
1667 runtime: TaskRuntime::Piped(Some(child)),
1668 io_handles: Some(io_handles),
1669 detached: false,
1670 child_exit_observed: false,
1671 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
1672 terminal_output_cache: None,
1673 pending_terminal_override: None,
1674 }),
1675 });
1676
1677 self.inner
1678 .tasks
1679 .lock()
1680 .map_err(|_| "background task registry lock poisoned".to_string())?
1681 .insert(task_id.clone(), task);
1682
1683 Ok(task_id)
1684 }
1685
1686 pub fn write_pty(
1687 &self,
1688 task_id: &str,
1689 session_id: &str,
1690 input: &[u8],
1691 ) -> Result<usize, String> {
1692 let task = self
1693 .task_for_session(task_id, session_id)
1694 .ok_or_else(|| "task_not_found".to_string())?;
1695
1696 let writer = {
1697 let state = task
1698 .state
1699 .lock()
1700 .map_err(|_| "background task lock poisoned".to_string())?;
1701 if state.metadata.mode != BgMode::Pty {
1702 return Err("task_not_pty".to_string());
1703 }
1704 if state.metadata.status.is_terminal() {
1705 return Err("task_exited".to_string());
1706 }
1707 match &state.runtime {
1708 TaskRuntime::Pty(Some(runtime)) => Arc::clone(&runtime.writer),
1709 TaskRuntime::Pty(None) => return Err("task_exited".to_string()),
1710 TaskRuntime::Piped(_) => return Err("task_not_pty".to_string()),
1711 }
1712 };
1713
1714 let mut writer = writer
1715 .lock()
1716 .map_err(|_| "PTY writer lock poisoned".to_string())?;
1717 writer
1718 .write_all(input)
1719 .map_err(|error| format!("failed to write to PTY: {error}"))?;
1720 writer
1721 .flush()
1722 .map_err(|error| format!("failed to flush PTY writer: {error}"))?;
1723 Ok(input.len())
1724 }
1725
1726 pub fn replay_session(&self, storage_dir: &Path, session_id: &str) -> Result<(), String> {
1727 self.replay_session_inner(storage_dir, session_id, None)
1728 }
1729
1730 pub fn replay_session_for_project(
1731 &self,
1732 storage_dir: &Path,
1733 session_id: &str,
1734 project_root: &Path,
1735 ) -> Result<(), String> {
1736 self.replay_session_inner(storage_dir, session_id, Some(project_root))
1737 }
1738
1739 fn replay_session_inner(
1740 &self,
1741 storage_dir: &Path,
1742 session_id: &str,
1743 project_root: Option<&Path>,
1744 ) -> Result<(), String> {
1745 self.start_watchdog();
1746 if !self.inner.persisted_gc_started.swap(true, Ordering::SeqCst) {
1747 if let Err(error) = self.maybe_gc_persisted(storage_dir) {
1748 crate::slog_warn!("failed to GC persisted background bash tasks: {error}");
1749 }
1750 }
1751
1752 let canonical_project = project_root.map(canonicalized_path);
1753 let tasks = match self.replay_session_from_db(session_id, project_root) {
1765 Some(Ok(tasks)) if !tasks.is_empty() => tasks,
1766 Some(Ok(_)) => {
1767 let disk_tasks = self.replay_session_from_disk(storage_dir, session_id)?;
1768 if !disk_tasks.is_empty() {
1769 crate::slog_info!(
1770 "bash task replay: 0 in DB for session {}, {} from disk fallback",
1771 session_id,
1772 disk_tasks.len()
1773 );
1774 }
1775 disk_tasks
1776 }
1777 Some(Err(error)) => {
1778 crate::slog_warn!(
1779 "bash task replay DB lookup failed for session {}; falling back to disk: {}",
1780 session_id,
1781 error
1782 );
1783 self.replay_session_from_disk(storage_dir, session_id)?
1784 }
1785 None => {
1786 self.replay_session_from_disk(storage_dir, session_id)?
1788 }
1789 };
1790
1791 for mut metadata in tasks {
1792 if project_root.is_none() && metadata.session_id != session_id {
1793 continue;
1794 }
1795 if let Some(canonical_project) = canonical_project.as_deref() {
1796 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
1797 if metadata_project.as_deref() != Some(canonical_project) {
1798 continue;
1799 }
1800 }
1801
1802 if validate_task_id(&metadata.task_id).is_err() {
1803 crate::slog_warn!(
1804 "ignoring persisted background task with invalid id {:?}",
1805 metadata.task_id
1806 );
1807 continue;
1808 }
1809 if self.task(&metadata.task_id).is_some() {
1813 continue;
1814 }
1815 let session_dir = session_tasks_dir(storage_dir, &metadata.session_id);
1816 let resolved = match resolve_task_layout(&session_dir, &metadata.task_id) {
1817 Ok(task) => task,
1818 Err(error) => {
1819 if Self::persisted_task_process_is_alive(&metadata) {
1820 crate::slog_warn!(
1821 "refusing to quarantine unresolved live background task {}: {error}",
1822 metadata.task_id
1823 );
1824 continue;
1825 }
1826 crate::slog_warn!(
1827 "quarantining unresolved background task {}: {error}",
1828 metadata.task_id
1829 );
1830 let _ = quarantine_task_layout(
1831 storage_dir,
1832 &session_dir,
1833 &metadata.task_id,
1834 "invalid",
1835 );
1836 continue;
1837 }
1838 };
1839 match read_task_at(&resolved) {
1840 Ok(disk)
1841 if disk.task_id == metadata.task_id
1842 && disk.session_id == metadata.session_id => {}
1843 Ok(_) | Err(_) => {
1844 if Self::persisted_task_process_is_alive(&metadata) {
1845 crate::slog_warn!(
1846 "refusing to quarantine mismatched live background task {}",
1847 metadata.task_id
1848 );
1849 continue;
1850 }
1851 let _ = quarantine_task_layout(
1852 storage_dir,
1853 &session_dir,
1854 &metadata.task_id,
1855 "mismatch",
1856 );
1857 continue;
1858 }
1859 }
1860 let paths = resolved.paths;
1861 let replay_task_id = metadata.task_id.clone();
1862 let delivery_session_id = (metadata.session_id != session_id).then_some(session_id);
1863 match metadata.status {
1864 BgTaskStatus::Starting => {
1865 let completion_was_delivered = metadata.completion_delivered;
1866 metadata.mark_terminal(
1867 BgTaskStatus::Failed,
1868 None,
1869 Some("spawn aborted".to_string()),
1870 );
1871 metadata.completion_delivered |= completion_was_delivered;
1872 let _ = self.persist_task(&paths, &metadata);
1873 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1874 self.insert_rehydrated_task(metadata, paths, true, delivery_session_id)?;
1875 }
1876 BgTaskStatus::Running | BgTaskStatus::Killing => {
1877 if metadata.mode == BgMode::Pty {
1878 if let Ok(Some(marker)) = read_exit_marker(&paths) {
1879 let completion_was_delivered = metadata.completion_delivered;
1880 metadata = terminal_metadata_from_marker(metadata, marker, None);
1881 metadata.completion_delivered |= completion_was_delivered;
1882 let _ = self.persist_task(&paths, &metadata);
1883 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1884 self.insert_rehydrated_task(
1885 metadata,
1886 paths,
1887 true,
1888 delivery_session_id,
1889 )?;
1890 } else if metadata.status.is_terminal() {
1891 self.insert_rehydrated_task(
1892 metadata,
1893 paths,
1894 true,
1895 delivery_session_id,
1896 )?;
1897 } else {
1898 let completion_was_delivered = metadata.completion_delivered;
1899 metadata.mark_terminal(
1900 BgTaskStatus::Killed,
1901 None,
1902 Some("pty_lost_on_bridge_restart".to_string()),
1903 );
1904 metadata.completion_delivered |= completion_was_delivered;
1905 let _ = self.persist_task(&paths, &metadata);
1906 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1907 self.insert_rehydrated_task(
1908 metadata,
1909 paths,
1910 true,
1911 delivery_session_id,
1912 )?;
1913 }
1914 } else if let Ok(Some(marker)) = read_exit_marker(&paths) {
1915 let reason = (metadata.status == BgTaskStatus::Killing).then(|| {
1916 "recovered from inconsistent killing state on replay".to_string()
1917 });
1918 if reason.is_some() {
1919 crate::slog_warn!("background task {} had killing state with exit marker; preferring marker",
1920 metadata.task_id);
1921 }
1922 let completion_was_delivered = metadata.completion_delivered;
1923 metadata = terminal_metadata_from_marker(metadata, marker, reason);
1924 metadata.completion_delivered |= completion_was_delivered;
1925 let _ = self.persist_task(&paths, &metadata);
1926 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1927 self.insert_rehydrated_task(metadata, paths, true, delivery_session_id)?;
1928 } else if metadata.status == BgTaskStatus::Killing {
1929 let _ = write_kill_marker_if_absent(&paths);
1930 let completion_was_delivered = metadata.completion_delivered;
1931 metadata.mark_terminal(
1932 BgTaskStatus::Killed,
1933 None,
1934 Some("recovered from inconsistent killing state on replay".to_string()),
1935 );
1936 metadata.completion_delivered |= completion_was_delivered;
1937 let _ = self.persist_task(&paths, &metadata);
1938 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1939 self.insert_rehydrated_task(metadata, paths, true, delivery_session_id)?;
1940 } else if Self::persisted_task_process_is_alive(&metadata) {
1941 self.insert_rehydrated_task(metadata, paths, true, delivery_session_id)?;
1942 } else {
1943 let completion_was_delivered = metadata.completion_delivered;
1944 metadata.mark_terminal(
1945 BgTaskStatus::FateUnknown,
1946 None,
1947 Some(restart_fate_unknown_reason(&metadata, &paths)),
1948 );
1949 metadata.completion_delivered |= completion_was_delivered;
1950 let _ = self.persist_task(&paths, &metadata);
1951 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1952 self.insert_rehydrated_task(metadata, paths, true, delivery_session_id)?;
1953 }
1954 }
1955 _ if metadata.status.is_terminal() => {
1956 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1962 self.insert_rehydrated_task(metadata, paths, true, delivery_session_id)?;
1963 }
1964 _ => {}
1965 }
1966 self.retarget_pending_completion(&replay_task_id, session_id);
1967 }
1968
1969 Ok(())
1970 }
1971
1972 fn replay_session_from_db(
1973 &self,
1974 session_id: &str,
1975 project_root: Option<&Path>,
1976 ) -> Option<Result<Vec<PersistedTask>, String>> {
1977 let pool = self
1978 .inner
1979 .db_pool
1980 .read()
1981 .ok()
1982 .and_then(|slot| slot.clone())?;
1983 let harness = self
1984 .inner
1985 .db_harness
1986 .read()
1987 .ok()
1988 .and_then(|slot| slot.clone())?;
1989 let conn = match pool.lock() {
1990 Ok(conn) => conn,
1991 Err(_) => return Some(Err("db mutex poisoned".to_string())),
1992 };
1993 let rows = if let Some(project_root) = project_root {
1994 let project_key = crate::path_identity::project_scope_key(project_root);
1995 crate::db::bash_tasks::list_replayable_bash_tasks_for_project(
1996 &conn,
1997 &harness,
1998 &project_key,
1999 )
2000 } else {
2001 crate::db::bash_tasks::list_bash_tasks_for_session(&conn, &harness, session_id)
2002 };
2003 Some(
2004 rows.map(|rows| rows.into_iter().map(PersistedTask::from).collect())
2005 .map_err(|error| error.to_string()),
2006 )
2007 }
2008
2009 fn replay_session_from_disk(
2010 &self,
2011 storage_dir: &Path,
2012 session_id: &str,
2013 ) -> Result<Vec<PersistedTask>, String> {
2014 let dir = session_tasks_dir(storage_dir, session_id);
2015 if !dir.exists() {
2016 return Ok(Vec::new());
2017 }
2018
2019 let (task_ids, invalid_entries) = discover_task_ids(&dir)
2020 .map_err(|error| format!("failed to discover background task layouts: {error}"))?;
2021 for entry in invalid_entries {
2022 if let Err(error) = quarantine_invalid_entry(storage_dir, &dir, &entry) {
2023 crate::slog_warn!(
2024 "failed to quarantine invalid background task entry {:?}: {error}",
2025 entry
2026 );
2027 }
2028 }
2029
2030 let mut tasks = Vec::new();
2031 for task_id in task_ids {
2032 let task = match resolve_task_layout(&dir, &task_id) {
2033 Ok(task) => task,
2034 Err(error)
2035 if error.kind() == std::io::ErrorKind::NotFound
2036 && uninitialized_layout_is_recent(
2037 &dir,
2038 &task_id,
2039 Duration::from_secs(5 * 60),
2040 )
2041 .unwrap_or(false) =>
2042 {
2043 continue;
2044 }
2045 Err(error) => {
2046 if self.db_has_live_process_for_task(&task_id) {
2047 crate::slog_warn!(
2048 "refusing to quarantine unresolved live background task {task_id} during replay: {error}"
2049 );
2050 continue;
2051 }
2052 crate::slog_warn!(
2053 "quarantining unresolved background task {task_id} during replay: {error}"
2054 );
2055 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
2056 continue;
2057 }
2058 };
2059 match read_task_at(&task) {
2060 Ok(metadata) if metadata.session_id == session_id => tasks.push(metadata),
2061 Ok(_) => {
2062 crate::slog_warn!(
2063 "quarantining background task {task_id} with mismatched session metadata"
2064 );
2065 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "mismatch");
2066 }
2067 Err(error) => {
2068 if self.db_has_live_process_for_task(&task_id) {
2069 crate::slog_warn!(
2070 "refusing to quarantine unreadable live background task {task_id} during replay: {error}"
2071 );
2072 continue;
2073 }
2074 crate::slog_warn!(
2075 "quarantining invalid background task metadata {task_id} during replay: {error}"
2076 );
2077 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
2078 }
2079 }
2080 }
2081 Ok(tasks)
2082 }
2083
2084 pub fn register_watch(
2085 &self,
2086 task_id: String,
2087 pattern: WatchPattern,
2088 once: bool,
2089 ) -> Result<String, &'static str> {
2090 let task = self.task(&task_id).ok_or("task_not_found")?;
2091 validate_task_id(&task_id).map_err(|_| "invalid_task_id")?;
2092 let (mode, terminal_at_registration) = task
2093 .state
2094 .lock()
2095 .map(|state| {
2096 (
2097 state.metadata.mode.clone(),
2098 state.metadata.status.is_terminal(),
2099 )
2100 })
2101 .map_err(|_| "background_task_lock_poisoned")?;
2102 let mut stdout = (mode == BgMode::Pipes)
2103 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
2104 .transpose()
2105 .map_err(|_| "artifact_refused")?;
2106 let mut stderr = (mode == BgMode::Pipes)
2107 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
2108 .transpose()
2109 .map_err(|_| "artifact_refused")?;
2110 let mut pty = (mode == BgMode::Pty)
2111 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
2112 .transpose()
2113 .map_err(|_| "artifact_refused")?;
2114
2115 let mut terminal_matches = Vec::new();
2116 let scanned_terminal = terminal_at_registration;
2117 let watch_id = {
2118 let mut registry = self
2119 .inner
2120 .watch_registry
2121 .lock()
2122 .map_err(|_| "watch_registry_poisoned")?;
2123 let watch_id = registry.register(task_id.clone(), pattern.clone(), once)?;
2124 match &mode {
2125 BgMode::Pipes => {
2126 let stdout_key = format!("{task_id}:stdout");
2127 let stderr_key = format!("{task_id}:stderr");
2128 if terminal_at_registration {
2129 registry.set_file_cursor(&stdout_key, 0);
2130 registry.set_file_cursor(&stderr_key, 0);
2131 terminal_matches.extend(registry.scan_file_new_bytes(
2132 &stdout_key,
2133 &task_id,
2134 stdout.as_mut().expect("pipe stdout opened"),
2135 ));
2136 terminal_matches.extend(registry.scan_file_new_bytes(
2137 &stderr_key,
2138 &task_id,
2139 stderr.as_mut().expect("pipe stderr opened"),
2140 ));
2141 } else {
2142 registry.prime_file_cursor(
2143 &stdout_key,
2144 stdout.as_ref().expect("pipe stdout opened"),
2145 );
2146 registry.prime_file_cursor(
2147 &stderr_key,
2148 stderr.as_ref().expect("pipe stderr opened"),
2149 );
2150 }
2151 }
2152 BgMode::Pty => {
2153 let pty_key = format!("{task_id}:pty");
2154 if terminal_at_registration {
2155 registry.set_file_cursor(&pty_key, 0);
2156 terminal_matches.extend(registry.scan_file_new_bytes(
2157 &pty_key,
2158 &task_id,
2159 pty.as_mut().expect("PTY artifact opened"),
2160 ));
2161 } else {
2162 registry.prime_file_cursor(
2163 &pty_key,
2164 pty.as_ref().expect("PTY artifact opened"),
2165 );
2166 }
2167 }
2168 }
2169 watch_id
2170 };
2171
2172 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
2173 self.persist_watch_registration(
2174 &task.session_id,
2175 &task_id,
2176 &watch_id,
2177 &pattern,
2178 once,
2179 stdout_offset,
2180 stderr_offset,
2181 pty_offset,
2182 );
2183
2184 if task.is_terminal() {
2185 if !scanned_terminal {
2186 terminal_matches = {
2187 let mut registry = self
2188 .inner
2189 .watch_registry
2190 .lock()
2191 .map_err(|_| "watch_registry_poisoned")?;
2192 match &mode {
2193 BgMode::Pipes => {
2194 let stdout_key = format!("{task_id}:stdout");
2195 let stderr_key = format!("{task_id}:stderr");
2196 registry.set_file_cursor(&stdout_key, 0);
2197 registry.set_file_cursor(&stderr_key, 0);
2198 let mut matches = registry.scan_file_new_bytes(
2199 &stdout_key,
2200 &task_id,
2201 stdout.as_mut().expect("pipe stdout opened"),
2202 );
2203 matches.extend(registry.scan_file_new_bytes(
2204 &stderr_key,
2205 &task_id,
2206 stderr.as_mut().expect("pipe stderr opened"),
2207 ));
2208 matches
2209 }
2210 BgMode::Pty => {
2211 let pty_key = format!("{task_id}:pty");
2212 registry.set_file_cursor(&pty_key, 0);
2213 registry.scan_file_new_bytes(
2214 &pty_key,
2215 &task_id,
2216 pty.as_mut().expect("PTY artifact opened"),
2217 )
2218 }
2219 }
2220 };
2221 }
2222
2223 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
2224 let (watch_controlled, watch_matched) = self.task_watch_state(&task_id);
2225 if terminal_matches.is_empty() && (!watch_controlled || watch_matched) {
2226 if watch_matched {
2227 let _ = task.set_completion_delivered(true, self);
2228 self.clear_task_watch_state(&task_id);
2229 self.delete_persisted_watches_for_task(&task.session_id, &task_id);
2231 }
2232 return Ok(watch_id);
2233 }
2234
2235 let completion = self
2236 .remove_pending_completion(&task_id)
2237 .or_else(|| self.completion_snapshot_for_task(&task));
2238 if terminal_matches.is_empty() {
2239 if let Some(completion) = completion.as_ref() {
2240 self.emit_bash_watch_exit(completion);
2241 }
2242 } else {
2243 for pattern_match in &terminal_matches {
2244 self.persist_watch_match(
2245 &task.session_id,
2246 &task_id,
2247 pattern_match,
2248 stdout_offset,
2249 stderr_offset,
2250 pty_offset,
2251 );
2252 self.emit_bash_pattern_match(&task.delivery_session_id, pattern_match.clone());
2253 }
2254 }
2255 let _ = task.set_completion_delivered(true, self);
2256 self.clear_task_watch_state(&task_id);
2257 self.delete_persisted_watches_for_task(&task.session_id, &task_id);
2259 }
2260
2261 Ok(watch_id)
2262 }
2263
2264 pub fn unregister_watch(&self, task_id: &str, watch_id: &str) {
2265 let session_id = self.task(task_id).map(|task| task.session_id.clone());
2266 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2267 registry.unregister(task_id, watch_id);
2268 }
2269 if let Some(session_id) = session_id {
2270 self.delete_persisted_watch(&session_id, task_id, watch_id);
2271 }
2272 }
2273
2274 pub fn active_watch_count(&self, task_id: &str) -> usize {
2275 self.inner
2276 .watch_registry
2277 .lock()
2278 .map(|registry| registry.active_count(task_id))
2279 .unwrap_or(0)
2280 }
2281
2282 fn task_watch_state(&self, task_id: &str) -> (bool, bool) {
2283 self.inner
2284 .watch_registry
2285 .lock()
2286 .map(|registry| {
2287 (
2288 registry.has_controlled_task(task_id),
2289 registry.has_matched_task(task_id),
2290 )
2291 })
2292 .unwrap_or((false, false))
2293 }
2294
2295 fn task_has_watch_control(&self, task_id: &str) -> bool {
2296 self.inner
2297 .watch_registry
2298 .lock()
2299 .map(|registry| registry.has_controlled_task(task_id))
2300 .unwrap_or(false)
2301 }
2302
2303 fn clear_task_watch_state(&self, task_id: &str) {
2304 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2305 registry.clear_task(task_id);
2306 }
2307 }
2308
2309 pub(crate) fn scan_task_watch_output(&self, task: &Arc<BgTask>) {
2310 let mode = match task.state.lock() {
2311 Ok(state) => state.metadata.mode.clone(),
2312 Err(_) => return,
2313 };
2314 let mut stdout = (mode == BgMode::Pipes)
2315 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
2316 .transpose()
2317 .ok()
2318 .flatten();
2319 let mut stderr = (mode == BgMode::Pipes)
2320 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
2321 .transpose()
2322 .ok()
2323 .flatten();
2324 let mut pty = (mode == BgMode::Pty)
2325 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
2326 .transpose()
2327 .ok()
2328 .flatten();
2329 let mut matches = Vec::new();
2330 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2331 match mode {
2332 BgMode::Pipes => {
2333 let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) else {
2334 return;
2335 };
2336 let stdout_key = format!("{}:stdout", task.task_id);
2337 let stderr_key = format!("{}:stderr", task.task_id);
2338 matches.extend(registry.scan_file_new_bytes(
2339 &stdout_key,
2340 &task.task_id,
2341 stdout,
2342 ));
2343 matches.extend(registry.scan_file_new_bytes(
2344 &stderr_key,
2345 &task.task_id,
2346 stderr,
2347 ));
2348 }
2349 BgMode::Pty => {
2350 let Some(pty) = pty.as_mut() else {
2351 return;
2352 };
2353 let pty_key = format!("{}:pty", task.task_id);
2354 matches.extend(registry.scan_file_new_bytes(&pty_key, &task.task_id, pty));
2355 }
2356 }
2357 }
2358 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
2359 if matches.is_empty() {
2360 if self.task_has_watch_control(&task.task_id) {
2363 self.persist_task_watch_cursors(
2364 &task.session_id,
2365 &task.task_id,
2366 stdout_offset,
2367 stderr_offset,
2368 pty_offset,
2369 );
2370 }
2371 return;
2372 }
2373 for pattern_match in matches {
2374 self.persist_watch_match(
2375 &task.session_id,
2376 &task.task_id,
2377 &pattern_match,
2378 stdout_offset,
2379 stderr_offset,
2380 pty_offset,
2381 );
2382 self.emit_bash_pattern_match(&task.delivery_session_id, pattern_match);
2383 }
2384 self.persist_task_watch_cursors(
2385 &task.session_id,
2386 &task.task_id,
2387 stdout_offset,
2388 stderr_offset,
2389 pty_offset,
2390 );
2391 }
2392
2393 pub fn status(
2394 &self,
2395 task_id: &str,
2396 session_id: &str,
2397 project_root: Option<&Path>,
2398 storage_dir: Option<&Path>,
2399 preview_bytes: usize,
2400 ) -> Option<BgTaskSnapshot> {
2401 validate_task_id(task_id).ok()?;
2402 let mut task = self.task_for_session(task_id, session_id);
2403 if task.is_none() {
2404 if let Some(storage_dir) = storage_dir {
2405 let _ = if let Some(project_root) = project_root {
2406 self.replay_session_for_project(storage_dir, session_id, project_root)
2407 } else {
2408 self.replay_session(storage_dir, session_id)
2409 };
2410 task = self.task_for_session(task_id, session_id);
2411 }
2412 }
2413 let Some(task) = task else {
2414 return self.status_relaxed(
2415 task_id,
2416 session_id,
2417 project_root?,
2418 storage_dir?,
2419 preview_bytes,
2420 );
2421 };
2422 let _ = self.poll_task(&task);
2423 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
2424 }
2425
2426 fn status_relaxed_task(
2427 &self,
2428 task_id: &str,
2429 project_root: &Path,
2430 storage_dir: &Path,
2431 ) -> Option<Arc<BgTask>> {
2432 validate_task_id(task_id).ok()?;
2433 let canonical_project = canonicalized_path(project_root);
2434 match self.lookup_relaxed_task_from_db(task_id, project_root) {
2435 Some(Ok(Some(metadata))) => {
2436 if let Some(task) = self.task(task_id) {
2437 let matches_project = task
2438 .state
2439 .lock()
2440 .map(|state| {
2441 state
2442 .metadata
2443 .project_root
2444 .as_deref()
2445 .map(canonicalized_path)
2446 .as_deref()
2447 == Some(canonical_project.as_path())
2448 })
2449 .unwrap_or(false);
2450 return matches_project.then_some(task);
2451 }
2452 let resolved = resolve_task_layout(
2453 &session_tasks_dir(storage_dir, &metadata.session_id),
2454 &metadata.task_id,
2455 )
2456 .ok()?;
2457 let disk = read_task_at(&resolved).ok()?;
2458 if disk.task_id != metadata.task_id || disk.session_id != metadata.session_id {
2459 return None;
2460 }
2461 if self
2462 .insert_rehydrated_task(metadata, resolved.paths, true, None)
2463 .is_err()
2464 {
2465 return None;
2466 }
2467 return self.task(task_id);
2468 }
2469 Some(Ok(None)) => {
2470 crate::slog_info!(
2471 "bash task relaxed DB miss for {}; falling back to disk",
2472 task_id
2473 );
2474 }
2475 Some(Err(error)) => {
2476 crate::slog_warn!(
2477 "bash task relaxed DB lookup failed for {}; falling back to disk: {}",
2478 task_id,
2479 error
2480 );
2481 }
2482 None => {
2483 crate::slog_info!(
2484 "bash task relaxed DB unavailable for {}; falling back to disk",
2485 task_id
2486 );
2487 }
2488 }
2489 let root = storage_dir.join("bash-tasks");
2490 let entries = fs::read_dir(&root).ok()?;
2491 for entry in entries.flatten() {
2492 let dir = entry.path();
2493 if !dir.is_dir() {
2494 continue;
2495 }
2496 let resolved = match resolve_task_layout(&dir, task_id) {
2497 Ok(task) => task,
2498 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
2499 Err(error) => {
2500 if self.db_has_live_process_for_task(task_id) {
2501 crate::slog_warn!(
2502 "refusing to quarantine unresolved live background task {task_id} during relaxed lookup: {error}"
2503 );
2504 continue;
2505 }
2506 crate::slog_warn!(
2507 "quarantining unresolved background task {task_id} during relaxed lookup: {error}"
2508 );
2509 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
2510 continue;
2511 }
2512 };
2513 let metadata = match read_task_at(&resolved) {
2514 Ok(metadata) => metadata,
2515 Err(error) => {
2516 if self.db_has_live_process_for_task(task_id) {
2517 crate::slog_warn!(
2518 "refusing to quarantine unreadable live background task {task_id} during relaxed lookup: {error}"
2519 );
2520 continue;
2521 }
2522 crate::slog_warn!(
2523 "quarantining invalid background task metadata {task_id} during relaxed lookup: {error}"
2524 );
2525 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
2526 continue;
2527 }
2528 };
2529 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
2530 if metadata_project.as_deref() != Some(canonical_project.as_path()) {
2531 continue;
2532 }
2533 if let Some(task) = self.task(task_id) {
2534 let matches_project = task
2535 .state
2536 .lock()
2537 .map(|state| {
2538 state
2539 .metadata
2540 .project_root
2541 .as_deref()
2542 .map(canonicalized_path)
2543 .as_deref()
2544 == Some(canonical_project.as_path())
2545 })
2546 .unwrap_or(false);
2547 return matches_project.then_some(task);
2548 }
2549 if self
2550 .insert_rehydrated_task(metadata, resolved.paths, true, None)
2551 .is_err()
2552 {
2553 return None;
2554 }
2555 return self.task(task_id);
2556 }
2557 None
2558 }
2559
2560 fn lookup_relaxed_task_from_db(
2561 &self,
2562 task_id: &str,
2563 project_root: &Path,
2564 ) -> Option<Result<Option<PersistedTask>, String>> {
2565 let pool = self
2566 .inner
2567 .db_pool
2568 .read()
2569 .ok()
2570 .and_then(|slot| slot.clone())?;
2571 let harness = self
2572 .inner
2573 .db_harness
2574 .read()
2575 .ok()
2576 .and_then(|slot| slot.clone())?;
2577 let conn = match pool.lock() {
2578 Ok(conn) => conn,
2579 Err(_) => return Some(Err("db mutex poisoned".to_string())),
2580 };
2581 let project_key = crate::path_identity::project_scope_key(project_root);
2582 Some(
2583 crate::db::bash_tasks::find_bash_task_for_project(
2584 &conn,
2585 &harness,
2586 &project_key,
2587 task_id,
2588 )
2589 .map(|row| row.map(PersistedTask::from))
2590 .map_err(|error| error.to_string()),
2591 )
2592 }
2593
2594 pub(super) fn status_relaxed(
2595 &self,
2596 task_id: &str,
2597 _session_id: &str,
2598 project_root: &Path,
2599 storage_dir: &Path,
2600 preview_bytes: usize,
2601 ) -> Option<BgTaskSnapshot> {
2602 let task = self.status_relaxed_task(task_id, project_root, storage_dir)?;
2603 let _ = self.poll_task(&task);
2604 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
2605 }
2606
2607 pub fn kill_relaxed(
2608 &self,
2609 task_id: &str,
2610 project_root: &Path,
2611 storage_dir: &Path,
2612 ) -> Result<BgTaskSnapshot, String> {
2613 let task = self
2614 .status_relaxed_task(task_id, project_root, storage_dir)
2615 .ok_or_else(|| format!("background task not found: {task_id}"))?;
2616 self.kill_with_status(task_id, &task.session_id, BgTaskStatus::Killed)
2617 }
2618
2619 pub fn maybe_gc_persisted(&self, storage_dir: &Path) -> Result<usize, String> {
2620 #[cfg(test)]
2621 self.inner.persisted_gc_runs.fetch_add(1, Ordering::SeqCst);
2622
2623 let mut deleted = 0usize;
2624
2625 let root = storage_dir.join("bash-tasks");
2626 if root.exists() {
2627 let session_dirs = fs::read_dir(&root).map_err(|e| {
2628 format!(
2629 "failed to read background task root {}: {e}",
2630 root.display()
2631 )
2632 })?;
2633 for session_entry in session_dirs.flatten() {
2634 let session_dir = session_entry.path();
2635 if !session_dir.is_dir() {
2636 continue;
2637 }
2638 let (task_ids, invalid_entries) = match discover_task_ids(&session_dir) {
2639 Ok(discovery) => discovery,
2640 Err(error) => {
2641 crate::slog_warn!(
2642 "failed to discover background task session {}: {error}",
2643 session_dir.display()
2644 );
2645 continue;
2646 }
2647 };
2648 for entry in invalid_entries {
2649 let _ = quarantine_invalid_entry(storage_dir, &session_dir, &entry);
2650 }
2651 for task_id in task_ids {
2652 let resolved = match resolve_task_layout(&session_dir, &task_id) {
2653 Ok(task) => task,
2654 Err(error)
2655 if error.kind() == std::io::ErrorKind::NotFound
2656 && uninitialized_layout_is_recent(
2657 &session_dir,
2658 &task_id,
2659 Duration::from_secs(5 * 60),
2660 )
2661 .unwrap_or(false) =>
2662 {
2663 continue;
2664 }
2665 Err(error) => {
2666 if self.db_has_live_process_for_task(&task_id) {
2667 crate::slog_warn!(
2668 "refusing to quarantine unresolved live background task {task_id} during GC: {error}"
2669 );
2670 continue;
2671 }
2672 crate::slog_warn!(
2673 "quarantining unresolved background task {task_id}: {error}"
2674 );
2675 quarantine_task_layout(storage_dir, &session_dir, &task_id, "invalid")
2676 .map_err(|error| error.to_string())?;
2677 continue;
2678 }
2679 };
2680 if modified_within(&resolved.paths.json, PERSISTED_GC_GRACE) {
2681 continue;
2682 }
2683 let metadata = match read_task_at(&resolved) {
2684 Ok(metadata) => metadata,
2685 Err(error) => {
2686 if self.db_has_live_process_for_task(&task_id) {
2687 crate::slog_warn!(
2688 "refusing to quarantine unreadable live background task {task_id} during GC: {error}"
2689 );
2690 continue;
2691 }
2692 crate::slog_warn!(
2693 "quarantining corrupt background task metadata {task_id}: {error}"
2694 );
2695 quarantine_task_layout(storage_dir, &session_dir, &task_id, "corrupt")
2696 .map_err(|error| error.to_string())?;
2697 continue;
2698 }
2699 };
2700 if !(metadata.status.is_terminal() && metadata.completion_delivered) {
2701 continue;
2702 }
2703 if Self::persisted_task_process_is_alive(&metadata)
2704 || self.db_has_live_process_for_task(&task_id)
2705 {
2706 crate::slog_warn!(
2707 "refusing to delete terminal background task bundle {task_id}: recorded process is still alive"
2708 );
2709 continue;
2710 }
2711 match delete_task_bundle(&resolved.paths) {
2712 Ok(()) => {
2713 self.delete_gc_task_from_db(&metadata);
2714 deleted += 1;
2715 log::debug!(
2716 "deleted persisted background task bundle {}",
2717 metadata.task_id
2718 );
2719 }
2720 Err(error) => {
2721 crate::slog_warn!(
2722 "failed to delete background task bundle {}: {error}",
2723 metadata.task_id
2724 );
2725 }
2726 }
2727 }
2728 }
2729 }
2730 gc_quarantine(storage_dir);
2731 Ok(deleted)
2732 }
2733
2734 pub fn list(&self, preview_bytes: usize) -> Vec<BgTaskSnapshot> {
2735 let tasks = self
2736 .inner
2737 .tasks
2738 .lock()
2739 .map(|tasks| tasks.values().cloned().collect::<Vec<_>>())
2740 .unwrap_or_default();
2741 tasks
2742 .into_iter()
2743 .map(|task| {
2744 let _ = self.poll_task(&task);
2745 self.snapshot_with_terminal_cache(&task, preview_bytes)
2746 })
2747 .collect()
2748 }
2749
2750 fn maybe_compress_snapshot(&self, task: &Arc<BgTask>, snapshot: &mut BgTaskSnapshot) {
2756 if !snapshot.info.status.is_terminal() || snapshot.info.mode == BgMode::Pty {
2757 return;
2758 }
2759 if let Some(cache) = self.ensure_terminal_output_cache(task) {
2760 snapshot.output_preview = cache.output_preview;
2761 snapshot.output_truncated = cache.output_truncated;
2762 }
2763 }
2764
2765 pub fn kill(&self, task_id: &str, session_id: &str) -> Result<BgTaskSnapshot, String> {
2766 self.kill_with_status(task_id, session_id, BgTaskStatus::Killed)
2767 }
2768
2769 pub fn kill_running_tasks_for_root(&self, project_root: &Path) -> usize {
2779 let canonical_root = canonicalized_path(project_root);
2780 let targets = self
2781 .inner
2782 .tasks
2783 .lock()
2784 .map(|tasks| {
2785 tasks
2786 .values()
2787 .filter_map(|task| {
2788 let state = task.state.lock().ok()?;
2789 let status = &state.metadata.status;
2790 let running = matches!(status, BgTaskStatus::Running)
2791 || (state.metadata.mode == BgMode::Pty
2792 && matches!(status, BgTaskStatus::Killing));
2793 if !running {
2794 return None;
2795 }
2796 let task_root = state
2797 .metadata
2798 .project_root
2799 .as_deref()
2800 .unwrap_or(&state.metadata.workdir);
2801 (canonicalized_path(task_root) == canonical_root)
2802 .then(|| (task.task_id.clone(), task.session_id.clone()))
2803 })
2804 .collect::<Vec<_>>()
2805 })
2806 .unwrap_or_default();
2807
2808 let mut killed = 0;
2809 for (task_id, session_id) in targets {
2810 match self.kill_with_status_reason(
2811 &task_id,
2812 &session_id,
2813 BgTaskStatus::Killed,
2814 Some(ROOT_RECLAIMED_REASON.to_string()),
2815 ) {
2816 Ok(_) => killed += 1,
2817 Err(error) => crate::slog_warn!(
2818 "failed to terminate background task {task_id} for reclaimed root {}: {error}",
2819 project_root.display()
2820 ),
2821 }
2822 }
2823 killed
2824 }
2825
2826 pub fn promote(&self, task_id: &str, session_id: &str) -> Result<bool, String> {
2827 let task = self
2828 .task_for_session(task_id, session_id)
2829 .ok_or_else(|| format!("background task not found: {task_id}"))?;
2830 let terminal_after_promote = {
2831 let mut state = task
2832 .state
2833 .lock()
2834 .map_err(|_| "background task lock poisoned".to_string())?;
2835 let updated = self
2836 .update_task_metadata(&task.paths, |metadata| {
2837 metadata.notify_on_completion = true;
2838 metadata.completion_delivered = false;
2839 })
2840 .map_err(|e| format!("failed to promote background task: {e}"))?;
2841 state.metadata = updated;
2842 state.metadata.status.is_terminal()
2843 };
2844 if terminal_after_promote {
2845 self.post_terminal_transition(&task, true)?;
2846 }
2847 Ok(true)
2848 }
2849
2850 pub(crate) fn kill_for_timeout(&self, task_id: &str, session_id: &str) -> Result<(), String> {
2851 self.kill_with_status(task_id, session_id, BgTaskStatus::TimedOut)
2852 .map(|_| ())
2853 }
2854
2855 pub fn cleanup_finished(&self, older_than: Duration) {
2856 let cutoff = Instant::now().checked_sub(older_than);
2857 let removable_paths: Vec<(String, TaskPaths)> =
2858 if let Ok(mut tasks) = self.inner.tasks.lock() {
2859 let removable = tasks
2860 .iter()
2861 .filter_map(|(task_id, task)| {
2862 let delivered_terminal = task
2863 .state
2864 .lock()
2865 .map(|state| {
2866 state.metadata.status.is_terminal()
2867 && state.metadata.completion_delivered
2868 })
2869 .unwrap_or(false);
2870 if !delivered_terminal {
2871 return None;
2872 }
2873
2874 let terminal_at = task.terminal_at.lock().ok().and_then(|at| *at);
2875 let expired = match (terminal_at, cutoff) {
2876 (Some(terminal_at), Some(cutoff)) => terminal_at <= cutoff,
2877 (Some(_), None) => true,
2878 (None, _) => false,
2879 };
2880 expired.then(|| task_id.clone())
2881 })
2882 .collect::<Vec<_>>();
2883
2884 removable
2885 .into_iter()
2886 .filter_map(|task_id| {
2887 tasks
2888 .remove(&task_id)
2889 .map(|task| (task_id, task.paths.clone()))
2890 })
2891 .collect()
2892 } else {
2893 Vec::new()
2894 };
2895
2896 for (task_id, paths) in removable_paths {
2897 match delete_task_bundle(&paths) {
2898 Ok(()) => log::debug!("deleted persisted background task bundle {task_id}"),
2899 Err(error) => crate::slog_warn!(
2900 "failed to delete persisted background task bundle {task_id}: {error}"
2901 ),
2902 }
2903 }
2904 }
2905
2906 pub fn drain_completions(&self) -> Vec<BgCompletion> {
2907 self.drain_completions_for_session(None)
2908 }
2909
2910 pub fn drain_completions_for_session(&self, session_id: Option<&str>) -> Vec<BgCompletion> {
2911 let completions = match self.inner.completions.lock() {
2912 Ok(completions) => completions,
2913 Err(_) => return Vec::new(),
2914 };
2915
2916 completions
2917 .iter()
2918 .filter(|completion| completion_matches_session(completion, session_id))
2919 .cloned()
2920 .collect()
2921 }
2922
2923 pub fn has_completions_for_session(&self, session_id: Option<&str>) -> bool {
2924 match self.inner.completions.lock() {
2925 Ok(completions) => completions
2926 .iter()
2927 .any(|completion| completion_matches_session(completion, session_id)),
2928 Err(_) => true,
2932 }
2933 }
2934
2935 pub fn ack_completions_for_session(
2936 &self,
2937 session_id: Option<&str>,
2938 task_ids: &[String],
2939 ) -> Vec<String> {
2940 if task_ids.is_empty() {
2941 return Vec::new();
2942 }
2943 let requested_task_ids = task_ids.iter().map(String::as_str).collect::<HashSet<_>>();
2944 let mut completion_sessions = HashMap::new();
2945 if let Ok(mut completions) = self.inner.completions.lock() {
2946 completions.retain(|completion| {
2947 let session_matches = session_id
2948 .map(|session_id| completion.session_id == session_id)
2949 .unwrap_or(true);
2950 if session_matches && requested_task_ids.contains(completion.task_id.as_str()) {
2951 completion_sessions
2952 .insert(completion.task_id.clone(), completion.session_id.clone());
2953 false
2954 } else {
2955 true
2956 }
2957 });
2958 }
2959
2960 let mut delivered = Vec::new();
2961 for task_id in task_ids {
2962 let task = if let Some(session_id) = session_id {
2963 self.task_for_session(task_id, session_id)
2964 .or_else(|| {
2965 self.task(task_id)
2966 .filter(|task| task.delivery_session_id == session_id)
2967 })
2968 .or_else(|| {
2969 completion_sessions
2970 .contains_key(task_id)
2971 .then(|| self.task(task_id))
2972 .flatten()
2973 })
2974 } else if let Some(completion_session_id) = completion_sessions.get(task_id) {
2975 self.task_for_session(task_id, completion_session_id)
2976 .or_else(|| self.task(task_id))
2977 } else {
2978 self.task(task_id)
2979 };
2980 if let Some(task) = task {
2981 let terminal = task
2982 .state
2983 .lock()
2984 .map(|state| state.metadata.status.is_terminal())
2985 .unwrap_or(false);
2986 self.ack_persisted_watches_for_task(&task.session_id, task_id, terminal);
2990 if terminal {
2991 self.clear_task_watch_state(task_id);
2992 if task.set_completion_delivered(true, self).is_ok() {
2993 delivered.push(task_id.clone());
2994 }
2995 } else {
2996 self.sync_memory_watches_after_ack(task_id);
2999 delivered.push(task_id.clone());
3000 }
3001 } else if let Some(session_id) = session_id {
3002 self.ack_persisted_watches_for_task(session_id, task_id, true);
3004 delivered.push(task_id.clone());
3005 }
3006 }
3007
3008 delivered
3009 }
3010
3011 fn sync_memory_watches_after_ack(&self, task_id: &str) {
3012 let Some((harness, pool)) = self.db_harness_and_pool() else {
3013 return;
3014 };
3015 let session_id = match self.task(task_id) {
3016 Some(task) => task.session_id.clone(),
3017 None => return,
3018 };
3019 let Ok(conn) = pool.lock() else {
3020 return;
3021 };
3022 let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
3023 &conn,
3024 &harness,
3025 &session_id,
3026 task_id,
3027 ) else {
3028 return;
3029 };
3030 let remaining: HashSet<String> = rows.into_iter().map(|row| row.watch_id).collect();
3031 if let Ok(mut registry) = self.inner.watch_registry.lock() {
3032 registry.retain_watch_ids(task_id, &remaining);
3033 }
3034 }
3035
3036 pub fn pending_completions_for_session(&self, session_id: &str) -> Vec<BgCompletion> {
3037 self.inner
3038 .completions
3039 .lock()
3040 .map(|completions| {
3041 completions
3042 .iter()
3043 .filter(|completion| completion.session_id == session_id)
3044 .cloned()
3045 .collect()
3046 })
3047 .unwrap_or_default()
3048 }
3049
3050 fn remove_pending_completion(&self, task_id: &str) -> Option<BgCompletion> {
3051 let mut completions = self.inner.completions.lock().ok()?;
3052 let idx = completions
3053 .iter()
3054 .position(|completion| completion.task_id == task_id)?;
3055 completions.remove(idx)
3056 }
3057
3058 fn retarget_pending_completion(&self, task_id: &str, session_id: &str) {
3059 if let Ok(mut completions) = self.inner.completions.lock() {
3060 if let Some(completion) = completions
3061 .iter_mut()
3062 .find(|completion| completion.task_id == task_id)
3063 {
3064 completion.session_id = session_id.to_string();
3065 }
3066 }
3067 }
3068
3069 fn completion_snapshot_for_task(&self, task: &Arc<BgTask>) -> Option<BgCompletion> {
3070 let snapshot = self.snapshot_with_terminal_cache(task, RUNNING_OUTPUT_PREVIEW_BYTES);
3071 if !snapshot.info.status.is_terminal() {
3072 return None;
3073 }
3074 let (output_preview, output_truncated) = if snapshot.info.mode == BgMode::Pty {
3075 (String::new(), false)
3076 } else {
3077 self.ensure_terminal_output_cache(task)
3078 .map(|cache| completion_preview_for_cache(&cache, snapshot.exit_code))
3079 .unwrap_or_else(|| (String::new(), false))
3080 };
3081 Some(BgCompletion {
3082 task_id: snapshot.info.task_id,
3083 session_id: task.delivery_session_id.clone(),
3084 status: snapshot.info.status,
3085 exit_code: snapshot.exit_code,
3086 command: snapshot.info.command,
3087 output_preview,
3088 output_truncated,
3089 original_tokens: None,
3090 compressed_tokens: None,
3091 tokens_skipped: false,
3092 status_reason: snapshot.info.status_reason,
3093 })
3094 }
3095
3096 pub fn detach(&self) {
3097 self.inner.shutdown.store(true, Ordering::SeqCst);
3098 if let Ok(mut tasks) = self.inner.tasks.lock() {
3099 for task in tasks.values() {
3100 if let Ok(mut state) = task.state.lock() {
3101 match &mut state.runtime {
3102 TaskRuntime::Piped(child) => *child = None,
3103 TaskRuntime::Pty(runtime) => *runtime = None,
3104 }
3105 state.detached = true;
3106 }
3107 }
3108 tasks.clear();
3109 }
3110 }
3111
3112 pub fn shutdown(&self) {
3113 let tasks = self
3114 .inner
3115 .tasks
3116 .lock()
3117 .map(|tasks| {
3118 tasks
3119 .values()
3120 .map(|task| (task.task_id.clone(), task.session_id.clone()))
3121 .collect::<Vec<_>>()
3122 })
3123 .unwrap_or_default();
3124 for (task_id, session_id) in tasks {
3125 let _ = self.kill(&task_id, &session_id);
3126 }
3127 }
3128
3129 pub(crate) fn poll_task(&self, task: &Arc<BgTask>) -> Result<(), String> {
3130 if let Ok(state) = task.state.lock() {
3131 if let TaskRuntime::Pty(Some(pty)) = &state.runtime {
3132 if !pty.exit_observed.load(Ordering::SeqCst) {
3140 return Ok(());
3141 }
3142 }
3143 }
3144 let marker = match read_exit_marker(&task.paths) {
3145 Ok(Some(marker)) => marker,
3146 Ok(None) => return Ok(()),
3147 Err(error) => return Err(format!("failed to read exit marker: {error}")),
3148 };
3149 self.finalize_from_marker(task, marker, None)
3150 }
3151
3152 pub(crate) fn reap_child(&self, task: &Arc<BgTask>) {
3153 let mut needs_completion = false;
3154 {
3155 let Ok(mut state) = task.state.lock() else {
3156 return;
3157 };
3158 match &mut state.runtime {
3159 TaskRuntime::Piped(child_slot) => {
3160 if let Some(child) = child_slot.as_mut() {
3161 if let Ok(Some(status)) = child.try_wait() {
3162 *child_slot = None;
3163 state.detached = true;
3164 state.child_exit_observed = true;
3165 if let Some(handles) = state.io_handles.as_mut() {
3166 if handles.artifact_len(TaskArtifact::Exit).unwrap_or(1) == 0 {
3167 let marker = status
3168 .code()
3169 .map(|code| code.to_string())
3170 .unwrap_or_else(|| "1".to_string());
3171 let _ = handles.write(TaskArtifact::Exit, marker.as_bytes());
3172 }
3173 }
3174 }
3175 } else if state.detached {
3176 let child_known_dead = state.child_exit_observed
3177 || state
3178 .metadata
3179 .child_pid
3180 .is_some_and(|pid| !is_process_alive(pid));
3181 if child_known_dead {
3182 needs_completion =
3183 self.fail_without_exit_marker_if_needed(task, &mut state);
3184 }
3185 }
3186 }
3187 TaskRuntime::Pty(Some(pty)) => {
3188 if pty.exit_observed.load(Ordering::SeqCst) {
3189 drop(state);
3190 let _ = self.poll_task(task);
3191 return;
3192 }
3193 }
3194 TaskRuntime::Pty(None) => {}
3195 }
3196 }
3197 if needs_completion {
3198 let _ = self.post_terminal_transition(task, true);
3199 }
3200 }
3201
3202 fn fail_without_exit_marker_if_needed(
3203 &self,
3204 task: &Arc<BgTask>,
3205 state: &mut BgTaskState,
3206 ) -> bool {
3207 if state.metadata.status.is_terminal() {
3208 return false;
3209 }
3210 if matches!(read_exit_marker(&task.paths), Ok(Some(_))) {
3211 return false;
3212 }
3213 let watch_controlled = self.task_has_watch_control(&task.task_id);
3214 let child_exit_observed = state.child_exit_observed;
3215 let updated = self.update_task_metadata(&task.paths, |metadata| {
3216 let (status, reason) = if child_exit_observed {
3217 (
3218 BgTaskStatus::Failed,
3219 "process exited without exit marker".to_string(),
3220 )
3221 } else {
3222 (
3223 BgTaskStatus::FateUnknown,
3224 restart_fate_unknown_reason(metadata, &task.paths),
3225 )
3226 };
3227 metadata.mark_terminal(status, None, Some(reason));
3228 if watch_controlled {
3229 metadata.completion_delivered = true;
3230 }
3231 });
3232 if let Ok(metadata) = updated {
3233 state.pending_terminal_override = None;
3234 state.metadata = metadata;
3235 task.mark_terminal_now();
3236 return true;
3237 }
3238 false
3239 }
3240
3241 pub(crate) fn running_tasks(&self) -> Vec<Arc<BgTask>> {
3242 self.inner
3243 .tasks
3244 .lock()
3245 .map(|tasks| {
3246 tasks
3247 .values()
3248 .filter(|task| task.is_running())
3249 .cloned()
3250 .collect()
3251 })
3252 .unwrap_or_default()
3253 }
3254
3255 fn insert_rehydrated_task(
3256 &self,
3257 metadata: PersistedTask,
3258 paths: TaskPaths,
3259 detached: bool,
3260 delivery_session_id: Option<&str>,
3261 ) -> Result<(), String> {
3262 let task_id = metadata.task_id.clone();
3263 let session_id = metadata.session_id.clone();
3264 let started = started_instant_from_unix_millis(metadata.started_at);
3265 let suppress_replayed_running_reminder = metadata.status == BgTaskStatus::Running;
3266 let mode = metadata.mode.clone();
3267 let task = Arc::new(BgTask {
3268 task_id: task_id.clone(),
3269 delivery_session_id: delivery_session_id.unwrap_or(&session_id).to_string(),
3270 session_id,
3271 paths: paths.clone(),
3272 artifact_root: canonical_artifact_root(&paths),
3273 started,
3274 last_reminder_at: Mutex::new(suppress_replayed_running_reminder.then(Instant::now)),
3275 terminal_at: Mutex::new(metadata.status.is_terminal().then(Instant::now)),
3276 state: Mutex::new(BgTaskState {
3277 metadata,
3278 runtime: if mode == BgMode::Pty {
3279 TaskRuntime::Pty(None)
3280 } else {
3281 TaskRuntime::Piped(None)
3282 },
3283 io_handles: None,
3284 detached,
3285 child_exit_observed: false,
3292 buffer: BgBuffer::registered(&paths, mode.clone()),
3293 terminal_output_cache: None,
3294 pending_terminal_override: None,
3295 }),
3296 });
3297 self.inner
3298 .tasks
3299 .lock()
3300 .map_err(|_| "background task registry lock poisoned".to_string())?
3301 .insert(task_id.clone(), Arc::clone(&task));
3302 self.rearm_persisted_watches(&task);
3306 Ok(())
3307 }
3308
3309 fn rearm_persisted_watches(&self, task: &Arc<BgTask>) {
3310 let Some((harness, pool)) = self.db_harness_and_pool() else {
3311 return;
3312 };
3313 let rows = {
3314 let Ok(conn) = pool.lock() else {
3315 return;
3316 };
3317 match crate::db::bash_watches::list_bash_pattern_watches_for_task(
3318 &conn,
3319 &harness,
3320 &task.session_id,
3321 &task.task_id,
3322 ) {
3323 Ok(rows) if !rows.is_empty() => rows,
3324 _ => return,
3325 }
3326 };
3327
3328 let mode = match task.state.lock() {
3329 Ok(state) => state.metadata.mode.clone(),
3330 Err(_) => return,
3331 };
3332 let terminal = task
3333 .state
3334 .lock()
3335 .map(|state| state.metadata.status.is_terminal())
3336 .unwrap_or(false);
3337 let completion_delivered = task
3338 .state
3339 .lock()
3340 .map(|state| state.metadata.completion_delivered)
3341 .unwrap_or(true);
3342
3343 let mut stdout = (mode == BgMode::Pipes)
3344 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
3345 .transpose()
3346 .ok()
3347 .flatten();
3348 let mut stderr = (mode == BgMode::Pipes)
3349 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
3350 .transpose()
3351 .ok()
3352 .flatten();
3353 let mut pty = (mode == BgMode::Pty)
3354 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
3355 .transpose()
3356 .ok()
3357 .flatten();
3358
3359 let mut pending_to_emit = Vec::new();
3360 let mut gap_matches = Vec::new();
3361 {
3362 let Ok(mut registry) = self.inner.watch_registry.lock() else {
3363 return;
3364 };
3365 let stdout_key = format!("{}:stdout", task.task_id);
3366 let stderr_key = format!("{}:stderr", task.task_id);
3367 let pty_key = format!("{}:pty", task.task_id);
3368
3369 let first = &rows[0];
3371 match mode {
3372 BgMode::Pipes => {
3373 registry.set_file_cursor(&stdout_key, first.stdout_offset.max(0) as u64);
3374 registry.set_file_cursor(&stderr_key, first.stderr_offset.max(0) as u64);
3375 }
3376 BgMode::Pty => {
3377 registry.set_file_cursor(&pty_key, first.pty_offset.max(0) as u64);
3378 }
3379 }
3380
3381 for row in &rows {
3382 let Ok(pattern) = WatchPattern::from_persisted(&row.pattern_kind, &row.pattern)
3383 else {
3384 crate::slog_warn!(
3385 "skipping unreadable persisted watch {}/{}",
3386 row.task_id,
3387 row.watch_id
3388 );
3389 continue;
3390 };
3391 if let Err(error) = registry.restore(
3392 row.watch_id.clone(),
3393 row.task_id.clone(),
3394 pattern,
3395 row.once,
3396 row.scanning,
3397 ) {
3398 crate::slog_warn!(
3399 "failed to restore watch {}/{}: {error}",
3400 row.task_id,
3401 row.watch_id
3402 );
3403 continue;
3404 }
3405 if row.pending_match {
3406 if let (Some(match_text), Some(match_offset), Some(context)) = (
3407 row.match_text.clone(),
3408 row.match_offset,
3409 row.match_context.clone(),
3410 ) {
3411 pending_to_emit.push(PatternMatch {
3412 watch_id: row.watch_id.clone(),
3413 task_id: row.task_id.clone(),
3414 match_text,
3415 match_offset: match_offset.max(0) as u64,
3416 context,
3417 once: row.once,
3418 });
3419 }
3420 }
3421 }
3422
3423 let should_gap_scan =
3427 rows.iter().any(|row| row.scanning) && !pending_to_emit.iter().any(|m| m.once);
3428 if should_gap_scan {
3429 match mode {
3430 BgMode::Pipes => {
3431 if let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) {
3432 gap_matches.extend(registry.scan_file_new_bytes(
3433 &stdout_key,
3434 &task.task_id,
3435 stdout,
3436 ));
3437 gap_matches.extend(registry.scan_file_new_bytes(
3438 &stderr_key,
3439 &task.task_id,
3440 stderr,
3441 ));
3442 }
3443 }
3444 BgMode::Pty => {
3445 if let Some(pty) = pty.as_mut() {
3446 gap_matches.extend(registry.scan_file_new_bytes(
3447 &pty_key,
3448 &task.task_id,
3449 pty,
3450 ));
3451 }
3452 }
3453 }
3454 }
3455 }
3456
3457 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
3458 for pattern_match in &gap_matches {
3459 self.persist_watch_match(
3460 &task.session_id,
3461 &task.task_id,
3462 pattern_match,
3463 stdout_offset,
3464 stderr_offset,
3465 pty_offset,
3466 );
3467 }
3468 if !gap_matches.is_empty() || rows.iter().any(|row| row.scanning) {
3469 self.persist_task_watch_cursors(
3470 &task.session_id,
3471 &task.task_id,
3472 stdout_offset,
3473 stderr_offset,
3474 pty_offset,
3475 );
3476 }
3477
3478 let emitted_pending = !pending_to_emit.is_empty();
3480 let to_emit = if emitted_pending {
3481 pending_to_emit
3482 } else {
3483 gap_matches
3484 };
3485 for pattern_match in to_emit {
3486 self.emit_bash_pattern_match(&task.delivery_session_id, pattern_match);
3487 }
3488
3489 if !terminal {
3490 return;
3491 }
3492
3493 let _ = self.remove_pending_completion(&task.task_id);
3496 let (watch_controlled, watch_matched) = self.task_watch_state(&task.task_id);
3497 if !watch_controlled {
3498 return;
3499 }
3500 if watch_matched {
3501 return;
3503 }
3504 if completion_delivered {
3505 self.clear_task_watch_state(&task.task_id);
3507 self.delete_persisted_watches_for_task(&task.session_id, &task.task_id);
3508 return;
3509 }
3510 if let Some(completion) = self.completion_snapshot_for_task(task) {
3511 self.emit_bash_watch_exit(&completion);
3512 }
3513 self.clear_task_watch_state(&task.task_id);
3515 }
3516
3517 fn kill_with_status(
3518 &self,
3519 task_id: &str,
3520 session_id: &str,
3521 terminal_status: BgTaskStatus,
3522 ) -> Result<BgTaskSnapshot, String> {
3523 self.kill_with_status_reason(task_id, session_id, terminal_status, None)
3524 }
3525
3526 fn kill_with_status_reason(
3527 &self,
3528 task_id: &str,
3529 session_id: &str,
3530 terminal_status: BgTaskStatus,
3531 reason: Option<String>,
3532 ) -> Result<BgTaskSnapshot, String> {
3533 let task = self
3534 .task_for_session(task_id, session_id)
3535 .ok_or_else(|| format!("background task not found: {task_id}"))?;
3536 let mut terminalized = false;
3537
3538 {
3539 let mut state = task
3540 .state
3541 .lock()
3542 .map_err(|_| "background task lock poisoned".to_string())?;
3543 if state.metadata.status.is_terminal() {
3544 state.pending_terminal_override = None;
3545 } else if let Ok(Some(marker)) = read_exit_marker(&task.paths) {
3546 state.metadata =
3547 terminal_metadata_from_marker(state.metadata.clone(), marker, reason.clone());
3548 if self.task_has_watch_control(&task.task_id) {
3549 state.metadata.completion_delivered = true;
3550 }
3551 state.pending_terminal_override = None;
3552 task.mark_terminal_now();
3553 match &mut state.runtime {
3554 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
3561 TaskRuntime::Pty(runtime) => *runtime = None,
3562 }
3563 state.detached = true;
3564 self.persist_task(&task.paths, &state.metadata)
3565 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
3566 terminalized = true;
3567 } else {
3568 let was_already_killing = state.metadata.status == BgTaskStatus::Killing;
3569 if !was_already_killing {
3570 state.metadata.status = BgTaskStatus::Killing;
3571 }
3572 if reason.is_some() {
3573 state.metadata.status_reason = reason.clone();
3574 }
3575 if !was_already_killing || reason.is_some() {
3576 self.persist_task(&task.paths, &state.metadata)
3577 .map_err(|e| format!("failed to persist killing state: {e}"))?;
3578 }
3579
3580 #[cfg(unix)]
3581 let pgid = state.metadata.pgid;
3582 #[cfg(windows)]
3583 let child_pid = state.metadata.child_pid;
3584 if !was_already_killing
3585 && state.metadata.mode == BgMode::Pty
3586 && terminal_status == BgTaskStatus::TimedOut
3587 {
3588 state.pending_terminal_override = Some(BgTaskStatus::TimedOut);
3589 }
3590
3591 #[cfg(windows)]
3592 let mut pty_forced_terminal_status: Option<BgTaskStatus> = None;
3593
3594 match &mut state.runtime {
3595 TaskRuntime::Piped(child_slot) => {
3596 #[cfg(unix)]
3597 if let Some(pgid) = pgid {
3598 terminate_pgid(pgid, child_slot.as_mut());
3599 }
3600 #[cfg(windows)]
3601 if let Some(child) = child_slot.as_mut() {
3602 super::process::terminate_process(child);
3603 } else if let Some(pid) = child_pid {
3604 terminate_pid(pid);
3605 }
3606 if let Some(child) = child_slot.as_mut() {
3607 let _ = child.wait();
3608 }
3609 *child_slot = None;
3610 state.detached = true;
3611
3612 if let Some(handles) = state.io_handles.as_mut() {
3613 handles.write(TaskArtifact::Exit, b"killed").map_err(|e| {
3614 format!("failed to write retained kill marker: {e}")
3615 })?;
3616 } else {
3617 write_kill_marker_if_absent(&task.paths)
3618 .map_err(|e| format!("failed to write kill marker: {e}"))?;
3619 }
3620
3621 let exit_code = terminal_exit_code_for_status(&terminal_status);
3622 state
3623 .metadata
3624 .mark_terminal(terminal_status, exit_code, reason.clone());
3625 if self.task_has_watch_control(&task.task_id) {
3626 state.metadata.completion_delivered = true;
3627 }
3628 state.pending_terminal_override = None;
3629 task.mark_terminal_now();
3630 self.persist_task(&task.paths, &state.metadata)
3631 .map_err(|e| format!("failed to persist killed state: {e}"))?;
3632 terminalized = true;
3633 }
3634 TaskRuntime::Pty(Some(pty)) => {
3635 pty.was_killed.store(true, Ordering::SeqCst);
3636 if let Err(error) = pty.killer.kill() {
3637 crate::slog_warn!(
3638 "[pty-kill] {task_id} ChildKiller::kill failed: {error}"
3639 );
3640 }
3641 if let Some(pid) = pty.child_pid {
3642 #[cfg(unix)]
3643 terminate_pgid(pid as i32, None);
3644 #[cfg(windows)]
3645 terminate_pid(pid);
3646 }
3647 drop(pty.master.take());
3648
3649 #[cfg(windows)]
3650 {
3651 let default_status = if terminal_status == BgTaskStatus::TimedOut {
3652 BgTaskStatus::TimedOut
3653 } else {
3654 BgTaskStatus::Killed
3655 };
3656 pty_forced_terminal_status = Some(
3657 state
3658 .pending_terminal_override
3659 .take()
3660 .unwrap_or(default_status),
3661 );
3662 }
3663 }
3664 TaskRuntime::Pty(None) => {}
3665 }
3666
3667 #[cfg(windows)]
3668 if let Some(target_status) = pty_forced_terminal_status {
3669 if !task.paths.exit.exists() {
3670 write_kill_marker_if_absent(&task.paths)
3671 .map_err(|e| format!("failed to write kill marker: {e}"))?;
3672 }
3673
3674 let exit_code = terminal_exit_code_for_status(&target_status);
3675 state
3676 .metadata
3677 .mark_terminal(target_status, exit_code, reason.clone());
3678 if self.task_has_watch_control(&task.task_id) {
3679 state.metadata.completion_delivered = true;
3680 }
3681 state.pending_terminal_override = None;
3682 task.mark_terminal_now();
3683 if let TaskRuntime::Pty(runtime) = &mut state.runtime {
3684 *runtime = None;
3685 }
3686 state.detached = true;
3687 self.persist_task(&task.paths, &state.metadata)
3688 .map_err(|e| format!("failed to persist killed PTY state: {e}"))?;
3689 terminalized = true;
3690 }
3691 }
3692 }
3693
3694 if terminalized {
3695 self.post_terminal_transition(&task, true)?;
3696 }
3697 Ok(self.snapshot_with_terminal_cache(&task, RUNNING_OUTPUT_PREVIEW_BYTES))
3698 }
3699
3700 fn finalize_from_marker(
3701 &self,
3702 task: &Arc<BgTask>,
3703 marker: ExitMarker,
3704 reason: Option<String>,
3705 ) -> Result<(), String> {
3706 let watch_controlled = self.task_has_watch_control(&task.task_id);
3707 let mut pty_reader_done = None;
3708 {
3709 let mut state = task
3710 .state
3711 .lock()
3712 .map_err(|_| "background task lock poisoned".to_string())?;
3713 if state.metadata.status.is_terminal() {
3714 state.pending_terminal_override = None;
3715 return Ok(());
3716 }
3717
3718 let pending_override = state.pending_terminal_override.take();
3719 let is_pty = state.metadata.mode == BgMode::Pty;
3720 let reason = reason.or_else(|| state.metadata.status_reason.clone());
3721 let updated = self
3722 .update_task_metadata(&task.paths, |metadata| {
3723 let mut new_metadata = if is_pty && marker == ExitMarker::Killed {
3724 let mut metadata = metadata.clone();
3725 let target_status = pending_override.unwrap_or(BgTaskStatus::Killed);
3726 let exit_code = terminal_exit_code_for_status(&target_status);
3727 metadata.mark_terminal(target_status, exit_code, reason.clone());
3728 metadata
3729 } else {
3730 terminal_metadata_from_marker(metadata.clone(), marker, reason.clone())
3731 };
3732 if watch_controlled {
3733 new_metadata.completion_delivered = true;
3734 }
3735 *metadata = new_metadata;
3736 })
3737 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
3738 state.metadata = updated;
3739 task.mark_terminal_now();
3740 match &mut state.runtime {
3741 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
3746 TaskRuntime::Pty(runtime) => {
3747 pty_reader_done = runtime
3748 .as_ref()
3749 .map(|runtime| Arc::clone(&runtime.reader_done));
3750 *runtime = None;
3751 }
3752 }
3753 state.detached = true;
3754 }
3755
3756 if let Some(reader_done) = pty_reader_done {
3757 let deadline = Instant::now() + Duration::from_millis(200);
3758 while !reader_done.load(Ordering::SeqCst) && Instant::now() < deadline {
3759 std::thread::sleep(Duration::from_millis(10));
3760 }
3761 }
3762
3763 self.scan_task_watch_output(task);
3766
3767 self.post_terminal_transition(task, true)
3768 }
3769
3770 fn enqueue_completion_if_needed(
3771 &self,
3772 metadata: &PersistedTask,
3773 paths: Option<&TaskPaths>,
3774 emit_frame: bool,
3775 ) {
3776 if metadata.status.is_terminal() && !metadata.completion_delivered {
3777 let cache =
3778 paths.and_then(|paths| self.render_terminal_output_from_paths(metadata, paths));
3779 self.enqueue_completion_from_parts(metadata, None, paths, emit_frame, cache.as_ref());
3780 }
3781 }
3782
3783 fn render_terminal_output_from_paths(
3784 &self,
3785 metadata: &PersistedTask,
3786 paths: &TaskPaths,
3787 ) -> Option<TerminalOutputCache> {
3788 if metadata.mode == BgMode::Pty {
3789 return None;
3790 }
3791 let mut buffer = BgBuffer::registered(paths, BgMode::Pipes);
3792 let disk_truncation = buffer.enforce_terminal_cap();
3793 Some(self.render_terminal_output(metadata, &buffer, disk_truncation, Some(paths)))
3794 }
3795
3796 fn enqueue_completion_from_parts(
3797 &self,
3798 metadata: &PersistedTask,
3799 buffer: Option<&BgBuffer>,
3800 paths: Option<&TaskPaths>,
3801 emit_frame: bool,
3802 terminal_render: Option<&TerminalOutputCache>,
3803 ) {
3804 if !metadata.status.is_terminal() {
3815 return;
3816 }
3817
3818 let owned_buffer = if buffer.is_none() && metadata.mode != BgMode::Pty {
3819 paths.map(|paths| BgBuffer::registered(paths, BgMode::Pipes))
3820 } else {
3821 None
3822 };
3823 let render_buffer = buffer.or(owned_buffer.as_ref());
3824 let owned_render = if terminal_render.is_none() {
3825 render_buffer.map(|buffer| {
3826 let mut capped_buffer = buffer.clone();
3827 let disk_truncation = capped_buffer.enforce_terminal_cap();
3828 self.render_terminal_output(metadata, &capped_buffer, disk_truncation, paths)
3829 })
3830 } else {
3831 None
3832 };
3833 let render = terminal_render.or(owned_render.as_ref());
3834
3835 let (mut output_preview, output_truncated) = render
3839 .map(|cache| completion_preview_for_cache(cache, metadata.exit_code))
3840 .unwrap_or_else(|| (String::new(), false));
3841 if metadata.status == BgTaskStatus::FateUnknown {
3842 if let Some(reason) = metadata.status_reason.as_deref() {
3843 output_preview = if output_preview.is_empty() {
3844 reason.to_string()
3845 } else {
3846 format!("{reason}\n{output_preview}")
3847 };
3848 }
3849 }
3850
3851 let token_counts = self.completion_token_counts(
3852 metadata,
3853 buffer,
3854 paths,
3855 render.map(|render| render.output_preview.as_str()),
3856 );
3857 let completion = BgCompletion {
3858 task_id: metadata.task_id.clone(),
3859 session_id: metadata.session_id.clone(),
3860 status: metadata.status.clone(),
3861 exit_code: metadata.exit_code,
3862 command: metadata.command.clone(),
3863 output_preview,
3864 output_truncated,
3865 original_tokens: token_counts.original_tokens,
3866 compressed_tokens: token_counts.compressed_tokens,
3867 tokens_skipped: token_counts.tokens_skipped,
3868 status_reason: metadata.status_reason.clone(),
3869 };
3870
3871 self.record_compression_event_if_applicable(metadata, &token_counts);
3882
3883 let (watch_controlled, watch_matched) = self.task_watch_state(&metadata.task_id);
3884 if watch_controlled {
3885 if emit_frame && !watch_matched {
3886 self.emit_bash_watch_exit(&completion);
3887 } else if watch_matched {
3888 if let Some(task) = self.task(&metadata.task_id) {
3893 let _ = task.set_completion_delivered(true, self);
3894 }
3895 }
3896 self.clear_task_watch_state(&metadata.task_id);
3898 return;
3899 }
3900
3901 if metadata.completion_delivered {
3911 return;
3912 }
3913
3914 let pushed = if let Ok(mut completions) = self.inner.completions.lock() {
3917 if completions
3918 .iter()
3919 .any(|existing| existing.task_id == metadata.task_id)
3920 {
3921 false
3922 } else {
3923 completions.push_back(completion.clone());
3924 true
3925 }
3926 } else {
3927 false
3928 };
3929
3930 if pushed && emit_frame {
3931 self.emit_bash_completed(completion);
3932 }
3933 }
3934
3935 fn record_compression_event_if_applicable(
3936 &self,
3937 metadata: &PersistedTask,
3938 token_counts: &CompletionTokenCounts,
3939 ) {
3940 if metadata.mode == BgMode::Pty {
3941 return;
3942 }
3943
3944 let (original_tokens, compressed_tokens, original_bytes, compressed_bytes) = match (
3945 token_counts.original_tokens,
3946 token_counts.compressed_tokens,
3947 token_counts.original_bytes,
3948 token_counts.compressed_bytes,
3949 ) {
3950 (
3951 Some(original_tokens),
3952 Some(compressed_tokens),
3953 Some(original_bytes),
3954 Some(compressed_bytes),
3955 ) => (
3956 original_tokens,
3957 compressed_tokens,
3958 original_bytes,
3959 compressed_bytes,
3960 ),
3961 _ => {
3962 crate::slog_warn!(
3963 "compression event skipped for {}: token counts unavailable (likely spill file missing or unreadable)",
3964 metadata.task_id
3965 );
3966 return;
3967 }
3968 };
3969
3970 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
3971 let Some(pool) = pool else {
3972 crate::slog_warn!(
3973 "compression event skipped for {}: db_pool not initialized — was configure run?",
3974 metadata.task_id
3975 );
3976 return;
3977 };
3978 let harness = self
3979 .inner
3980 .db_harness
3981 .read()
3982 .ok()
3983 .and_then(|slot| slot.clone());
3984 let Some(harness) = harness else {
3985 crate::slog_warn!(
3986 "compression event insert skipped for {}: harness not configured",
3987 metadata.task_id
3988 );
3989 return;
3990 };
3991
3992 let project_root = metadata
3993 .project_root
3994 .as_deref()
3995 .unwrap_or(&metadata.workdir);
3996 let project_key = crate::path_identity::project_scope_key(project_root);
3997 let row = crate::db::compression_events::CompressionEventRow {
3998 harness: &harness,
3999 session_id: Some(&metadata.session_id),
4000 project_key: &project_key,
4001 tool: "bash",
4002 task_id: Some(&metadata.task_id),
4003 command: Some(&metadata.command),
4004 compressor: if metadata.compressed {
4005 "registry"
4006 } else {
4007 "none"
4008 },
4009 original_bytes,
4010 compressed_bytes,
4011 original_tokens,
4012 compressed_tokens,
4013 created_at: unix_millis() as i64,
4014 };
4015
4016 let conn = match pool.lock() {
4017 Ok(conn) => conn,
4018 Err(_) => {
4019 crate::slog_warn!(
4020 "compression event insert failed for {}: db mutex poisoned",
4021 metadata.task_id
4022 );
4023 return;
4024 }
4025 };
4026 match crate::db::compression_events::insert_compression_event(&conn, &row) {
4027 Ok(Some(row_id)) => {
4028 self.inner
4032 .compression_aggregates
4033 .record_successful_insert(&conn, &row, row_id);
4034 crate::slog_debug!(
4038 "compression event recorded for {} (project={}, session={}, {} → {} tokens)",
4039 metadata.task_id,
4040 project_key,
4041 metadata.session_id,
4042 original_tokens,
4043 compressed_tokens
4044 );
4045 }
4046 Ok(None) => {
4047 crate::slog_debug!(
4048 "duplicate compression event ignored for {} (project={}, session={})",
4049 metadata.task_id,
4050 project_key,
4051 metadata.session_id
4052 );
4053 }
4054 Err(error) => {
4055 crate::slog_warn!(
4056 "compression event insert failed for {}: {}",
4057 metadata.task_id,
4058 error
4059 );
4060 }
4061 }
4062 }
4063
4064 fn emit_bash_pattern_match(&self, session_id: &str, pattern_match: PatternMatch) {
4065 let Ok(progress_sender) = self
4066 .inner
4067 .progress_sender
4068 .lock()
4069 .map(|sender| sender.clone())
4070 else {
4071 return;
4072 };
4073 if let Some(sender) = progress_sender.as_ref() {
4074 sender(PushFrame::BashPatternMatch(BashPatternMatchFrame::new(
4075 pattern_match.task_id,
4076 session_id.to_string(),
4077 pattern_match.watch_id,
4078 pattern_match.match_text,
4079 pattern_match.match_offset,
4080 pattern_match.context,
4081 pattern_match.once,
4082 )));
4083 }
4084 }
4085
4086 fn emit_bash_watch_exit(&self, completion: &BgCompletion) {
4087 let Ok(progress_sender) = self
4088 .inner
4089 .progress_sender
4090 .lock()
4091 .map(|sender| sender.clone())
4092 else {
4093 return;
4094 };
4095 let Some(sender) = progress_sender.as_ref() else {
4096 return;
4097 };
4098 let status = completion_status_text(&completion.status, completion.exit_code);
4099 let preview = completion.output_preview.trim_end();
4100 let context = if preview.is_empty() {
4101 format!("task {} exited ({status})", completion.task_id)
4102 } else {
4103 format!(
4104 "task {} exited ({status})
4105{preview}",
4106 completion.task_id
4107 )
4108 };
4109 sender(PushFrame::BashPatternMatch(
4110 BashPatternMatchFrame::task_exit(
4111 completion.task_id.clone(),
4112 completion.session_id.clone(),
4113 format!("exited ({status})"),
4114 context,
4115 ),
4116 ));
4117 }
4118
4119 fn emit_bash_completed(&self, completion: BgCompletion) {
4120 let Ok(progress_sender) = self
4121 .inner
4122 .progress_sender
4123 .lock()
4124 .map(|sender| sender.clone())
4125 else {
4126 return;
4127 };
4128 let Some(sender) = progress_sender.as_ref() else {
4129 return;
4130 };
4131 let mut frame = BashCompletedFrame::new(
4139 completion.task_id,
4140 completion.session_id,
4141 completion.status,
4142 completion.exit_code,
4143 completion.command,
4144 completion.output_preview,
4145 completion.output_truncated,
4146 completion.original_tokens,
4147 completion.compressed_tokens,
4148 completion.tokens_skipped,
4149 );
4150 frame.status_reason = completion.status_reason;
4151 sender(PushFrame::BashCompleted(frame));
4152 }
4153
4154 fn completion_token_counts(
4155 &self,
4156 metadata: &PersistedTask,
4157 buffer: Option<&BgBuffer>,
4158 paths: Option<&TaskPaths>,
4159 rendered_output: Option<&str>,
4160 ) -> CompletionTokenCounts {
4161 if metadata.mode == BgMode::Pty {
4162 return CompletionTokenCounts::skipped();
4163 }
4164
4165 let raw = match buffer {
4166 Some(buffer) => buffer.read_for_token_count(TOKENIZE_CAP_BYTES_PER_STREAM),
4167 None => paths
4168 .map(|paths| {
4169 read_for_token_count_from_disk(metadata, paths, TOKENIZE_CAP_BYTES_PER_STREAM)
4170 })
4171 .unwrap_or(TokenCountInput::Skipped),
4172 };
4173
4174 let TokenCountInput::Text(raw_output) = raw else {
4175 return CompletionTokenCounts::skipped();
4176 };
4177
4178 let original_tokens = token_count_u32(&raw_output);
4179 let original_bytes = raw_output.len() as i64;
4180 let compressed_output = rendered_output.unwrap_or(&raw_output);
4181 let compressed_tokens = token_count_u32(compressed_output);
4182 let compressed_bytes = compressed_output.len() as i64;
4183 CompletionTokenCounts {
4184 original_tokens: Some(original_tokens),
4185 compressed_tokens: Some(compressed_tokens),
4186 original_bytes: Some(original_bytes),
4187 compressed_bytes: Some(compressed_bytes),
4188 tokens_skipped: false,
4189 }
4190 }
4191
4192 pub(crate) fn maybe_emit_long_running_reminder(&self, task: &Arc<BgTask>) {
4193 if !self
4194 .inner
4195 .long_running_reminder_enabled
4196 .load(Ordering::SeqCst)
4197 {
4198 return;
4199 }
4200 let interval_ms = self
4201 .inner
4202 .long_running_reminder_interval_ms
4203 .load(Ordering::SeqCst);
4204 if interval_ms == 0 {
4205 return;
4206 }
4207 let interval = Duration::from_millis(interval_ms);
4208 let now = Instant::now();
4209 let Ok(mut last_reminder_at) = task.last_reminder_at.lock() else {
4210 return;
4211 };
4212 let since = last_reminder_at.unwrap_or(task.started);
4213 if now.duration_since(since) < interval {
4214 return;
4215 }
4216 let command = task
4217 .state
4218 .lock()
4219 .map(|state| state.metadata.command.clone())
4220 .unwrap_or_default();
4221 *last_reminder_at = Some(now);
4222 self.emit_bash_long_running(BashLongRunningFrame::new(
4223 task.task_id.clone(),
4224 task.session_id.clone(),
4225 command,
4226 task.started.elapsed().as_millis() as u64,
4227 ));
4228 }
4229
4230 fn emit_bash_long_running(&self, frame: BashLongRunningFrame) {
4231 let Ok(progress_sender) = self
4232 .inner
4233 .progress_sender
4234 .lock()
4235 .map(|sender| sender.clone())
4236 else {
4237 return;
4238 };
4239 if let Some(sender) = progress_sender.as_ref() {
4240 sender(PushFrame::BashLongRunning(frame));
4241 }
4242 }
4243
4244 fn task(&self, task_id: &str) -> Option<Arc<BgTask>> {
4245 validate_task_id(task_id).ok()?;
4246 self.inner
4247 .tasks
4248 .lock()
4249 .ok()
4250 .and_then(|tasks| tasks.get(task_id).cloned())
4251 }
4252
4253 fn task_for_session(&self, task_id: &str, session_id: &str) -> Option<Arc<BgTask>> {
4254 self.task(task_id)
4255 .filter(|task| task.session_id == session_id)
4256 }
4257
4258 pub fn try_health_counts(&self) -> Option<BgTaskHealthCounts> {
4259 let running = self
4260 .inner
4261 .tasks
4262 .try_lock()
4263 .ok()
4264 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())?;
4265 let pending_completions = self.inner.completions.try_lock().ok().map(|q| q.len())?;
4266 Some(BgTaskHealthCounts {
4267 running,
4268 pending_completions,
4269 })
4270 }
4271
4272 pub fn estimated_memory(&self) -> crate::memory::MemoryEstimate {
4276 let tasks = match self.inner.tasks.try_lock() {
4277 Ok(tasks) => tasks.values().cloned().collect::<Vec<_>>(),
4278 Err(_) => return crate::memory::MemoryEstimate::busy(),
4279 };
4280 let mut bytes = 0u64;
4281 let mut terminal_output_caches = 0usize;
4282 let mut sessions = HashSet::new();
4283 for task in &tasks {
4284 sessions.insert(task.session_id.clone());
4285 let state = match task.state.try_lock() {
4286 Ok(state) => state,
4287 Err(_) => return crate::memory::MemoryEstimate::busy(),
4288 };
4289 if let Some(cache) = state.terminal_output_cache.as_ref() {
4290 terminal_output_caches = terminal_output_caches.saturating_add(1);
4291 bytes = bytes.saturating_add(terminal_output_cache_estimated_bytes(cache));
4292 }
4293 }
4294 let completion_count = match self.inner.completions.try_lock() {
4295 Ok(completions) => {
4296 for completion in completions.iter() {
4297 sessions.insert(completion.session_id.clone());
4298 bytes = bytes.saturating_add(completion_estimated_bytes(completion));
4299 }
4300 completions.len()
4301 }
4302 Err(_) => return crate::memory::MemoryEstimate::busy(),
4303 };
4304
4305 crate::memory::MemoryEstimate::estimated(bytes)
4306 .count("tasks", tasks.len())
4307 .count("sessions", sessions.len())
4308 .count("terminal_output_caches", terminal_output_caches)
4309 .count("completion_caches", completion_count)
4310 .count_u64("output_ring_bytes", 0)
4311 }
4312
4313 fn running_count(&self) -> usize {
4314 self.inner
4315 .tasks
4316 .lock()
4317 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())
4318 .unwrap_or(0)
4319 }
4320
4321 fn start_watchdog(&self) {
4322 if !self.inner.watchdog_started.swap(true, Ordering::SeqCst) {
4323 super::watchdog::start(self.clone());
4324 }
4325 }
4326
4327 #[cfg(test)]
4328 pub fn task_json_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
4329 self.task_for_session(task_id, session_id)
4330 .map(|task| task.paths.json.clone())
4331 }
4332
4333 #[cfg(test)]
4334 pub fn task_exit_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
4335 self.task_for_session(task_id, session_id)
4336 .map(|task| task.paths.exit.clone())
4337 }
4338}
4339
4340#[cfg(unix)]
4341fn should_capture_pipeline_status(
4342 spawn_plan: &SpawnPlan,
4343 has_pipeline: bool,
4344 shell: &Path,
4345) -> bool {
4346 if spawn_plan.is_native_launcher() {
4347 return false;
4350 }
4351 has_pipeline && super::process::pipeline_shell_kind(shell).is_some()
4352}
4353
4354fn canonical_artifact_root(paths: &TaskPaths) -> PathBuf {
4355 fs::canonicalize(&paths.io_dir).unwrap_or_else(|_| paths.io_dir.clone())
4356}
4357
4358fn restart_fate_unknown_reason(metadata: &PersistedTask, paths: &TaskPaths) -> String {
4359 let output = match metadata.mode {
4360 BgMode::Pipes => &paths.stdout,
4361 BgMode::Pty => &paths.pty,
4362 };
4363 format!(
4364 "task {}: daemon restarted, process fate unknown, last output at {}",
4365 metadata.task_id,
4366 output.display()
4367 )
4368}
4369
4370fn append_pipeline_warning(
4375 cache: &mut TerminalOutputCache,
4376 metadata: &PersistedTask,
4377 paths: Option<&TaskPaths>,
4378) {
4379 if metadata.exit_code != Some(0) {
4380 return;
4381 }
4382 let Some(paths) = paths else {
4383 return;
4384 };
4385 if metadata.pipeline_segments.len() < 2 {
4386 return;
4387 }
4388 let Ok(mut status_file) = open_task_artifact(paths, TaskArtifact::PipelineStatus) else {
4389 return;
4390 };
4391 let Ok(status_bytes) = status_file.read_all() else {
4392 return;
4393 };
4394 let Some(statuses) = String::from_utf8_lossy(&status_bytes)
4395 .lines()
4396 .map(|line| line.trim().parse::<i32>().ok())
4397 .collect::<Option<Vec<_>>>()
4398 else {
4399 return;
4400 };
4401 if statuses.len() != metadata.pipeline_segments.len() {
4402 return;
4403 }
4404 let Some((failing_index, failing_code)) = statuses
4405 .iter()
4406 .enumerate()
4407 .take(statuses.len().saturating_sub(1))
4408 .find(|(_, code)| **code != 0)
4409 .map(|(index, code)| (index, *code))
4410 else {
4411 return;
4412 };
4413 let Some(final_segment) = metadata.pipeline_segments.last() else {
4414 return;
4415 };
4416 let failing_segment = &metadata.pipeline_segments[failing_index];
4417 let footer = format!(
4418 "note: `{}` (segment {} of {}) exited {}; the pipeline's exit code is `{}`'s.",
4419 failing_segment,
4420 failing_index + 1,
4421 metadata.pipeline_segments.len(),
4422 failing_code,
4423 final_segment,
4424 );
4425 if cache.output_preview.trim().is_empty() {
4426 cache.output_preview = footer;
4427 } else {
4428 cache.output_preview = format!("{}\n{}", cache.output_preview.trim_end(), footer,);
4429 }
4430}
4431
4432fn normalize_piped_display_output(text: &mut String) {
4435 if !text.contains('\r') {
4436 return;
4437 }
4438
4439 let mut rendered = String::with_capacity(text.len());
4440 let mut line = Vec::new();
4441 let mut column = 0;
4442 let mut chars = text.chars().peekable();
4443
4444 while let Some(character) = chars.next() {
4445 match character {
4446 '\r' if chars.peek() == Some(&'\n') => {
4447 chars.next();
4448 for character in &line {
4449 rendered.push(*character);
4450 }
4451 rendered.push('\n');
4452 line.clear();
4453 column = 0;
4454 }
4455 '\r' => column = 0,
4456 '\n' => {
4457 for character in &line {
4458 rendered.push(*character);
4459 }
4460 rendered.push('\n');
4461 line.clear();
4462 column = 0;
4463 }
4464 character => {
4465 if column < line.len() {
4466 line[column] = character;
4467 } else {
4468 line.resize(column, ' ');
4469 line.push(character);
4470 }
4471 column += 1;
4472 }
4473 }
4474 }
4475
4476 for character in &line {
4477 rendered.push(*character);
4478 }
4479 *text = rendered;
4480}
4481
4482fn render_compressed_with_recovery(
4483 buffer: &BgBuffer,
4484 mut compressed: CompressionResult,
4485 input_truncated: bool,
4486 disk_truncation: DiskTruncation,
4487 artifact_access: ArtifactRecoveryAccess,
4488) -> TerminalOutputCache {
4489 let had_trailing_newline = compressed.text.ends_with('\n');
4497 let mut text = strip_plain_truncation_marker_lines(&compressed.text)
4498 .trim_end()
4499 .to_string();
4500 if had_trailing_newline && !text.is_empty() {
4501 text.push('\n');
4502 }
4503 compressed.text = text;
4504
4505 let output_path = buffer.output_path().map(|path| path.display().to_string());
4506 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
4507 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
4508 let mut recovery = RecoveryContext {
4509 dropped_by_class: compressed.dropped_by_class,
4510 had_inner_drop: compressed.had_inner_drop,
4511 offset_hint_eligible: compressed.offset_hint_eligible,
4512 offset_start_line: compressed.offset_start_line,
4513 byte_truncated: input_truncated,
4514 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
4515 output_path: output_path.clone(),
4516 stderr_path: stderr_path.clone(),
4517 include_stderr_path,
4518 artifact_access: artifact_access.clone(),
4519 };
4520
4521 let (output_preview, output_truncated) =
4522 render_body_with_recovery_marker(&compressed.text, &mut recovery);
4523 TerminalOutputCache {
4524 output_preview,
4525 output_truncated,
4526 kind: TerminalOutputKind::Compressed,
4527 output_path,
4528 stderr_path,
4529 artifact_access,
4530 recovery: Some(recovery),
4531 }
4532}
4533
4534fn render_body_with_recovery_marker(body: &str, recovery: &mut RecoveryContext) -> (String, bool) {
4535 render_body_with_recovery_marker_at_cap(
4536 body,
4537 recovery,
4538 FINAL_OUTPUT_CAP_BYTES,
4539 cap_final_output,
4540 cap_final_output_with_marker,
4541 )
4542}
4543
4544fn render_raw_body_with_recovery_marker(
4545 body: &str,
4546 recovery: &mut RecoveryContext,
4547) -> (String, bool) {
4548 render_body_with_recovery_marker_at_cap(
4549 body,
4550 recovery,
4551 RAW_PASSTHROUGH_CAP_BYTES,
4552 |input| {
4553 super::output::cap_head_tail(
4554 input,
4555 RAW_PASSTHROUGH_CAP_BYTES,
4556 RAW_PASSTHROUGH_HEAD_BYTES,
4557 RAW_PASSTHROUGH_TAIL_BYTES,
4558 )
4559 },
4560 |input, marker| {
4561 super::output::cap_head_tail_with_marker(
4562 input,
4563 RAW_PASSTHROUGH_CAP_BYTES,
4564 RAW_PASSTHROUGH_HEAD_BYTES,
4565 RAW_PASSTHROUGH_TAIL_BYTES,
4566 marker,
4567 )
4568 },
4569 )
4570}
4571
4572fn render_body_with_recovery_marker_at_cap<F, G>(
4573 body: &str,
4574 recovery: &mut RecoveryContext,
4575 cap_bytes: usize,
4576 cap_plain: F,
4577 cap_with_marker: G,
4578) -> (String, bool)
4579where
4580 F: Fn(&str) -> super::output::CappedText,
4581 G: Fn(&str, &str) -> super::output::CappedText,
4582{
4583 let needs_marker = recovery.has_visible_drop();
4584 if body.len() > cap_bytes {
4585 recovery.byte_truncated = true;
4586 if let Some(marker) = recovery_marker(recovery) {
4587 let capped = cap_with_marker(body, &marker);
4588 return (capped.text, true);
4589 }
4590 let capped = cap_plain(body);
4591 return (capped.text, capped.truncated || needs_marker);
4592 }
4593
4594 if !needs_marker {
4595 return (body.to_string(), false);
4596 }
4597
4598 let Some(marker) = recovery_marker(recovery) else {
4599 return (body.to_string(), true);
4600 };
4601 let with_marker = append_recovery_marker(body, &marker);
4602 if with_marker.len() <= cap_bytes {
4603 return (with_marker, true);
4604 }
4605
4606 recovery.byte_truncated = true;
4607 let marker = recovery_marker(recovery).unwrap_or(marker);
4608 let capped = cap_with_marker(body, &marker);
4609 (capped.text, true)
4610}
4611
4612fn append_recovery_marker(body: &str, marker: &str) -> String {
4613 if body.is_empty() {
4614 return marker.to_string();
4615 }
4616 let mut output = body.trim_end().to_string();
4617 output.push('\n');
4618 output.push_str(marker);
4619 output
4620}
4621
4622fn recovery_marker(recovery: &RecoveryContext) -> Option<String> {
4623 let mut parts = Vec::new();
4624 for (class, count) in &recovery.dropped_by_class {
4625 let label = if *count == 1 {
4626 class.singular()
4627 } else {
4628 class.plural()
4629 };
4630 parts.push(format!("+{count} more {label}"));
4631 }
4632 if recovery.byte_truncated {
4633 parts.push("truncated output".to_string());
4634 }
4635 let disk_truncated_prefix_bytes = recovery.disk_truncated_prefix_bytes;
4636 if disk_truncated_prefix_bytes > 0 {
4637 parts.push(format!(
4638 "truncated {disk_truncated_prefix_bytes} bytes from saved output prefix"
4639 ));
4640 } else if recovery.had_inner_drop && parts.is_empty() {
4641 parts.push("omitted output".to_string());
4642 }
4643
4644 if parts.is_empty() {
4645 return None;
4646 }
4647
4648 let hint = recovery_hint(recovery);
4649 Some(format!("[{}; {hint}]", parts.join(", ")))
4650}
4651
4652fn bash_status_recovery_hint(access: &ArtifactRecoveryAccess) -> String {
4653 let task_id = serde_json::to_string(&access.task_id)
4654 .unwrap_or_else(|_| format!("\"{}\"", access.task_id));
4655 format!("use bash_status({{taskId: {task_id}}})")
4656}
4657
4658fn recovery_hint(recovery: &RecoveryContext) -> String {
4659 if !recovery.artifact_access.readable {
4660 return bash_status_recovery_hint(&recovery.artifact_access);
4661 }
4662
4663 if recovery.offset_hint_eligible
4667 && !recovery.byte_truncated
4668 && recovery.dropped_by_class.is_empty()
4669 && !recovery.include_stderr_path
4670 {
4671 if let (Some(path), Some(line)) =
4672 (recovery.output_path.as_deref(), recovery.offset_start_line)
4673 {
4674 return format!("see remaining: tail -n +{line} {}", quote_path(path));
4675 }
4676 }
4677
4678 let mut paths = Vec::new();
4679 if let Some(path) = recovery.output_path.as_deref() {
4680 paths.push(path);
4681 }
4682 if recovery.include_stderr_path {
4683 if let Some(path) = recovery.stderr_path.as_deref() {
4684 if !paths.contains(&path) {
4685 paths.push(path);
4686 }
4687 }
4688 }
4689
4690 if paths.is_empty() {
4691 return "full output unavailable".to_string();
4692 }
4693
4694 let reads = paths
4695 .into_iter()
4696 .map(|path| format!("read {}", quote_path(path)))
4697 .collect::<Vec<_>>()
4698 .join(" and ");
4699 if recovery.disk_truncated_prefix_bytes > 0 {
4700 format!("retained output: {reads}")
4701 } else {
4702 format!("full output: {reads}")
4703 }
4704}
4705
4706fn strip_plain_truncation_marker_lines(input: &str) -> String {
4707 input
4708 .lines()
4709 .filter(|line| !is_plain_truncation_marker(line.trim()))
4710 .collect::<Vec<_>>()
4711 .join("\n")
4712}
4713
4714fn strip_recovery_marker_lines(input: &str) -> String {
4715 input
4716 .lines()
4717 .filter(|line| !is_recovery_marker(line.trim()))
4718 .collect::<Vec<_>>()
4719 .join("\n")
4720}
4721
4722fn is_plain_truncation_marker(line: &str) -> bool {
4723 let Some(rest) = line.strip_prefix("...<truncated ") else {
4724 return false;
4725 };
4726 let Some(bytes) = rest.strip_suffix(" bytes>...") else {
4727 return false;
4728 };
4729 !bytes.is_empty() && bytes.chars().all(|ch| ch.is_ascii_digit())
4730}
4731
4732fn is_recovery_marker(line: &str) -> bool {
4733 line.starts_with('[')
4734 && line.ends_with(']')
4735 && (line.contains("full output: read ")
4736 || line.contains("retained output: read ")
4737 || line.contains("see remaining: tail -n +")
4738 || line.contains("use bash_status({taskId:")
4739 || line.contains("full output unavailable"))
4740}
4741
4742fn structured_output_pointer(
4743 total_bytes: u64,
4744 output_path: &str,
4745 truncated_prefix_bytes: u64,
4746 artifact_access: &ArtifactRecoveryAccess,
4747) -> String {
4748 if artifact_access.readable {
4749 return if truncated_prefix_bytes > 0 {
4750 retained_json_output_pointer(total_bytes, output_path, truncated_prefix_bytes)
4751 } else {
4752 json_output_pointer(total_bytes, output_path)
4753 };
4754 }
4755
4756 let kb = total_bytes.div_ceil(1024);
4757 let hint = bash_status_recovery_hint(artifact_access);
4758 if truncated_prefix_bytes > 0 {
4759 format!(
4760 "[JSON output {kb} KB; truncated {truncated_prefix_bytes} bytes from saved output prefix; retained output: {hint}]"
4761 )
4762 } else {
4763 format!("[JSON output {kb} KB; full output: {hint}]")
4764 }
4765}
4766
4767fn render_structured_output(
4768 command: &str,
4769 buffer: &BgBuffer,
4770 disk_truncation: DiskTruncation,
4771 artifact_access: ArtifactRecoveryAccess,
4772) -> Option<TerminalOutputCache> {
4773 if !is_gh_structured_command(command) {
4774 return None;
4775 }
4776
4777 let output_path = buffer
4778 .output_path()
4779 .map(|path| path.display().to_string())?;
4780 let stdout_bytes = buffer.stream_len(StreamKind::Stdout);
4781 if stdout_bytes == 0 {
4782 return None;
4783 }
4784
4785 if stdout_bytes > STRUCTURED_OUTPUT_CAP_BYTES as u64 {
4786 if !stream_starts_like_json(buffer, StreamKind::Stdout) {
4787 return None;
4788 }
4789 let output_preview = structured_output_pointer(
4790 stdout_bytes,
4791 &output_path,
4792 disk_truncation.total_prefix_bytes(),
4793 &artifact_access,
4794 );
4795 return Some(TerminalOutputCache {
4796 output_preview,
4797 output_truncated: true,
4798 kind: TerminalOutputKind::Structured,
4799 output_path: Some(output_path),
4800 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
4801 artifact_access,
4802 recovery: None,
4803 });
4804 }
4805
4806 let stdout = buffer.read_stream_bounded(StreamKind::Stdout, STRUCTURED_OUTPUT_CAP_BYTES);
4807 if stdout.truncated || !is_structured_body(&stdout.text) {
4808 return None;
4809 }
4810
4811 Some(TerminalOutputCache {
4812 output_preview: stdout.text,
4813 output_truncated: false,
4814 kind: TerminalOutputKind::Structured,
4815 output_path: Some(output_path),
4816 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
4817 artifact_access,
4818 recovery: None,
4819 })
4820}
4821
4822fn render_raw_passthrough(
4823 buffer: &BgBuffer,
4824 disk_truncation: DiskTruncation,
4825 artifact_access: ArtifactRecoveryAccess,
4826) -> TerminalOutputCache {
4827 let raw = buffer.read_combined_head_tail(
4828 RAW_PASSTHROUGH_CAP_BYTES,
4829 RAW_PASSTHROUGH_HEAD_BYTES,
4830 RAW_PASSTHROUGH_TAIL_BYTES,
4831 );
4832 let output_path = buffer.output_path().map(|path| path.display().to_string());
4833 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
4834 if !raw.truncated && disk_truncation.total_prefix_bytes() == 0 {
4835 return TerminalOutputCache {
4836 output_preview: raw.text,
4837 output_truncated: false,
4838 kind: TerminalOutputKind::Raw,
4839 output_path,
4840 stderr_path,
4841 artifact_access,
4842 recovery: None,
4843 };
4844 }
4845
4846 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
4847 let mut recovery = RecoveryContext {
4848 dropped_by_class: BTreeMap::new(),
4849 had_inner_drop: false,
4850 offset_hint_eligible: false,
4851 offset_start_line: None,
4852 byte_truncated: raw.truncated,
4853 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
4854 output_path: output_path.clone(),
4855 stderr_path: stderr_path.clone(),
4856 include_stderr_path,
4857 artifact_access: artifact_access.clone(),
4858 };
4859 let (output_preview, output_truncated) =
4860 render_raw_body_with_recovery_marker(&raw.text, &mut recovery);
4861 TerminalOutputCache {
4862 output_preview,
4863 output_truncated,
4864 kind: TerminalOutputKind::Raw,
4865 output_path,
4866 stderr_path,
4867 artifact_access,
4868 recovery: Some(recovery),
4869 }
4870}
4871
4872fn completion_preview_for_cache(
4873 cache: &TerminalOutputCache,
4874 exit_code: Option<i32>,
4875) -> (String, bool) {
4876 let exit_ok = exit_code == Some(0);
4879 let threshold = completion_preview_threshold(exit_ok);
4880 if cache.kind == TerminalOutputKind::Structured && cache.output_preview.len() > threshold {
4881 if let Some(path) = cache.output_path.as_deref() {
4882 return (
4883 structured_output_pointer(
4884 cache.output_preview.len() as u64,
4885 path,
4886 0,
4887 &cache.artifact_access,
4888 ),
4889 true,
4890 );
4891 }
4892 return (cache.output_preview.clone(), cache.output_truncated);
4893 }
4894
4895 if let Some(recovery) = cache.recovery.as_ref() {
4896 if cache.output_preview.len() <= threshold {
4897 return (cache.output_preview.clone(), cache.output_truncated);
4898 }
4899 let body = strip_recovery_marker_lines(&cache.output_preview);
4900 let mut completion_recovery = recovery.clone();
4901 completion_recovery.byte_truncated = true;
4902 if let Some(marker) = recovery_marker(&completion_recovery) {
4903 let capped = cap_completion_output_with_marker(&body, &marker, exit_ok);
4904 return (capped.text, true);
4905 }
4906 }
4907
4908 let capped = cap_completion_output(&cache.output_preview, exit_ok);
4909 (capped.text, cache.output_truncated || capped.truncated)
4910}
4911
4912fn is_gh_structured_command(command: &str) -> bool {
4913 let Some(normalized) = crate::compress::plain_command_for_structured_output(command) else {
4914 return false;
4915 };
4916 let tokens = shell_words_for_flags(&normalized);
4917 let Some(head) = tokens.first() else {
4918 return false;
4919 };
4920 let head_name = Path::new(head)
4921 .file_name()
4922 .and_then(|name| name.to_str())
4923 .unwrap_or(head);
4924 if !(head_name == "gh" || head_name.eq_ignore_ascii_case("gh.exe")) {
4925 return false;
4926 }
4927 tokens.iter().any(|token| {
4928 matches!(token.as_str(), "--json" | "--jq" | "--template")
4929 || token.starts_with("--json=")
4930 || token.starts_with("--jq=")
4931 || token.starts_with("--template=")
4932 })
4933}
4934
4935fn shell_words_for_flags(command: &str) -> Vec<String> {
4936 let mut words = Vec::new();
4937 let mut current = String::new();
4938 let mut in_single = false;
4939 let mut in_double = false;
4940 let mut escaped = false;
4941
4942 for ch in command.chars() {
4943 if escaped {
4944 current.push(ch);
4945 escaped = false;
4946 continue;
4947 }
4948 if ch == '\\' && !in_single {
4949 escaped = true;
4950 continue;
4951 }
4952 if ch == '\'' && !in_double {
4953 in_single = !in_single;
4954 continue;
4955 }
4956 if ch == '"' && !in_single {
4957 in_double = !in_double;
4958 continue;
4959 }
4960 if ch.is_whitespace() && !in_single && !in_double {
4961 if !current.is_empty() {
4962 words.push(std::mem::take(&mut current));
4963 }
4964 continue;
4965 }
4966 if matches!(ch, ';' | '&' | '|') && !in_single && !in_double {
4967 if !current.is_empty() {
4968 words.push(std::mem::take(&mut current));
4969 }
4970 continue;
4971 }
4972 current.push(ch);
4973 }
4974 if !current.is_empty() {
4975 words.push(current);
4976 }
4977 words
4978}
4979
4980fn is_structured_body(body: &str) -> bool {
4981 let trimmed = body.trim();
4982 if trimmed.is_empty() {
4983 return false;
4984 }
4985 if serde_json::from_str::<serde_json::Value>(trimmed).is_ok() {
4986 return true;
4987 }
4988
4989 let mut saw_line = false;
4990 for line in trimmed
4991 .lines()
4992 .map(str::trim)
4993 .filter(|line| !line.is_empty())
4994 {
4995 saw_line = true;
4996 if serde_json::from_str::<serde_json::Value>(line).is_err() {
4997 return false;
4998 }
4999 }
5000 saw_line
5001}
5002
5003fn stream_starts_like_json(buffer: &BgBuffer, stream: StreamKind) -> bool {
5004 buffer
5005 .read_stream_bounded(stream, 512)
5006 .text
5007 .chars()
5008 .find(|ch| !ch.is_whitespace())
5009 .is_some_and(|ch| matches!(ch, '{' | '[' | '"' | '-' | '0'..='9' | 't' | 'f' | 'n'))
5010}
5011
5012struct CompletionTokenCounts {
5013 original_tokens: Option<u32>,
5014 compressed_tokens: Option<u32>,
5015 original_bytes: Option<i64>,
5016 compressed_bytes: Option<i64>,
5017 tokens_skipped: bool,
5018}
5019
5020impl CompletionTokenCounts {
5021 fn skipped() -> Self {
5022 Self {
5023 original_tokens: None,
5024 compressed_tokens: None,
5025 original_bytes: None,
5026 compressed_bytes: None,
5027 tokens_skipped: true,
5028 }
5029 }
5030}
5031
5032fn completion_status_text(status: &BgTaskStatus, exit_code: Option<i32>) -> String {
5033 match status {
5034 BgTaskStatus::TimedOut => "timed out".to_string(),
5035 BgTaskStatus::Killed => "killed".to_string(),
5036 _ => exit_code
5037 .map(|code| format!("exit {code}"))
5038 .unwrap_or_else(|| format!("{status:?}").to_lowercase()),
5039 }
5040}
5041
5042fn token_count_u32(text: &str) -> u32 {
5043 aft_tokenizer::count_tokens(text)
5044 .try_into()
5045 .unwrap_or(u32::MAX)
5046}
5047
5048impl Default for BgTaskRegistry {
5049 fn default() -> Self {
5050 Self::new(Arc::new(Mutex::new(None)))
5051 }
5052}
5053
5054fn modified_within(path: &Path, grace: Duration) -> bool {
5055 fs::metadata(path)
5056 .and_then(|metadata| metadata.modified())
5057 .ok()
5058 .and_then(|modified| SystemTime::now().duration_since(modified).ok())
5059 .map(|age| age < grace)
5060 .unwrap_or(false)
5061}
5062
5063fn canonicalized_path(path: &Path) -> PathBuf {
5064 fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf())
5065}
5066
5067fn started_instant_from_unix_millis(started_at: u64) -> Instant {
5068 let now_ms = SystemTime::now()
5069 .duration_since(UNIX_EPOCH)
5070 .ok()
5071 .map(|duration| duration.as_millis() as u64)
5072 .unwrap_or(started_at);
5073 let elapsed_ms = now_ms.saturating_sub(started_at);
5074 Instant::now()
5075 .checked_sub(Duration::from_millis(elapsed_ms))
5076 .unwrap_or_else(Instant::now)
5077}
5078
5079fn gc_quarantine(storage_dir: &Path) {
5080 let quarantine_root = storage_dir.join("bash-tasks-quarantine");
5081 let Ok(session_dirs) = fs::read_dir(&quarantine_root) else {
5082 return;
5083 };
5084 for session_entry in session_dirs.flatten() {
5085 let session_quarantine_dir = session_entry.path();
5086 if !session_quarantine_dir.is_dir() {
5087 continue;
5088 }
5089 let entries = match fs::read_dir(&session_quarantine_dir) {
5090 Ok(entries) => entries,
5091 Err(error) => {
5092 crate::slog_warn!(
5093 "failed to read background task quarantine dir {}: {error}",
5094 session_quarantine_dir.display()
5095 );
5096 continue;
5097 }
5098 };
5099 for entry in entries.flatten() {
5100 let path = entry.path();
5101 if modified_within(&path, QUARANTINE_GC_GRACE) {
5102 continue;
5103 }
5104 let result = if path.is_dir() {
5105 fs::remove_dir_all(&path)
5106 } else {
5107 fs::remove_file(&path)
5108 };
5109 match result {
5110 Ok(()) => log::debug!(
5111 "deleted old background task quarantine entry {}",
5112 path.display()
5113 ),
5114 Err(error) => crate::slog_warn!(
5115 "failed to delete old background task quarantine entry {}: {error}",
5116 path.display()
5117 ),
5118 }
5119 }
5120 let _ = fs::remove_dir(&session_quarantine_dir);
5121 }
5122 let _ = fs::remove_dir(&quarantine_root);
5123}
5124
5125fn read_for_token_count_from_disk(
5126 metadata: &PersistedTask,
5127 paths: &TaskPaths,
5128 max_bytes_per_stream: usize,
5129) -> TokenCountInput {
5130 if metadata.mode == BgMode::Pty {
5131 return TokenCountInput::Skipped;
5132 }
5133 let stdout = read_file_tail_capped(paths, TaskArtifact::Stdout, max_bytes_per_stream);
5140 let stderr = read_file_tail_capped(paths, TaskArtifact::Stderr, max_bytes_per_stream);
5141 match (stdout, stderr) {
5142 (Ok(stdout), Ok(stderr)) => TokenCountInput::Text(combine_streams(
5143 String::from_utf8_lossy(&stdout).as_ref(),
5144 String::from_utf8_lossy(&stderr).as_ref(),
5145 )),
5146 (Ok(stdout), Err(_)) => TokenCountInput::Text(combine_streams(
5147 String::from_utf8_lossy(&stdout).as_ref(),
5148 "",
5149 )),
5150 (Err(_), Ok(stderr)) => TokenCountInput::Text(combine_streams(
5151 "",
5152 String::from_utf8_lossy(&stderr).as_ref(),
5153 )),
5154 (Err(_), Err(_)) => TokenCountInput::Skipped,
5155 }
5156}
5157
5158fn read_file_tail_capped(
5159 paths: &TaskPaths,
5160 artifact: TaskArtifact,
5161 max_bytes: usize,
5162) -> std::io::Result<Vec<u8>> {
5163 let mut file = open_task_artifact(paths, artifact)?;
5164 file.tail(max_bytes).map(|(bytes, _)| bytes)
5165}
5166
5167impl BgTask {
5168 fn snapshot(&self, preview_bytes: usize) -> BgTaskSnapshot {
5169 let state = self
5170 .state
5171 .lock()
5172 .unwrap_or_else(|poison| poison.into_inner());
5173 self.snapshot_locked(&state, preview_bytes)
5174 }
5175
5176 fn snapshot_locked(&self, state: &BgTaskState, preview_bytes: usize) -> BgTaskSnapshot {
5177 let metadata = &state.metadata;
5178 let duration_ms = metadata.duration_ms.or_else(|| {
5179 metadata
5180 .status
5181 .is_terminal()
5182 .then(|| self.started.elapsed().as_millis() as u64)
5183 });
5184 let (output_preview, output_truncated) = if metadata.mode == BgMode::Pty {
5185 (String::new(), false)
5186 } else if metadata.status.is_terminal() {
5187 state
5188 .terminal_output_cache
5189 .as_ref()
5190 .map(|cache| (cache.output_preview.clone(), cache.output_truncated))
5191 .unwrap_or_else(|| (String::new(), false))
5192 } else if preview_bytes == 0 {
5193 (String::new(), false)
5194 } else {
5195 state.buffer.read_tail(preview_bytes)
5196 };
5197 BgTaskSnapshot {
5198 info: BgTaskInfo {
5199 task_id: self.task_id.clone(),
5200 status: metadata.status.clone(),
5201 command: metadata.command.clone(),
5202 mode: metadata.mode.clone(),
5203 started_at: metadata.started_at,
5204 duration_ms,
5205 status_reason: metadata.status_reason.clone(),
5206 },
5207 exit_code: metadata.exit_code,
5208 child_pid: metadata.child_pid,
5209 workdir: metadata.workdir.display().to_string(),
5210 output_preview,
5211 output_truncated,
5212 output_path: state
5213 .buffer
5214 .output_path()
5215 .map(|path| path.display().to_string()),
5216 stderr_path: state
5217 .buffer
5218 .stderr_path()
5219 .map(|path| path.display().to_string()),
5220 pty_rows: (metadata.mode == BgMode::Pty).then_some(metadata.pty_rows.unwrap_or(24)),
5221 pty_cols: (metadata.mode == BgMode::Pty).then_some(metadata.pty_cols.unwrap_or(80)),
5222 pty_screen: None,
5223 scanner_report: metadata.scanner_report.clone(),
5224 sandbox_native: metadata.sandbox_native,
5225 sandbox_unavailable: metadata.sandbox_native
5226 && open_task_artifact(&self.paths, TaskArtifact::SandboxUnavailable)
5227 .and_then(|mut file| file.read_all())
5228 .is_ok_and(|bytes| bytes == b"sandbox_unavailable"),
5229 }
5230 }
5231
5232 pub(crate) fn is_running(&self) -> bool {
5233 self.state
5234 .lock()
5235 .map(|state| {
5236 state.metadata.status == BgTaskStatus::Running
5237 || (state.metadata.mode == BgMode::Pty
5238 && state.metadata.status == BgTaskStatus::Killing)
5239 })
5240 .unwrap_or(false)
5241 }
5242
5243 fn is_terminal(&self) -> bool {
5244 self.state
5245 .lock()
5246 .map(|state| state.metadata.status.is_terminal())
5247 .unwrap_or(false)
5248 }
5249
5250 fn mark_terminal_now(&self) {
5251 if let Ok(mut terminal_at) = self.terminal_at.lock() {
5252 if terminal_at.is_none() {
5253 *terminal_at = Some(Instant::now());
5254 }
5255 }
5256 }
5257
5258 fn set_completion_delivered(
5259 &self,
5260 delivered: bool,
5261 registry: &BgTaskRegistry,
5262 ) -> Result<(), String> {
5263 let mut state = self
5264 .state
5265 .lock()
5266 .map_err(|_| "background task lock poisoned".to_string())?;
5267 let updated = registry
5268 .update_task_metadata(&self.paths, |metadata| {
5269 metadata.completion_delivered = delivered;
5270 })
5271 .map_err(|e| format!("failed to update completion delivery: {e}"))?;
5272 state.metadata = updated;
5273 Ok(())
5274 }
5275}
5276
5277#[cfg(unix)]
5298fn reap_piped_child(child_slot: &mut Option<Child>) {
5299 if let Some(mut child) = child_slot.take() {
5300 if matches!(child.try_wait(), Ok(None)) {
5301 let _ = child.wait();
5302 }
5303 }
5304}
5305
5306#[cfg(windows)]
5311fn reap_piped_child(child_slot: &mut Option<Child>) {
5312 *child_slot = None;
5313}
5314
5315fn terminal_metadata_from_marker(
5316 mut metadata: PersistedTask,
5317 marker: ExitMarker,
5318 reason: Option<String>,
5319) -> PersistedTask {
5320 match marker {
5321 ExitMarker::Code(code) => {
5322 let status = if code == 0 {
5323 BgTaskStatus::Completed
5324 } else {
5325 BgTaskStatus::Failed
5326 };
5327 metadata.mark_terminal(status, Some(code), reason);
5328 }
5329 ExitMarker::Killed => metadata.mark_terminal(
5330 BgTaskStatus::Killed,
5331 terminal_exit_code_for_status(&BgTaskStatus::Killed),
5332 reason,
5333 ),
5334 }
5335 metadata
5336}
5337
5338fn terminal_exit_code_for_status(status: &BgTaskStatus) -> Option<i32> {
5339 match status {
5340 BgTaskStatus::TimedOut => Some(124),
5341 BgTaskStatus::Killed => Some(137),
5342 _ => None,
5343 }
5344}
5345
5346fn attach_sandbox_metadata(metadata: &mut PersistedTask, spawn_plan: &SpawnPlan) {
5347 metadata.sandbox_native = spawn_plan.is_native_launcher();
5348 metadata.sandbox_temp_dir = spawn_plan.temp_dir().map(Path::to_path_buf);
5349}
5350
5351#[cfg(unix)]
5352pub(crate) fn resolve_posix_shell() -> PathBuf {
5353 static POSIX_SHELL: OnceLock<PathBuf> = OnceLock::new();
5354 POSIX_SHELL
5355 .get_or_init(|| {
5356 std::env::var_os("BASH")
5357 .filter(|value| !value.is_empty())
5358 .map(PathBuf::from)
5359 .filter(|path| path.exists())
5360 .or_else(|| which::which("bash").ok())
5361 .or_else(|| which::which("zsh").ok())
5362 .unwrap_or_else(|| PathBuf::from("/bin/sh"))
5363 })
5364 .clone()
5365}
5366
5367#[cfg(windows)]
5368fn detached_shell_command_for(
5369 shell: crate::windows_shell::WindowsShell,
5370 command: &str,
5371 exit_path: &Path,
5372 paths: &TaskPaths,
5373 creation_flags: u32,
5374) -> Result<Command, String> {
5375 use crate::windows_shell::WindowsShell;
5376 let wrapper_body = shell.wrapper_script_bytes(command, exit_path);
5389 let wrapper_ext = match shell {
5390 WindowsShell::Pwsh | WindowsShell::Powershell => "ps1",
5391 WindowsShell::Cmd => "bat",
5392 WindowsShell::Posix(_) => "sh",
5396 };
5397 let wrapper_path = paths.dir.join(format!(
5398 "{}.{}",
5399 paths
5400 .json
5401 .file_stem()
5402 .and_then(|s| s.to_str())
5403 .unwrap_or("wrapper"),
5404 wrapper_ext
5405 ));
5406 fs::write(&wrapper_path, wrapper_body)
5407 .map_err(|e| format!("failed to write background bash wrapper script: {e}"))?;
5408
5409 let mut cmd = Command::new(shell.binary().as_ref());
5410 match shell {
5411 WindowsShell::Pwsh | WindowsShell::Powershell => {
5412 cmd.args([
5415 "-NoLogo",
5416 "-NoProfile",
5417 "-NonInteractive",
5418 "-ExecutionPolicy",
5419 "Bypass",
5420 "-File",
5421 ]);
5422 cmd.arg(&wrapper_path);
5423 }
5424 WindowsShell::Cmd => {
5425 cmd.args(["/D", "/C"]);
5432 cmd.arg(&wrapper_path);
5433 }
5434 WindowsShell::Posix(_) => {
5435 cmd.arg(&wrapper_path);
5440 }
5441 }
5442
5443 cmd.creation_flags(creation_flags);
5447 Ok(cmd)
5448}
5449
5450fn spawn_detached_child(
5466 spawn_plan: &SpawnPlan,
5467 command: &str,
5468 shell: super::BashShell,
5469 shell_path: &Path,
5470 paths: &TaskPaths,
5471 workdir: &Path,
5472 env: &HashMap<String, String>,
5473 io_handles: &mut TaskIoHandles,
5474 capture_pipeline_status: bool,
5475) -> Result<std::process::Child, String> {
5476 #[cfg(windows)]
5477 let _ = capture_pipeline_status;
5478 #[cfg(not(windows))]
5479 let _ = (command, shell);
5480 #[cfg(not(windows))]
5481 {
5482 use std::os::fd::AsRawFd;
5483
5484 let stdout = io_handles
5485 .clone_file(TaskArtifact::Stdout)
5486 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
5487 let stderr = io_handles
5488 .clone_file(TaskArtifact::Stderr)
5489 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
5490 let prepared = spawn_plan
5491 .prepared_task()
5492 .ok_or_else(|| "background task payload was not prepared".to_string())?;
5493 let payload = prepared.invocation()?;
5494 let exit = io_handles
5495 .inheritable_file(TaskArtifact::Exit)
5496 .map_err(|e| format!("failed to inherit exit marker handle: {e}"))?;
5497 let failure = io_handles
5498 .inheritable_file(TaskArtifact::SandboxUnavailable)
5499 .map_err(|e| format!("failed to inherit sandbox failure marker handle: {e}"))?;
5500 let pipeline_status = capture_pipeline_status
5501 .then(|| io_handles.inheritable_file(TaskArtifact::PipelineStatus))
5502 .transpose()
5503 .map_err(|e| format!("failed to inherit pipeline status handle: {e}"))?;
5504 let shell_path = spawn_plan.host_shell_path().unwrap_or(shell_path);
5505 let pipeline_shell = super::process::pipeline_shell_kind(shell_path).unwrap_or("");
5506 let pipeline_status_fd = if capture_pipeline_status {
5507 crate::sandbox_spawn::CHILD_PIPE_STATUS_FD.to_string()
5508 } else {
5509 String::new()
5510 };
5511 let args = vec![
5512 OsString::from("-c"),
5513 payload.wrapper_text.clone(),
5514 OsString::from("aft-payload-wrapper"),
5515 shell_path.as_os_str().to_os_string(),
5516 payload.command_text.clone(),
5517 OsString::from(crate::sandbox_spawn::CHILD_EXIT_FD.to_string()),
5518 OsString::from(pipeline_status_fd),
5519 OsString::from(pipeline_shell),
5520 ];
5521 let (mut child_command, profile_handle) = crate::sandbox_spawn::detached_command_for_plan(
5522 spawn_plan,
5523 std::ffi::OsStr::new("/bin/sh"),
5524 &args,
5525 &paths.json,
5526 crate::sandbox_spawn::CHILD_EXIT_FD,
5527 crate::sandbox_spawn::CHILD_FAILURE_FD,
5528 )?;
5529 crate::sandbox_spawn::apply_marker_fd_allowlist(
5530 &mut child_command,
5531 exit.as_raw_fd(),
5532 failure.as_raw_fd(),
5533 pipeline_status.as_ref().map(|file| file.as_raw_fd()),
5534 )?;
5535 child_command
5536 .current_dir(workdir)
5537 .envs(env)
5538 .stdin(Stdio::null())
5539 .stdout(Stdio::from(stdout))
5540 .stderr(Stdio::from(stderr));
5541 crate::sandbox_spawn::apply_sandbox_environment(spawn_plan, &mut child_command, env);
5542 let child = child_command
5543 .spawn()
5544 .map_err(|e| format!("failed to spawn background bash command: {e}"));
5545 drop((payload, exit, failure, pipeline_status, profile_handle));
5546 child
5547 }
5548 #[cfg(windows)]
5549 {
5550 let _ = shell_path;
5551 use crate::windows_shell::shell_candidates;
5552 match spawn_plan {
5553 SpawnPlan::Unsandboxed | SpawnPlan::Host { .. } => {}
5554 SpawnPlan::Refused { code, .. } => return Err((*code).to_string()),
5555 SpawnPlan::Launcher { .. } => return Err("sandbox_unavailable".to_string()),
5556 }
5557 let candidates: Vec<crate::windows_shell::WindowsShell> = if shell.is_powershell() {
5568 vec![crate::windows_shell::WindowsShell::Pwsh]
5569 } else {
5570 shell_candidates()
5571 };
5572 const FLAG_CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200;
5585 const FLAG_CREATE_BREAKAWAY_FROM_JOB: u32 = 0x0100_0000;
5586 const FLAG_CREATE_NO_WINDOW: u32 = 0x0800_0000;
5587 let with_breakaway =
5588 FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP | FLAG_CREATE_BREAKAWAY_FROM_JOB;
5589 let without_breakaway = FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP;
5590 let mut last_error: Option<String> = None;
5591 for (idx, shell) in candidates.iter().enumerate() {
5592 for &flags in &[with_breakaway, without_breakaway] {
5596 let stdout = io_handles
5599 .clone_file(TaskArtifact::Stdout)
5600 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
5601 let stderr = io_handles
5602 .clone_file(TaskArtifact::Stderr)
5603 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
5604 let mut cmd =
5605 detached_shell_command_for(shell.clone(), command, &paths.exit, paths, flags)?;
5606 cmd.current_dir(workdir)
5607 .envs(env)
5608 .stdin(Stdio::null())
5609 .stdout(Stdio::from(stdout))
5610 .stderr(Stdio::from(stderr));
5611 match cmd.spawn() {
5612 Ok(child) => {
5613 if idx > 0 {
5614 crate::slog_warn!("background bash spawn fell back to {} after {} earlier candidate(s) failed; \
5615 the cached PATH probe disagreed with runtime spawn — likely PATH \
5616 inheritance, antivirus / AppLocker / Defender ASR, or sandbox policy.",
5617 shell.binary(),
5618 idx);
5619 }
5620 if flags == without_breakaway {
5621 crate::slog_warn!(
5622 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected \
5623 (likely a restrictive Job Object — CI sandbox or MDM policy). \
5624 Spawned without breakaway; the bg task will be torn down if the \
5625 AFT process group is killed."
5626 );
5627 }
5628 return Ok(child);
5629 }
5630 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
5631 crate::slog_warn!("background bash spawn: {} returned NotFound at runtime — trying next candidate",
5632 shell.binary());
5633 last_error = Some(format!("{}: {e}", shell.binary()));
5634 break;
5637 }
5638 Err(e) if flags == with_breakaway && e.raw_os_error() == Some(5) => {
5639 crate::slog_warn!(
5641 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected with \
5642 Access Denied — retrying {} without breakaway",
5643 shell.binary()
5644 );
5645 last_error = Some(format!("{}: {e}", shell.binary()));
5646 continue;
5647 }
5648 Err(e) => {
5649 return Err(format!(
5650 "failed to spawn background bash command via {}: {e}",
5651 shell.binary()
5652 ));
5653 }
5654 }
5655 }
5656 }
5657 Err(format!(
5658 "failed to spawn background bash command: no Windows shell could be spawned. \
5659 Last error: {}. PATH-probed candidates: {:?}",
5660 last_error.unwrap_or_else(|| "no candidates were attempted".to_string()),
5661 candidates.iter().map(|s| s.binary()).collect::<Vec<_>>()
5662 ))
5663 }
5664}
5665
5666#[cfg(test)]
5667fn random_slug() -> String {
5668 let mut bytes = [0u8; 8];
5676 getrandom::fill(&mut bytes).unwrap_or_else(|_| {
5678 let t = SystemTime::now()
5680 .duration_since(UNIX_EPOCH)
5681 .map(|d| d.as_nanos() as u64)
5682 .unwrap_or(0);
5683 let p = u64::from(std::process::id());
5684 bytes.copy_from_slice(&(t ^ p.rotate_left(32)).to_le_bytes());
5685 });
5686 let hex: String = bytes.iter().map(|b| format!("{b:02x}")).collect();
5688 format!("bash-{hex}")
5689}
5690
5691#[cfg(test)]
5692mod tests {
5693 use std::collections::HashMap;
5694 use std::fs;
5695 use std::io::Write;
5696 #[cfg(unix)]
5697 use std::os::unix::fs::PermissionsExt;
5698 use std::sync::atomic::{AtomicBool, AtomicUsize};
5699 use std::sync::{Arc, Mutex};
5700 use std::time::{Duration, Instant, SystemTime};
5701
5702 use super::*;
5703 use crate::bash_background::persistence::{read_task, task_paths, write_task};
5704
5705 #[cfg(unix)]
5706 const QUICK_SUCCESS_COMMAND: &str = "true";
5707 #[cfg(windows)]
5708 const QUICK_SUCCESS_COMMAND: &str = "cmd /c exit 0";
5709
5710 #[cfg(unix)]
5711 const LONG_RUNNING_COMMAND: &str = "sleep 5";
5712
5713 #[cfg(unix)]
5714 #[test]
5715 fn launcher_plans_disable_pipeline_status_capture() {
5716 let launcher = SpawnPlan::launcher_for_test(
5717 crate::sandbox_profile::SandboxProfile {
5718 v: crate::sandbox_profile::SANDBOX_PROFILE_VERSION,
5719 writable_roots: Vec::new(),
5720 write_deny: Vec::new(),
5721 write_deny_nested: Vec::new(),
5722 read_allow: Vec::new(),
5723 read_deny: Vec::new(),
5724 socket_deny: Vec::new(),
5725 cache_roots: Vec::new(),
5726 temp_dir: PathBuf::from("/tmp/aft-test-sandbox"),
5727 },
5728 PathBuf::from("/bin/true"),
5729 );
5730 assert!(!should_capture_pipeline_status(
5731 &launcher,
5732 true,
5733 Path::new("/bin/bash")
5734 ));
5735 assert!(should_capture_pipeline_status(
5736 &SpawnPlan::Unsandboxed,
5737 true,
5738 Path::new("/bin/bash")
5739 ));
5740 }
5741
5742 #[cfg(windows)]
5743 const LONG_RUNNING_COMMAND: &str = "cmd /c timeout /t 5 /nobreak > nul";
5744
5745 #[test]
5746 fn bash_memory_estimate_is_zero_when_empty_and_nonzero_for_completion_cache() {
5747 let registry = BgTaskRegistry::default();
5748 assert_eq!(registry.estimated_memory().estimated_bytes, Some(0));
5749 registry
5750 .inner
5751 .completions
5752 .lock()
5753 .unwrap()
5754 .push_back(BgCompletion {
5755 task_id: "bash-memory".to_string(),
5756 session_id: "session-memory".to_string(),
5757 status: BgTaskStatus::Completed,
5758 exit_code: Some(0),
5759 command: "printf memory".to_string(),
5760 output_preview: "resident completion output".to_string(),
5761 output_truncated: false,
5762 original_tokens: None,
5763 compressed_tokens: None,
5764 tokens_skipped: false,
5765 status_reason: None,
5766 });
5767 let estimate = registry.estimated_memory();
5768 assert!(estimate.estimated_bytes.unwrap() > 0);
5769 assert_eq!(estimate.counts["completion_caches"], 1);
5770 assert_eq!(estimate.counts["sessions"], 1);
5771 }
5772
5773 #[test]
5774 fn gh_structured_detection_rejects_piped_commands() {
5775 assert!(is_gh_structured_command(
5776 "gh issue list --json number,title"
5777 ));
5778 assert!(is_gh_structured_command(
5779 "cd repo && gh issue list --json number,title"
5780 ));
5781
5782 assert!(!is_gh_structured_command(
5783 "gh issue list --json number,title | jq '.[]'"
5784 ));
5785 assert!(!is_gh_structured_command(
5786 "gh issue list --json number,title |"
5787 ));
5788 }
5789
5790 fn insert_terminal_piped_task(
5791 registry: &BgTaskRegistry,
5792 dir: &tempfile::TempDir,
5793 command: &str,
5794 stdout: &str,
5795 stderr: &str,
5796 compressed: bool,
5797 ) -> (String, Arc<BgTask>) {
5798 let task_id = random_slug();
5799 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5800 fs::create_dir_all(&paths.dir).unwrap();
5801 fs::write(&paths.stdout, stdout).unwrap();
5802 fs::write(&paths.stderr, stderr).unwrap();
5803 let mut metadata = PersistedTask::starting(
5804 task_id.clone(),
5805 "session".to_string(),
5806 command.to_string(),
5807 dir.path().to_path_buf(),
5808 Some(dir.path().to_path_buf()),
5809 Some(30_000),
5810 true,
5811 compressed,
5812 );
5813 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
5814 write_task(&paths.json, &metadata).unwrap();
5815 registry
5816 .insert_rehydrated_task(metadata, paths, true, None)
5817 .expect("insert terminal task");
5818 let task = registry.task_for_session(&task_id, "session").unwrap();
5819 (task_id, task)
5820 }
5821
5822 #[test]
5823 fn bash_zero_preview_running_status_skips_output_read_while_explicit_preview_reads() {
5824 let registry = BgTaskRegistry::default();
5825 let dir = tempfile::tempdir().unwrap();
5826 let task_id = random_slug();
5827 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5828 fs::create_dir_all(&paths.dir).unwrap();
5829 fs::write(&paths.stdout, "live output\n").unwrap();
5830 fs::write(&paths.stderr, "").unwrap();
5831 let stdout_path = paths.stdout.clone();
5832 let mut metadata = PersistedTask::starting(
5833 task_id.clone(),
5834 "session".to_string(),
5835 "sleep 60".to_string(),
5836 dir.path().to_path_buf(),
5837 Some(dir.path().to_path_buf()),
5838 Some(30_000),
5839 true,
5840 false,
5841 );
5842 metadata.status = BgTaskStatus::Running;
5843 write_task(&paths.json, &metadata).unwrap();
5844 registry
5845 .insert_rehydrated_task(metadata, paths, false, None)
5846 .expect("insert running task");
5847
5848 crate::bash_background::buffer::reset_tail_read_count(&stdout_path);
5849 for _ in 0..5 {
5850 let snapshot = registry
5851 .status(&task_id, "session", Some(dir.path()), Some(dir.path()), 0)
5852 .expect("running snapshot");
5853 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
5854 assert!(snapshot.output_preview.is_empty());
5855 }
5856 assert_eq!(
5857 crate::bash_background::buffer::tail_read_count(&stdout_path),
5858 0
5859 );
5860
5861 let snapshot = registry
5862 .status(
5863 &task_id,
5864 "session",
5865 Some(dir.path()),
5866 Some(dir.path()),
5867 RUNNING_OUTPUT_PREVIEW_BYTES,
5868 )
5869 .expect("explicit running snapshot");
5870 assert_eq!(snapshot.output_preview, "live output\n");
5871 assert_eq!(
5872 crate::bash_background::buffer::tail_read_count(&stdout_path),
5873 1
5874 );
5875 }
5876
5877 #[test]
5878 fn artifact_read_capability_requires_exact_canonical_path_and_session() {
5879 let registry = BgTaskRegistry::default();
5880 let dir = tempfile::tempdir().unwrap();
5881 let (_task_id, task) = insert_terminal_piped_task(
5882 ®istry,
5883 &dir,
5884 "printf output",
5885 "stdout\n",
5886 "stderr\n",
5887 true,
5888 );
5889 fs::write(&task.paths.exit, "0\n").unwrap();
5890
5891 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stdout));
5892 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stderr));
5893 assert!(registry.is_session_owned_artifact_path("session", &task.paths.exit));
5894 assert!(!registry.is_session_owned_artifact_path("different-session", &task.paths.stdout));
5895 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.json));
5896
5897 let unregistered = task.paths.dir.join("unregistered-output");
5898 fs::write(&unregistered, "not a task artifact\n").unwrap();
5899 assert!(!registry.is_session_owned_artifact_path("session", &unregistered));
5900 }
5901
5902 #[cfg(unix)]
5903 #[test]
5904 fn artifact_directory_symlink_does_not_create_a_prefix_exception() {
5905 let registry = BgTaskRegistry::default();
5906 let dir = tempfile::tempdir().unwrap();
5907 let project = dir.path().join("project");
5908 fs::create_dir_all(&project).unwrap();
5909 let (_task_id, task) =
5910 insert_terminal_piped_task(®istry, &dir, "printf output", "stdout\n", "", true);
5911 let link = project.join("task-artifacts");
5912 std::os::unix::fs::symlink(&task.paths.dir, &link).unwrap();
5913 let unregistered = task.paths.dir.join("unregistered-output");
5914 fs::write(&unregistered, "not registered\n").unwrap();
5915
5916 assert!(!registry.is_session_owned_artifact_path("session", &link));
5917 assert!(
5918 !registry.is_session_owned_artifact_path("session", &link.join("unregistered-output"))
5919 );
5920 assert!(registry.is_session_owned_artifact_path(
5921 "session",
5922 &link.join(task.paths.stdout.file_name().unwrap())
5923 ));
5924
5925 let outside = dir.path().join("outside-secret");
5926 fs::write(&outside, "must stay private\n").unwrap();
5927 fs::remove_file(&task.paths.stdout).unwrap();
5928 std::os::unix::fs::symlink(&outside, &task.paths.stdout).unwrap();
5929 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.stdout));
5930 }
5931
5932 #[test]
5933 fn recovery_footer_uses_bash_status_when_artifact_is_not_registered() {
5934 let registry = BgTaskRegistry::default();
5935 let dir = tempfile::tempdir().unwrap();
5936 let task_id = "bash-1111111111111111";
5937 let paths = task_paths(dir.path(), "session", task_id).unwrap();
5938 fs::create_dir_all(&paths.dir).unwrap();
5939 fs::write(
5940 &paths.stdout,
5941 format!("{}tail\n", "output-line\n".repeat(2_000)),
5942 )
5943 .unwrap();
5944 fs::write(&paths.stderr, "").unwrap();
5945 let mut metadata = PersistedTask::starting(
5946 task_id.to_string(),
5947 "session".to_string(),
5948 "printf output".to_string(),
5949 dir.path().to_path_buf(),
5950 Some(dir.path().to_path_buf()),
5951 Some(30_000),
5952 true,
5953 true,
5954 );
5955 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
5956 write_task(&paths.json, &metadata).unwrap();
5957
5958 let cache = registry
5959 .render_terminal_output_from_paths(&metadata, &paths)
5960 .expect("terminal render");
5961
5962 assert!(cache
5963 .output_preview
5964 .contains("use bash_status({taskId: \"bash-1111111111111111\"})"));
5965 assert!(!cache.output_preview.contains("full output: read "));
5966 }
5967
5968 fn insert_terminal_pty_task(
5969 registry: &BgTaskRegistry,
5970 dir: &tempfile::TempDir,
5971 pty_output: &str,
5972 ) -> (String, Arc<BgTask>) {
5973 let task_id = random_slug();
5974 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5975 fs::create_dir_all(&paths.dir).unwrap();
5976 fs::write(&paths.pty, pty_output).unwrap();
5977 let mut metadata = PersistedTask::starting(
5978 task_id.clone(),
5979 "session".to_string(),
5980 "python".to_string(),
5981 dir.path().to_path_buf(),
5982 Some(dir.path().to_path_buf()),
5983 Some(30_000),
5984 true,
5985 true,
5986 );
5987 metadata.mode = BgMode::Pty;
5988 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
5989 write_task(&paths.json, &metadata).unwrap();
5990 registry
5991 .insert_rehydrated_task(metadata, paths, true, None)
5992 .expect("insert terminal pty task");
5993 let task = registry.task_for_session(&task_id, "session").unwrap();
5994 (task_id, task)
5995 }
5996
5997 #[cfg(unix)]
5998 fn wait_for_terminal_snapshot(
5999 registry: &BgTaskRegistry,
6000 task_id: &str,
6001 session_id: &str,
6002 project: &Path,
6003 storage: &Path,
6004 ) -> BgTaskSnapshot {
6005 let started = Instant::now();
6006 loop {
6007 let snapshot = registry
6008 .status(task_id, session_id, Some(project), Some(storage), 4096)
6009 .expect("spawned task should be visible to status");
6010 if snapshot.info.status.is_terminal() {
6011 return snapshot;
6012 }
6013 assert!(
6014 started.elapsed() < Duration::from_secs(10),
6015 "timed out waiting for task {task_id} to finish; last status={:?}",
6016 snapshot.info.status
6017 );
6018 std::thread::sleep(Duration::from_millis(50));
6019 }
6020 }
6021
6022 fn write_running_project_task(storage: &Path, project: &Path, session: &str, task_id: &str) {
6023 let paths = task_paths(storage, session, task_id).unwrap();
6024 let mut metadata = PersistedTask::starting(
6025 task_id.to_string(),
6026 session.to_string(),
6027 "sleep 60".to_string(),
6028 project.to_path_buf(),
6029 Some(project.to_path_buf()),
6030 Some(30_000),
6031 true,
6032 true,
6033 );
6034 metadata.status = BgTaskStatus::Running;
6035 metadata.child_pid = Some(std::process::id());
6041 write_task(&paths.json, &metadata).unwrap();
6042 fs::write(&paths.stdout, "still running\n").unwrap();
6043 fs::write(&paths.stderr, "").unwrap();
6044 }
6045
6046 #[test]
6047 fn status_replay_filters_same_session_by_project_root() {
6048 let project_a = tempfile::tempdir().unwrap();
6049 let project_b = tempfile::tempdir().unwrap();
6050 let storage = tempfile::tempdir().unwrap();
6051 let session = "shared-session";
6052 let task_id = "bash-2222222222222222";
6053 write_running_project_task(storage.path(), project_a.path(), session, task_id);
6054
6055 let actor_b = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6056 assert!(actor_b
6057 .status(
6058 task_id,
6059 session,
6060 Some(project_b.path()),
6061 Some(storage.path()),
6062 1024,
6063 )
6064 .is_none());
6065 assert!(actor_b.task_for_session(task_id, session).is_none());
6066
6067 let actor_a = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6068 let snapshot = actor_a
6069 .status(
6070 task_id,
6071 session,
6072 Some(project_a.path()),
6073 Some(storage.path()),
6074 1024,
6075 )
6076 .expect("owning project should replay its task");
6077 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
6078 }
6079
6080 #[cfg(unix)]
6081 #[test]
6082 fn multiline_pipeline_stdout_persists_all_lines_after_terminal_status() {
6083 let cases = [
6084 (
6085 "long-first",
6086 "sleep 0.5; printf 'one\\n' | cat\nprintf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
6087 vec!["one", "1", "three"],
6088 ),
6089 (
6090 "short-first",
6091 "printf 'one\\n' | cat\nsleep 0.2; printf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
6092 vec!["one", "1", "three"],
6093 ),
6094 (
6095 "failing-middle",
6096 "sleep 0.2; printf 'one\\n' | cat\nfalse; printf 'after-false\\n' | cat\nprintf 'three\\n' | cat",
6097 vec!["one", "after-false", "three"],
6098 ),
6099 ];
6100
6101 for (name, command, expected_lines) in cases {
6102 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6103 let dir = tempfile::tempdir().unwrap();
6104 let session_id = format!("session-{name}");
6105 let task_id = registry
6106 .spawn(
6107 SpawnPlan::Unsandboxed,
6108 command,
6109 session_id.clone(),
6110 dir.path().to_path_buf(),
6111 HashMap::new(),
6112 Some(Duration::from_secs(30)),
6113 dir.path().to_path_buf(),
6114 10,
6115 true,
6116 true,
6117 Some(dir.path().to_path_buf()),
6118 )
6119 .unwrap();
6120
6121 let snapshot = wait_for_terminal_snapshot(
6122 ®istry,
6123 &task_id,
6124 &session_id,
6125 dir.path(),
6126 dir.path(),
6127 );
6128 assert_eq!(
6129 snapshot.info.status,
6130 BgTaskStatus::Completed,
6131 "{name}: task should complete; snapshot={snapshot:?}"
6132 );
6133 assert_eq!(
6134 snapshot.exit_code,
6135 Some(0),
6136 "{name}: script should use the final command's exit code"
6137 );
6138
6139 let stdout = String::from_utf8(
6140 registry
6141 .read_artifact(&task_id, &session_id, TaskArtifact::Stdout)
6142 .expect("read validated stdout artifact"),
6143 )
6144 .expect("stdout is UTF-8");
6145 let lines: Vec<&str> = stdout.lines().collect();
6146 assert_eq!(
6147 lines, expected_lines,
6148 "{name}: raw stdout artifact must include every newline-separated command's output"
6149 );
6150 }
6151 }
6152
6153 #[test]
6154 fn recognizes_all_recovery_marker_forms() {
6155 assert!(is_recovery_marker(
6156 "[truncated output; full output: read \"/tmp/out\"]"
6157 ));
6158 assert!(is_recovery_marker(
6159 "[omitted output; see remaining: tail -n +42 \"/tmp/out\"]"
6160 ));
6161 assert!(is_recovery_marker(
6162 "[truncated output; full output unavailable]"
6163 ));
6164 assert!(is_recovery_marker(
6165 r#"[truncated 123 bytes from saved output prefix; retained output: read "/tmp/out"]"#
6166 ));
6167 }
6168
6169 #[test]
6170 fn recovery_marker_reports_disk_prefix_truncation_as_retained_output() {
6171 let recovery = RecoveryContext {
6172 dropped_by_class: BTreeMap::new(),
6173 had_inner_drop: false,
6174 offset_hint_eligible: false,
6175 offset_start_line: None,
6176 byte_truncated: false,
6177 disk_truncated_prefix_bytes: 4096,
6178 output_path: Some("/tmp/stdout".to_string()),
6179 stderr_path: None,
6180 include_stderr_path: false,
6181 artifact_access: ArtifactRecoveryAccess {
6182 task_id: "bash-test".to_string(),
6183 readable: true,
6184 },
6185 };
6186
6187 let marker = recovery_marker(&recovery).expect("disk truncation must emit marker");
6188
6189 assert!(marker.contains("truncated 4096 bytes from saved output prefix"));
6190 assert!(marker.contains(r#"retained output: read "/tmp/stdout""#));
6191 assert!(!marker.contains("full output: read"));
6192 }
6193
6194 #[test]
6195 fn killed_exit_marker_sets_nonzero_sentinel_exit_code() {
6196 let metadata = PersistedTask::starting(
6197 "task".to_string(),
6198 "session".to_string(),
6199 "cargo test".to_string(),
6200 PathBuf::from("/tmp"),
6201 None,
6202 None,
6203 true,
6204 true,
6205 );
6206
6207 let terminal = terminal_metadata_from_marker(metadata, ExitMarker::Killed, None);
6208
6209 assert_eq!(terminal.status, BgTaskStatus::Killed);
6210 assert_eq!(terminal.exit_code, Some(137));
6211 }
6212
6213 #[test]
6214 fn terminal_status_polls_use_cached_render_once_and_off_lock() {
6215 let registry = BgTaskRegistry::default();
6216 let dir = tempfile::tempdir().unwrap();
6217 let (_task_id, task) = insert_terminal_piped_task(
6218 ®istry,
6219 &dir,
6220 "custom-tool --verbose",
6221 &"stdout line\n".repeat(200_000),
6222 "",
6223 true,
6224 );
6225 let calls = Arc::new(AtomicUsize::new(0));
6226 let saw_unlocked_state = Arc::new(AtomicBool::new(false));
6227 let task_holder = Arc::new(Mutex::new(Some(Arc::clone(&task))));
6228 let calls_for_closure = Arc::clone(&calls);
6229 let unlocked_for_closure = Arc::clone(&saw_unlocked_state);
6230 let task_for_closure = Arc::clone(&task_holder);
6231 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
6232 calls_for_closure.fetch_add(1, Ordering::SeqCst);
6233 if let Some(task) = task_for_closure.lock().unwrap().as_ref() {
6234 if task.state.try_lock().is_ok() {
6235 unlocked_for_closure.store(true, Ordering::SeqCst);
6236 }
6237 }
6238 CompressionResult::new(format!("compressed {} bytes", output.len()))
6239 });
6240
6241 let first = registry
6242 .status(
6243 &task.task_id,
6244 "session",
6245 None,
6246 Some(dir.path()),
6247 RUNNING_OUTPUT_PREVIEW_BYTES,
6248 )
6249 .unwrap();
6250 let second = registry
6251 .status(
6252 &task.task_id,
6253 "session",
6254 None,
6255 Some(dir.path()),
6256 RUNNING_OUTPUT_PREVIEW_BYTES,
6257 )
6258 .unwrap();
6259 let listed = registry.list(RUNNING_OUTPUT_PREVIEW_BYTES);
6260
6261 assert_eq!(
6262 calls.load(Ordering::SeqCst),
6263 1,
6264 "terminal render must be cached"
6265 );
6266 assert!(
6267 saw_unlocked_state.load(Ordering::SeqCst),
6268 "compressor must run after releasing the task state lock"
6269 );
6270 assert!(first.output_preview.starts_with("compressed "));
6271 assert_eq!(second.output_preview, first.output_preview);
6272 assert_eq!(listed[0].output_preview, first.output_preview);
6273 }
6274
6275 #[test]
6276 fn completion_preview_success_keeps_tail_only() {
6277 let registry = BgTaskRegistry::default();
6282 let dir = tempfile::tempdir().unwrap();
6283 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
6284 let (_task_id, task) =
6285 insert_terminal_piped_task(®istry, &dir, "cat big.log", &output, "", false);
6286
6287 registry.post_terminal_transition(&task, true).unwrap();
6288 let completions = registry.drain_completions_for_session(Some("session"));
6289 assert_eq!(completions.len(), 1);
6290 let preview = &completions[0].output_preview;
6291 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
6292 assert!(!preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
6293 assert!(completions[0].output_truncated);
6294 }
6295
6296 #[test]
6297 fn completion_preview_failure_keeps_head_and_tail() {
6298 let registry = BgTaskRegistry::default();
6301 let dir = tempfile::tempdir().unwrap();
6302 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
6303 let task_id = random_slug();
6304 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
6305 fs::create_dir_all(&paths.dir).unwrap();
6306 fs::write(&paths.stdout, &output).unwrap();
6307 fs::write(&paths.stderr, "").unwrap();
6308 let mut metadata = PersistedTask::starting(
6309 task_id.clone(),
6310 "session".to_string(),
6311 "cat big.log".to_string(),
6312 dir.path().to_path_buf(),
6313 Some(dir.path().to_path_buf()),
6314 Some(30_000),
6315 true,
6316 false,
6317 );
6318 metadata.mark_terminal(BgTaskStatus::Failed, Some(1), None);
6319 write_task(&paths.json, &metadata).unwrap();
6320 registry
6321 .insert_rehydrated_task(metadata, paths, true, None)
6322 .expect("insert terminal task");
6323 let task = registry.task_for_session(&task_id, "session").unwrap();
6324
6325 registry.post_terminal_transition(&task, true).unwrap();
6326 let completions = registry.drain_completions_for_session(Some("session"));
6327 assert_eq!(completions.len(), 1);
6328 let preview = &completions[0].output_preview;
6329 assert!(preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
6330 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
6331 }
6332
6333 #[test]
6334 fn has_completions_for_session_matches_pending_delivery() {
6335 let registry = BgTaskRegistry::default();
6336 assert!(!registry.has_completions_for_session(Some("session")));
6337 assert!(!registry.has_completions_for_session(None));
6338
6339 let dir = tempfile::tempdir().unwrap();
6340 let (_task_id, task) =
6341 insert_terminal_piped_task(®istry, &dir, QUICK_SUCCESS_COMMAND, "done\n", "", false);
6342 registry.post_terminal_transition(&task, true).unwrap();
6343
6344 assert!(registry.has_completions_for_session(Some("session")));
6345 assert!(registry.has_completions_for_session(None));
6346 assert!(!registry.has_completions_for_session(Some("other-session")));
6347
6348 let completions = registry.drain_completions_for_session(Some("session"));
6349 assert_eq!(completions.len(), 1);
6350 assert_eq!(completions[0].task_id, task.task_id);
6351 }
6352
6353 #[test]
6354 fn structured_gh_json_survives_intact_and_ignores_stderr() {
6355 let registry = BgTaskRegistry::default();
6356 let dir = tempfile::tempdir().unwrap();
6357 let calls = Arc::new(AtomicUsize::new(0));
6358 let calls_for_closure = Arc::clone(&calls);
6359 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
6360 calls_for_closure.fetch_add(1, Ordering::SeqCst);
6361 CompressionResult::new(output)
6362 });
6363 let (task_id, _task) = insert_terminal_piped_task(
6364 ®istry,
6365 &dir,
6366 "gh pr view 123 --json body",
6367 "{\"body\":\"hello\"}",
6368 "warning: stderr must not join json",
6369 true,
6370 );
6371
6372 let snapshot = registry
6373 .status(
6374 &task_id,
6375 "session",
6376 None,
6377 Some(dir.path()),
6378 RUNNING_OUTPUT_PREVIEW_BYTES,
6379 )
6380 .unwrap();
6381
6382 assert_eq!(snapshot.output_preview, "{\"body\":\"hello\"}");
6383 assert!(!snapshot.output_preview.contains("warning"));
6384 assert!(!snapshot.output_truncated);
6385 assert_eq!(
6386 calls.load(Ordering::SeqCst),
6387 0,
6388 "structured JSON bypasses compression"
6389 );
6390 }
6391
6392 #[test]
6393 fn registry_emits_single_recovery_marker_for_class_drops() {
6394 let registry = BgTaskRegistry::default();
6395 let dir = tempfile::tempdir().unwrap();
6396 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
6397 let mut dropped = BTreeMap::new();
6398 dropped.insert(DropClass::Error, 18);
6399 dropped.insert(DropClass::Warning, 6);
6400 CompressionResult::with_class_drops("kept diagnostic", dropped)
6401 });
6402 let (task_id, task) =
6403 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
6404
6405 let snapshot = registry
6406 .status(
6407 &task_id,
6408 "session",
6409 None,
6410 Some(dir.path()),
6411 RUNNING_OUTPUT_PREVIEW_BYTES,
6412 )
6413 .unwrap();
6414
6415 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
6416 assert!(snapshot.output_preview.contains("+18 more errors"));
6417 assert!(snapshot.output_preview.contains("+6 more warnings"));
6418 assert!(snapshot
6419 .output_preview
6420 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
6421 assert!(!snapshot.output_preview.contains("tail -n +"));
6422 assert!(snapshot.output_truncated);
6423 }
6424
6425 #[test]
6426 fn registry_marker_reports_semantic_and_byte_drops_once() {
6427 let registry = BgTaskRegistry::default();
6428 let dir = tempfile::tempdir().unwrap();
6429 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
6430 let mut dropped = BTreeMap::new();
6431 dropped.insert(DropClass::Error, 1);
6432 CompressionResult::with_class_drops(
6433 format!("HEAD-SIGNAL\n{}TAIL-SIGNAL", "middle\n".repeat(8_000)),
6434 dropped,
6435 )
6436 });
6437 let (task_id, _task) =
6438 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
6439
6440 let snapshot = registry
6441 .status(
6442 &task_id,
6443 "session",
6444 None,
6445 Some(dir.path()),
6446 RUNNING_OUTPUT_PREVIEW_BYTES,
6447 )
6448 .unwrap();
6449
6450 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
6451 assert!(snapshot.output_preview.contains("+1 more error"));
6452 assert!(snapshot.output_preview.contains("truncated output"));
6453 assert!(snapshot.output_preview.contains("HEAD-SIGNAL"));
6454 assert!(snapshot.output_preview.contains("TAIL-SIGNAL"));
6455 assert!(!snapshot.output_preview.contains("...<truncated"));
6456 assert!(snapshot.output_truncated);
6457 }
6458
6459 #[test]
6460 fn cargo_stderr_class_drops_name_both_capture_paths() {
6461 let registry = BgTaskRegistry::default();
6462 let dir = tempfile::tempdir().unwrap();
6463 let filter_registry = crate::compress::toml_filter::FilterRegistry::default();
6464 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
6465 crate::compress::compress_with_registry_exit_code(
6466 command,
6467 &output,
6468 exit_code,
6469 &filter_registry,
6470 )
6471 });
6472 let stderr = (0..22)
6473 .map(|index| {
6474 format!(
6475 "error: cargo failure {index}\n --> src/lib.rs:{}:1\n |\n{} | boom\n",
6476 index + 1,
6477 index + 1
6478 )
6479 })
6480 .collect::<Vec<_>>()
6481 .join("\n");
6482 let (task_id, task) = insert_terminal_piped_task(
6483 ®istry,
6484 &dir,
6485 "cargo check",
6486 "Finished dev [unoptimized] target(s) in 0.01s\n",
6487 &stderr,
6488 true,
6489 );
6490
6491 let snapshot = registry
6492 .status(
6493 &task_id,
6494 "session",
6495 None,
6496 Some(dir.path()),
6497 RUNNING_OUTPUT_PREVIEW_BYTES,
6498 )
6499 .unwrap();
6500
6501 assert!(snapshot.output_preview.contains("+2 more errors"));
6502 assert!(snapshot
6503 .output_preview
6504 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
6505 assert!(snapshot
6506 .output_preview
6507 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
6508 assert!(!snapshot.output_preview.contains("tail -n +"));
6509 }
6510
6511 #[test]
6512 fn over_ceiling_structured_json_uses_pointer_not_partial_json() {
6513 let registry = BgTaskRegistry::default();
6514 let dir = tempfile::tempdir().unwrap();
6515 let body = format!("{{\"body\":\"{}\"}}", "x".repeat(60 * 1024));
6516 let (task_id, task) = insert_terminal_piped_task(
6517 ®istry,
6518 &dir,
6519 "cd /repo && gh pr view 123 --json body",
6520 &body,
6521 "",
6522 true,
6523 );
6524
6525 let snapshot = registry
6526 .status(
6527 &task_id,
6528 "session",
6529 None,
6530 Some(dir.path()),
6531 RUNNING_OUTPUT_PREVIEW_BYTES,
6532 )
6533 .unwrap();
6534
6535 assert!(snapshot.output_preview.starts_with("[JSON output "));
6536 assert!(snapshot
6537 .output_preview
6538 .contains(&task.paths.stdout.display().to_string()));
6539 assert!(!snapshot.output_preview.contains(&"x".repeat(1024)));
6540 assert!(snapshot.output_truncated);
6541 }
6542
6543 #[test]
6544 fn toml_strip_tail_cap_uses_full_output_hint_not_offset_hint() {
6545 let registry = BgTaskRegistry::default();
6546 let dir = tempfile::tempdir().unwrap();
6547 let filter_registry = crate::compress::toml_filter::build_registry(
6548 crate::compress::builtin_filters::ALL,
6549 None,
6550 None,
6551 );
6552 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
6553 crate::compress::compress_with_registry_exit_code(
6554 command,
6555 &output,
6556 exit_code,
6557 &filter_registry,
6558 )
6559 });
6560 let stdout = format!(
6561 "make[1]: Entering directory `/tmp`\n{}",
6562 (0..100)
6563 .map(|index| format!("compile line {index}"))
6564 .collect::<Vec<_>>()
6565 .join("\n")
6566 );
6567 let (task_id, task) =
6568 insert_terminal_piped_task(®istry, &dir, "make all", &stdout, "", true);
6569
6570 let snapshot = registry
6571 .status(
6572 &task_id,
6573 "session",
6574 None,
6575 Some(dir.path()),
6576 RUNNING_OUTPUT_PREVIEW_BYTES,
6577 )
6578 .unwrap();
6579
6580 assert!(snapshot.output_preview.contains("compile line 99"));
6581 assert!(snapshot.output_preview.contains(&format!(
6582 "full output: read \"{}\"",
6583 task.paths.stdout.display()
6584 )));
6585 assert!(!snapshot
6586 .output_preview
6587 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
6588 assert!(!snapshot.output_preview.contains("tail -n +"));
6589 }
6590
6591 #[test]
6592 fn compressed_false_raw_passthrough_uses_wider_head_tail_cap() {
6593 let registry = BgTaskRegistry::default();
6594 let dir = tempfile::tempdir().unwrap();
6595 let output = format!("RAW-HEAD\n{}RAW-TAIL\n", "raw-middle\n".repeat(8_000));
6596 let (task_id, task) =
6597 insert_terminal_piped_task(®istry, &dir, "cat raw.log", &output, "RAW-ERR\n", false);
6598
6599 let snapshot = registry
6600 .status(
6601 &task_id,
6602 "session",
6603 None,
6604 Some(dir.path()),
6605 RUNNING_OUTPUT_PREVIEW_BYTES,
6606 )
6607 .unwrap();
6608
6609 assert!(snapshot.output_preview.contains("RAW-HEAD"));
6610 assert!(snapshot.output_preview.contains("RAW-TAIL"));
6611 assert!(snapshot.output_preview.contains("truncated output"));
6612 assert!(snapshot
6613 .output_preview
6614 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
6615 assert!(snapshot
6616 .output_preview
6617 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
6618 assert!(!snapshot.output_preview.contains("tail -n +"));
6619 assert!(snapshot.output_preview.len() > 16 * 1024);
6620 assert!(snapshot.output_truncated);
6621 }
6622
6623 #[test]
6624 fn pty_terminal_snapshot_bypasses_line_compression() {
6625 let registry = BgTaskRegistry::default();
6626 let dir = tempfile::tempdir().unwrap();
6627 let calls = Arc::new(AtomicUsize::new(0));
6628 let calls_for_closure = Arc::clone(&calls);
6629 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
6630 calls_for_closure.fetch_add(1, Ordering::SeqCst);
6631 CompressionResult::new(output)
6632 });
6633 let (task_id, _task) = insert_terminal_pty_task(®istry, &dir, "raw\u{1b}[31m pty bytes");
6634
6635 let snapshot = registry
6636 .status(
6637 &task_id,
6638 "session",
6639 None,
6640 Some(dir.path()),
6641 RUNNING_OUTPUT_PREVIEW_BYTES,
6642 )
6643 .unwrap();
6644
6645 assert_eq!(snapshot.info.mode, BgMode::Pty);
6646 assert_eq!(snapshot.output_preview, "");
6647 assert_eq!(calls.load(Ordering::SeqCst), 0);
6648 }
6649
6650 #[test]
6651 fn pty_dimensions_are_persisted_and_returned_in_snapshot() {
6652 let registry = BgTaskRegistry::default();
6653 let dir = tempfile::tempdir().unwrap();
6654 let task_id = registry
6655 .spawn_pty(
6656 SpawnPlan::Unsandboxed,
6657 QUICK_SUCCESS_COMMAND,
6658 "session".to_string(),
6659 dir.path().to_path_buf(),
6660 HashMap::new(),
6661 Some(Duration::from_secs(30)),
6662 dir.path().to_path_buf(),
6663 10,
6664 true,
6665 false,
6666 Some(dir.path().to_path_buf()),
6667 50,
6668 120,
6669 )
6670 .unwrap();
6671
6672 let resolved =
6673 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
6674 let metadata = read_task_at(&resolved).unwrap();
6675 assert_eq!(
6676 metadata.schema_version,
6677 crate::bash_background::persistence::SCHEMA_VERSION
6678 );
6679 assert_eq!(metadata.mode, BgMode::Pty);
6680 assert_eq!(metadata.pty_rows, Some(50));
6681 assert_eq!(metadata.pty_cols, Some(120));
6682
6683 let snapshot = registry
6684 .status(&task_id, "session", None, Some(dir.path()), 1024)
6685 .unwrap();
6686 assert_eq!(snapshot.pty_rows, Some(50));
6687 assert_eq!(snapshot.pty_cols, Some(120));
6688 }
6689
6690 fn spawn_dead_child() -> std::process::Child {
6695 #[cfg(unix)]
6696 let mut cmd = std::process::Command::new("true");
6697 #[cfg(windows)]
6698 let mut cmd = {
6699 let mut c = std::process::Command::new("cmd");
6700 c.args(["/c", "exit", "0"]);
6701 c
6702 };
6703 cmd.stdin(std::process::Stdio::null());
6704 cmd.stdout(std::process::Stdio::null());
6705 cmd.stderr(std::process::Stdio::null());
6706 let mut child = cmd.spawn().expect("spawn replacement child for reap test");
6707 let started = Instant::now();
6716 loop {
6717 match child.try_wait() {
6718 Ok(Some(_)) => break,
6719 Ok(None) => {
6720 if started.elapsed() > Duration::from_secs(5) {
6721 panic!("dead-child stand-in did not exit within 5s");
6722 }
6723 std::thread::sleep(Duration::from_millis(10));
6724 }
6725 Err(error) => panic!("dead-child try_wait failed: {error}"),
6726 }
6727 }
6728 child
6729 }
6730
6731 #[test]
6732 fn ack_marks_delivered_even_when_completion_was_already_consumed_locally() {
6733 let registry = BgTaskRegistry::default();
6734 let dir = tempfile::tempdir().unwrap();
6735 let task_id = registry
6736 .spawn(
6737 SpawnPlan::Unsandboxed,
6738 LONG_RUNNING_COMMAND,
6739 "session".to_string(),
6740 dir.path().to_path_buf(),
6741 HashMap::new(),
6742 Some(Duration::from_secs(30)),
6743 dir.path().to_path_buf(),
6744 10,
6745 true,
6746 false,
6747 Some(dir.path().to_path_buf()),
6748 )
6749 .unwrap();
6750 registry
6751 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6752 .unwrap();
6753 assert_eq!(
6754 registry
6755 .drain_completions_for_session(Some("session"))
6756 .len(),
6757 1
6758 );
6759
6760 registry.inner.completions.lock().unwrap().clear();
6763
6764 assert_eq!(
6765 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
6766 vec![task_id.clone()]
6767 );
6768 assert!(registry
6769 .drain_completions_for_session(Some("session"))
6770 .is_empty());
6771
6772 let resolved =
6773 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
6774 let metadata = read_task_at(&resolved).unwrap();
6775 assert!(metadata.completion_delivered);
6776
6777 let replayed = BgTaskRegistry::default();
6778 replayed
6779 .replay_session_inner(dir.path(), "session", None)
6780 .unwrap();
6781 assert!(replayed
6782 .drain_completions_for_session(Some("session"))
6783 .is_empty());
6784 }
6785
6786 #[test]
6787 fn reclaimed_root_kills_running_task_and_persists_reason() {
6788 let registry = BgTaskRegistry::default();
6789 let root = tempfile::tempdir().unwrap();
6790 let storage = tempfile::tempdir().unwrap();
6791 let task_id = registry
6792 .spawn(
6793 SpawnPlan::Unsandboxed,
6794 LONG_RUNNING_COMMAND,
6795 "session".to_string(),
6796 root.path().to_path_buf(),
6797 HashMap::new(),
6798 Some(Duration::from_secs(30)),
6799 storage.path().to_path_buf(),
6800 10,
6801 true,
6802 false,
6803 Some(root.path().to_path_buf()),
6804 )
6805 .unwrap();
6806 let pid = registry
6807 .status(
6808 &task_id,
6809 "session",
6810 Some(root.path()),
6811 Some(storage.path()),
6812 0,
6813 )
6814 .unwrap()
6815 .child_pid
6816 .unwrap();
6817 assert!(is_process_alive(pid));
6818
6819 assert_eq!(registry.kill_running_tasks_for_root(root.path()), 1);
6820 let deadline = Instant::now() + Duration::from_secs(5);
6821 while is_process_alive(pid) {
6822 assert!(
6823 Instant::now() < deadline,
6824 "reclaimed task process survived kill"
6825 );
6826 std::thread::sleep(Duration::from_millis(20));
6827 }
6828
6829 let snapshot = registry
6830 .status(
6831 &task_id,
6832 "session",
6833 Some(root.path()),
6834 Some(storage.path()),
6835 0,
6836 )
6837 .unwrap();
6838 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
6839 assert_eq!(
6840 snapshot.info.status_reason.as_deref(),
6841 Some(ROOT_RECLAIMED_REASON)
6842 );
6843 let persisted = read_task(
6844 ®istry
6845 .task_json_path(&task_id, "session")
6846 .expect("reclaimed task metadata path"),
6847 )
6848 .expect("persisted reclaimed task");
6849 assert_eq!(
6850 persisted.status_reason.as_deref(),
6851 Some(ROOT_RECLAIMED_REASON)
6852 );
6853 let completion = registry
6854 .drain_completions_for_session(Some("session"))
6855 .pop()
6856 .expect("reclaimed task completion");
6857 assert_eq!(
6858 completion.status_reason.as_deref(),
6859 Some(ROOT_RECLAIMED_REASON)
6860 );
6861 registry.detach();
6862 }
6863
6864 #[test]
6865 fn reclaimed_root_kills_pty_task_and_preserves_reason() {
6866 let registry = BgTaskRegistry::default();
6867 let root = tempfile::tempdir().unwrap();
6868 let storage = tempfile::tempdir().unwrap();
6869 let command = if cfg!(windows) {
6870 "Start-Sleep -Seconds 30"
6871 } else {
6872 "sleep 30"
6873 };
6874 let task_id = registry
6875 .spawn_pty(
6876 SpawnPlan::Unsandboxed,
6877 command,
6878 "session".to_string(),
6879 root.path().to_path_buf(),
6880 HashMap::new(),
6881 Some(Duration::from_secs(60)),
6882 storage.path().to_path_buf(),
6883 10,
6884 true,
6885 false,
6886 Some(root.path().to_path_buf()),
6887 24,
6888 80,
6889 )
6890 .unwrap();
6891 let pid = registry
6892 .status(
6893 &task_id,
6894 "session",
6895 Some(root.path()),
6896 Some(storage.path()),
6897 0,
6898 )
6899 .unwrap()
6900 .child_pid
6901 .unwrap();
6902 assert!(is_process_alive(pid));
6903
6904 assert_eq!(registry.kill_running_tasks_for_root(root.path()), 1);
6905 let deadline = Instant::now() + Duration::from_secs(10);
6906 loop {
6907 let snapshot = registry
6908 .status(
6909 &task_id,
6910 "session",
6911 Some(root.path()),
6912 Some(storage.path()),
6913 0,
6914 )
6915 .unwrap();
6916 if snapshot.info.status.is_terminal() {
6917 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
6918 assert_eq!(
6919 snapshot.info.status_reason.as_deref(),
6920 Some(ROOT_RECLAIMED_REASON)
6921 );
6922 break;
6923 }
6924 assert!(
6925 Instant::now() < deadline,
6926 "reclaimed PTY task did not terminate"
6927 );
6928 std::thread::sleep(Duration::from_millis(20));
6929 }
6930 assert!(!is_process_alive(pid));
6931 let completion = loop {
6936 if let Some(completion) = registry
6937 .drain_completions_for_session(Some("session"))
6938 .pop()
6939 {
6940 break completion;
6941 }
6942 assert!(
6943 Instant::now() < deadline,
6944 "reclaimed PTY completion never arrived"
6945 );
6946 std::thread::sleep(Duration::from_millis(20));
6947 };
6948 assert_eq!(
6949 completion.status_reason.as_deref(),
6950 Some(ROOT_RECLAIMED_REASON)
6951 );
6952 registry.detach();
6953 }
6954
6955 #[test]
6956 fn register_watch_rejects_unknown_task() {
6957 let registry = BgTaskRegistry::default();
6958
6959 let result = registry.register_watch(
6960 "missing-task".to_string(),
6961 WatchPattern::Substring("READY".into()),
6962 true,
6963 );
6964
6965 assert_eq!(result, Err("task_not_found"));
6966 }
6967
6968 #[test]
6969 fn register_watch_on_terminal_task_scans_existing_output() {
6970 let frames = Arc::new(Mutex::new(Vec::new()));
6971 let captured = Arc::clone(&frames);
6972 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
6973 captured.lock().unwrap().push(frame);
6974 })
6975 as Box<dyn Fn(PushFrame) + Send + Sync>);
6976 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
6977 let dir = tempfile::tempdir().unwrap();
6978 let task_id = registry
6979 .spawn(
6980 SpawnPlan::Unsandboxed,
6981 LONG_RUNNING_COMMAND,
6982 "session".to_string(),
6983 dir.path().to_path_buf(),
6984 HashMap::new(),
6985 Some(Duration::from_secs(30)),
6986 dir.path().to_path_buf(),
6987 10,
6988 true,
6989 false,
6990 Some(dir.path().to_path_buf()),
6991 )
6992 .unwrap();
6993 registry
6994 .inner
6995 .shutdown
6996 .store(true, std::sync::atomic::Ordering::SeqCst);
6997 let task = registry.task_for_session(&task_id, "session").unwrap();
6998 std::fs::write(&task.paths.stdout, "READY\n").unwrap();
6999 registry
7000 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
7001 .unwrap();
7002 frames.lock().unwrap().clear();
7003 registry.inner.completions.lock().unwrap().clear();
7004
7005 registry
7006 .register_watch(
7007 task_id.clone(),
7008 WatchPattern::Substring("READY".into()),
7009 true,
7010 )
7011 .unwrap();
7012
7013 let frames = frames.lock().unwrap();
7014 let frame = frames
7015 .iter()
7016 .find_map(|frame| match frame {
7017 PushFrame::BashPatternMatch(frame) => Some(frame),
7018 _ => None,
7019 })
7020 .expect("terminal watch registration should emit pattern frame");
7021 assert_eq!(frame.reason, "pattern_match");
7022 assert_eq!(frame.task_id, task_id);
7023 assert_eq!(frame.session_id, "session");
7024 assert_eq!(frame.match_text, "READY");
7025 assert_eq!(frame.match_offset, 0);
7026 assert_eq!(registry.active_watch_count(&frame.task_id), 0);
7027 let metadata = read_task(&task.paths.json).unwrap();
7028 assert!(metadata.completion_delivered);
7029 }
7030
7031 #[test]
7032 fn cleanup_finished_removes_terminal_tasks_older_than_threshold() {
7033 let registry = BgTaskRegistry::default();
7034 let dir = tempfile::tempdir().unwrap();
7035 let task_id = registry
7036 .spawn(
7037 SpawnPlan::Unsandboxed,
7038 QUICK_SUCCESS_COMMAND,
7039 "session".to_string(),
7040 dir.path().to_path_buf(),
7041 HashMap::new(),
7042 Some(Duration::from_secs(30)),
7043 dir.path().to_path_buf(),
7044 10,
7045 true,
7046 false,
7047 Some(dir.path().to_path_buf()),
7048 )
7049 .unwrap();
7050 registry
7051 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
7052 .unwrap();
7053 let completions = registry.drain_completions_for_session(Some("session"));
7054 assert_eq!(completions.len(), 1);
7055 assert_eq!(
7056 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
7057 vec![task_id.clone()]
7058 );
7059
7060 registry.cleanup_finished(Duration::ZERO);
7061
7062 assert!(registry.inner.tasks.lock().unwrap().is_empty());
7063 }
7064
7065 #[test]
7066 fn cleanup_finished_retains_undelivered_terminals() {
7067 let registry = BgTaskRegistry::default();
7068 let dir = tempfile::tempdir().unwrap();
7069 let task_id = registry
7070 .spawn(
7071 SpawnPlan::Unsandboxed,
7072 QUICK_SUCCESS_COMMAND,
7073 "session".to_string(),
7074 dir.path().to_path_buf(),
7075 HashMap::new(),
7076 Some(Duration::from_secs(30)),
7077 dir.path().to_path_buf(),
7078 10,
7079 true,
7080 false,
7081 Some(dir.path().to_path_buf()),
7082 )
7083 .unwrap();
7084 registry
7085 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
7086 .unwrap();
7087
7088 registry.cleanup_finished(Duration::ZERO);
7089
7090 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
7091 }
7092
7093 #[test]
7101 fn reap_child_marks_failed_when_child_exits_without_exit_marker() {
7102 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7103 let dir = tempfile::tempdir().unwrap();
7104 let task_id = registry
7105 .spawn(
7106 SpawnPlan::Unsandboxed,
7107 QUICK_SUCCESS_COMMAND,
7108 "session".to_string(),
7109 dir.path().to_path_buf(),
7110 HashMap::new(),
7111 Some(Duration::from_secs(30)),
7112 dir.path().to_path_buf(),
7113 10,
7114 true,
7115 false,
7116 Some(dir.path().to_path_buf()),
7117 )
7118 .unwrap();
7119
7120 let task = registry.task_for_session(&task_id, "session").unwrap();
7121
7122 let started = Instant::now();
7127 loop {
7128 let exited = {
7129 let mut state = task.state.lock().unwrap();
7130 match &mut state.runtime {
7131 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
7132 _ => true,
7133 }
7134 };
7135 if exited {
7136 break;
7137 }
7138 assert!(
7139 started.elapsed() < Duration::from_secs(5),
7140 "child should exit quickly"
7141 );
7142 std::thread::sleep(Duration::from_millis(20));
7143 }
7144
7145 registry
7153 .inner
7154 .shutdown
7155 .store(true, std::sync::atomic::Ordering::SeqCst);
7156 std::thread::sleep(Duration::from_millis(550));
7160
7161 let _ = std::fs::remove_file(&task.paths.exit);
7164
7165 {
7180 let mut state = task.state.lock().unwrap();
7181 state.metadata.status = BgTaskStatus::Running;
7182 state.metadata.status_reason = None;
7183 state.metadata.exit_code = None;
7184 state.metadata.finished_at = None;
7185 state.metadata.duration_ms = None;
7186 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
7189 .expect("persist reset Running metadata for reap_child test");
7190 if matches!(state.runtime, TaskRuntime::Piped(None)) {
7194 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
7195 }
7196 }
7197 *task.terminal_at.lock().unwrap() = None;
7200
7201 assert!(
7204 task.is_running(),
7205 "precondition: metadata.status == Running"
7206 );
7207 assert!(
7208 !task.paths.exit.exists(),
7209 "precondition: exit marker absent"
7210 );
7211
7212 registry.reap_child(&task);
7217
7218 {
7219 let state = task.state.lock().unwrap();
7220 assert_eq!(
7221 state.metadata.status,
7222 BgTaskStatus::Running,
7223 "first reap must leave status Running while waiting one pass for marker"
7224 );
7225 assert_eq!(
7226 state.metadata.status_reason, None,
7227 "first reap must not record a failure reason"
7228 );
7229 assert!(
7230 matches!(state.runtime, TaskRuntime::Piped(None)),
7231 "child handle must be released after first reap"
7232 );
7233 assert!(
7234 state.detached,
7235 "task must be marked detached after first reap"
7236 );
7237 }
7238
7239 registry.reap_child(&task);
7243
7244 let state = task.state.lock().unwrap();
7245 assert!(
7246 state.metadata.status.is_terminal(),
7247 "second reap must transition to terminal when PID dead and no marker. Got status={:?}",
7248 state.metadata.status
7249 );
7250 assert_eq!(
7251 state.metadata.status,
7252 BgTaskStatus::Failed,
7253 "must specifically be Failed (not Killed): status={:?}",
7254 state.metadata.status
7255 );
7256 assert_eq!(
7257 state.metadata.status_reason.as_deref(),
7258 Some("process exited without exit marker"),
7259 "reason must match replay path's wording: {:?}",
7260 state.metadata.status_reason
7261 );
7262 assert!(
7263 matches!(state.runtime, TaskRuntime::Piped(None)),
7264 "child handle must stay released after second reap"
7265 );
7266 assert!(
7267 state.detached,
7268 "task must remain detached after second reap"
7269 );
7270 }
7271
7272 #[test]
7277 fn reap_child_preserves_running_when_exit_marker_exists() {
7278 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7279 let dir = tempfile::tempdir().unwrap();
7280 let task_id = registry
7281 .spawn(
7282 SpawnPlan::Unsandboxed,
7283 QUICK_SUCCESS_COMMAND,
7284 "session".to_string(),
7285 dir.path().to_path_buf(),
7286 HashMap::new(),
7287 Some(Duration::from_secs(30)),
7288 dir.path().to_path_buf(),
7289 10,
7290 true,
7291 false,
7292 Some(dir.path().to_path_buf()),
7293 )
7294 .unwrap();
7295
7296 let task = registry.task_for_session(&task_id, "session").unwrap();
7297
7298 let started = Instant::now();
7301 loop {
7302 let exited = {
7303 let mut state = task.state.lock().unwrap();
7304 match &mut state.runtime {
7305 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
7306 _ => true,
7307 }
7308 };
7309 if exited && task.paths.exit.exists() {
7310 break;
7311 }
7312 assert!(
7313 started.elapsed() < Duration::from_secs(5),
7314 "child should exit and write marker quickly"
7315 );
7316 std::thread::sleep(Duration::from_millis(20));
7317 }
7318
7319 registry
7325 .inner
7326 .shutdown
7327 .store(true, std::sync::atomic::Ordering::SeqCst);
7328 std::thread::sleep(Duration::from_millis(550));
7329
7330 {
7336 let mut state = task.state.lock().unwrap();
7337 state.metadata.status = BgTaskStatus::Running;
7338 state.metadata.status_reason = None;
7339 if matches!(state.runtime, TaskRuntime::Piped(None)) {
7340 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
7341 }
7342 }
7343 *task.terminal_at.lock().unwrap() = None;
7344 if !task.paths.exit.exists() {
7347 std::fs::write(&task.paths.exit, "0").expect("write replacement exit marker");
7348 }
7349
7350 registry.reap_child(&task);
7354
7355 let state = task.state.lock().unwrap();
7356 assert!(
7357 matches!(state.runtime, TaskRuntime::Piped(None)),
7358 "child handle still released even when marker exists"
7359 );
7360 assert!(
7361 state.detached,
7362 "task still marked detached even when marker exists"
7363 );
7364 assert_eq!(
7369 state.metadata.status,
7370 BgTaskStatus::Running,
7371 "reap_child must defer to poll_task when marker exists"
7372 );
7373 }
7374
7375 #[cfg(unix)]
7379 fn pid_stat(pid: u32) -> Option<String> {
7380 let output = std::process::Command::new("ps")
7381 .args(["-o", "stat=", "-p", &pid.to_string()])
7382 .output()
7383 .ok()?;
7384 if !output.status.success() {
7385 return None;
7386 }
7387 let stat = String::from_utf8_lossy(&output.stdout).trim().to_string();
7388 if stat.is_empty() {
7389 None
7390 } else {
7391 Some(stat)
7392 }
7393 }
7394
7395 #[cfg(unix)]
7397 fn is_zombie(pid: u32) -> bool {
7398 pid_stat(pid).is_some_and(|stat| stat.starts_with('Z'))
7399 }
7400
7401 #[cfg(unix)]
7407 fn spawn_unreaped_zombie() -> std::process::Child {
7408 let child = std::process::Command::new("true")
7409 .stdin(std::process::Stdio::null())
7410 .stdout(std::process::Stdio::null())
7411 .stderr(std::process::Stdio::null())
7412 .spawn()
7413 .expect("spawn zombie stand-in");
7414 let pid = child.id();
7415 let started = Instant::now();
7416 while !is_zombie(pid) {
7417 assert!(
7418 started.elapsed() < Duration::from_secs(5),
7419 "stand-in child should become a zombie within 5s"
7420 );
7421 std::thread::sleep(Duration::from_millis(10));
7422 }
7423 child
7425 }
7426
7427 #[cfg(unix)]
7437 #[test]
7438 fn finalize_from_marker_reaps_child_no_zombie() {
7439 use std::sync::atomic::Ordering;
7440
7441 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7442 let dir = tempfile::tempdir().unwrap();
7443 let task_id = registry
7444 .spawn(
7445 SpawnPlan::Unsandboxed,
7446 QUICK_SUCCESS_COMMAND,
7447 "session".to_string(),
7448 dir.path().to_path_buf(),
7449 HashMap::new(),
7450 Some(Duration::from_secs(30)),
7451 dir.path().to_path_buf(),
7452 10,
7453 true,
7454 false,
7455 Some(dir.path().to_path_buf()),
7456 )
7457 .unwrap();
7458
7459 registry.inner.shutdown.store(true, Ordering::SeqCst);
7463 std::thread::sleep(Duration::from_millis(550));
7464
7465 let task = registry.task_for_session(&task_id, "session").unwrap();
7466
7467 let started = Instant::now();
7471 while !task.paths.exit.exists() {
7472 assert!(
7473 started.elapsed() < Duration::from_secs(5),
7474 "exit marker should land quickly for `true`"
7475 );
7476 std::thread::sleep(Duration::from_millis(20));
7477 }
7478
7479 let zombie_pid;
7485 {
7486 let mut state = task.state.lock().unwrap();
7487 state.metadata.status = BgTaskStatus::Running;
7488 state.metadata.status_reason = None;
7489 state.metadata.exit_code = None;
7490 state.metadata.finished_at = None;
7491 state.metadata.duration_ms = None;
7492 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
7493 .expect("persist reset Running metadata");
7494 let zombie = spawn_unreaped_zombie();
7495 zombie_pid = zombie.id();
7496 state.runtime = TaskRuntime::Piped(Some(zombie));
7497 }
7498 *task.terminal_at.lock().unwrap() = None;
7499
7500 assert!(
7502 is_zombie(zombie_pid),
7503 "precondition: stand-in child {zombie_pid} must be a zombie before finalize"
7504 );
7505
7506 registry.poll_task(&task).unwrap();
7509
7510 {
7511 let state = task.state.lock().unwrap();
7512 assert!(
7513 matches!(state.runtime, TaskRuntime::Piped(None)),
7514 "child handle must be released after marker finalize"
7515 );
7516 assert!(
7517 state.metadata.status.is_terminal(),
7518 "task must be terminal after marker finalize: {:?}",
7519 state.metadata.status
7520 );
7521 }
7522
7523 assert!(
7526 !is_zombie(zombie_pid),
7527 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
7528 after the exit-marker terminal transition"
7529 );
7530 }
7531
7532 #[cfg(unix)]
7536 #[test]
7537 fn kill_with_existing_marker_reaps_child_no_zombie() {
7538 use std::sync::atomic::Ordering;
7539
7540 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7541 let dir = tempfile::tempdir().unwrap();
7542 let task_id = registry
7543 .spawn(
7544 SpawnPlan::Unsandboxed,
7545 QUICK_SUCCESS_COMMAND,
7546 "session".to_string(),
7547 dir.path().to_path_buf(),
7548 HashMap::new(),
7549 Some(Duration::from_secs(30)),
7550 dir.path().to_path_buf(),
7551 10,
7552 true,
7553 false,
7554 Some(dir.path().to_path_buf()),
7555 )
7556 .unwrap();
7557
7558 registry.inner.shutdown.store(true, Ordering::SeqCst);
7559 std::thread::sleep(Duration::from_millis(550));
7560
7561 let task = registry.task_for_session(&task_id, "session").unwrap();
7562
7563 let started = Instant::now();
7564 while !task.paths.exit.exists() {
7565 assert!(
7566 started.elapsed() < Duration::from_secs(5),
7567 "exit marker should land quickly for `true`"
7568 );
7569 std::thread::sleep(Duration::from_millis(20));
7570 }
7571
7572 let zombie_pid;
7573 {
7574 let mut state = task.state.lock().unwrap();
7575 state.metadata.status = BgTaskStatus::Running;
7576 state.metadata.status_reason = None;
7577 state.metadata.exit_code = None;
7578 state.metadata.finished_at = None;
7579 state.metadata.duration_ms = None;
7580 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
7581 .expect("persist reset Running metadata");
7582 let zombie = spawn_unreaped_zombie();
7583 zombie_pid = zombie.id();
7584 state.runtime = TaskRuntime::Piped(Some(zombie));
7585 }
7586 *task.terminal_at.lock().unwrap() = None;
7587
7588 assert!(
7589 is_zombie(zombie_pid),
7590 "precondition: stand-in child {zombie_pid} must be a zombie before kill"
7591 );
7592
7593 registry
7595 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
7596 .expect("kill should succeed");
7597
7598 {
7599 let state = task.state.lock().unwrap();
7600 assert!(
7601 matches!(state.runtime, TaskRuntime::Piped(None)),
7602 "child handle must be released after marker-aware kill"
7603 );
7604 assert!(state.metadata.status.is_terminal());
7605 }
7606
7607 assert!(
7608 !is_zombie(zombie_pid),
7609 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
7610 after a marker-aware kill"
7611 );
7612 }
7613
7614 #[test]
7615 fn cleanup_finished_keeps_running_tasks() {
7616 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7617 let dir = tempfile::tempdir().unwrap();
7618 let task_id = registry
7619 .spawn(
7620 SpawnPlan::Unsandboxed,
7621 LONG_RUNNING_COMMAND,
7622 "session".to_string(),
7623 dir.path().to_path_buf(),
7624 HashMap::new(),
7625 Some(Duration::from_secs(30)),
7626 dir.path().to_path_buf(),
7627 10,
7628 true,
7629 false,
7630 Some(dir.path().to_path_buf()),
7631 )
7632 .unwrap();
7633
7634 registry.cleanup_finished(Duration::ZERO);
7635
7636 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
7637 let _ = registry.kill(&task_id, "session");
7638 }
7639
7640 #[cfg(unix)]
7641 #[test]
7642 fn rehydrating_sandboxed_task_never_respawns_persisted_command() {
7643 let project = tempfile::tempdir().unwrap();
7644 let storage = tempfile::tempdir().unwrap();
7645 let sandbox_temp = storage.path().join("sandbox-temp");
7646 fs::create_dir(&sandbox_temp).unwrap();
7647 let launcher_script = project.path().join("sandbox-launch");
7648 let launcher = PathBuf::from("/bin/sh");
7649 fs::write(
7650 &launcher_script,
7651 "while [ \"$#\" -gt 0 ]; do\n if [ \"$1\" = -- ]; then\n shift\n exec \"$@\"\n fi\n shift\ndone\nexit 78\n",
7652 )
7653 .unwrap();
7654 let mut permissions = fs::metadata(&launcher_script).unwrap().permissions();
7655 permissions.set_mode(0o700);
7656 fs::set_permissions(&launcher_script, permissions).unwrap();
7657
7658 let profile = crate::sandbox_profile::SandboxProfile::build(
7659 vec![project.path().to_path_buf()],
7660 Vec::new(),
7661 Vec::new(),
7662 Vec::new(),
7663 Vec::new(),
7664 Vec::new(),
7665 Vec::new(),
7666 sandbox_temp,
7667 )
7668 .unwrap();
7669 let plan = SpawnPlan::launcher_for_test(profile, launcher);
7670 let spawn_marker = project.path().join("spawn-count");
7671 let stop_marker = project.path().join("stop-command");
7672 let quote =
7673 |path: &Path| format!("'{}'", path.display().to_string().replace('\'', "'\\''"));
7674 let command = format!(
7675 "printf 'spawn\\n' >> {}; while [ ! -e {} ]; do sleep 0.05; done",
7676 quote(&spawn_marker),
7677 quote(&stop_marker)
7678 );
7679
7680 let original = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7681 let task_id = original
7682 .spawn(
7683 plan,
7684 &command,
7685 "sandbox-rehydrate".to_string(),
7686 project.path().to_path_buf(),
7687 HashMap::new(),
7688 Some(Duration::from_secs(30)),
7689 storage.path().to_path_buf(),
7690 10,
7691 true,
7692 false,
7693 Some(project.path().to_path_buf()),
7694 )
7695 .unwrap();
7696 let started = Instant::now();
7697 while !spawn_marker.exists() {
7698 assert!(
7699 started.elapsed() < Duration::from_secs(20),
7700 "original sandboxed task did not start"
7701 );
7702 std::thread::sleep(Duration::from_millis(10));
7703 }
7704 original.detach();
7705
7706 let restarted = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7707 restarted
7708 .replay_session(storage.path(), "sandbox-rehydrate")
7709 .unwrap();
7710 let replayed = restarted
7711 .status(
7712 &task_id,
7713 "sandbox-rehydrate",
7714 Some(project.path()),
7715 Some(storage.path()),
7716 4096,
7717 )
7718 .expect("rehydrated sandbox task");
7719 assert_eq!(replayed.info.status, BgTaskStatus::Running);
7720 assert!(replayed.sandbox_native);
7721
7722 std::thread::sleep(Duration::from_millis(650));
7723 assert_eq!(
7724 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
7725 1,
7726 "registry replay must observe the persisted process without spawning its command"
7727 );
7728
7729 fs::write(&stop_marker, "stop").unwrap();
7730 let terminal = wait_for_terminal_snapshot(
7731 &restarted,
7732 &task_id,
7733 "sandbox-rehydrate",
7734 project.path(),
7735 storage.path(),
7736 );
7737 assert_eq!(terminal.info.status, BgTaskStatus::Completed);
7738 assert_eq!(
7739 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
7740 1
7741 );
7742 restarted.detach();
7743 }
7744
7745 #[cfg(windows)]
7746 fn wait_for_file(path: &Path) -> String {
7747 let started = Instant::now();
7754 loop {
7755 if let Ok(content) = fs::read_to_string(path) {
7756 if !content.trim().is_empty() {
7757 return content;
7758 }
7759 }
7760 assert!(
7761 started.elapsed() < Duration::from_secs(30),
7762 "timed out waiting for non-empty {}",
7763 path.display()
7764 );
7765 std::thread::sleep(Duration::from_millis(100));
7766 }
7767 }
7768
7769 #[cfg(windows)]
7770 fn spawn_windows_registry_command(
7771 command: &str,
7772 ) -> (BgTaskRegistry, tempfile::TempDir, String) {
7773 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
7774 let dir = tempfile::tempdir().unwrap();
7775 let task_id = registry
7776 .spawn(
7777 SpawnPlan::Unsandboxed,
7778 command,
7779 "session".to_string(),
7780 dir.path().to_path_buf(),
7781 HashMap::new(),
7782 Some(Duration::from_secs(30)),
7783 dir.path().to_path_buf(),
7784 10,
7785 false,
7786 false,
7787 Some(dir.path().to_path_buf()),
7788 )
7789 .unwrap();
7790 (registry, dir, task_id)
7791 }
7792
7793 #[cfg(windows)]
7794 #[test]
7795 fn windows_spawn_writes_exit_marker_for_zero_exit() {
7796 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 0");
7797 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
7798
7799 let content = wait_for_file(&exit_path);
7800
7801 assert_eq!(content.trim(), "0");
7802 }
7803
7804 #[cfg(windows)]
7805 #[test]
7806 fn windows_spawn_writes_exit_marker_for_nonzero_exit() {
7807 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 42");
7808 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
7809
7810 let content = wait_for_file(&exit_path);
7811
7812 assert_eq!(content.trim(), "42");
7813 }
7814
7815 #[cfg(windows)]
7816 #[test]
7817 fn windows_spawn_captures_stdout_to_disk() {
7818 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c echo hello");
7819 let task = registry.task_for_session(&task_id, "session").unwrap();
7820 let stdout_path = task.paths.stdout.clone();
7821 let exit_path = task.paths.exit.clone();
7822
7823 let _ = wait_for_file(&exit_path);
7824 let stdout = fs::read_to_string(stdout_path).expect("read stdout");
7825
7826 assert!(stdout.contains("hello"), "stdout was {stdout:?}");
7827 }
7828
7829 #[cfg(windows)]
7830 #[test]
7831 fn windows_spawn_uses_pwsh_when_available() {
7832 let candidates = crate::windows_shell::shell_candidates_with(
7836 |binary| match binary {
7837 "pwsh.exe" => Some(std::path::PathBuf::from(r"C:\pwsh\pwsh.exe")),
7838 "powershell.exe" => Some(std::path::PathBuf::from(r"C:\ps\powershell.exe")),
7839 _ => None,
7840 },
7841 || None,
7842 );
7843 let shell = candidates.first().expect("at least one candidate").clone();
7844 assert_eq!(shell, crate::windows_shell::WindowsShell::Pwsh);
7845 assert_eq!(shell.binary().as_ref(), "pwsh.exe");
7846 }
7847
7848 #[cfg(windows)]
7851 #[test]
7852 fn windows_shell_cmd_wrapper_writes_marker_via_temp_rename() {
7853 let exit_path = Path::new(r"C:\Temp\bash-test.exit");
7854 let script =
7855 crate::windows_shell::WindowsShell::Cmd.wrapper_script("cmd /c exit 42", exit_path);
7856
7857 assert!(
7858 script.contains("set CODE=%ERRORLEVEL%"),
7859 "wrapper must capture the child exit code: {script}"
7860 );
7861 assert!(
7862 script.contains("exit /B %CODE%"),
7863 "wrapper must propagate the child exit code: {script}"
7864 );
7865 assert!(
7870 script.contains("bash-test.exit"),
7871 "wrapper must target the exit marker path: {script}"
7872 );
7873 assert!(
7874 script.contains("move /Y"),
7875 "wrapper must write the marker atomically via temp-file + rename: {script}"
7876 );
7877 }
7878
7879 #[cfg(windows)]
7885 #[test]
7886 fn windows_shell_cmd_bg_command_uses_minimal_cmd_flags() {
7887 use crate::windows_shell::WindowsShell;
7888 let cmd = WindowsShell::Cmd.bg_command("echo wrapped");
7889 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
7890 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
7891 assert_eq!(
7892 args_strs,
7893 vec!["/D", "/S", "/C", "echo wrapped"],
7894 "Cmd::bg_command must prepend /D /S /C"
7895 );
7896 }
7897
7898 #[cfg(windows)]
7901 #[test]
7902 fn windows_shell_pwsh_bg_command_uses_standard_args() {
7903 use crate::windows_shell::WindowsShell;
7904 let cmd = WindowsShell::Pwsh.bg_command("Get-Date");
7905 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
7906 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
7907 assert!(
7908 args_strs.contains(&"-Command"),
7909 "Pwsh::bg_command must use -Command: {args_strs:?}"
7910 );
7911 assert!(
7912 args_strs.contains(&"Get-Date"),
7913 "Pwsh::bg_command must include the user command body"
7914 );
7915 }
7916
7917 fn registry_with_db_and_frames(
7918 storage: &Path,
7919 ) -> (
7920 BgTaskRegistry,
7921 Arc<Mutex<Connection>>,
7922 Arc<Mutex<Vec<PushFrame>>>,
7923 ) {
7924 let frames = Arc::new(Mutex::new(Vec::new()));
7925 let captured = Arc::clone(&frames);
7926 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
7927 captured.lock().unwrap().push(frame);
7928 })
7929 as Box<dyn Fn(PushFrame) + Send + Sync>);
7930 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
7931 registry.set_harness(Harness::Opencode);
7932 let conn = crate::db::open(&storage.join("aft.db")).expect("open test DB");
7933 let shared = Arc::new(Mutex::new(conn));
7934 registry.set_db_pool(shared.clone());
7935 (registry, shared, frames)
7936 }
7937
7938 fn pattern_match_frames(frames: &Mutex<Vec<PushFrame>>) -> Vec<BashPatternMatchFrame> {
7939 frames
7940 .lock()
7941 .unwrap()
7942 .iter()
7943 .filter_map(|frame| match frame {
7944 PushFrame::BashPatternMatch(frame) => Some(frame.clone()),
7945 _ => None,
7946 })
7947 .collect()
7948 }
7949
7950 #[cfg(unix)]
7951 #[test]
7952 fn gc_refuses_to_delete_or_quarantine_a_recorded_live_process() {
7953 let dir = tempfile::tempdir().unwrap();
7954 let storage = dir.path();
7955 let (registry, db, _frames) = registry_with_db_and_frames(storage);
7956 let task_id = "bash-0000000000000198";
7957 let paths = task_paths(storage, "session", task_id).unwrap();
7958 let mut running = PersistedTask::starting(
7959 task_id.to_string(),
7960 "session".to_string(),
7961 "live-process-canary".to_string(),
7962 storage.to_path_buf(),
7963 Some(storage.to_path_buf()),
7964 None,
7965 true,
7966 false,
7967 );
7968 running.mark_running(std::process::id(), std::process::id() as i32);
7972 write_task(&paths.json, &running).unwrap();
7973 fs::write(&paths.stdout, b"").unwrap();
7974 fs::write(&paths.stderr, b"").unwrap();
7975 {
7976 let conn = db.lock().unwrap();
7977 crate::db::bash_tasks::upsert_bash_task(
7978 &conn,
7979 &running.to_bash_task_row("opencode", &paths).unwrap(),
7980 )
7981 .unwrap();
7982 }
7983 let running_json = fs::read(&paths.json).unwrap();
7984 let mut terminal: PersistedTask = serde_json::from_slice(&running_json).unwrap();
7985 terminal.mark_terminal(BgTaskStatus::Completed, Some(0), None);
7986 terminal.completion_delivered = true;
7987 write_task_at(
7988 &resolve_task_layout(&paths.session_dir, task_id).unwrap(),
7989 &terminal,
7990 )
7991 .unwrap();
7992 let old = SystemTime::now()
7993 .checked_sub(Duration::from_secs(25 * 60 * 60))
7994 .unwrap();
7995 filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
7996
7997 assert_eq!(registry.maybe_gc_persisted(storage).unwrap(), 0);
7998 assert!(paths.io_dir.exists(), "GC deleted a live task bundle");
7999
8000 fs::write(&paths.json, b"{corrupt").unwrap();
8001 filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
8002 assert_eq!(registry.maybe_gc_persisted(storage).unwrap(), 0);
8003 assert!(
8004 paths.io_dir.exists(),
8005 "GC quarantined a live task with unreadable metadata"
8006 );
8007
8008 fs::write(&paths.json, running_json).unwrap();
8009 }
8010
8011 #[test]
8012 fn pattern_watch_survives_registry_teardown_and_rehydrate() {
8013 let dir = tempfile::tempdir().unwrap();
8014 let storage = dir.path();
8015 let (registry, _db, frames) = registry_with_db_and_frames(storage);
8016 let task_id = registry
8017 .spawn(
8018 SpawnPlan::Unsandboxed,
8019 LONG_RUNNING_COMMAND,
8020 "session".to_string(),
8021 storage.to_path_buf(),
8022 HashMap::new(),
8023 Some(Duration::from_secs(30)),
8024 storage.to_path_buf(),
8025 10,
8026 true,
8027 false,
8028 Some(storage.to_path_buf()),
8029 )
8030 .unwrap();
8031 registry
8032 .register_watch(
8033 task_id.clone(),
8034 WatchPattern::Substring("READY".into()),
8035 true,
8036 )
8037 .unwrap();
8038 let task = registry.task_for_session(&task_id, "session").unwrap();
8039 registry.clear_task_watch_state(&task_id);
8041 assert_eq!(registry.active_watch_count(&task_id), 0);
8042
8043 std::fs::OpenOptions::new()
8044 .append(true)
8045 .open(&task.paths.stdout)
8046 .unwrap()
8047 .write_all(b"READY\n")
8048 .unwrap();
8049 frames.lock().unwrap().clear();
8050
8051 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
8052 registry
8054 .inner
8055 .shutdown
8056 .store(true, std::sync::atomic::Ordering::SeqCst);
8057 replayed
8058 .replay_session_inner(storage, "session", None)
8059 .unwrap();
8060
8061 let matches = pattern_match_frames(&replay_frames);
8062 assert!(
8063 matches.iter().any(|frame| {
8064 frame.task_id == task_id
8065 && frame.reason == "pattern_match"
8066 && frame.match_text == "READY"
8067 }),
8068 "rehydrate should deliver gap match: {matches:?}"
8069 );
8070 }
8071
8072 #[test]
8073 fn pattern_watch_gap_match_between_teardown_and_rehydrate_delivers_once() {
8074 let dir = tempfile::tempdir().unwrap();
8075 let storage = dir.path();
8076 let (registry, _db, frames) = registry_with_db_and_frames(storage);
8077 let task_id = registry
8078 .spawn(
8079 SpawnPlan::Unsandboxed,
8080 LONG_RUNNING_COMMAND,
8081 "session".to_string(),
8082 storage.to_path_buf(),
8083 HashMap::new(),
8084 Some(Duration::from_secs(30)),
8085 storage.to_path_buf(),
8086 10,
8087 true,
8088 false,
8089 Some(storage.to_path_buf()),
8090 )
8091 .unwrap();
8092 registry
8093 .register_watch(
8094 task_id.clone(),
8095 WatchPattern::Substring("GAP-HIT".into()),
8096 true,
8097 )
8098 .unwrap();
8099 let task = registry.task_for_session(&task_id, "session").unwrap();
8100 let cursor_before = registry.watch_stream_cursors(&task_id).0;
8101 registry.clear_task_watch_state(&task_id);
8102
8103 std::fs::OpenOptions::new()
8105 .append(true)
8106 .open(&task.paths.stdout)
8107 .unwrap()
8108 .write_all(b"prefix GAP-HIT suffix\n")
8109 .unwrap();
8110 frames.lock().unwrap().clear();
8111
8112 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
8113 registry
8114 .inner
8115 .shutdown
8116 .store(true, std::sync::atomic::Ordering::SeqCst);
8117 replayed
8118 .replay_session_inner(storage, "session", None)
8119 .unwrap();
8120
8121 let matches: Vec<_> = pattern_match_frames(&replay_frames)
8122 .into_iter()
8123 .filter(|frame| frame.task_id == task_id && frame.match_text.contains("GAP-HIT"))
8124 .collect();
8125 assert_eq!(
8126 matches.len(),
8127 1,
8128 "gap match must deliver exactly once: {matches:?}"
8129 );
8130 assert!(
8131 matches[0].match_offset >= cursor_before,
8132 "match offset should be at/after the persisted cursor ({cursor_before}), got {}",
8133 matches[0].match_offset
8134 );
8135 }
8136
8137 #[test]
8138 fn pattern_watch_acked_match_does_not_redeliver_after_restart() {
8139 let dir = tempfile::tempdir().unwrap();
8140 let storage = dir.path();
8141 let (registry, db, frames) = registry_with_db_and_frames(storage);
8142 let task_id = registry
8143 .spawn(
8144 SpawnPlan::Unsandboxed,
8145 LONG_RUNNING_COMMAND,
8146 "session".to_string(),
8147 storage.to_path_buf(),
8148 HashMap::new(),
8149 Some(Duration::from_secs(30)),
8150 storage.to_path_buf(),
8151 10,
8152 true,
8153 false,
8154 Some(storage.to_path_buf()),
8155 )
8156 .unwrap();
8157 registry
8158 .register_watch(
8159 task_id.clone(),
8160 WatchPattern::Substring("READY".into()),
8161 true,
8162 )
8163 .unwrap();
8164 let task = registry.task_for_session(&task_id, "session").unwrap();
8165 std::fs::OpenOptions::new()
8166 .append(true)
8167 .open(&task.paths.stdout)
8168 .unwrap()
8169 .write_all(b"READY\n")
8170 .unwrap();
8171 registry.scan_task_watch_output(&task);
8172 let delivered = pattern_match_frames(&frames);
8173 assert!(
8174 delivered
8175 .iter()
8176 .any(|frame| frame.task_id == task_id && frame.match_text == "READY"),
8177 "live path should deliver match: {delivered:?}"
8178 );
8179 assert!(registry
8181 .ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id))
8182 .contains(&task_id));
8183 {
8184 let conn = db.lock().unwrap();
8185 let rows = crate::db::bash_watches::list_bash_pattern_watches_for_task(
8186 &conn, "opencode", "session", &task_id,
8187 )
8188 .unwrap();
8189 assert!(
8190 rows.is_empty(),
8191 "acked once-watch rows must be deleted: {rows:?}"
8192 );
8193 }
8194
8195 frames.lock().unwrap().clear();
8196 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
8197 registry
8198 .inner
8199 .shutdown
8200 .store(true, std::sync::atomic::Ordering::SeqCst);
8201 replayed
8202 .replay_session_inner(storage, "session", None)
8203 .unwrap();
8204 let matches = pattern_match_frames(&replay_frames)
8205 .into_iter()
8206 .filter(|frame| frame.task_id == task_id)
8207 .collect::<Vec<_>>();
8208 assert!(
8209 matches.is_empty(),
8210 "acked match must not re-deliver after restart: {matches:?}"
8211 );
8212 }
8213
8214 #[test]
8215 fn pattern_watch_rows_are_removed_when_task_is_gc_deleted() {
8216 let dir = tempfile::tempdir().unwrap();
8217 let storage = dir.path();
8218 let (registry, db, _frames) = registry_with_db_and_frames(storage);
8219 let task_id = "bash-aaaaaaaaaaaaaaaa";
8220 let paths = task_paths(storage, "session", task_id).unwrap();
8221 let mut metadata = PersistedTask::starting(
8222 task_id.to_string(),
8223 "session".to_string(),
8224 "true".to_string(),
8225 storage.to_path_buf(),
8226 Some(storage.to_path_buf()),
8227 None,
8228 true,
8229 true,
8230 );
8231 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
8232 metadata.completion_delivered = true;
8233 write_task(&paths.json, &metadata).unwrap();
8234 {
8235 let conn = db.lock().unwrap();
8236 crate::db::bash_tasks::upsert_bash_task(
8237 &conn,
8238 &metadata.to_bash_task_row("opencode", &paths).unwrap(),
8239 )
8240 .unwrap();
8241 crate::db::bash_watches::upsert_bash_pattern_watch(
8242 &conn,
8243 &BashPatternWatchRow {
8244 harness: "opencode".into(),
8245 session_id: "session".into(),
8246 task_id: task_id.into(),
8247 watch_id: "watch-00000001".into(),
8248 pattern_kind: "substring".into(),
8249 pattern: "x".into(),
8250 once: true,
8251 created_at: 1,
8252 stdout_offset: 0,
8253 stderr_offset: 0,
8254 pty_offset: 0,
8255 scanning: true,
8256 pending_match: false,
8257 match_text: None,
8258 match_offset: None,
8259 match_context: None,
8260 },
8261 )
8262 .unwrap();
8263 }
8264 let old = SystemTime::now()
8265 .checked_sub(Duration::from_secs(25 * 60 * 60))
8266 .unwrap();
8267 filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
8268
8269 let deleted = registry.maybe_gc_persisted(storage).unwrap();
8270 assert!(
8271 deleted >= 1,
8272 "expected GC to delete the terminal task bundle"
8273 );
8274 let conn = db.lock().unwrap();
8275 let watches = crate::db::bash_watches::list_bash_pattern_watches_for_task(
8276 &conn, "opencode", "session", task_id,
8277 )
8278 .unwrap();
8279 assert!(
8280 watches.is_empty(),
8281 "task GC must remove watch rows: {watches:?}"
8282 );
8283 }
8284}