1use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
2#[cfg(not(windows))]
3use std::ffi::OsString;
4use std::fs;
5use std::io::Write;
6use std::path::{Path, PathBuf};
7#[cfg(windows)]
8use std::process::Command;
9use std::process::{Child, Stdio};
10use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
11#[cfg(unix)]
12use std::sync::OnceLock;
13use std::sync::{Arc, Mutex, RwLock};
14use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
15
16use rusqlite::Connection;
17use serde::Serialize;
18
19use crate::bash_permissions::PermissionAsk;
20use crate::compress::caps::DropClass;
21use crate::compress::CompressionResult;
22use crate::context::SharedProgressSender;
23use crate::db::compression_events::CompressionAggregateCache;
24use crate::harness::Harness;
25use crate::protocol::{BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, PushFrame};
26use crate::sandbox_spawn::SpawnPlan;
27
28#[cfg(windows)]
29use std::os::windows::process::CommandExt;
30
31use super::buffer::{combine_streams, BgBuffer, DiskTruncation, StreamKind, TokenCountInput};
32use super::output::{
33 cap_completion_output, cap_completion_output_with_marker, cap_final_output,
34 cap_final_output_with_marker, completion_preview_threshold, json_output_pointer, quote_path,
35 retained_json_output_pointer, COMPRESS_INPUT_CAP_BYTES, COMPRESS_INPUT_HEAD_BYTES,
36 COMPRESS_INPUT_TAIL_BYTES, FINAL_OUTPUT_CAP_BYTES, RAW_PASSTHROUGH_CAP_BYTES,
37 RAW_PASSTHROUGH_HEAD_BYTES, RAW_PASSTHROUGH_TAIL_BYTES, RUNNING_OUTPUT_PREVIEW_BYTES,
38 STRUCTURED_OUTPUT_CAP_BYTES,
39};
40use super::persistence::{
41 allocate_task_layout, delete_resolved_task, delete_task_bundle, discover_task_ids,
42 open_task_artifact, quarantine_invalid_entry, quarantine_task_layout, read_exit_marker,
43 read_task_at, resolve_task_layout, session_tasks_dir, uninitialized_layout_is_recent,
44 unix_millis, update_task_at, validate_task_id, write_kill_marker_if_absent, write_task_at,
45 BgMode, ExitMarker, PersistedTask, TaskArtifact, TaskIoHandles, TaskPaths,
46};
47use super::process::is_process_alive;
48#[cfg(unix)]
49use super::process::terminate_pgid;
50#[cfg(windows)]
51use super::process::terminate_pid;
52use super::pty_process::spawn_pty_for_command;
53use super::pty_runtime::PtyRuntime;
54use super::watches::{PatternMatch, WatchPattern, WatchRegistry};
55use super::{BgTaskInfo, BgTaskStatus};
56use crate::db::bash_watches::BashPatternWatchRow;
57const DEFAULT_BG_TIMEOUT: Duration = Duration::from_secs(30 * 60);
60const STALE_RUNNING_AFTER: Duration = Duration::from_secs(24 * 60 * 60);
61const PERSISTED_GC_GRACE: Duration = Duration::from_secs(24 * 60 * 60);
62const QUARANTINE_GC_GRACE: Duration = Duration::from_secs(30 * 24 * 60 * 60);
63
64const TOKENIZE_CAP_BYTES_PER_STREAM: usize = 128 * 1024;
65
66#[derive(Debug, Clone, Serialize)]
67pub struct BgCompletion {
68 pub task_id: String,
69 #[serde(skip_serializing)]
72 pub session_id: String,
73 pub status: BgTaskStatus,
74 pub exit_code: Option<i32>,
75 pub command: String,
76 #[serde(default, skip_serializing_if = "String::is_empty")]
82 pub output_preview: String,
83 #[serde(default, skip_serializing_if = "is_false")]
88 pub output_truncated: bool,
89 #[serde(default, skip_serializing_if = "Option::is_none")]
92 pub original_tokens: Option<u32>,
93 #[serde(default, skip_serializing_if = "Option::is_none")]
96 pub compressed_tokens: Option<u32>,
97 #[serde(default, skip_serializing_if = "is_false")]
99 pub tokens_skipped: bool,
100}
101
102fn is_false(v: &bool) -> bool {
103 !*v
104}
105
106#[derive(Debug, Clone, Serialize)]
107pub struct BgTaskSnapshot {
108 #[serde(flatten)]
109 pub info: BgTaskInfo,
110 pub exit_code: Option<i32>,
111 pub child_pid: Option<u32>,
112 pub workdir: String,
113 pub output_preview: String,
114 pub output_truncated: bool,
115 pub output_path: Option<String>,
116 pub stderr_path: Option<String>,
117 #[serde(skip_serializing_if = "Option::is_none")]
118 pub pty_rows: Option<u16>,
119 #[serde(skip_serializing_if = "Option::is_none")]
120 pub pty_cols: Option<u16>,
121 #[serde(skip_serializing_if = "Option::is_none")]
122 pub pty_screen: Option<String>,
123 #[serde(default, skip_serializing_if = "Vec::is_empty")]
124 pub scanner_report: Vec<PermissionAsk>,
125 #[serde(default, skip_serializing_if = "is_false")]
126 pub sandbox_native: bool,
127 #[serde(default, skip_serializing_if = "is_false")]
128 pub sandbox_unavailable: bool,
129}
130
131#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
132pub struct BgTaskHealthCounts {
133 pub running: usize,
134 pub pending_completions: usize,
135}
136
137#[derive(Debug, Clone, Copy, PartialEq, Eq)]
138enum TerminalOutputKind {
139 Compressed,
140 Raw,
141 Structured,
142}
143
144#[derive(Debug, Clone, PartialEq, Eq)]
145struct TerminalOutputCache {
146 output_preview: String,
147 output_truncated: bool,
148 kind: TerminalOutputKind,
149 output_path: Option<String>,
150 stderr_path: Option<String>,
151 artifact_access: ArtifactRecoveryAccess,
152 recovery: Option<RecoveryContext>,
153}
154
155#[derive(Debug, Clone, PartialEq, Eq)]
156struct ArtifactRecoveryAccess {
157 task_id: String,
158 readable: bool,
159}
160
161#[derive(Debug, Clone, PartialEq, Eq)]
162struct RecoveryContext {
163 dropped_by_class: BTreeMap<DropClass, usize>,
164 had_inner_drop: bool,
165 offset_hint_eligible: bool,
166 offset_start_line: Option<usize>,
167 byte_truncated: bool,
168 disk_truncated_prefix_bytes: u64,
169 output_path: Option<String>,
170 stderr_path: Option<String>,
171 include_stderr_path: bool,
172 artifact_access: ArtifactRecoveryAccess,
173}
174
175fn optional_string_bytes(value: Option<&String>) -> u64 {
176 value
177 .map(|value| crate::memory::usize_to_u64(value.len()))
178 .unwrap_or(0)
179}
180
181fn terminal_output_cache_estimated_bytes(cache: &TerminalOutputCache) -> u64 {
182 let recovery_bytes = cache
183 .recovery
184 .as_ref()
185 .map(|recovery| {
186 crate::memory::usize_to_u64(recovery.dropped_by_class.len())
187 .saturating_mul(
188 (std::mem::size_of::<DropClass>() + std::mem::size_of::<usize>()) as u64,
189 )
190 .saturating_add(optional_string_bytes(recovery.output_path.as_ref()))
191 .saturating_add(optional_string_bytes(recovery.stderr_path.as_ref()))
192 .saturating_add(crate::memory::usize_to_u64(
193 recovery.artifact_access.task_id.len(),
194 ))
195 })
196 .unwrap_or(0);
197 (std::mem::size_of::<TerminalOutputCache>() as u64)
198 .saturating_add(crate::memory::usize_to_u64(cache.output_preview.len()))
199 .saturating_add(optional_string_bytes(cache.output_path.as_ref()))
200 .saturating_add(optional_string_bytes(cache.stderr_path.as_ref()))
201 .saturating_add(crate::memory::usize_to_u64(
202 cache.artifact_access.task_id.len(),
203 ))
204 .saturating_add(recovery_bytes)
205}
206
207fn completion_estimated_bytes(completion: &BgCompletion) -> u64 {
208 (std::mem::size_of::<BgCompletion>() as u64)
209 .saturating_add(crate::memory::usize_to_u64(completion.task_id.len()))
210 .saturating_add(crate::memory::usize_to_u64(completion.session_id.len()))
211 .saturating_add(crate::memory::usize_to_u64(completion.command.len()))
212 .saturating_add(crate::memory::usize_to_u64(completion.output_preview.len()))
213}
214
215impl RecoveryContext {
216 fn has_visible_drop(&self) -> bool {
217 self.byte_truncated
218 || self.disk_truncated_prefix_bytes > 0
219 || self.had_inner_drop
220 || !self.dropped_by_class.is_empty()
221 }
222}
223
224#[derive(Clone)]
225pub struct BgTaskRegistry {
226 pub(crate) inner: Arc<RegistryInner>,
227}
228
229pub(crate) struct RegistryInner {
230 pub(crate) tasks: Mutex<HashMap<String, Arc<BgTask>>>,
231 pub(crate) completions: Mutex<VecDeque<BgCompletion>>,
232 pub(crate) progress_sender: SharedProgressSender,
233 watchdog_started: AtomicBool,
234 pub(crate) shutdown: AtomicBool,
235 pub(crate) long_running_reminder_enabled: AtomicBool,
236 pub(crate) long_running_reminder_interval_ms: AtomicU64,
237 persisted_gc_started: AtomicBool,
238 #[cfg(test)]
239 persisted_gc_runs: AtomicU64,
240 pub(crate) compressor:
246 Mutex<Option<Box<dyn Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync>>>,
247 pub(crate) db_pool: RwLock<Option<Arc<Mutex<Connection>>>>,
248 pub(crate) db_harness: RwLock<Option<String>>,
249 pub(crate) compression_aggregates: Arc<CompressionAggregateCache>,
250 pub(crate) wake_tx: crossbeam_channel::Sender<()>,
251 pub(crate) wake_rx: crossbeam_channel::Receiver<()>,
252 pub(crate) watch_registry: Mutex<WatchRegistry>,
253 wait_detach_sessions: Mutex<HashSet<String>>,
254 active_wait_sessions: Mutex<HashMap<String, usize>>,
255}
256
257pub(crate) struct BgTask {
258 pub(crate) task_id: String,
259 pub(crate) session_id: String,
260 pub(crate) paths: TaskPaths,
261 artifact_root: PathBuf,
262 pub(crate) started: Instant,
263 pub(crate) last_reminder_at: Mutex<Option<Instant>>,
264 pub(crate) terminal_at: Mutex<Option<Instant>>,
265 pub(crate) state: Mutex<BgTaskState>,
266}
267
268pub(crate) enum TaskRuntime {
269 Piped(Option<Child>),
270 Pty(Option<PtyRuntime>),
271}
272
273pub(crate) struct BgTaskState {
274 pub(crate) metadata: PersistedTask,
275 pub(crate) runtime: TaskRuntime,
276 pub(crate) io_handles: Option<TaskIoHandles>,
279 pub(crate) detached: bool,
280 pub(crate) child_exit_observed: bool,
291 pub(crate) buffer: BgBuffer,
292 terminal_output_cache: Option<TerminalOutputCache>,
293 pub(crate) pending_terminal_override: Option<BgTaskStatus>,
295}
296
297fn completion_matches_session(completion: &BgCompletion, session_id: Option<&str>) -> bool {
298 session_id
299 .map(|session_id| completion.session_id == session_id)
300 .unwrap_or(true)
301}
302
303impl BgTaskRegistry {
304 pub fn new(progress_sender: SharedProgressSender) -> Self {
305 let (wake_tx, wake_rx) = crossbeam_channel::bounded(1);
306 Self {
307 inner: Arc::new(RegistryInner {
308 tasks: Mutex::new(HashMap::new()),
309 completions: Mutex::new(VecDeque::new()),
310 progress_sender,
311 watchdog_started: AtomicBool::new(false),
312 shutdown: AtomicBool::new(false),
313 long_running_reminder_enabled: AtomicBool::new(true),
314 long_running_reminder_interval_ms: AtomicU64::new(600_000),
315 persisted_gc_started: AtomicBool::new(false),
316 #[cfg(test)]
317 persisted_gc_runs: AtomicU64::new(0),
318 compressor: Mutex::new(None),
319 db_pool: RwLock::new(None),
320 db_harness: RwLock::new(None),
321 compression_aggregates: Arc::new(CompressionAggregateCache::default()),
322 wake_tx,
323 wake_rx,
324 watch_registry: Mutex::new(WatchRegistry::default()),
325 wait_detach_sessions: Mutex::new(HashSet::new()),
326 active_wait_sessions: Mutex::new(HashMap::new()),
327 }),
328 }
329 }
330
331 pub fn is_session_owned_artifact_path(&self, session_id: &str, path: &Path) -> bool {
339 let Ok(requested) = fs::canonicalize(path) else {
340 return false;
341 };
342 let Ok(tasks) = self.inner.tasks.lock() else {
343 return false;
344 };
345
346 tasks.values().any(|task| {
347 task.session_id == session_id
348 && [
349 &task.paths.stdout,
350 &task.paths.stderr,
351 &task.paths.exit,
352 &task.paths.pty,
353 ]
354 .into_iter()
355 .filter_map(|known| known.file_name())
356 .any(|name| task.artifact_root.join(name) == requested)
357 })
358 }
359
360 pub fn read_artifact_path(
361 &self,
362 session_id: &str,
363 path: &Path,
364 ) -> Option<Result<Vec<u8>, String>> {
365 let requested = fs::canonicalize(path).ok()?;
366 let tasks = self.inner.tasks.lock().ok()?;
367 let (task, artifact) = tasks.values().find_map(|task| {
368 if task.session_id != session_id {
369 return None;
370 }
371 TaskArtifact::ALL.into_iter().find_map(|artifact| {
372 let expected = task
373 .paths
374 .artifact_path(artifact)
375 .file_name()
376 .map(|name| task.artifact_root.join(name));
377 (expected.as_deref() == Some(requested.as_path()))
378 .then(|| (Arc::clone(task), artifact))
379 })
380 })?;
381 drop(tasks);
382 Some(self.read_artifact(&task.task_id, session_id, artifact))
383 }
384
385 pub fn read_artifact_range(
386 &self,
387 task_id: &str,
388 session_id: &str,
389 artifact: TaskArtifact,
390 offset: u64,
391 ) -> Result<(Vec<u8>, u64), String> {
392 validate_task_id(task_id).map_err(|error| error.to_string())?;
393 let task = self
394 .task_for_session(task_id, session_id)
395 .ok_or_else(|| "task_not_found".to_string())?;
396 let mut file = open_task_artifact(&task.paths, artifact)
397 .map_err(|error| format!("artifact_refused: {error}"))?;
398 let len = file
399 .len()
400 .map_err(|error| format!("artifact_refused: {error}"))?;
401 let start = offset.min(len);
402 let bytes = file
403 .read_range(start, len.saturating_sub(start))
404 .map_err(|error| format!("artifact_refused: {error}"))?;
405 Ok((bytes, len))
406 }
407
408 pub fn read_artifact(
409 &self,
410 task_id: &str,
411 session_id: &str,
412 artifact: TaskArtifact,
413 ) -> Result<Vec<u8>, String> {
414 validate_task_id(task_id).map_err(|error| error.to_string())?;
415 let task = self
416 .task_for_session(task_id, session_id)
417 .ok_or_else(|| "task_not_found".to_string())?;
418 let mut file = open_task_artifact(&task.paths, artifact)
419 .map_err(|error| format!("artifact_refused: {error}"))?;
420 file.read_all()
421 .map_err(|error| format!("artifact_refused: {error}"))
422 }
423
424 pub fn set_harness(&self, harness: Harness) {
425 if let Ok(mut slot) = self.inner.db_harness.write() {
426 *slot = Some(harness.storage_segment());
427 }
428 }
429
430 pub fn set_db_pool(&self, conn: Arc<Mutex<Connection>>) {
431 if let Ok(mut slot) = self.inner.db_pool.write() {
432 *slot = Some(conn);
433 }
434 self.inner.compression_aggregates.clear();
435 }
436
437 pub fn clear_db_pool(&self) {
438 if let Ok(mut slot) = self.inner.db_pool.write() {
439 *slot = None;
440 }
441 self.inner.compression_aggregates.clear();
442 }
443
444 pub(crate) fn compression_aggregate_cache(&self) -> Arc<CompressionAggregateCache> {
445 Arc::clone(&self.inner.compression_aggregates)
446 }
447
448 pub fn begin_wait_mode_session(&self, session_id: &str) {
449 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
450 *active.entry(session_id.to_string()).or_insert(0) += 1;
451 }
452 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
453 detach.remove(session_id);
454 }
455 }
456
457 pub fn end_wait_mode_session(&self, session_id: &str) {
458 if let Ok(mut active) = self.inner.active_wait_sessions.lock() {
459 match active.get_mut(session_id) {
460 Some(count) if *count > 1 => *count -= 1,
461 Some(_) => {
462 active.remove(session_id);
463 }
464 None => {}
465 }
466 }
467 if let Ok(mut detach) = self.inner.wait_detach_sessions.lock() {
468 detach.remove(session_id);
469 }
470 }
471
472 pub fn signal_wait_mode_detach(&self, session_id: &str) -> bool {
473 let is_waiting = self
474 .inner
475 .active_wait_sessions
476 .lock()
477 .map(|active| active.get(session_id).copied().unwrap_or(0) > 0)
478 .unwrap_or(false);
479 if !is_waiting {
480 return false;
481 }
482 self.inner
483 .wait_detach_sessions
484 .lock()
485 .map(|mut detach| detach.insert(session_id.to_string()))
486 .unwrap_or(false)
487 }
488
489 pub fn active_wait_session_count(&self) -> usize {
492 self.inner
493 .active_wait_sessions
494 .lock()
495 .map(|active| active.len())
496 .unwrap_or(0)
497 }
498
499 pub fn take_wait_mode_detach(&self, session_id: &str) -> bool {
500 self.inner
501 .wait_detach_sessions
502 .lock()
503 .map(|mut detach| detach.remove(session_id))
504 .unwrap_or(false)
505 }
506
507 pub fn set_compressor<F>(&self, compressor: F)
512 where
513 F: Fn(&str, String) -> CompressionResult + Send + Sync + 'static,
514 {
515 self.set_compressor_with_exit_code(move |command, output, _exit_code| {
516 compressor(command, output)
517 });
518 }
519
520 pub fn set_compressor_with_exit_code<F>(&self, compressor: F)
521 where
522 F: Fn(&str, String, Option<i32>) -> CompressionResult + Send + Sync + 'static,
523 {
524 if let Ok(mut slot) = self.inner.compressor.lock() {
525 *slot = Some(Box::new(compressor));
526 }
527 }
528
529 pub(crate) fn compress_output(
532 &self,
533 command: &str,
534 output: String,
535 exit_code: Option<i32>,
536 ) -> CompressionResult {
537 let Ok(slot) = self.inner.compressor.lock() else {
538 return CompressionResult::new(output);
539 };
540 match slot.as_ref() {
541 Some(compressor) => compressor(command, output, exit_code),
542 None => CompressionResult::new(output),
543 }
544 }
545
546 fn ensure_terminal_output_cache(&self, task: &Arc<BgTask>) -> Option<TerminalOutputCache> {
547 let (metadata, buffer) = {
548 let state = task.state.lock().ok()?;
549 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
550 return None;
551 }
552 if let Some(cache) = state.terminal_output_cache.clone() {
553 return Some(cache);
554 }
555 (state.metadata.clone(), state.buffer.clone())
556 };
557
558 let mut cap_buffer = buffer.clone();
559 let disk_truncation = cap_buffer.enforce_terminal_cap();
560 let cache = self.render_terminal_output(&metadata, &cap_buffer, disk_truncation);
561 let mut state = task.state.lock().ok()?;
562 if !state.metadata.status.is_terminal() || state.metadata.mode == BgMode::Pty {
563 return None;
564 }
565 if let Some(existing) = state.terminal_output_cache.clone() {
566 return Some(existing);
567 }
568 state.terminal_output_cache = Some(cache.clone());
569 Some(cache)
570 }
571
572 fn render_terminal_output(
573 &self,
574 metadata: &PersistedTask,
575 buffer: &BgBuffer,
576 disk_truncation: DiskTruncation,
577 ) -> TerminalOutputCache {
578 let output_readable = buffer
579 .output_path()
580 .is_some_and(|path| self.is_session_owned_artifact_path(&metadata.session_id, &path));
581 let stderr_readable = buffer
582 .stderr_path()
583 .map(|path| self.is_session_owned_artifact_path(&metadata.session_id, path))
584 .unwrap_or(true);
585 let artifact_access = ArtifactRecoveryAccess {
586 task_id: metadata.task_id.clone(),
587 readable: output_readable && stderr_readable,
588 };
589
590 if metadata.mode == BgMode::Pty {
591 return TerminalOutputCache {
592 output_preview: String::new(),
593 output_truncated: false,
594 kind: TerminalOutputKind::Raw,
595 output_path: buffer.output_path().map(|path| path.display().to_string()),
596 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
597 artifact_access,
598 recovery: None,
599 };
600 }
601
602 if let Some(structured) = render_structured_output(
603 &metadata.command,
604 buffer,
605 disk_truncation,
606 artifact_access.clone(),
607 ) {
608 return structured;
609 }
610
611 if !metadata.compressed {
612 return render_raw_passthrough(buffer, disk_truncation, artifact_access);
613 }
614
615 let raw = buffer.read_combined_head_tail(
616 COMPRESS_INPUT_CAP_BYTES,
617 COMPRESS_INPUT_HEAD_BYTES,
618 COMPRESS_INPUT_TAIL_BYTES,
619 );
620 let compressed = self.compress_output(&metadata.command, raw.text, metadata.exit_code);
621 render_compressed_with_recovery(
622 buffer,
623 compressed,
624 raw.truncated,
625 disk_truncation,
626 artifact_access,
627 )
628 }
629
630 fn snapshot_with_terminal_cache(
631 &self,
632 task: &Arc<BgTask>,
633 preview_bytes: usize,
634 ) -> BgTaskSnapshot {
635 let mut snapshot = task.snapshot(preview_bytes);
636 self.maybe_compress_snapshot(task, &mut snapshot);
637 snapshot
638 }
639
640 fn post_terminal_transition(&self, task: &Arc<BgTask>, emit_frame: bool) -> Result<(), String> {
641 let (metadata, buffer) = {
642 let state = task
643 .state
644 .lock()
645 .map_err(|_| "background task lock poisoned".to_string())?;
646 if !state.metadata.status.is_terminal() {
647 return Ok(());
648 }
649 (state.metadata.clone(), state.buffer.clone())
650 };
651
652 let cache = self.ensure_terminal_output_cache(task);
653 self.enqueue_completion_from_parts(
654 &metadata,
655 Some(&buffer),
656 None,
657 emit_frame,
658 cache.as_ref(),
659 );
660 Ok(())
661 }
662
663 fn persist_task(&self, paths: &TaskPaths, metadata: &PersistedTask) -> std::io::Result<()> {
664 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
665 write_task_at(&task, metadata)?;
666 self.dual_write_task(paths, metadata);
667 Ok(())
668 }
669
670 fn update_task_metadata<F>(
671 &self,
672 paths: &TaskPaths,
673 update: F,
674 ) -> std::io::Result<PersistedTask>
675 where
676 F: FnOnce(&mut PersistedTask),
677 {
678 let task = resolve_task_layout(&paths.session_dir, &paths.task_id)?;
679 let metadata = update_task_at(&task, update)?;
680 self.dual_write_task(paths, &metadata);
681 Ok(metadata)
682 }
683
684 fn dual_write_task(&self, paths: &TaskPaths, metadata: &PersistedTask) {
685 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
686 let Some(pool) = pool else {
687 return;
688 };
689 let harness = self
690 .inner
691 .db_harness
692 .read()
693 .ok()
694 .and_then(|slot| slot.clone());
695 let Some(harness) = harness else {
696 crate::slog_warn!(
697 "dual-write bash_task to DB skipped for {}: harness not configured",
698 metadata.task_id
699 );
700 return;
701 };
702 let row = match metadata.to_bash_task_row(&harness, paths) {
703 Ok(row) => row,
704 Err(error) => {
705 crate::slog_warn!(
706 "dual-write bash_task to DB failed for {}: {}",
707 metadata.task_id,
708 error
709 );
710 return;
711 }
712 };
713 let conn = match pool.lock() {
714 Ok(conn) => conn,
715 Err(_) => {
716 crate::slog_warn!(
717 "dual-write bash_task to DB failed for {}: db mutex poisoned",
718 metadata.task_id
719 );
720 return;
721 }
722 };
723 if let Err(error) = crate::db::bash_tasks::upsert_bash_task(&conn, &row) {
724 crate::slog_warn!(
725 "dual-write bash_task to DB failed for {}: {}",
726 metadata.task_id,
727 error
728 );
729 }
730 }
731
732 fn delete_gc_task_from_db(&self, metadata: &PersistedTask) {
733 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
734 let Some(pool) = pool else {
735 return;
736 };
737 let harness = self
738 .inner
739 .db_harness
740 .read()
741 .ok()
742 .and_then(|slot| slot.clone());
743 let Some(harness) = harness else {
744 crate::slog_warn!(
745 "GC bash_task DB delete skipped for {}: harness not configured",
746 metadata.task_id
747 );
748 return;
749 };
750 let conn = match pool.lock() {
751 Ok(conn) => conn,
752 Err(_) => {
753 crate::slog_warn!(
754 "GC bash_task DB delete failed for {}: db mutex poisoned",
755 metadata.task_id
756 );
757 return;
758 }
759 };
760 if let Err(error) = crate::db::bash_tasks::delete_delivered_terminal_bash_task(
761 &conn,
762 &harness,
763 &metadata.session_id,
764 &metadata.task_id,
765 ) {
766 crate::slog_warn!(
767 "GC bash_task DB delete failed for {}: {}",
768 metadata.task_id,
769 error
770 );
771 }
772 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
774 &conn,
775 &harness,
776 &metadata.session_id,
777 &metadata.task_id,
778 ) {
779 crate::slog_warn!(
780 "GC bash_pattern_watches DB delete failed for {}: {}",
781 metadata.task_id,
782 error
783 );
784 }
785 }
786
787 fn db_harness_and_pool(&self) -> Option<(String, Arc<Mutex<Connection>>)> {
788 let pool = self
789 .inner
790 .db_pool
791 .read()
792 .ok()
793 .and_then(|slot| slot.clone())?;
794 let harness = self
795 .inner
796 .db_harness
797 .read()
798 .ok()
799 .and_then(|slot| slot.clone())?;
800 Some((harness, pool))
801 }
802
803 fn persist_watch_registration(
804 &self,
805 session_id: &str,
806 task_id: &str,
807 watch_id: &str,
808 pattern: &WatchPattern,
809 once: bool,
810 stdout_offset: u64,
811 stderr_offset: u64,
812 pty_offset: u64,
813 ) {
814 let Some((harness, pool)) = self.db_harness_and_pool() else {
815 return;
816 };
817 let Ok(conn) = pool.lock() else {
818 return;
819 };
820 let row = BashPatternWatchRow {
821 harness,
822 session_id: session_id.to_string(),
823 task_id: task_id.to_string(),
824 watch_id: watch_id.to_string(),
825 pattern_kind: pattern.kind_name().to_string(),
826 pattern: pattern.pattern_text().to_string(),
827 once,
828 created_at: unix_millis() as i64,
829 stdout_offset: stdout_offset as i64,
830 stderr_offset: stderr_offset as i64,
831 pty_offset: pty_offset as i64,
832 scanning: true,
833 pending_match: false,
834 match_text: None,
835 match_offset: None,
836 match_context: None,
837 };
838 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
839 crate::slog_warn!(
840 "persist bash_pattern_watch failed for {task_id}/{watch_id}: {error}"
841 );
842 }
843 }
844
845 fn delete_persisted_watch(&self, session_id: &str, task_id: &str, watch_id: &str) {
846 let Some((harness, pool)) = self.db_harness_and_pool() else {
847 return;
848 };
849 let Ok(conn) = pool.lock() else {
850 return;
851 };
852 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watch(
853 &conn, &harness, session_id, task_id, watch_id,
854 ) {
855 crate::slog_warn!("delete bash_pattern_watch failed for {task_id}/{watch_id}: {error}");
856 }
857 }
858
859 fn delete_persisted_watches_for_task(&self, session_id: &str, task_id: &str) {
860 let Some((harness, pool)) = self.db_harness_and_pool() else {
861 return;
862 };
863 let Ok(conn) = pool.lock() else {
864 return;
865 };
866 if let Err(error) = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
867 &conn, &harness, session_id, task_id,
868 ) {
869 crate::slog_warn!("delete bash_pattern_watches for {task_id} failed: {error}");
870 }
871 }
872
873 fn persist_watch_match(
874 &self,
875 session_id: &str,
876 task_id: &str,
877 pattern_match: &PatternMatch,
878 stdout_offset: u64,
879 stderr_offset: u64,
880 pty_offset: u64,
881 ) {
882 let Some((harness, pool)) = self.db_harness_and_pool() else {
883 return;
884 };
885 let Ok(conn) = pool.lock() else {
886 return;
887 };
888 let Ok(Some(mut row)) = crate::db::bash_watches::get_bash_pattern_watch(
889 &conn,
890 &harness,
891 session_id,
892 task_id,
893 &pattern_match.watch_id,
894 ) else {
895 return;
896 };
897 row.stdout_offset = stdout_offset as i64;
898 row.stderr_offset = stderr_offset as i64;
899 row.pty_offset = pty_offset as i64;
900 row.pending_match = true;
901 row.match_text = Some(pattern_match.match_text.clone());
902 row.match_offset = Some(pattern_match.match_offset as i64);
903 row.match_context = Some(pattern_match.context.clone());
904 if pattern_match.once {
905 row.scanning = false;
908 }
909 if let Err(error) = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row) {
910 crate::slog_warn!(
911 "persist bash_pattern_watch match failed for {}/{}: {error}",
912 task_id,
913 pattern_match.watch_id
914 );
915 }
916 }
917
918 fn persist_task_watch_cursors(
919 &self,
920 session_id: &str,
921 task_id: &str,
922 stdout_offset: u64,
923 stderr_offset: u64,
924 pty_offset: u64,
925 ) {
926 let Some((harness, pool)) = self.db_harness_and_pool() else {
927 return;
928 };
929 let Ok(conn) = pool.lock() else {
930 return;
931 };
932 if let Err(error) = crate::db::bash_watches::update_watch_offsets_for_task(
933 &conn,
934 &harness,
935 session_id,
936 task_id,
937 stdout_offset as i64,
938 stderr_offset as i64,
939 pty_offset as i64,
940 ) {
941 crate::slog_warn!("persist bash_pattern_watch cursors failed for {task_id}: {error}");
942 }
943 }
944
945 fn watch_stream_cursors(&self, task_id: &str) -> (u64, u64, u64) {
946 let Ok(registry) = self.inner.watch_registry.lock() else {
947 return (0, 0, 0);
948 };
949 let stdout = registry
950 .file_cursor(&format!("{task_id}:stdout"))
951 .unwrap_or(0);
952 let stderr = registry
953 .file_cursor(&format!("{task_id}:stderr"))
954 .unwrap_or(0);
955 let pty = registry.file_cursor(&format!("{task_id}:pty")).unwrap_or(0);
956 (stdout, stderr, pty)
957 }
958
959 fn ack_persisted_watches_for_task(&self, session_id: &str, task_id: &str, task_terminal: bool) {
962 let Some((harness, pool)) = self.db_harness_and_pool() else {
963 return;
964 };
965 let Ok(conn) = pool.lock() else {
966 return;
967 };
968 if task_terminal {
969 let _ = crate::db::bash_watches::delete_bash_pattern_watches_for_task(
970 &conn, &harness, session_id, task_id,
971 );
972 return;
973 }
974 let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
975 &conn, &harness, session_id, task_id,
976 ) else {
977 return;
978 };
979 for mut row in rows {
980 if row.once && (!row.scanning || row.pending_match) {
981 let _ = crate::db::bash_watches::delete_bash_pattern_watch(
982 &conn,
983 &harness,
984 session_id,
985 task_id,
986 &row.watch_id,
987 );
988 continue;
989 }
990 if row.pending_match {
991 row.pending_match = false;
992 row.match_text = None;
993 row.match_offset = None;
994 row.match_context = None;
995 let _ = crate::db::bash_watches::upsert_bash_pattern_watch(&conn, &row);
996 }
997 }
998 }
999
1000 pub fn record_scanner_report(
1001 &self,
1002 task_id: &str,
1003 session_id: &str,
1004 scanner_report: Vec<PermissionAsk>,
1005 ) -> Result<(), String> {
1006 if scanner_report.is_empty() {
1007 return Ok(());
1008 }
1009 let task = self.task_for_session(task_id, session_id).ok_or_else(|| {
1010 "background task not found while recording scanner report".to_string()
1011 })?;
1012 let metadata = {
1013 let mut state = task
1014 .state
1015 .lock()
1016 .map_err(|_| "background task lock poisoned".to_string())?;
1017 state.metadata.scanner_report = scanner_report;
1018 state.metadata.clone()
1019 };
1020 self.persist_task(&task.paths, &metadata)
1021 .map_err(|error| format!("failed to persist scanner report: {error}"))
1022 }
1023
1024 pub fn configure_long_running_reminders(&self, enabled: bool, interval_ms: u64) {
1025 self.inner
1026 .long_running_reminder_enabled
1027 .store(enabled, Ordering::SeqCst);
1028 self.inner
1029 .long_running_reminder_interval_ms
1030 .store(interval_ms, Ordering::SeqCst);
1031 }
1032
1033 #[cfg(unix)]
1034 #[allow(clippy::too_many_arguments)]
1035 pub fn spawn(
1036 &self,
1037 spawn_plan: SpawnPlan,
1038 command: &str,
1039 session_id: String,
1040 workdir: PathBuf,
1041 env: HashMap<String, String>,
1042 timeout: Option<Duration>,
1043 storage_dir: PathBuf,
1044 max_running: usize,
1045 notify_on_completion: bool,
1046 compressed: bool,
1047 project_root: Option<PathBuf>,
1048 ) -> Result<String, String> {
1049 self.start_watchdog();
1050
1051 let running = self.running_count();
1052 if running >= max_running {
1053 #[cfg(unix)]
1054 if let Some(prepared) = spawn_plan.prepared_task() {
1055 let _ = delete_resolved_task(&prepared.resolved_task());
1056 }
1057 return Err(format!(
1058 "background bash task limit exceeded: {running} running (max {max_running})"
1059 ));
1060 }
1061
1062 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1063 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1064 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1065 (spawn_plan.clone(), prepared.resolved_task())
1066 } else {
1067 let task = allocate_task_layout(&storage_dir, &session_id)
1068 .map_err(|error| format!("failed to create background task layout: {error}"))?;
1069 let shell = resolve_posix_shell();
1070 let root = project_root.as_deref().unwrap_or(&workdir);
1071 let environment =
1072 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1073 let prepared = match crate::sandbox_spawn::prepare_task_payload(
1074 &task,
1075 command.as_bytes(),
1076 root,
1077 &workdir,
1078 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1079 &shell,
1080 &environment,
1081 ) {
1082 Ok(prepared) => prepared,
1083 Err(error) => {
1084 let _ = delete_resolved_task(&task);
1085 return Err(error);
1086 }
1087 };
1088 let task = prepared.resolved_task();
1089 (spawn_plan.with_prepared_task(prepared), task)
1090 };
1091 let task_id = task_layout.paths.task_id.clone();
1092 let paths = task_layout.paths.clone();
1093
1094 if self.task(&task_id).is_some() {
1095 let _ = delete_resolved_task(&task_layout);
1096 return Err("background task id collided with a live task".to_string());
1097 }
1098
1099 let mut metadata = PersistedTask::starting(
1100 task_id.clone(),
1101 session_id.clone(),
1102 command.to_string(),
1103 workdir.clone(),
1104 project_root,
1105 timeout_ms,
1106 notify_on_completion,
1107 compressed,
1108 );
1109 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1110 if let Err(error) = write_task_at(&task_layout, &metadata) {
1111 let _ = delete_resolved_task(&task_layout);
1112 return Err(format!(
1113 "failed to persist background task metadata: {error}"
1114 ));
1115 }
1116 self.dual_write_task(&paths, &metadata);
1117
1118 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pipes)
1119 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
1120 let child = match spawn_detached_child(
1121 &spawn_plan,
1122 command,
1123 &paths,
1124 &workdir,
1125 &env,
1126 &mut io_handles,
1127 ) {
1128 Ok(child) => child,
1129 Err(error) => {
1130 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
1131 let _ = delete_task_bundle(&paths);
1132 return Err(error);
1133 }
1134 };
1135
1136 let child_pid = child.id();
1137 metadata.mark_running(child_pid, child_pid as i32);
1138 self.persist_task(&paths, &metadata)
1139 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1140
1141 let task = Arc::new(BgTask {
1142 task_id: task_id.clone(),
1143 session_id,
1144 paths: paths.clone(),
1145 artifact_root: canonical_artifact_root(&paths),
1146 started: Instant::now(),
1147 last_reminder_at: Mutex::new(None),
1148 terminal_at: Mutex::new(None),
1149 state: Mutex::new(BgTaskState {
1150 metadata,
1151 runtime: TaskRuntime::Piped(Some(child)),
1152 io_handles: Some(io_handles),
1153 detached: false,
1154 child_exit_observed: false,
1155 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
1156 terminal_output_cache: None,
1157 pending_terminal_override: None,
1158 }),
1159 });
1160
1161 self.inner
1162 .tasks
1163 .lock()
1164 .map_err(|_| "background task registry lock poisoned".to_string())?
1165 .insert(task_id.clone(), task);
1166
1167 Ok(task_id)
1168 }
1169
1170 #[allow(clippy::too_many_arguments)]
1171 pub fn spawn_pty(
1172 &self,
1173 spawn_plan: SpawnPlan,
1174 command: &str,
1175 session_id: String,
1176 workdir: PathBuf,
1177 env: HashMap<String, String>,
1178 timeout: Option<Duration>,
1179 storage_dir: PathBuf,
1180 max_running: usize,
1181 notify_on_completion: bool,
1182 compressed: bool,
1183 project_root: Option<PathBuf>,
1184 rows: u16,
1185 cols: u16,
1186 ) -> Result<String, String> {
1187 self.start_watchdog();
1188
1189 let running = self.running_count();
1190 if running >= max_running {
1191 #[cfg(unix)]
1192 if let Some(prepared) = spawn_plan.prepared_task() {
1193 let _ = delete_resolved_task(&prepared.resolved_task());
1194 }
1195 return Err(format!(
1196 "background bash task limit exceeded: {running} running (max {max_running})"
1197 ));
1198 }
1199
1200 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1201 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1202 #[cfg(unix)]
1203 let (spawn_plan, task_layout) = if let Some(prepared) = spawn_plan.prepared_task() {
1204 (spawn_plan.clone(), prepared.resolved_task())
1205 } else {
1206 let task = allocate_task_layout(&storage_dir, &session_id)
1207 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1208 let shell = super::resolved_shell_path(true);
1209 let root = project_root.as_deref().unwrap_or(&workdir);
1210 let environment =
1211 crate::sandbox_spawn::approved_payload_environment(&env, &std::env::temp_dir());
1212 let prepared = match crate::sandbox_spawn::prepare_task_payload(
1213 &task,
1214 command.as_bytes(),
1215 root,
1216 &workdir,
1217 &crate::sandbox_spawn::AuthenticatedPrincipal::FirstParty,
1218 &shell,
1219 &environment,
1220 ) {
1221 Ok(prepared) => prepared,
1222 Err(error) => {
1223 let _ = delete_resolved_task(&task);
1224 return Err(error);
1225 }
1226 };
1227 let task = prepared.resolved_task();
1228 (spawn_plan.with_prepared_task(prepared), task)
1229 };
1230 #[cfg(windows)]
1231 let task_layout = allocate_task_layout(&storage_dir, &session_id)
1232 .map_err(|error| format!("failed to create PTY task layout: {error}"))?;
1233 let task_id = task_layout.paths.task_id.clone();
1234 let paths = task_layout.paths.clone();
1235
1236 let mut metadata = PersistedTask::starting(
1237 task_id.clone(),
1238 session_id.clone(),
1239 command.to_string(),
1240 workdir.clone(),
1241 project_root,
1242 timeout_ms,
1243 notify_on_completion,
1244 compressed,
1245 );
1246 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1247 metadata.mode = BgMode::Pty;
1248 metadata.pty_rows = Some(rows);
1249 metadata.pty_cols = Some(cols);
1250 if let Err(error) = write_task_at(&task_layout, &metadata) {
1251 let _ = delete_resolved_task(&task_layout);
1252 return Err(format!(
1253 "failed to persist background task metadata: {error}"
1254 ));
1255 }
1256 self.dual_write_task(&paths, &metadata);
1257 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pty)
1258 .map_err(|error| format!("failed to pre-open PTY output handles: {error}"))?;
1259
1260 let runtime = match spawn_pty_for_command(
1261 &spawn_plan,
1262 &task_id,
1263 &session_id,
1264 command,
1265 &paths,
1266 &workdir,
1267 &env,
1268 rows,
1269 cols,
1270 self.inner.wake_tx.clone(),
1271 &mut io_handles,
1272 ) {
1273 Ok(runtime) => runtime,
1274 Err(error) => {
1275 crate::slog_warn!(
1276 "failed to spawn PTY background bash task {task_id}; deleting partial bundle: {error}"
1277 );
1278 let _ = delete_task_bundle(&paths);
1279 return Err(error);
1280 }
1281 };
1282
1283 if let Some(child_pid) = runtime.child_pid {
1284 metadata.mark_running(child_pid, child_pid as i32);
1285 } else {
1286 metadata.status = BgTaskStatus::Running;
1287 metadata.pgid = None;
1288 }
1289 self.persist_task(&paths, &metadata)
1290 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1291
1292 let task = Arc::new(BgTask {
1293 task_id: task_id.clone(),
1294 session_id,
1295 paths: paths.clone(),
1296 artifact_root: canonical_artifact_root(&paths),
1297 started: Instant::now(),
1298 last_reminder_at: Mutex::new(None),
1299 terminal_at: Mutex::new(None),
1300 state: Mutex::new(BgTaskState {
1301 metadata,
1302 runtime: TaskRuntime::Pty(Some(runtime)),
1303 io_handles: Some(io_handles),
1304 detached: false,
1305 child_exit_observed: false,
1306 buffer: BgBuffer::registered(&paths, BgMode::Pty),
1307 terminal_output_cache: None,
1308 pending_terminal_override: None,
1309 }),
1310 });
1311
1312 self.inner
1313 .tasks
1314 .lock()
1315 .map_err(|_| "background task registry lock poisoned".to_string())?
1316 .insert(task_id.clone(), task);
1317
1318 Ok(task_id)
1319 }
1320
1321 #[cfg(windows)]
1322 #[allow(clippy::too_many_arguments)]
1323 pub fn spawn(
1324 &self,
1325 spawn_plan: SpawnPlan,
1326 command: &str,
1327 session_id: String,
1328 workdir: PathBuf,
1329 env: HashMap<String, String>,
1330 timeout: Option<Duration>,
1331 storage_dir: PathBuf,
1332 max_running: usize,
1333 notify_on_completion: bool,
1334 compressed: bool,
1335 project_root: Option<PathBuf>,
1336 ) -> Result<String, String> {
1337 self.start_watchdog();
1338
1339 let running = self.running_count();
1340 if running >= max_running {
1341 #[cfg(unix)]
1342 if let Some(prepared) = spawn_plan.prepared_task() {
1343 let _ = delete_resolved_task(&prepared.resolved_task());
1344 }
1345 return Err(format!(
1346 "background bash task limit exceeded: {running} running (max {max_running})"
1347 ));
1348 }
1349
1350 let timeout = timeout.or(Some(DEFAULT_BG_TIMEOUT));
1351 let timeout_ms = timeout.map(|timeout| timeout.as_millis() as u64);
1352 let task_layout = allocate_task_layout(&storage_dir, &session_id)
1353 .map_err(|error| format!("failed to create background task layout: {error}"))?;
1354 let task_id = task_layout.paths.task_id.clone();
1355 let paths = task_layout.paths.clone();
1356
1357 let mut metadata = PersistedTask::starting(
1358 task_id.clone(),
1359 session_id.clone(),
1360 command.to_string(),
1361 workdir.clone(),
1362 project_root,
1363 timeout_ms,
1364 notify_on_completion,
1365 compressed,
1366 );
1367 attach_sandbox_metadata(&mut metadata, &spawn_plan);
1368 if let Err(error) = write_task_at(&task_layout, &metadata) {
1369 let _ = delete_resolved_task(&task_layout);
1370 return Err(format!(
1371 "failed to persist background task metadata: {error}"
1372 ));
1373 }
1374 self.dual_write_task(&paths, &metadata);
1375 let mut io_handles = TaskIoHandles::create(&task_layout, BgMode::Pipes)
1376 .map_err(|error| format!("failed to pre-open task output handles: {error}"))?;
1377
1378 let child = match spawn_detached_child(
1379 &spawn_plan,
1380 command,
1381 &paths,
1382 &workdir,
1383 &env,
1384 &mut io_handles,
1385 ) {
1386 Ok(child) => child,
1387 Err(error) => {
1388 crate::slog_warn!("failed to spawn background bash task {task_id}; deleting partial bundle: {error}");
1389 let _ = delete_task_bundle(&paths);
1390 return Err(error);
1391 }
1392 };
1393
1394 let child_pid = child.id();
1395 metadata.status = BgTaskStatus::Running;
1396 metadata.child_pid = Some(child_pid);
1397 metadata.pgid = None;
1398 self.persist_task(&paths, &metadata)
1399 .map_err(|e| format!("failed to persist running background task metadata: {e}"))?;
1400
1401 let task = Arc::new(BgTask {
1402 task_id: task_id.clone(),
1403 session_id,
1404 paths: paths.clone(),
1405 artifact_root: canonical_artifact_root(&paths),
1406 started: Instant::now(),
1407 last_reminder_at: Mutex::new(None),
1408 terminal_at: Mutex::new(None),
1409 state: Mutex::new(BgTaskState {
1410 metadata,
1411 runtime: TaskRuntime::Piped(Some(child)),
1412 io_handles: Some(io_handles),
1413 detached: false,
1414 child_exit_observed: false,
1415 buffer: BgBuffer::registered(&paths, BgMode::Pipes),
1416 terminal_output_cache: None,
1417 pending_terminal_override: None,
1418 }),
1419 });
1420
1421 self.inner
1422 .tasks
1423 .lock()
1424 .map_err(|_| "background task registry lock poisoned".to_string())?
1425 .insert(task_id.clone(), task);
1426
1427 Ok(task_id)
1428 }
1429
1430 pub fn write_pty(
1431 &self,
1432 task_id: &str,
1433 session_id: &str,
1434 input: &[u8],
1435 ) -> Result<usize, String> {
1436 let task = self
1437 .task_for_session(task_id, session_id)
1438 .ok_or_else(|| "task_not_found".to_string())?;
1439
1440 let writer = {
1441 let state = task
1442 .state
1443 .lock()
1444 .map_err(|_| "background task lock poisoned".to_string())?;
1445 if state.metadata.mode != BgMode::Pty {
1446 return Err("task_not_pty".to_string());
1447 }
1448 if state.metadata.status.is_terminal() {
1449 return Err("task_exited".to_string());
1450 }
1451 match &state.runtime {
1452 TaskRuntime::Pty(Some(runtime)) => Arc::clone(&runtime.writer),
1453 TaskRuntime::Pty(None) => return Err("task_exited".to_string()),
1454 TaskRuntime::Piped(_) => return Err("task_not_pty".to_string()),
1455 }
1456 };
1457
1458 let mut writer = writer
1459 .lock()
1460 .map_err(|_| "PTY writer lock poisoned".to_string())?;
1461 writer
1462 .write_all(input)
1463 .map_err(|error| format!("failed to write to PTY: {error}"))?;
1464 writer
1465 .flush()
1466 .map_err(|error| format!("failed to flush PTY writer: {error}"))?;
1467 Ok(input.len())
1468 }
1469
1470 pub fn replay_session(&self, storage_dir: &Path, session_id: &str) -> Result<(), String> {
1471 self.replay_session_inner(storage_dir, session_id, None)
1472 }
1473
1474 pub fn replay_session_for_project(
1475 &self,
1476 storage_dir: &Path,
1477 session_id: &str,
1478 project_root: &Path,
1479 ) -> Result<(), String> {
1480 self.replay_session_inner(storage_dir, session_id, Some(project_root))
1481 }
1482
1483 fn replay_session_inner(
1484 &self,
1485 storage_dir: &Path,
1486 session_id: &str,
1487 project_root: Option<&Path>,
1488 ) -> Result<(), String> {
1489 self.start_watchdog();
1490 if !self.inner.persisted_gc_started.swap(true, Ordering::SeqCst) {
1491 if let Err(error) = self.maybe_gc_persisted(storage_dir) {
1492 crate::slog_warn!("failed to GC persisted background bash tasks: {error}");
1493 }
1494 }
1495
1496 let canonical_project = project_root.map(canonicalized_path);
1497 let tasks = match self.replay_session_from_db(session_id) {
1509 Some(Ok(tasks)) if !tasks.is_empty() => tasks,
1510 Some(Ok(_)) => {
1511 let disk_tasks = self.replay_session_from_disk(storage_dir, session_id)?;
1512 if !disk_tasks.is_empty() {
1513 crate::slog_info!(
1514 "bash task replay: 0 in DB for session {}, {} from disk fallback",
1515 session_id,
1516 disk_tasks.len()
1517 );
1518 }
1519 disk_tasks
1520 }
1521 Some(Err(error)) => {
1522 crate::slog_warn!(
1523 "bash task replay DB lookup failed for session {}; falling back to disk: {}",
1524 session_id,
1525 error
1526 );
1527 self.replay_session_from_disk(storage_dir, session_id)?
1528 }
1529 None => {
1530 self.replay_session_from_disk(storage_dir, session_id)?
1532 }
1533 };
1534
1535 for mut metadata in tasks {
1536 if metadata.session_id != session_id {
1537 continue;
1538 }
1539 if let Some(canonical_project) = canonical_project.as_deref() {
1540 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
1541 if metadata_project.as_deref() != Some(canonical_project) {
1542 continue;
1543 }
1544 }
1545
1546 if validate_task_id(&metadata.task_id).is_err() {
1547 crate::slog_warn!(
1548 "ignoring persisted background task with invalid id {:?}",
1549 metadata.task_id
1550 );
1551 continue;
1552 }
1553 let session_dir = session_tasks_dir(storage_dir, session_id);
1554 let resolved = match resolve_task_layout(&session_dir, &metadata.task_id) {
1555 Ok(task) => task,
1556 Err(error) => {
1557 crate::slog_warn!(
1558 "quarantining unresolved background task {}: {error}",
1559 metadata.task_id
1560 );
1561 let _ = quarantine_task_layout(
1562 storage_dir,
1563 &session_dir,
1564 &metadata.task_id,
1565 "invalid",
1566 );
1567 continue;
1568 }
1569 };
1570 match read_task_at(&resolved) {
1571 Ok(disk) if disk.task_id == metadata.task_id && disk.session_id == session_id => {}
1572 Ok(_) | Err(_) => {
1573 let _ = quarantine_task_layout(
1574 storage_dir,
1575 &session_dir,
1576 &metadata.task_id,
1577 "mismatch",
1578 );
1579 continue;
1580 }
1581 }
1582 let paths = resolved.paths;
1583 match metadata.status {
1584 BgTaskStatus::Starting => {
1585 let completion_was_delivered = metadata.completion_delivered;
1586 metadata.mark_terminal(
1587 BgTaskStatus::Failed,
1588 None,
1589 Some("spawn aborted".to_string()),
1590 );
1591 metadata.completion_delivered |= completion_was_delivered;
1592 let _ = self.persist_task(&paths, &metadata);
1593 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1594 self.insert_rehydrated_task(metadata, paths, true)?;
1595 }
1596 BgTaskStatus::Running | BgTaskStatus::Killing => {
1597 if metadata.mode == BgMode::Pty {
1598 if let Ok(Some(marker)) = read_exit_marker(&paths) {
1599 let completion_was_delivered = metadata.completion_delivered;
1600 metadata = terminal_metadata_from_marker(metadata, marker, None);
1601 metadata.completion_delivered |= completion_was_delivered;
1602 let _ = self.persist_task(&paths, &metadata);
1603 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1604 self.insert_rehydrated_task(metadata, paths, true)?;
1605 } else if metadata.status.is_terminal() {
1606 self.insert_rehydrated_task(metadata, paths, true)?;
1607 } else {
1608 let completion_was_delivered = metadata.completion_delivered;
1609 metadata.mark_terminal(
1610 BgTaskStatus::Killed,
1611 None,
1612 Some("pty_lost_on_bridge_restart".to_string()),
1613 );
1614 metadata.completion_delivered |= completion_was_delivered;
1615 let _ = self.persist_task(&paths, &metadata);
1616 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1617 self.insert_rehydrated_task(metadata, paths, true)?;
1618 }
1619 } else if self.running_metadata_is_stale(&metadata) {
1620 let completion_was_delivered = metadata.completion_delivered;
1621 metadata.mark_terminal(
1622 BgTaskStatus::Killed,
1623 None,
1624 Some("orphaned (>24h)".to_string()),
1625 );
1626 metadata.completion_delivered |= completion_was_delivered;
1627 let _ = write_kill_marker_if_absent(&paths);
1628 let _ = self.persist_task(&paths, &metadata);
1629 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1630 self.insert_rehydrated_task(metadata, paths, true)?;
1631 } else if let Ok(Some(marker)) = read_exit_marker(&paths) {
1632 let reason = (metadata.status == BgTaskStatus::Killing).then(|| {
1633 "recovered from inconsistent killing state on replay".to_string()
1634 });
1635 if reason.is_some() {
1636 crate::slog_warn!("background task {} had killing state with exit marker; preferring marker",
1637 metadata.task_id);
1638 }
1639 let completion_was_delivered = metadata.completion_delivered;
1640 metadata = terminal_metadata_from_marker(metadata, marker, reason);
1641 metadata.completion_delivered |= completion_was_delivered;
1642 let _ = self.persist_task(&paths, &metadata);
1643 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1644 self.insert_rehydrated_task(metadata, paths, true)?;
1645 } else if metadata.status == BgTaskStatus::Killing {
1646 let _ = write_kill_marker_if_absent(&paths);
1647 let completion_was_delivered = metadata.completion_delivered;
1648 metadata.mark_terminal(
1649 BgTaskStatus::Killed,
1650 None,
1651 Some("recovered from inconsistent killing state on replay".to_string()),
1652 );
1653 metadata.completion_delivered |= completion_was_delivered;
1654 let _ = self.persist_task(&paths, &metadata);
1655 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1656 self.insert_rehydrated_task(metadata, paths, true)?;
1657 } else if metadata.child_pid.is_some_and(|pid| !is_process_alive(pid)) {
1658 let completion_was_delivered = metadata.completion_delivered;
1659 metadata.mark_terminal(
1660 BgTaskStatus::Failed,
1661 None,
1662 Some("process exited without exit marker".to_string()),
1663 );
1664 metadata.completion_delivered |= completion_was_delivered;
1665 let _ = self.persist_task(&paths, &metadata);
1666 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1667 self.insert_rehydrated_task(metadata, paths, true)?;
1668 } else {
1669 self.insert_rehydrated_task(metadata, paths, true)?;
1670 }
1671 }
1672 _ if metadata.status.is_terminal() => {
1673 self.enqueue_completion_if_needed(&metadata, Some(&paths), false);
1679 self.insert_rehydrated_task(metadata, paths, true)?;
1680 }
1681 _ => {}
1682 }
1683 }
1684
1685 Ok(())
1686 }
1687
1688 fn replay_session_from_db(
1689 &self,
1690 session_id: &str,
1691 ) -> Option<Result<Vec<PersistedTask>, String>> {
1692 let pool = self
1693 .inner
1694 .db_pool
1695 .read()
1696 .ok()
1697 .and_then(|slot| slot.clone())?;
1698 let harness = self
1699 .inner
1700 .db_harness
1701 .read()
1702 .ok()
1703 .and_then(|slot| slot.clone())?;
1704 let conn = match pool.lock() {
1705 Ok(conn) => conn,
1706 Err(_) => return Some(Err("db mutex poisoned".to_string())),
1707 };
1708 Some(
1709 crate::db::bash_tasks::list_bash_tasks_for_session(&conn, &harness, session_id)
1710 .map(|rows| rows.into_iter().map(PersistedTask::from).collect())
1711 .map_err(|error| error.to_string()),
1712 )
1713 }
1714
1715 fn replay_session_from_disk(
1716 &self,
1717 storage_dir: &Path,
1718 session_id: &str,
1719 ) -> Result<Vec<PersistedTask>, String> {
1720 let dir = session_tasks_dir(storage_dir, session_id);
1721 if !dir.exists() {
1722 return Ok(Vec::new());
1723 }
1724
1725 let (task_ids, invalid_entries) = discover_task_ids(&dir)
1726 .map_err(|error| format!("failed to discover background task layouts: {error}"))?;
1727 for entry in invalid_entries {
1728 if let Err(error) = quarantine_invalid_entry(storage_dir, &dir, &entry) {
1729 crate::slog_warn!(
1730 "failed to quarantine invalid background task entry {:?}: {error}",
1731 entry
1732 );
1733 }
1734 }
1735
1736 let mut tasks = Vec::new();
1737 for task_id in task_ids {
1738 let task = match resolve_task_layout(&dir, &task_id) {
1739 Ok(task) => task,
1740 Err(error)
1741 if error.kind() == std::io::ErrorKind::NotFound
1742 && uninitialized_layout_is_recent(
1743 &dir,
1744 &task_id,
1745 Duration::from_secs(5 * 60),
1746 )
1747 .unwrap_or(false) =>
1748 {
1749 continue;
1750 }
1751 Err(error) => {
1752 crate::slog_warn!(
1753 "quarantining unresolved background task {task_id} during replay: {error}"
1754 );
1755 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
1756 continue;
1757 }
1758 };
1759 match read_task_at(&task) {
1760 Ok(metadata) if metadata.session_id == session_id => tasks.push(metadata),
1761 Ok(_) => {
1762 crate::slog_warn!(
1763 "quarantining background task {task_id} with mismatched session metadata"
1764 );
1765 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "mismatch");
1766 }
1767 Err(error) => {
1768 crate::slog_warn!(
1769 "quarantining invalid background task metadata {task_id} during replay: {error}"
1770 );
1771 let _ = quarantine_task_layout(storage_dir, &dir, &task_id, "invalid");
1772 }
1773 }
1774 }
1775 Ok(tasks)
1776 }
1777
1778 pub fn register_watch(
1779 &self,
1780 task_id: String,
1781 pattern: WatchPattern,
1782 once: bool,
1783 ) -> Result<String, &'static str> {
1784 let task = self.task(&task_id).ok_or("task_not_found")?;
1785 validate_task_id(&task_id).map_err(|_| "invalid_task_id")?;
1786 let (mode, terminal_at_registration) = task
1787 .state
1788 .lock()
1789 .map(|state| {
1790 (
1791 state.metadata.mode.clone(),
1792 state.metadata.status.is_terminal(),
1793 )
1794 })
1795 .map_err(|_| "background_task_lock_poisoned")?;
1796 let mut stdout = (mode == BgMode::Pipes)
1797 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
1798 .transpose()
1799 .map_err(|_| "artifact_refused")?;
1800 let mut stderr = (mode == BgMode::Pipes)
1801 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
1802 .transpose()
1803 .map_err(|_| "artifact_refused")?;
1804 let mut pty = (mode == BgMode::Pty)
1805 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
1806 .transpose()
1807 .map_err(|_| "artifact_refused")?;
1808
1809 let mut terminal_matches = Vec::new();
1810 let scanned_terminal = terminal_at_registration;
1811 let watch_id = {
1812 let mut registry = self
1813 .inner
1814 .watch_registry
1815 .lock()
1816 .map_err(|_| "watch_registry_poisoned")?;
1817 let watch_id = registry.register(task_id.clone(), pattern.clone(), once)?;
1818 match &mode {
1819 BgMode::Pipes => {
1820 let stdout_key = format!("{task_id}:stdout");
1821 let stderr_key = format!("{task_id}:stderr");
1822 if terminal_at_registration {
1823 registry.set_file_cursor(&stdout_key, 0);
1824 registry.set_file_cursor(&stderr_key, 0);
1825 terminal_matches.extend(registry.scan_file_new_bytes(
1826 &stdout_key,
1827 &task_id,
1828 stdout.as_mut().expect("pipe stdout opened"),
1829 ));
1830 terminal_matches.extend(registry.scan_file_new_bytes(
1831 &stderr_key,
1832 &task_id,
1833 stderr.as_mut().expect("pipe stderr opened"),
1834 ));
1835 } else {
1836 registry.prime_file_cursor(
1837 &stdout_key,
1838 stdout.as_ref().expect("pipe stdout opened"),
1839 );
1840 registry.prime_file_cursor(
1841 &stderr_key,
1842 stderr.as_ref().expect("pipe stderr opened"),
1843 );
1844 }
1845 }
1846 BgMode::Pty => {
1847 let pty_key = format!("{task_id}:pty");
1848 if terminal_at_registration {
1849 registry.set_file_cursor(&pty_key, 0);
1850 terminal_matches.extend(registry.scan_file_new_bytes(
1851 &pty_key,
1852 &task_id,
1853 pty.as_mut().expect("PTY artifact opened"),
1854 ));
1855 } else {
1856 registry.prime_file_cursor(
1857 &pty_key,
1858 pty.as_ref().expect("PTY artifact opened"),
1859 );
1860 }
1861 }
1862 }
1863 watch_id
1864 };
1865
1866 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
1867 self.persist_watch_registration(
1868 &task.session_id,
1869 &task_id,
1870 &watch_id,
1871 &pattern,
1872 once,
1873 stdout_offset,
1874 stderr_offset,
1875 pty_offset,
1876 );
1877
1878 if task.is_terminal() {
1879 if !scanned_terminal {
1880 terminal_matches = {
1881 let mut registry = self
1882 .inner
1883 .watch_registry
1884 .lock()
1885 .map_err(|_| "watch_registry_poisoned")?;
1886 match &mode {
1887 BgMode::Pipes => {
1888 let stdout_key = format!("{task_id}:stdout");
1889 let stderr_key = format!("{task_id}:stderr");
1890 registry.set_file_cursor(&stdout_key, 0);
1891 registry.set_file_cursor(&stderr_key, 0);
1892 let mut matches = registry.scan_file_new_bytes(
1893 &stdout_key,
1894 &task_id,
1895 stdout.as_mut().expect("pipe stdout opened"),
1896 );
1897 matches.extend(registry.scan_file_new_bytes(
1898 &stderr_key,
1899 &task_id,
1900 stderr.as_mut().expect("pipe stderr opened"),
1901 ));
1902 matches
1903 }
1904 BgMode::Pty => {
1905 let pty_key = format!("{task_id}:pty");
1906 registry.set_file_cursor(&pty_key, 0);
1907 registry.scan_file_new_bytes(
1908 &pty_key,
1909 &task_id,
1910 pty.as_mut().expect("PTY artifact opened"),
1911 )
1912 }
1913 }
1914 };
1915 }
1916
1917 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task_id);
1918 let (watch_controlled, watch_matched) = self.task_watch_state(&task_id);
1919 if terminal_matches.is_empty() && (!watch_controlled || watch_matched) {
1920 if watch_matched {
1921 let _ = task.set_completion_delivered(true, self);
1922 self.clear_task_watch_state(&task_id);
1923 self.delete_persisted_watches_for_task(&task.session_id, &task_id);
1925 }
1926 return Ok(watch_id);
1927 }
1928
1929 let completion = self
1930 .remove_pending_completion(&task_id)
1931 .or_else(|| self.completion_snapshot_for_task(&task));
1932 if terminal_matches.is_empty() {
1933 if let Some(completion) = completion.as_ref() {
1934 self.emit_bash_watch_exit(completion);
1935 }
1936 } else {
1937 for pattern_match in &terminal_matches {
1938 self.persist_watch_match(
1939 &task.session_id,
1940 &task_id,
1941 pattern_match,
1942 stdout_offset,
1943 stderr_offset,
1944 pty_offset,
1945 );
1946 self.emit_bash_pattern_match(&task.session_id, pattern_match.clone());
1947 }
1948 }
1949 let _ = task.set_completion_delivered(true, self);
1950 self.clear_task_watch_state(&task_id);
1951 self.delete_persisted_watches_for_task(&task.session_id, &task_id);
1953 }
1954
1955 Ok(watch_id)
1956 }
1957
1958 pub fn unregister_watch(&self, task_id: &str, watch_id: &str) {
1959 let session_id = self.task(task_id).map(|task| task.session_id.clone());
1960 if let Ok(mut registry) = self.inner.watch_registry.lock() {
1961 registry.unregister(task_id, watch_id);
1962 }
1963 if let Some(session_id) = session_id {
1964 self.delete_persisted_watch(&session_id, task_id, watch_id);
1965 }
1966 }
1967
1968 pub fn active_watch_count(&self, task_id: &str) -> usize {
1969 self.inner
1970 .watch_registry
1971 .lock()
1972 .map(|registry| registry.active_count(task_id))
1973 .unwrap_or(0)
1974 }
1975
1976 fn task_watch_state(&self, task_id: &str) -> (bool, bool) {
1977 self.inner
1978 .watch_registry
1979 .lock()
1980 .map(|registry| {
1981 (
1982 registry.has_controlled_task(task_id),
1983 registry.has_matched_task(task_id),
1984 )
1985 })
1986 .unwrap_or((false, false))
1987 }
1988
1989 fn task_has_watch_control(&self, task_id: &str) -> bool {
1990 self.inner
1991 .watch_registry
1992 .lock()
1993 .map(|registry| registry.has_controlled_task(task_id))
1994 .unwrap_or(false)
1995 }
1996
1997 fn clear_task_watch_state(&self, task_id: &str) {
1998 if let Ok(mut registry) = self.inner.watch_registry.lock() {
1999 registry.clear_task(task_id);
2000 }
2001 }
2002
2003 pub(crate) fn scan_task_watch_output(&self, task: &Arc<BgTask>) {
2004 let mode = match task.state.lock() {
2005 Ok(state) => state.metadata.mode.clone(),
2006 Err(_) => return,
2007 };
2008 let mut stdout = (mode == BgMode::Pipes)
2009 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
2010 .transpose()
2011 .ok()
2012 .flatten();
2013 let mut stderr = (mode == BgMode::Pipes)
2014 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
2015 .transpose()
2016 .ok()
2017 .flatten();
2018 let mut pty = (mode == BgMode::Pty)
2019 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
2020 .transpose()
2021 .ok()
2022 .flatten();
2023 let mut matches = Vec::new();
2024 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2025 match mode {
2026 BgMode::Pipes => {
2027 let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) else {
2028 return;
2029 };
2030 let stdout_key = format!("{}:stdout", task.task_id);
2031 let stderr_key = format!("{}:stderr", task.task_id);
2032 matches.extend(registry.scan_file_new_bytes(
2033 &stdout_key,
2034 &task.task_id,
2035 stdout,
2036 ));
2037 matches.extend(registry.scan_file_new_bytes(
2038 &stderr_key,
2039 &task.task_id,
2040 stderr,
2041 ));
2042 }
2043 BgMode::Pty => {
2044 let Some(pty) = pty.as_mut() else {
2045 return;
2046 };
2047 let pty_key = format!("{}:pty", task.task_id);
2048 matches.extend(registry.scan_file_new_bytes(&pty_key, &task.task_id, pty));
2049 }
2050 }
2051 }
2052 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
2053 if matches.is_empty() {
2054 if self.task_has_watch_control(&task.task_id) {
2057 self.persist_task_watch_cursors(
2058 &task.session_id,
2059 &task.task_id,
2060 stdout_offset,
2061 stderr_offset,
2062 pty_offset,
2063 );
2064 }
2065 return;
2066 }
2067 for pattern_match in matches {
2068 self.persist_watch_match(
2069 &task.session_id,
2070 &task.task_id,
2071 &pattern_match,
2072 stdout_offset,
2073 stderr_offset,
2074 pty_offset,
2075 );
2076 self.emit_bash_pattern_match(&task.session_id, pattern_match);
2077 }
2078 self.persist_task_watch_cursors(
2079 &task.session_id,
2080 &task.task_id,
2081 stdout_offset,
2082 stderr_offset,
2083 pty_offset,
2084 );
2085 }
2086
2087 pub fn status(
2088 &self,
2089 task_id: &str,
2090 session_id: &str,
2091 project_root: Option<&Path>,
2092 storage_dir: Option<&Path>,
2093 preview_bytes: usize,
2094 ) -> Option<BgTaskSnapshot> {
2095 validate_task_id(task_id).ok()?;
2096 let mut task = self.task_for_session(task_id, session_id);
2097 if task.is_none() {
2098 if let Some(storage_dir) = storage_dir {
2099 let _ = if let Some(project_root) = project_root {
2100 self.replay_session_for_project(storage_dir, session_id, project_root)
2101 } else {
2102 self.replay_session(storage_dir, session_id)
2103 };
2104 task = self.task_for_session(task_id, session_id);
2105 }
2106 }
2107 let Some(task) = task else {
2108 return self.status_relaxed(
2109 task_id,
2110 session_id,
2111 project_root?,
2112 storage_dir?,
2113 preview_bytes,
2114 );
2115 };
2116 let _ = self.poll_task(&task);
2117 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
2118 }
2119
2120 fn status_relaxed_task(
2121 &self,
2122 task_id: &str,
2123 project_root: &Path,
2124 storage_dir: &Path,
2125 ) -> Option<Arc<BgTask>> {
2126 validate_task_id(task_id).ok()?;
2127 let canonical_project = canonicalized_path(project_root);
2128 match self.lookup_relaxed_task_from_db(task_id, project_root) {
2129 Some(Ok(Some(metadata))) => {
2130 if let Some(task) = self.task(task_id) {
2131 let matches_project = task
2132 .state
2133 .lock()
2134 .map(|state| {
2135 state
2136 .metadata
2137 .project_root
2138 .as_deref()
2139 .map(canonicalized_path)
2140 .as_deref()
2141 == Some(canonical_project.as_path())
2142 })
2143 .unwrap_or(false);
2144 return matches_project.then_some(task);
2145 }
2146 let resolved = resolve_task_layout(
2147 &session_tasks_dir(storage_dir, &metadata.session_id),
2148 &metadata.task_id,
2149 )
2150 .ok()?;
2151 let disk = read_task_at(&resolved).ok()?;
2152 if disk.task_id != metadata.task_id || disk.session_id != metadata.session_id {
2153 return None;
2154 }
2155 if self
2156 .insert_rehydrated_task(metadata, resolved.paths, true)
2157 .is_err()
2158 {
2159 return None;
2160 }
2161 return self.task(task_id);
2162 }
2163 Some(Ok(None)) => {
2164 crate::slog_info!(
2165 "bash task relaxed DB miss for {}; falling back to disk",
2166 task_id
2167 );
2168 }
2169 Some(Err(error)) => {
2170 crate::slog_warn!(
2171 "bash task relaxed DB lookup failed for {}; falling back to disk: {}",
2172 task_id,
2173 error
2174 );
2175 }
2176 None => {
2177 crate::slog_info!(
2178 "bash task relaxed DB unavailable for {}; falling back to disk",
2179 task_id
2180 );
2181 }
2182 }
2183 let root = storage_dir.join("bash-tasks");
2184 let entries = fs::read_dir(&root).ok()?;
2185 for entry in entries.flatten() {
2186 let dir = entry.path();
2187 if !dir.is_dir() {
2188 continue;
2189 }
2190 let resolved = match resolve_task_layout(&dir, task_id) {
2191 Ok(task) => task,
2192 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
2193 Err(error) => {
2194 crate::slog_warn!(
2195 "quarantining unresolved background task {task_id} during relaxed lookup: {error}"
2196 );
2197 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
2198 continue;
2199 }
2200 };
2201 let metadata = match read_task_at(&resolved) {
2202 Ok(metadata) => metadata,
2203 Err(error) => {
2204 crate::slog_warn!(
2205 "quarantining invalid background task metadata {task_id} during relaxed lookup: {error}"
2206 );
2207 let _ = quarantine_task_layout(storage_dir, &dir, task_id, "invalid");
2208 continue;
2209 }
2210 };
2211 let metadata_project = metadata.project_root.as_deref().map(canonicalized_path);
2212 if metadata_project.as_deref() != Some(canonical_project.as_path()) {
2213 continue;
2214 }
2215 if let Some(task) = self.task(task_id) {
2216 let matches_project = task
2217 .state
2218 .lock()
2219 .map(|state| {
2220 state
2221 .metadata
2222 .project_root
2223 .as_deref()
2224 .map(canonicalized_path)
2225 .as_deref()
2226 == Some(canonical_project.as_path())
2227 })
2228 .unwrap_or(false);
2229 return matches_project.then_some(task);
2230 }
2231 if self
2232 .insert_rehydrated_task(metadata, resolved.paths, true)
2233 .is_err()
2234 {
2235 return None;
2236 }
2237 return self.task(task_id);
2238 }
2239 None
2240 }
2241
2242 fn lookup_relaxed_task_from_db(
2243 &self,
2244 task_id: &str,
2245 project_root: &Path,
2246 ) -> Option<Result<Option<PersistedTask>, String>> {
2247 let pool = self
2248 .inner
2249 .db_pool
2250 .read()
2251 .ok()
2252 .and_then(|slot| slot.clone())?;
2253 let harness = self
2254 .inner
2255 .db_harness
2256 .read()
2257 .ok()
2258 .and_then(|slot| slot.clone())?;
2259 let conn = match pool.lock() {
2260 Ok(conn) => conn,
2261 Err(_) => return Some(Err("db mutex poisoned".to_string())),
2262 };
2263 let project_key = crate::path_identity::project_scope_key(project_root);
2264 Some(
2265 crate::db::bash_tasks::find_bash_task_for_project(
2266 &conn,
2267 &harness,
2268 &project_key,
2269 task_id,
2270 )
2271 .map(|row| row.map(PersistedTask::from))
2272 .map_err(|error| error.to_string()),
2273 )
2274 }
2275
2276 pub(super) fn status_relaxed(
2277 &self,
2278 task_id: &str,
2279 _session_id: &str,
2280 project_root: &Path,
2281 storage_dir: &Path,
2282 preview_bytes: usize,
2283 ) -> Option<BgTaskSnapshot> {
2284 let task = self.status_relaxed_task(task_id, project_root, storage_dir)?;
2285 let _ = self.poll_task(&task);
2286 Some(self.snapshot_with_terminal_cache(&task, preview_bytes))
2287 }
2288
2289 pub fn kill_relaxed(
2290 &self,
2291 task_id: &str,
2292 project_root: &Path,
2293 storage_dir: &Path,
2294 ) -> Result<BgTaskSnapshot, String> {
2295 let task = self
2296 .status_relaxed_task(task_id, project_root, storage_dir)
2297 .ok_or_else(|| format!("background task not found: {task_id}"))?;
2298 self.kill_with_status(task_id, &task.session_id, BgTaskStatus::Killed)
2299 }
2300
2301 pub fn maybe_gc_persisted(&self, storage_dir: &Path) -> Result<usize, String> {
2302 #[cfg(test)]
2303 self.inner.persisted_gc_runs.fetch_add(1, Ordering::SeqCst);
2304
2305 let mut deleted = 0usize;
2306
2307 let root = storage_dir.join("bash-tasks");
2308 if root.exists() {
2309 let session_dirs = fs::read_dir(&root).map_err(|e| {
2310 format!(
2311 "failed to read background task root {}: {e}",
2312 root.display()
2313 )
2314 })?;
2315 for session_entry in session_dirs.flatten() {
2316 let session_dir = session_entry.path();
2317 if !session_dir.is_dir() {
2318 continue;
2319 }
2320 let (task_ids, invalid_entries) = match discover_task_ids(&session_dir) {
2321 Ok(discovery) => discovery,
2322 Err(error) => {
2323 crate::slog_warn!(
2324 "failed to discover background task session {}: {error}",
2325 session_dir.display()
2326 );
2327 continue;
2328 }
2329 };
2330 for entry in invalid_entries {
2331 let _ = quarantine_invalid_entry(storage_dir, &session_dir, &entry);
2332 }
2333 for task_id in task_ids {
2334 let resolved = match resolve_task_layout(&session_dir, &task_id) {
2335 Ok(task) => task,
2336 Err(error)
2337 if error.kind() == std::io::ErrorKind::NotFound
2338 && uninitialized_layout_is_recent(
2339 &session_dir,
2340 &task_id,
2341 Duration::from_secs(5 * 60),
2342 )
2343 .unwrap_or(false) =>
2344 {
2345 continue;
2346 }
2347 Err(error) => {
2348 crate::slog_warn!(
2349 "quarantining unresolved background task {task_id}: {error}"
2350 );
2351 quarantine_task_layout(storage_dir, &session_dir, &task_id, "invalid")
2352 .map_err(|error| error.to_string())?;
2353 continue;
2354 }
2355 };
2356 if modified_within(&resolved.paths.json, PERSISTED_GC_GRACE) {
2357 continue;
2358 }
2359 let metadata = match read_task_at(&resolved) {
2360 Ok(metadata) => metadata,
2361 Err(error) => {
2362 crate::slog_warn!(
2363 "quarantining corrupt background task metadata {task_id}: {error}"
2364 );
2365 quarantine_task_layout(storage_dir, &session_dir, &task_id, "corrupt")
2366 .map_err(|error| error.to_string())?;
2367 continue;
2368 }
2369 };
2370 if !(metadata.status.is_terminal() && metadata.completion_delivered) {
2371 continue;
2372 }
2373 match delete_task_bundle(&resolved.paths) {
2374 Ok(()) => {
2375 self.delete_gc_task_from_db(&metadata);
2376 deleted += 1;
2377 log::debug!(
2378 "deleted persisted background task bundle {}",
2379 metadata.task_id
2380 );
2381 }
2382 Err(error) => {
2383 crate::slog_warn!(
2384 "failed to delete background task bundle {}: {error}",
2385 metadata.task_id
2386 );
2387 }
2388 }
2389 }
2390 }
2391 }
2392 gc_quarantine(storage_dir);
2393 Ok(deleted)
2394 }
2395
2396 pub fn list(&self, preview_bytes: usize) -> Vec<BgTaskSnapshot> {
2397 let tasks = self
2398 .inner
2399 .tasks
2400 .lock()
2401 .map(|tasks| tasks.values().cloned().collect::<Vec<_>>())
2402 .unwrap_or_default();
2403 tasks
2404 .into_iter()
2405 .map(|task| {
2406 let _ = self.poll_task(&task);
2407 self.snapshot_with_terminal_cache(&task, preview_bytes)
2408 })
2409 .collect()
2410 }
2411
2412 fn maybe_compress_snapshot(&self, task: &Arc<BgTask>, snapshot: &mut BgTaskSnapshot) {
2418 if !snapshot.info.status.is_terminal() || snapshot.info.mode == BgMode::Pty {
2419 return;
2420 }
2421 if let Some(cache) = self.ensure_terminal_output_cache(task) {
2422 snapshot.output_preview = cache.output_preview;
2423 snapshot.output_truncated = cache.output_truncated;
2424 }
2425 }
2426
2427 pub fn kill(&self, task_id: &str, session_id: &str) -> Result<BgTaskSnapshot, String> {
2428 self.kill_with_status(task_id, session_id, BgTaskStatus::Killed)
2429 }
2430
2431 pub fn promote(&self, task_id: &str, session_id: &str) -> Result<bool, String> {
2432 let task = self
2433 .task_for_session(task_id, session_id)
2434 .ok_or_else(|| format!("background task not found: {task_id}"))?;
2435 let terminal_after_promote = {
2436 let mut state = task
2437 .state
2438 .lock()
2439 .map_err(|_| "background task lock poisoned".to_string())?;
2440 let updated = self
2441 .update_task_metadata(&task.paths, |metadata| {
2442 metadata.notify_on_completion = true;
2443 metadata.completion_delivered = false;
2444 })
2445 .map_err(|e| format!("failed to promote background task: {e}"))?;
2446 state.metadata = updated;
2447 state.metadata.status.is_terminal()
2448 };
2449 if terminal_after_promote {
2450 self.post_terminal_transition(&task, true)?;
2451 }
2452 Ok(true)
2453 }
2454
2455 pub(crate) fn kill_for_timeout(&self, task_id: &str, session_id: &str) -> Result<(), String> {
2456 self.kill_with_status(task_id, session_id, BgTaskStatus::TimedOut)
2457 .map(|_| ())
2458 }
2459
2460 pub fn cleanup_finished(&self, older_than: Duration) {
2461 let cutoff = Instant::now().checked_sub(older_than);
2462 let removable_paths: Vec<(String, TaskPaths)> =
2463 if let Ok(mut tasks) = self.inner.tasks.lock() {
2464 let removable = tasks
2465 .iter()
2466 .filter_map(|(task_id, task)| {
2467 let delivered_terminal = task
2468 .state
2469 .lock()
2470 .map(|state| {
2471 state.metadata.status.is_terminal()
2472 && state.metadata.completion_delivered
2473 })
2474 .unwrap_or(false);
2475 if !delivered_terminal {
2476 return None;
2477 }
2478
2479 let terminal_at = task.terminal_at.lock().ok().and_then(|at| *at);
2480 let expired = match (terminal_at, cutoff) {
2481 (Some(terminal_at), Some(cutoff)) => terminal_at <= cutoff,
2482 (Some(_), None) => true,
2483 (None, _) => false,
2484 };
2485 expired.then(|| task_id.clone())
2486 })
2487 .collect::<Vec<_>>();
2488
2489 removable
2490 .into_iter()
2491 .filter_map(|task_id| {
2492 tasks
2493 .remove(&task_id)
2494 .map(|task| (task_id, task.paths.clone()))
2495 })
2496 .collect()
2497 } else {
2498 Vec::new()
2499 };
2500
2501 for (task_id, paths) in removable_paths {
2502 match delete_task_bundle(&paths) {
2503 Ok(()) => log::debug!("deleted persisted background task bundle {task_id}"),
2504 Err(error) => crate::slog_warn!(
2505 "failed to delete persisted background task bundle {task_id}: {error}"
2506 ),
2507 }
2508 }
2509 }
2510
2511 pub fn drain_completions(&self) -> Vec<BgCompletion> {
2512 self.drain_completions_for_session(None)
2513 }
2514
2515 pub fn drain_completions_for_session(&self, session_id: Option<&str>) -> Vec<BgCompletion> {
2516 let completions = match self.inner.completions.lock() {
2517 Ok(completions) => completions,
2518 Err(_) => return Vec::new(),
2519 };
2520
2521 completions
2522 .iter()
2523 .filter(|completion| completion_matches_session(completion, session_id))
2524 .cloned()
2525 .collect()
2526 }
2527
2528 pub fn has_completions_for_session(&self, session_id: Option<&str>) -> bool {
2529 match self.inner.completions.lock() {
2530 Ok(completions) => completions
2531 .iter()
2532 .any(|completion| completion_matches_session(completion, session_id)),
2533 Err(_) => true,
2537 }
2538 }
2539
2540 pub fn ack_completions_for_session(
2541 &self,
2542 session_id: Option<&str>,
2543 task_ids: &[String],
2544 ) -> Vec<String> {
2545 if task_ids.is_empty() {
2546 return Vec::new();
2547 }
2548 let requested_task_ids = task_ids.iter().map(String::as_str).collect::<HashSet<_>>();
2549 let mut completion_sessions = HashMap::new();
2550 if let Ok(mut completions) = self.inner.completions.lock() {
2551 completions.retain(|completion| {
2552 let session_matches = session_id
2553 .map(|session_id| completion.session_id == session_id)
2554 .unwrap_or(true);
2555 if session_matches && requested_task_ids.contains(completion.task_id.as_str()) {
2556 completion_sessions
2557 .insert(completion.task_id.clone(), completion.session_id.clone());
2558 false
2559 } else {
2560 true
2561 }
2562 });
2563 }
2564
2565 let mut delivered = Vec::new();
2566 for task_id in task_ids {
2567 let task = if let Some(session_id) = session_id {
2568 self.task_for_session(task_id, session_id)
2569 } else if let Some(completion_session_id) = completion_sessions.get(task_id) {
2570 self.task_for_session(task_id, completion_session_id)
2571 } else {
2572 self.task(task_id)
2573 };
2574 if let Some(task) = task {
2575 let terminal = task
2576 .state
2577 .lock()
2578 .map(|state| state.metadata.status.is_terminal())
2579 .unwrap_or(false);
2580 self.ack_persisted_watches_for_task(&task.session_id, task_id, terminal);
2584 if terminal {
2585 self.clear_task_watch_state(task_id);
2586 if task.set_completion_delivered(true, self).is_ok() {
2587 delivered.push(task_id.clone());
2588 }
2589 } else {
2590 self.sync_memory_watches_after_ack(task_id);
2593 delivered.push(task_id.clone());
2594 }
2595 } else if let Some(session_id) = session_id {
2596 self.ack_persisted_watches_for_task(session_id, task_id, true);
2598 delivered.push(task_id.clone());
2599 }
2600 }
2601
2602 delivered
2603 }
2604
2605 fn sync_memory_watches_after_ack(&self, task_id: &str) {
2606 let Some((harness, pool)) = self.db_harness_and_pool() else {
2607 return;
2608 };
2609 let session_id = match self.task(task_id) {
2610 Some(task) => task.session_id.clone(),
2611 None => return,
2612 };
2613 let Ok(conn) = pool.lock() else {
2614 return;
2615 };
2616 let Ok(rows) = crate::db::bash_watches::list_bash_pattern_watches_for_task(
2617 &conn,
2618 &harness,
2619 &session_id,
2620 task_id,
2621 ) else {
2622 return;
2623 };
2624 let remaining: HashSet<String> = rows.into_iter().map(|row| row.watch_id).collect();
2625 if let Ok(mut registry) = self.inner.watch_registry.lock() {
2626 registry.retain_watch_ids(task_id, &remaining);
2627 }
2628 }
2629
2630 pub fn pending_completions_for_session(&self, session_id: &str) -> Vec<BgCompletion> {
2631 self.inner
2632 .completions
2633 .lock()
2634 .map(|completions| {
2635 completions
2636 .iter()
2637 .filter(|completion| completion.session_id == session_id)
2638 .cloned()
2639 .collect()
2640 })
2641 .unwrap_or_default()
2642 }
2643
2644 fn remove_pending_completion(&self, task_id: &str) -> Option<BgCompletion> {
2645 let mut completions = self.inner.completions.lock().ok()?;
2646 let idx = completions
2647 .iter()
2648 .position(|completion| completion.task_id == task_id)?;
2649 completions.remove(idx)
2650 }
2651
2652 fn completion_snapshot_for_task(&self, task: &Arc<BgTask>) -> Option<BgCompletion> {
2653 let snapshot = self.snapshot_with_terminal_cache(task, RUNNING_OUTPUT_PREVIEW_BYTES);
2654 if !snapshot.info.status.is_terminal() {
2655 return None;
2656 }
2657 let (output_preview, output_truncated) = if snapshot.info.mode == BgMode::Pty {
2658 (String::new(), false)
2659 } else {
2660 self.ensure_terminal_output_cache(task)
2661 .map(|cache| completion_preview_for_cache(&cache, snapshot.exit_code))
2662 .unwrap_or_else(|| (String::new(), false))
2663 };
2664 Some(BgCompletion {
2665 task_id: snapshot.info.task_id,
2666 session_id: task.session_id.clone(),
2667 status: snapshot.info.status,
2668 exit_code: snapshot.exit_code,
2669 command: snapshot.info.command,
2670 output_preview,
2671 output_truncated,
2672 original_tokens: None,
2673 compressed_tokens: None,
2674 tokens_skipped: false,
2675 })
2676 }
2677
2678 pub fn detach(&self) {
2679 self.inner.shutdown.store(true, Ordering::SeqCst);
2680 if let Ok(mut tasks) = self.inner.tasks.lock() {
2681 for task in tasks.values() {
2682 if let Ok(mut state) = task.state.lock() {
2683 match &mut state.runtime {
2684 TaskRuntime::Piped(child) => *child = None,
2685 TaskRuntime::Pty(runtime) => *runtime = None,
2686 }
2687 state.detached = true;
2688 }
2689 }
2690 tasks.clear();
2691 }
2692 }
2693
2694 pub fn shutdown(&self) {
2695 let tasks = self
2696 .inner
2697 .tasks
2698 .lock()
2699 .map(|tasks| {
2700 tasks
2701 .values()
2702 .map(|task| (task.task_id.clone(), task.session_id.clone()))
2703 .collect::<Vec<_>>()
2704 })
2705 .unwrap_or_default();
2706 for (task_id, session_id) in tasks {
2707 let _ = self.kill(&task_id, &session_id);
2708 }
2709 }
2710
2711 pub(crate) fn poll_task(&self, task: &Arc<BgTask>) -> Result<(), String> {
2712 if let Ok(state) = task.state.lock() {
2713 if let TaskRuntime::Pty(Some(pty)) = &state.runtime {
2714 if !pty.exit_observed.load(Ordering::SeqCst) {
2722 return Ok(());
2723 }
2724 }
2725 }
2726 let marker = match read_exit_marker(&task.paths) {
2727 Ok(Some(marker)) => marker,
2728 Ok(None) => return Ok(()),
2729 Err(error) => return Err(format!("failed to read exit marker: {error}")),
2730 };
2731 self.finalize_from_marker(task, marker, None)
2732 }
2733
2734 pub(crate) fn reap_child(&self, task: &Arc<BgTask>) {
2735 let mut needs_completion = false;
2736 {
2737 let Ok(mut state) = task.state.lock() else {
2738 return;
2739 };
2740 match &mut state.runtime {
2741 TaskRuntime::Piped(child_slot) => {
2742 if let Some(child) = child_slot.as_mut() {
2743 if let Ok(Some(status)) = child.try_wait() {
2744 *child_slot = None;
2745 state.detached = true;
2746 state.child_exit_observed = true;
2747 if let Some(handles) = state.io_handles.as_mut() {
2748 if handles.artifact_len(TaskArtifact::Exit).unwrap_or(1) == 0 {
2749 let marker = status
2750 .code()
2751 .map(|code| code.to_string())
2752 .unwrap_or_else(|| "1".to_string());
2753 let _ = handles.write(TaskArtifact::Exit, marker.as_bytes());
2754 }
2755 }
2756 }
2757 } else if state.detached {
2758 let child_known_dead = state.child_exit_observed
2759 || state
2760 .metadata
2761 .child_pid
2762 .is_some_and(|pid| !is_process_alive(pid));
2763 if child_known_dead {
2764 needs_completion =
2765 self.fail_without_exit_marker_if_needed(task, &mut state);
2766 }
2767 }
2768 }
2769 TaskRuntime::Pty(Some(pty)) => {
2770 if pty.exit_observed.load(Ordering::SeqCst) {
2771 drop(state);
2772 let _ = self.poll_task(task);
2773 return;
2774 }
2775 }
2776 TaskRuntime::Pty(None) => {}
2777 }
2778 }
2779 if needs_completion {
2780 let _ = self.post_terminal_transition(task, true);
2781 }
2782 }
2783
2784 fn fail_without_exit_marker_if_needed(
2785 &self,
2786 task: &Arc<BgTask>,
2787 state: &mut BgTaskState,
2788 ) -> bool {
2789 if state.metadata.status.is_terminal() {
2790 return false;
2791 }
2792 if matches!(read_exit_marker(&task.paths), Ok(Some(_))) {
2793 return false;
2794 }
2795 let watch_controlled = self.task_has_watch_control(&task.task_id);
2796 let updated = self.update_task_metadata(&task.paths, |metadata| {
2797 metadata.mark_terminal(
2798 BgTaskStatus::Failed,
2799 None,
2800 Some("process exited without exit marker".to_string()),
2801 );
2802 if watch_controlled {
2803 metadata.completion_delivered = true;
2804 }
2805 });
2806 if let Ok(metadata) = updated {
2807 state.pending_terminal_override = None;
2808 state.metadata = metadata;
2809 task.mark_terminal_now();
2810 return true;
2811 }
2812 false
2813 }
2814
2815 pub(crate) fn running_tasks(&self) -> Vec<Arc<BgTask>> {
2816 self.inner
2817 .tasks
2818 .lock()
2819 .map(|tasks| {
2820 tasks
2821 .values()
2822 .filter(|task| task.is_running())
2823 .cloned()
2824 .collect()
2825 })
2826 .unwrap_or_default()
2827 }
2828
2829 fn insert_rehydrated_task(
2830 &self,
2831 metadata: PersistedTask,
2832 paths: TaskPaths,
2833 detached: bool,
2834 ) -> Result<(), String> {
2835 let task_id = metadata.task_id.clone();
2836 let session_id = metadata.session_id.clone();
2837 let started = started_instant_from_unix_millis(metadata.started_at);
2838 let suppress_replayed_running_reminder = metadata.status == BgTaskStatus::Running;
2839 let mode = metadata.mode.clone();
2840 let task = Arc::new(BgTask {
2841 task_id: task_id.clone(),
2842 session_id,
2843 paths: paths.clone(),
2844 artifact_root: canonical_artifact_root(&paths),
2845 started,
2846 last_reminder_at: Mutex::new(suppress_replayed_running_reminder.then(Instant::now)),
2847 terminal_at: Mutex::new(metadata.status.is_terminal().then(Instant::now)),
2848 state: Mutex::new(BgTaskState {
2849 metadata,
2850 runtime: if mode == BgMode::Pty {
2851 TaskRuntime::Pty(None)
2852 } else {
2853 TaskRuntime::Piped(None)
2854 },
2855 io_handles: None,
2856 detached,
2857 child_exit_observed: false,
2864 buffer: BgBuffer::registered(&paths, mode.clone()),
2865 terminal_output_cache: None,
2866 pending_terminal_override: None,
2867 }),
2868 });
2869 self.inner
2870 .tasks
2871 .lock()
2872 .map_err(|_| "background task registry lock poisoned".to_string())?
2873 .insert(task_id.clone(), Arc::clone(&task));
2874 self.rearm_persisted_watches(&task);
2878 Ok(())
2879 }
2880
2881 fn rearm_persisted_watches(&self, task: &Arc<BgTask>) {
2882 let Some((harness, pool)) = self.db_harness_and_pool() else {
2883 return;
2884 };
2885 let rows = {
2886 let Ok(conn) = pool.lock() else {
2887 return;
2888 };
2889 match crate::db::bash_watches::list_bash_pattern_watches_for_task(
2890 &conn,
2891 &harness,
2892 &task.session_id,
2893 &task.task_id,
2894 ) {
2895 Ok(rows) if !rows.is_empty() => rows,
2896 _ => return,
2897 }
2898 };
2899
2900 let mode = match task.state.lock() {
2901 Ok(state) => state.metadata.mode.clone(),
2902 Err(_) => return,
2903 };
2904 let terminal = task
2905 .state
2906 .lock()
2907 .map(|state| state.metadata.status.is_terminal())
2908 .unwrap_or(false);
2909 let completion_delivered = task
2910 .state
2911 .lock()
2912 .map(|state| state.metadata.completion_delivered)
2913 .unwrap_or(true);
2914
2915 let mut stdout = (mode == BgMode::Pipes)
2916 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stdout))
2917 .transpose()
2918 .ok()
2919 .flatten();
2920 let mut stderr = (mode == BgMode::Pipes)
2921 .then(|| open_task_artifact(&task.paths, TaskArtifact::Stderr))
2922 .transpose()
2923 .ok()
2924 .flatten();
2925 let mut pty = (mode == BgMode::Pty)
2926 .then(|| open_task_artifact(&task.paths, TaskArtifact::Pty))
2927 .transpose()
2928 .ok()
2929 .flatten();
2930
2931 let mut pending_to_emit = Vec::new();
2932 let mut gap_matches = Vec::new();
2933 {
2934 let Ok(mut registry) = self.inner.watch_registry.lock() else {
2935 return;
2936 };
2937 let stdout_key = format!("{}:stdout", task.task_id);
2938 let stderr_key = format!("{}:stderr", task.task_id);
2939 let pty_key = format!("{}:pty", task.task_id);
2940
2941 let first = &rows[0];
2943 match mode {
2944 BgMode::Pipes => {
2945 registry.set_file_cursor(&stdout_key, first.stdout_offset.max(0) as u64);
2946 registry.set_file_cursor(&stderr_key, first.stderr_offset.max(0) as u64);
2947 }
2948 BgMode::Pty => {
2949 registry.set_file_cursor(&pty_key, first.pty_offset.max(0) as u64);
2950 }
2951 }
2952
2953 for row in &rows {
2954 let Ok(pattern) = WatchPattern::from_persisted(&row.pattern_kind, &row.pattern)
2955 else {
2956 crate::slog_warn!(
2957 "skipping unreadable persisted watch {}/{}",
2958 row.task_id,
2959 row.watch_id
2960 );
2961 continue;
2962 };
2963 if let Err(error) = registry.restore(
2964 row.watch_id.clone(),
2965 row.task_id.clone(),
2966 pattern,
2967 row.once,
2968 row.scanning,
2969 ) {
2970 crate::slog_warn!(
2971 "failed to restore watch {}/{}: {error}",
2972 row.task_id,
2973 row.watch_id
2974 );
2975 continue;
2976 }
2977 if row.pending_match {
2978 if let (Some(match_text), Some(match_offset), Some(context)) = (
2979 row.match_text.clone(),
2980 row.match_offset,
2981 row.match_context.clone(),
2982 ) {
2983 pending_to_emit.push(PatternMatch {
2984 watch_id: row.watch_id.clone(),
2985 task_id: row.task_id.clone(),
2986 match_text,
2987 match_offset: match_offset.max(0) as u64,
2988 context,
2989 once: row.once,
2990 });
2991 }
2992 }
2993 }
2994
2995 let should_gap_scan =
2999 rows.iter().any(|row| row.scanning) && !pending_to_emit.iter().any(|m| m.once);
3000 if should_gap_scan {
3001 match mode {
3002 BgMode::Pipes => {
3003 if let (Some(stdout), Some(stderr)) = (stdout.as_mut(), stderr.as_mut()) {
3004 gap_matches.extend(registry.scan_file_new_bytes(
3005 &stdout_key,
3006 &task.task_id,
3007 stdout,
3008 ));
3009 gap_matches.extend(registry.scan_file_new_bytes(
3010 &stderr_key,
3011 &task.task_id,
3012 stderr,
3013 ));
3014 }
3015 }
3016 BgMode::Pty => {
3017 if let Some(pty) = pty.as_mut() {
3018 gap_matches.extend(registry.scan_file_new_bytes(
3019 &pty_key,
3020 &task.task_id,
3021 pty,
3022 ));
3023 }
3024 }
3025 }
3026 }
3027 }
3028
3029 let (stdout_offset, stderr_offset, pty_offset) = self.watch_stream_cursors(&task.task_id);
3030 for pattern_match in &gap_matches {
3031 self.persist_watch_match(
3032 &task.session_id,
3033 &task.task_id,
3034 pattern_match,
3035 stdout_offset,
3036 stderr_offset,
3037 pty_offset,
3038 );
3039 }
3040 if !gap_matches.is_empty() || rows.iter().any(|row| row.scanning) {
3041 self.persist_task_watch_cursors(
3042 &task.session_id,
3043 &task.task_id,
3044 stdout_offset,
3045 stderr_offset,
3046 pty_offset,
3047 );
3048 }
3049
3050 let emitted_pending = !pending_to_emit.is_empty();
3052 let to_emit = if emitted_pending {
3053 pending_to_emit
3054 } else {
3055 gap_matches
3056 };
3057 let emitted_once_ids: Vec<String> = to_emit
3058 .iter()
3059 .filter(|m| m.once)
3060 .map(|m| m.watch_id.clone())
3061 .collect();
3062 for pattern_match in to_emit {
3063 self.emit_bash_pattern_match(&task.session_id, pattern_match);
3064 }
3065 if emitted_pending {
3069 for watch_id in emitted_once_ids {
3070 self.delete_persisted_watch(&task.session_id, &task.task_id, &watch_id);
3071 }
3072 }
3073
3074 if !terminal {
3075 return;
3076 }
3077
3078 let _ = self.remove_pending_completion(&task.task_id);
3081 let (watch_controlled, watch_matched) = self.task_watch_state(&task.task_id);
3082 if !watch_controlled {
3083 return;
3084 }
3085 if watch_matched {
3086 return;
3088 }
3089 if completion_delivered {
3090 self.clear_task_watch_state(&task.task_id);
3092 self.delete_persisted_watches_for_task(&task.session_id, &task.task_id);
3093 return;
3094 }
3095 if let Some(completion) = self.completion_snapshot_for_task(task) {
3096 self.emit_bash_watch_exit(&completion);
3097 }
3098 self.clear_task_watch_state(&task.task_id);
3100 }
3101
3102 fn kill_with_status(
3103 &self,
3104 task_id: &str,
3105 session_id: &str,
3106 terminal_status: BgTaskStatus,
3107 ) -> Result<BgTaskSnapshot, String> {
3108 let task = self
3109 .task_for_session(task_id, session_id)
3110 .ok_or_else(|| format!("background task not found: {task_id}"))?;
3111 let mut terminalized = false;
3112
3113 {
3114 let mut state = task
3115 .state
3116 .lock()
3117 .map_err(|_| "background task lock poisoned".to_string())?;
3118 if state.metadata.status.is_terminal() {
3119 state.pending_terminal_override = None;
3120 } else if let Ok(Some(marker)) = read_exit_marker(&task.paths) {
3121 state.metadata =
3122 terminal_metadata_from_marker(state.metadata.clone(), marker, None);
3123 if self.task_has_watch_control(&task.task_id) {
3124 state.metadata.completion_delivered = true;
3125 }
3126 state.pending_terminal_override = None;
3127 task.mark_terminal_now();
3128 match &mut state.runtime {
3129 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
3136 TaskRuntime::Pty(runtime) => *runtime = None,
3137 }
3138 state.detached = true;
3139 self.persist_task(&task.paths, &state.metadata)
3140 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
3141 terminalized = true;
3142 } else {
3143 let was_already_killing = state.metadata.status == BgTaskStatus::Killing;
3144 if !was_already_killing {
3145 state.metadata.status = BgTaskStatus::Killing;
3146 self.persist_task(&task.paths, &state.metadata)
3147 .map_err(|e| format!("failed to persist killing state: {e}"))?;
3148 }
3149
3150 #[cfg(unix)]
3151 let pgid = state.metadata.pgid;
3152 #[cfg(windows)]
3153 let child_pid = state.metadata.child_pid;
3154 if !was_already_killing
3155 && state.metadata.mode == BgMode::Pty
3156 && terminal_status == BgTaskStatus::TimedOut
3157 {
3158 state.pending_terminal_override = Some(BgTaskStatus::TimedOut);
3159 }
3160
3161 #[cfg(windows)]
3162 let mut pty_forced_terminal_status: Option<BgTaskStatus> = None;
3163
3164 match &mut state.runtime {
3165 TaskRuntime::Piped(child_slot) => {
3166 #[cfg(unix)]
3167 if let Some(pgid) = pgid {
3168 terminate_pgid(pgid, child_slot.as_mut());
3169 }
3170 #[cfg(windows)]
3171 if let Some(child) = child_slot.as_mut() {
3172 super::process::terminate_process(child);
3173 } else if let Some(pid) = child_pid {
3174 terminate_pid(pid);
3175 }
3176 if let Some(child) = child_slot.as_mut() {
3177 let _ = child.wait();
3178 }
3179 *child_slot = None;
3180 state.detached = true;
3181
3182 if let Some(handles) = state.io_handles.as_mut() {
3183 handles.write(TaskArtifact::Exit, b"killed").map_err(|e| {
3184 format!("failed to write retained kill marker: {e}")
3185 })?;
3186 } else {
3187 write_kill_marker_if_absent(&task.paths)
3188 .map_err(|e| format!("failed to write kill marker: {e}"))?;
3189 }
3190
3191 let exit_code = terminal_exit_code_for_status(&terminal_status);
3192 state
3193 .metadata
3194 .mark_terminal(terminal_status, exit_code, None);
3195 if self.task_has_watch_control(&task.task_id) {
3196 state.metadata.completion_delivered = true;
3197 }
3198 state.pending_terminal_override = None;
3199 task.mark_terminal_now();
3200 self.persist_task(&task.paths, &state.metadata)
3201 .map_err(|e| format!("failed to persist killed state: {e}"))?;
3202 terminalized = true;
3203 }
3204 TaskRuntime::Pty(Some(pty)) => {
3205 pty.was_killed.store(true, Ordering::SeqCst);
3206 if let Err(error) = pty.killer.kill() {
3207 crate::slog_warn!(
3208 "[pty-kill] {task_id} ChildKiller::kill failed: {error}"
3209 );
3210 }
3211 if let Some(pid) = pty.child_pid {
3212 #[cfg(unix)]
3213 terminate_pgid(pid as i32, None);
3214 #[cfg(windows)]
3215 terminate_pid(pid);
3216 }
3217 drop(pty.master.take());
3218
3219 #[cfg(windows)]
3220 {
3221 let default_status = if terminal_status == BgTaskStatus::TimedOut {
3222 BgTaskStatus::TimedOut
3223 } else {
3224 BgTaskStatus::Killed
3225 };
3226 pty_forced_terminal_status = Some(
3227 state
3228 .pending_terminal_override
3229 .take()
3230 .unwrap_or(default_status),
3231 );
3232 }
3233 }
3234 TaskRuntime::Pty(None) => {}
3235 }
3236
3237 #[cfg(windows)]
3238 if let Some(target_status) = pty_forced_terminal_status {
3239 if !task.paths.exit.exists() {
3240 write_kill_marker_if_absent(&task.paths)
3241 .map_err(|e| format!("failed to write kill marker: {e}"))?;
3242 }
3243
3244 let exit_code = terminal_exit_code_for_status(&target_status);
3245 state.metadata.mark_terminal(target_status, exit_code, None);
3246 if self.task_has_watch_control(&task.task_id) {
3247 state.metadata.completion_delivered = true;
3248 }
3249 state.pending_terminal_override = None;
3250 task.mark_terminal_now();
3251 if let TaskRuntime::Pty(runtime) = &mut state.runtime {
3252 *runtime = None;
3253 }
3254 state.detached = true;
3255 self.persist_task(&task.paths, &state.metadata)
3256 .map_err(|e| format!("failed to persist killed PTY state: {e}"))?;
3257 terminalized = true;
3258 }
3259 }
3260 }
3261
3262 if terminalized {
3263 self.post_terminal_transition(&task, true)?;
3264 }
3265 Ok(self.snapshot_with_terminal_cache(&task, RUNNING_OUTPUT_PREVIEW_BYTES))
3266 }
3267
3268 fn finalize_from_marker(
3269 &self,
3270 task: &Arc<BgTask>,
3271 marker: ExitMarker,
3272 reason: Option<String>,
3273 ) -> Result<(), String> {
3274 let watch_controlled = self.task_has_watch_control(&task.task_id);
3275 let mut pty_reader_done = None;
3276 {
3277 let mut state = task
3278 .state
3279 .lock()
3280 .map_err(|_| "background task lock poisoned".to_string())?;
3281 if state.metadata.status.is_terminal() {
3282 state.pending_terminal_override = None;
3283 return Ok(());
3284 }
3285
3286 let pending_override = state.pending_terminal_override.take();
3287 let is_pty = state.metadata.mode == BgMode::Pty;
3288 let updated = self
3289 .update_task_metadata(&task.paths, |metadata| {
3290 let mut new_metadata = if is_pty && marker == ExitMarker::Killed {
3291 let mut metadata = metadata.clone();
3292 let target_status = pending_override.unwrap_or(BgTaskStatus::Killed);
3293 let exit_code = terminal_exit_code_for_status(&target_status);
3294 metadata.mark_terminal(target_status, exit_code, reason);
3295 metadata
3296 } else {
3297 terminal_metadata_from_marker(metadata.clone(), marker, reason)
3298 };
3299 if watch_controlled {
3300 new_metadata.completion_delivered = true;
3301 }
3302 *metadata = new_metadata;
3303 })
3304 .map_err(|e| format!("failed to persist terminal state: {e}"))?;
3305 state.metadata = updated;
3306 task.mark_terminal_now();
3307 match &mut state.runtime {
3308 TaskRuntime::Piped(child_slot) => reap_piped_child(child_slot),
3313 TaskRuntime::Pty(runtime) => {
3314 pty_reader_done = runtime
3315 .as_ref()
3316 .map(|runtime| Arc::clone(&runtime.reader_done));
3317 *runtime = None;
3318 }
3319 }
3320 state.detached = true;
3321 }
3322
3323 if let Some(reader_done) = pty_reader_done {
3324 let deadline = Instant::now() + Duration::from_millis(200);
3325 while !reader_done.load(Ordering::SeqCst) && Instant::now() < deadline {
3326 std::thread::sleep(Duration::from_millis(10));
3327 }
3328 }
3329
3330 self.scan_task_watch_output(task);
3333
3334 self.post_terminal_transition(task, true)
3335 }
3336
3337 fn enqueue_completion_if_needed(
3338 &self,
3339 metadata: &PersistedTask,
3340 paths: Option<&TaskPaths>,
3341 emit_frame: bool,
3342 ) {
3343 if metadata.status.is_terminal() && !metadata.completion_delivered {
3344 let cache =
3345 paths.and_then(|paths| self.render_terminal_output_from_paths(metadata, paths));
3346 self.enqueue_completion_from_parts(metadata, None, paths, emit_frame, cache.as_ref());
3347 }
3348 }
3349
3350 fn render_terminal_output_from_paths(
3351 &self,
3352 metadata: &PersistedTask,
3353 paths: &TaskPaths,
3354 ) -> Option<TerminalOutputCache> {
3355 if metadata.mode == BgMode::Pty {
3356 return None;
3357 }
3358 let mut buffer = BgBuffer::registered(paths, BgMode::Pipes);
3359 let disk_truncation = buffer.enforce_terminal_cap();
3360 Some(self.render_terminal_output(metadata, &buffer, disk_truncation))
3361 }
3362
3363 fn enqueue_completion_from_parts(
3364 &self,
3365 metadata: &PersistedTask,
3366 buffer: Option<&BgBuffer>,
3367 paths: Option<&TaskPaths>,
3368 emit_frame: bool,
3369 terminal_render: Option<&TerminalOutputCache>,
3370 ) {
3371 if !metadata.status.is_terminal() {
3382 return;
3383 }
3384
3385 let owned_buffer = if buffer.is_none() && metadata.mode != BgMode::Pty {
3386 paths.map(|paths| BgBuffer::registered(paths, BgMode::Pipes))
3387 } else {
3388 None
3389 };
3390 let render_buffer = buffer.or(owned_buffer.as_ref());
3391 let owned_render = if terminal_render.is_none() {
3392 render_buffer.map(|buffer| {
3393 let mut capped_buffer = buffer.clone();
3394 let disk_truncation = capped_buffer.enforce_terminal_cap();
3395 self.render_terminal_output(metadata, &capped_buffer, disk_truncation)
3396 })
3397 } else {
3398 None
3399 };
3400 let render = terminal_render.or(owned_render.as_ref());
3401
3402 let (output_preview, output_truncated) = render
3406 .map(|cache| completion_preview_for_cache(cache, metadata.exit_code))
3407 .unwrap_or_else(|| (String::new(), false));
3408
3409 let token_counts = self.completion_token_counts(
3410 metadata,
3411 buffer,
3412 paths,
3413 render.map(|render| render.output_preview.as_str()),
3414 );
3415 let completion = BgCompletion {
3416 task_id: metadata.task_id.clone(),
3417 session_id: metadata.session_id.clone(),
3418 status: metadata.status.clone(),
3419 exit_code: metadata.exit_code,
3420 command: metadata.command.clone(),
3421 output_preview,
3422 output_truncated,
3423 original_tokens: token_counts.original_tokens,
3424 compressed_tokens: token_counts.compressed_tokens,
3425 tokens_skipped: token_counts.tokens_skipped,
3426 };
3427
3428 self.record_compression_event_if_applicable(metadata, &token_counts);
3439
3440 let (watch_controlled, watch_matched) = self.task_watch_state(&metadata.task_id);
3441 if watch_controlled {
3442 if emit_frame && !watch_matched {
3443 self.emit_bash_watch_exit(&completion);
3444 } else if watch_matched {
3445 if let Some(task) = self.task(&metadata.task_id) {
3450 let _ = task.set_completion_delivered(true, self);
3451 }
3452 }
3453 self.clear_task_watch_state(&metadata.task_id);
3455 return;
3456 }
3457
3458 if metadata.completion_delivered {
3468 return;
3469 }
3470
3471 let pushed = if let Ok(mut completions) = self.inner.completions.lock() {
3474 if completions
3475 .iter()
3476 .any(|existing| existing.task_id == metadata.task_id)
3477 {
3478 false
3479 } else {
3480 completions.push_back(completion.clone());
3481 true
3482 }
3483 } else {
3484 false
3485 };
3486
3487 if pushed && emit_frame {
3488 self.emit_bash_completed(completion);
3489 }
3490 }
3491
3492 fn record_compression_event_if_applicable(
3493 &self,
3494 metadata: &PersistedTask,
3495 token_counts: &CompletionTokenCounts,
3496 ) {
3497 if metadata.mode == BgMode::Pty {
3498 return;
3499 }
3500
3501 let (original_tokens, compressed_tokens, original_bytes, compressed_bytes) = match (
3502 token_counts.original_tokens,
3503 token_counts.compressed_tokens,
3504 token_counts.original_bytes,
3505 token_counts.compressed_bytes,
3506 ) {
3507 (
3508 Some(original_tokens),
3509 Some(compressed_tokens),
3510 Some(original_bytes),
3511 Some(compressed_bytes),
3512 ) => (
3513 original_tokens,
3514 compressed_tokens,
3515 original_bytes,
3516 compressed_bytes,
3517 ),
3518 _ => {
3519 crate::slog_warn!(
3520 "compression event skipped for {}: token counts unavailable (likely spill file missing or unreadable)",
3521 metadata.task_id
3522 );
3523 return;
3524 }
3525 };
3526
3527 let pool = self.inner.db_pool.read().ok().and_then(|slot| slot.clone());
3528 let Some(pool) = pool else {
3529 crate::slog_warn!(
3530 "compression event skipped for {}: db_pool not initialized — was configure run?",
3531 metadata.task_id
3532 );
3533 return;
3534 };
3535 let harness = self
3536 .inner
3537 .db_harness
3538 .read()
3539 .ok()
3540 .and_then(|slot| slot.clone());
3541 let Some(harness) = harness else {
3542 crate::slog_warn!(
3543 "compression event insert skipped for {}: harness not configured",
3544 metadata.task_id
3545 );
3546 return;
3547 };
3548
3549 let project_root = metadata
3550 .project_root
3551 .as_deref()
3552 .unwrap_or(&metadata.workdir);
3553 let project_key = crate::path_identity::project_scope_key(project_root);
3554 let row = crate::db::compression_events::CompressionEventRow {
3555 harness: &harness,
3556 session_id: Some(&metadata.session_id),
3557 project_key: &project_key,
3558 tool: "bash",
3559 task_id: Some(&metadata.task_id),
3560 command: Some(&metadata.command),
3561 compressor: if metadata.compressed {
3562 "registry"
3563 } else {
3564 "none"
3565 },
3566 original_bytes,
3567 compressed_bytes,
3568 original_tokens,
3569 compressed_tokens,
3570 created_at: unix_millis() as i64,
3571 };
3572
3573 let conn = match pool.lock() {
3574 Ok(conn) => conn,
3575 Err(_) => {
3576 crate::slog_warn!(
3577 "compression event insert failed for {}: db mutex poisoned",
3578 metadata.task_id
3579 );
3580 return;
3581 }
3582 };
3583 match crate::db::compression_events::insert_compression_event(&conn, &row) {
3584 Ok(Some(row_id)) => {
3585 self.inner
3589 .compression_aggregates
3590 .record_successful_insert(&conn, &row, row_id);
3591 crate::slog_debug!(
3595 "compression event recorded for {} (project={}, session={}, {} → {} tokens)",
3596 metadata.task_id,
3597 project_key,
3598 metadata.session_id,
3599 original_tokens,
3600 compressed_tokens
3601 );
3602 }
3603 Ok(None) => {
3604 crate::slog_debug!(
3605 "duplicate compression event ignored for {} (project={}, session={})",
3606 metadata.task_id,
3607 project_key,
3608 metadata.session_id
3609 );
3610 }
3611 Err(error) => {
3612 crate::slog_warn!(
3613 "compression event insert failed for {}: {}",
3614 metadata.task_id,
3615 error
3616 );
3617 }
3618 }
3619 }
3620
3621 fn emit_bash_pattern_match(&self, session_id: &str, pattern_match: PatternMatch) {
3622 let Ok(progress_sender) = self
3623 .inner
3624 .progress_sender
3625 .lock()
3626 .map(|sender| sender.clone())
3627 else {
3628 return;
3629 };
3630 if let Some(sender) = progress_sender.as_ref() {
3631 sender(PushFrame::BashPatternMatch(BashPatternMatchFrame::new(
3632 pattern_match.task_id,
3633 session_id.to_string(),
3634 pattern_match.watch_id,
3635 pattern_match.match_text,
3636 pattern_match.match_offset,
3637 pattern_match.context,
3638 pattern_match.once,
3639 )));
3640 }
3641 }
3642
3643 fn emit_bash_watch_exit(&self, completion: &BgCompletion) {
3644 let Ok(progress_sender) = self
3645 .inner
3646 .progress_sender
3647 .lock()
3648 .map(|sender| sender.clone())
3649 else {
3650 return;
3651 };
3652 let Some(sender) = progress_sender.as_ref() else {
3653 return;
3654 };
3655 let status = completion_status_text(&completion.status, completion.exit_code);
3656 let preview = completion.output_preview.trim_end();
3657 let context = if preview.is_empty() {
3658 format!("task {} exited ({status})", completion.task_id)
3659 } else {
3660 format!(
3661 "task {} exited ({status})
3662{preview}",
3663 completion.task_id
3664 )
3665 };
3666 sender(PushFrame::BashPatternMatch(
3667 BashPatternMatchFrame::task_exit(
3668 completion.task_id.clone(),
3669 completion.session_id.clone(),
3670 format!("exited ({status})"),
3671 context,
3672 ),
3673 ));
3674 }
3675
3676 fn emit_bash_completed(&self, completion: BgCompletion) {
3677 let Ok(progress_sender) = self
3678 .inner
3679 .progress_sender
3680 .lock()
3681 .map(|sender| sender.clone())
3682 else {
3683 return;
3684 };
3685 let Some(sender) = progress_sender.as_ref() else {
3686 return;
3687 };
3688 sender(PushFrame::BashCompleted(BashCompletedFrame::new(
3696 completion.task_id,
3697 completion.session_id,
3698 completion.status,
3699 completion.exit_code,
3700 completion.command,
3701 completion.output_preview,
3702 completion.output_truncated,
3703 completion.original_tokens,
3704 completion.compressed_tokens,
3705 completion.tokens_skipped,
3706 )));
3707 }
3708
3709 fn completion_token_counts(
3710 &self,
3711 metadata: &PersistedTask,
3712 buffer: Option<&BgBuffer>,
3713 paths: Option<&TaskPaths>,
3714 rendered_output: Option<&str>,
3715 ) -> CompletionTokenCounts {
3716 if metadata.mode == BgMode::Pty {
3717 return CompletionTokenCounts::skipped();
3718 }
3719
3720 let raw = match buffer {
3721 Some(buffer) => buffer.read_for_token_count(TOKENIZE_CAP_BYTES_PER_STREAM),
3722 None => paths
3723 .map(|paths| {
3724 read_for_token_count_from_disk(metadata, paths, TOKENIZE_CAP_BYTES_PER_STREAM)
3725 })
3726 .unwrap_or(TokenCountInput::Skipped),
3727 };
3728
3729 let TokenCountInput::Text(raw_output) = raw else {
3730 return CompletionTokenCounts::skipped();
3731 };
3732
3733 let original_tokens = token_count_u32(&raw_output);
3734 let original_bytes = raw_output.len() as i64;
3735 let compressed_output = rendered_output.unwrap_or(&raw_output);
3736 let compressed_tokens = token_count_u32(compressed_output);
3737 let compressed_bytes = compressed_output.len() as i64;
3738 CompletionTokenCounts {
3739 original_tokens: Some(original_tokens),
3740 compressed_tokens: Some(compressed_tokens),
3741 original_bytes: Some(original_bytes),
3742 compressed_bytes: Some(compressed_bytes),
3743 tokens_skipped: false,
3744 }
3745 }
3746
3747 pub(crate) fn maybe_emit_long_running_reminder(&self, task: &Arc<BgTask>) {
3748 if !self
3749 .inner
3750 .long_running_reminder_enabled
3751 .load(Ordering::SeqCst)
3752 {
3753 return;
3754 }
3755 let interval_ms = self
3756 .inner
3757 .long_running_reminder_interval_ms
3758 .load(Ordering::SeqCst);
3759 if interval_ms == 0 {
3760 return;
3761 }
3762 let interval = Duration::from_millis(interval_ms);
3763 let now = Instant::now();
3764 let Ok(mut last_reminder_at) = task.last_reminder_at.lock() else {
3765 return;
3766 };
3767 let since = last_reminder_at.unwrap_or(task.started);
3768 if now.duration_since(since) < interval {
3769 return;
3770 }
3771 let command = task
3772 .state
3773 .lock()
3774 .map(|state| state.metadata.command.clone())
3775 .unwrap_or_default();
3776 *last_reminder_at = Some(now);
3777 self.emit_bash_long_running(BashLongRunningFrame::new(
3778 task.task_id.clone(),
3779 task.session_id.clone(),
3780 command,
3781 task.started.elapsed().as_millis() as u64,
3782 ));
3783 }
3784
3785 fn emit_bash_long_running(&self, frame: BashLongRunningFrame) {
3786 let Ok(progress_sender) = self
3787 .inner
3788 .progress_sender
3789 .lock()
3790 .map(|sender| sender.clone())
3791 else {
3792 return;
3793 };
3794 if let Some(sender) = progress_sender.as_ref() {
3795 sender(PushFrame::BashLongRunning(frame));
3796 }
3797 }
3798
3799 fn task(&self, task_id: &str) -> Option<Arc<BgTask>> {
3800 validate_task_id(task_id).ok()?;
3801 self.inner
3802 .tasks
3803 .lock()
3804 .ok()
3805 .and_then(|tasks| tasks.get(task_id).cloned())
3806 }
3807
3808 fn task_for_session(&self, task_id: &str, session_id: &str) -> Option<Arc<BgTask>> {
3809 self.task(task_id)
3810 .filter(|task| task.session_id == session_id)
3811 }
3812
3813 pub fn try_health_counts(&self) -> Option<BgTaskHealthCounts> {
3814 let running = self
3815 .inner
3816 .tasks
3817 .try_lock()
3818 .ok()
3819 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())?;
3820 let pending_completions = self.inner.completions.try_lock().ok().map(|q| q.len())?;
3821 Some(BgTaskHealthCounts {
3822 running,
3823 pending_completions,
3824 })
3825 }
3826
3827 pub fn estimated_memory(&self) -> crate::memory::MemoryEstimate {
3831 let tasks = match self.inner.tasks.try_lock() {
3832 Ok(tasks) => tasks.values().cloned().collect::<Vec<_>>(),
3833 Err(_) => return crate::memory::MemoryEstimate::busy(),
3834 };
3835 let mut bytes = 0u64;
3836 let mut terminal_output_caches = 0usize;
3837 let mut sessions = HashSet::new();
3838 for task in &tasks {
3839 sessions.insert(task.session_id.clone());
3840 let state = match task.state.try_lock() {
3841 Ok(state) => state,
3842 Err(_) => return crate::memory::MemoryEstimate::busy(),
3843 };
3844 if let Some(cache) = state.terminal_output_cache.as_ref() {
3845 terminal_output_caches = terminal_output_caches.saturating_add(1);
3846 bytes = bytes.saturating_add(terminal_output_cache_estimated_bytes(cache));
3847 }
3848 }
3849 let completion_count = match self.inner.completions.try_lock() {
3850 Ok(completions) => {
3851 for completion in completions.iter() {
3852 sessions.insert(completion.session_id.clone());
3853 bytes = bytes.saturating_add(completion_estimated_bytes(completion));
3854 }
3855 completions.len()
3856 }
3857 Err(_) => return crate::memory::MemoryEstimate::busy(),
3858 };
3859
3860 crate::memory::MemoryEstimate::estimated(bytes)
3861 .count("tasks", tasks.len())
3862 .count("sessions", sessions.len())
3863 .count("terminal_output_caches", terminal_output_caches)
3864 .count("completion_caches", completion_count)
3865 .count_u64("output_ring_bytes", 0)
3866 }
3867
3868 fn running_count(&self) -> usize {
3869 self.inner
3870 .tasks
3871 .lock()
3872 .map(|tasks| tasks.values().filter(|task| task.is_running()).count())
3873 .unwrap_or(0)
3874 }
3875
3876 fn start_watchdog(&self) {
3877 if !self.inner.watchdog_started.swap(true, Ordering::SeqCst) {
3878 super::watchdog::start(self.clone());
3879 }
3880 }
3881
3882 fn running_metadata_is_stale(&self, metadata: &PersistedTask) -> bool {
3883 unix_millis().saturating_sub(metadata.started_at) > STALE_RUNNING_AFTER.as_millis() as u64
3884 }
3885
3886 #[cfg(test)]
3887 pub fn task_json_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
3888 self.task_for_session(task_id, session_id)
3889 .map(|task| task.paths.json.clone())
3890 }
3891
3892 #[cfg(test)]
3893 pub fn task_exit_path(&self, task_id: &str, session_id: &str) -> Option<PathBuf> {
3894 self.task_for_session(task_id, session_id)
3895 .map(|task| task.paths.exit.clone())
3896 }
3897}
3898
3899fn canonical_artifact_root(paths: &TaskPaths) -> PathBuf {
3900 fs::canonicalize(&paths.io_dir).unwrap_or_else(|_| paths.io_dir.clone())
3901}
3902
3903fn render_compressed_with_recovery(
3904 buffer: &BgBuffer,
3905 mut compressed: CompressionResult,
3906 input_truncated: bool,
3907 disk_truncation: DiskTruncation,
3908 artifact_access: ArtifactRecoveryAccess,
3909) -> TerminalOutputCache {
3910 let had_trailing_newline = compressed.text.ends_with('\n');
3918 let mut text = strip_plain_truncation_marker_lines(&compressed.text)
3919 .trim_end()
3920 .to_string();
3921 if had_trailing_newline && !text.is_empty() {
3922 text.push('\n');
3923 }
3924 compressed.text = text;
3925
3926 let output_path = buffer.output_path().map(|path| path.display().to_string());
3927 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
3928 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
3929 let mut recovery = RecoveryContext {
3930 dropped_by_class: compressed.dropped_by_class,
3931 had_inner_drop: compressed.had_inner_drop,
3932 offset_hint_eligible: compressed.offset_hint_eligible,
3933 offset_start_line: compressed.offset_start_line,
3934 byte_truncated: input_truncated,
3935 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
3936 output_path: output_path.clone(),
3937 stderr_path: stderr_path.clone(),
3938 include_stderr_path,
3939 artifact_access: artifact_access.clone(),
3940 };
3941
3942 let (output_preview, output_truncated) =
3943 render_body_with_recovery_marker(&compressed.text, &mut recovery);
3944 TerminalOutputCache {
3945 output_preview,
3946 output_truncated,
3947 kind: TerminalOutputKind::Compressed,
3948 output_path,
3949 stderr_path,
3950 artifact_access,
3951 recovery: Some(recovery),
3952 }
3953}
3954
3955fn render_body_with_recovery_marker(body: &str, recovery: &mut RecoveryContext) -> (String, bool) {
3956 render_body_with_recovery_marker_at_cap(
3957 body,
3958 recovery,
3959 FINAL_OUTPUT_CAP_BYTES,
3960 cap_final_output,
3961 cap_final_output_with_marker,
3962 )
3963}
3964
3965fn render_raw_body_with_recovery_marker(
3966 body: &str,
3967 recovery: &mut RecoveryContext,
3968) -> (String, bool) {
3969 render_body_with_recovery_marker_at_cap(
3970 body,
3971 recovery,
3972 RAW_PASSTHROUGH_CAP_BYTES,
3973 |input| {
3974 super::output::cap_head_tail(
3975 input,
3976 RAW_PASSTHROUGH_CAP_BYTES,
3977 RAW_PASSTHROUGH_HEAD_BYTES,
3978 RAW_PASSTHROUGH_TAIL_BYTES,
3979 )
3980 },
3981 |input, marker| {
3982 super::output::cap_head_tail_with_marker(
3983 input,
3984 RAW_PASSTHROUGH_CAP_BYTES,
3985 RAW_PASSTHROUGH_HEAD_BYTES,
3986 RAW_PASSTHROUGH_TAIL_BYTES,
3987 marker,
3988 )
3989 },
3990 )
3991}
3992
3993fn render_body_with_recovery_marker_at_cap<F, G>(
3994 body: &str,
3995 recovery: &mut RecoveryContext,
3996 cap_bytes: usize,
3997 cap_plain: F,
3998 cap_with_marker: G,
3999) -> (String, bool)
4000where
4001 F: Fn(&str) -> super::output::CappedText,
4002 G: Fn(&str, &str) -> super::output::CappedText,
4003{
4004 let needs_marker = recovery.has_visible_drop();
4005 if body.len() > cap_bytes {
4006 recovery.byte_truncated = true;
4007 if let Some(marker) = recovery_marker(recovery) {
4008 let capped = cap_with_marker(body, &marker);
4009 return (capped.text, true);
4010 }
4011 let capped = cap_plain(body);
4012 return (capped.text, capped.truncated || needs_marker);
4013 }
4014
4015 if !needs_marker {
4016 return (body.to_string(), false);
4017 }
4018
4019 let Some(marker) = recovery_marker(recovery) else {
4020 return (body.to_string(), true);
4021 };
4022 let with_marker = append_recovery_marker(body, &marker);
4023 if with_marker.len() <= cap_bytes {
4024 return (with_marker, true);
4025 }
4026
4027 recovery.byte_truncated = true;
4028 let marker = recovery_marker(recovery).unwrap_or(marker);
4029 let capped = cap_with_marker(body, &marker);
4030 (capped.text, true)
4031}
4032
4033fn append_recovery_marker(body: &str, marker: &str) -> String {
4034 if body.is_empty() {
4035 return marker.to_string();
4036 }
4037 let mut output = body.trim_end().to_string();
4038 output.push('\n');
4039 output.push_str(marker);
4040 output
4041}
4042
4043fn recovery_marker(recovery: &RecoveryContext) -> Option<String> {
4044 let mut parts = Vec::new();
4045 for (class, count) in &recovery.dropped_by_class {
4046 let label = if *count == 1 {
4047 class.singular()
4048 } else {
4049 class.plural()
4050 };
4051 parts.push(format!("+{count} more {label}"));
4052 }
4053 if recovery.byte_truncated {
4054 parts.push("truncated output".to_string());
4055 }
4056 let disk_truncated_prefix_bytes = recovery.disk_truncated_prefix_bytes;
4057 if disk_truncated_prefix_bytes > 0 {
4058 parts.push(format!(
4059 "truncated {disk_truncated_prefix_bytes} bytes from saved output prefix"
4060 ));
4061 } else if recovery.had_inner_drop && parts.is_empty() {
4062 parts.push("omitted output".to_string());
4063 }
4064
4065 if parts.is_empty() {
4066 return None;
4067 }
4068
4069 let hint = recovery_hint(recovery);
4070 Some(format!("[{}; {hint}]", parts.join(", ")))
4071}
4072
4073fn bash_status_recovery_hint(access: &ArtifactRecoveryAccess) -> String {
4074 let task_id = serde_json::to_string(&access.task_id)
4075 .unwrap_or_else(|_| format!("\"{}\"", access.task_id));
4076 format!("use bash_status({{taskId: {task_id}}})")
4077}
4078
4079fn recovery_hint(recovery: &RecoveryContext) -> String {
4080 if !recovery.artifact_access.readable {
4081 return bash_status_recovery_hint(&recovery.artifact_access);
4082 }
4083
4084 if recovery.offset_hint_eligible
4088 && !recovery.byte_truncated
4089 && recovery.dropped_by_class.is_empty()
4090 && !recovery.include_stderr_path
4091 {
4092 if let (Some(path), Some(line)) =
4093 (recovery.output_path.as_deref(), recovery.offset_start_line)
4094 {
4095 return format!("see remaining: tail -n +{line} {}", quote_path(path));
4096 }
4097 }
4098
4099 let mut paths = Vec::new();
4100 if let Some(path) = recovery.output_path.as_deref() {
4101 paths.push(path);
4102 }
4103 if recovery.include_stderr_path {
4104 if let Some(path) = recovery.stderr_path.as_deref() {
4105 if !paths.contains(&path) {
4106 paths.push(path);
4107 }
4108 }
4109 }
4110
4111 if paths.is_empty() {
4112 return "full output unavailable".to_string();
4113 }
4114
4115 let reads = paths
4116 .into_iter()
4117 .map(|path| format!("read {}", quote_path(path)))
4118 .collect::<Vec<_>>()
4119 .join(" and ");
4120 if recovery.disk_truncated_prefix_bytes > 0 {
4121 format!("retained output: {reads}")
4122 } else {
4123 format!("full output: {reads}")
4124 }
4125}
4126
4127fn strip_plain_truncation_marker_lines(input: &str) -> String {
4128 input
4129 .lines()
4130 .filter(|line| !is_plain_truncation_marker(line.trim()))
4131 .collect::<Vec<_>>()
4132 .join("\n")
4133}
4134
4135fn strip_recovery_marker_lines(input: &str) -> String {
4136 input
4137 .lines()
4138 .filter(|line| !is_recovery_marker(line.trim()))
4139 .collect::<Vec<_>>()
4140 .join("\n")
4141}
4142
4143fn is_plain_truncation_marker(line: &str) -> bool {
4144 let Some(rest) = line.strip_prefix("...<truncated ") else {
4145 return false;
4146 };
4147 let Some(bytes) = rest.strip_suffix(" bytes>...") else {
4148 return false;
4149 };
4150 !bytes.is_empty() && bytes.chars().all(|ch| ch.is_ascii_digit())
4151}
4152
4153fn is_recovery_marker(line: &str) -> bool {
4154 line.starts_with('[')
4155 && line.ends_with(']')
4156 && (line.contains("full output: read ")
4157 || line.contains("retained output: read ")
4158 || line.contains("see remaining: tail -n +")
4159 || line.contains("use bash_status({taskId:")
4160 || line.contains("full output unavailable"))
4161}
4162
4163fn structured_output_pointer(
4164 total_bytes: u64,
4165 output_path: &str,
4166 truncated_prefix_bytes: u64,
4167 artifact_access: &ArtifactRecoveryAccess,
4168) -> String {
4169 if artifact_access.readable {
4170 return if truncated_prefix_bytes > 0 {
4171 retained_json_output_pointer(total_bytes, output_path, truncated_prefix_bytes)
4172 } else {
4173 json_output_pointer(total_bytes, output_path)
4174 };
4175 }
4176
4177 let kb = total_bytes.div_ceil(1024);
4178 let hint = bash_status_recovery_hint(artifact_access);
4179 if truncated_prefix_bytes > 0 {
4180 format!(
4181 "[JSON output {kb} KB; truncated {truncated_prefix_bytes} bytes from saved output prefix; retained output: {hint}]"
4182 )
4183 } else {
4184 format!("[JSON output {kb} KB; full output: {hint}]")
4185 }
4186}
4187
4188fn render_structured_output(
4189 command: &str,
4190 buffer: &BgBuffer,
4191 disk_truncation: DiskTruncation,
4192 artifact_access: ArtifactRecoveryAccess,
4193) -> Option<TerminalOutputCache> {
4194 if !is_gh_structured_command(command) {
4195 return None;
4196 }
4197
4198 let output_path = buffer
4199 .output_path()
4200 .map(|path| path.display().to_string())?;
4201 let stdout_bytes = buffer.stream_len(StreamKind::Stdout);
4202 if stdout_bytes == 0 {
4203 return None;
4204 }
4205
4206 if stdout_bytes > STRUCTURED_OUTPUT_CAP_BYTES as u64 {
4207 if !stream_starts_like_json(buffer, StreamKind::Stdout) {
4208 return None;
4209 }
4210 let output_preview = structured_output_pointer(
4211 stdout_bytes,
4212 &output_path,
4213 disk_truncation.total_prefix_bytes(),
4214 &artifact_access,
4215 );
4216 return Some(TerminalOutputCache {
4217 output_preview,
4218 output_truncated: true,
4219 kind: TerminalOutputKind::Structured,
4220 output_path: Some(output_path),
4221 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
4222 artifact_access,
4223 recovery: None,
4224 });
4225 }
4226
4227 let stdout = buffer.read_stream_bounded(StreamKind::Stdout, STRUCTURED_OUTPUT_CAP_BYTES);
4228 if stdout.truncated || !is_structured_body(&stdout.text) {
4229 return None;
4230 }
4231
4232 Some(TerminalOutputCache {
4233 output_preview: stdout.text,
4234 output_truncated: false,
4235 kind: TerminalOutputKind::Structured,
4236 output_path: Some(output_path),
4237 stderr_path: buffer.stderr_path().map(|path| path.display().to_string()),
4238 artifact_access,
4239 recovery: None,
4240 })
4241}
4242
4243fn render_raw_passthrough(
4244 buffer: &BgBuffer,
4245 disk_truncation: DiskTruncation,
4246 artifact_access: ArtifactRecoveryAccess,
4247) -> TerminalOutputCache {
4248 let raw = buffer.read_combined_head_tail(
4249 RAW_PASSTHROUGH_CAP_BYTES,
4250 RAW_PASSTHROUGH_HEAD_BYTES,
4251 RAW_PASSTHROUGH_TAIL_BYTES,
4252 );
4253 let output_path = buffer.output_path().map(|path| path.display().to_string());
4254 let stderr_path = buffer.stderr_path().map(|path| path.display().to_string());
4255 if !raw.truncated && disk_truncation.total_prefix_bytes() == 0 {
4256 return TerminalOutputCache {
4257 output_preview: raw.text,
4258 output_truncated: false,
4259 kind: TerminalOutputKind::Raw,
4260 output_path,
4261 stderr_path,
4262 artifact_access,
4263 recovery: None,
4264 };
4265 }
4266
4267 let include_stderr_path = buffer.stream_len(StreamKind::Stderr) > 0;
4268 let mut recovery = RecoveryContext {
4269 dropped_by_class: BTreeMap::new(),
4270 had_inner_drop: false,
4271 offset_hint_eligible: false,
4272 offset_start_line: None,
4273 byte_truncated: raw.truncated,
4274 disk_truncated_prefix_bytes: disk_truncation.total_prefix_bytes(),
4275 output_path: output_path.clone(),
4276 stderr_path: stderr_path.clone(),
4277 include_stderr_path,
4278 artifact_access: artifact_access.clone(),
4279 };
4280 let (output_preview, output_truncated) =
4281 render_raw_body_with_recovery_marker(&raw.text, &mut recovery);
4282 TerminalOutputCache {
4283 output_preview,
4284 output_truncated,
4285 kind: TerminalOutputKind::Raw,
4286 output_path,
4287 stderr_path,
4288 artifact_access,
4289 recovery: Some(recovery),
4290 }
4291}
4292
4293fn completion_preview_for_cache(
4294 cache: &TerminalOutputCache,
4295 exit_code: Option<i32>,
4296) -> (String, bool) {
4297 let exit_ok = exit_code == Some(0);
4300 let threshold = completion_preview_threshold(exit_ok);
4301 if cache.kind == TerminalOutputKind::Structured && cache.output_preview.len() > threshold {
4302 if let Some(path) = cache.output_path.as_deref() {
4303 return (
4304 structured_output_pointer(
4305 cache.output_preview.len() as u64,
4306 path,
4307 0,
4308 &cache.artifact_access,
4309 ),
4310 true,
4311 );
4312 }
4313 return (cache.output_preview.clone(), cache.output_truncated);
4314 }
4315
4316 if let Some(recovery) = cache.recovery.as_ref() {
4317 if cache.output_preview.len() <= threshold {
4318 return (cache.output_preview.clone(), cache.output_truncated);
4319 }
4320 let body = strip_recovery_marker_lines(&cache.output_preview);
4321 let mut completion_recovery = recovery.clone();
4322 completion_recovery.byte_truncated = true;
4323 if let Some(marker) = recovery_marker(&completion_recovery) {
4324 let capped = cap_completion_output_with_marker(&body, &marker, exit_ok);
4325 return (capped.text, true);
4326 }
4327 }
4328
4329 let capped = cap_completion_output(&cache.output_preview, exit_ok);
4330 (capped.text, cache.output_truncated || capped.truncated)
4331}
4332
4333fn is_gh_structured_command(command: &str) -> bool {
4334 let Some(normalized) = crate::compress::plain_command_for_structured_output(command) else {
4335 return false;
4336 };
4337 let tokens = shell_words_for_flags(&normalized);
4338 let Some(head) = tokens.first() else {
4339 return false;
4340 };
4341 let head_name = Path::new(head)
4342 .file_name()
4343 .and_then(|name| name.to_str())
4344 .unwrap_or(head);
4345 if !(head_name == "gh" || head_name.eq_ignore_ascii_case("gh.exe")) {
4346 return false;
4347 }
4348 tokens.iter().any(|token| {
4349 matches!(token.as_str(), "--json" | "--jq" | "--template")
4350 || token.starts_with("--json=")
4351 || token.starts_with("--jq=")
4352 || token.starts_with("--template=")
4353 })
4354}
4355
4356fn shell_words_for_flags(command: &str) -> Vec<String> {
4357 let mut words = Vec::new();
4358 let mut current = String::new();
4359 let mut in_single = false;
4360 let mut in_double = false;
4361 let mut escaped = false;
4362
4363 for ch in command.chars() {
4364 if escaped {
4365 current.push(ch);
4366 escaped = false;
4367 continue;
4368 }
4369 if ch == '\\' && !in_single {
4370 escaped = true;
4371 continue;
4372 }
4373 if ch == '\'' && !in_double {
4374 in_single = !in_single;
4375 continue;
4376 }
4377 if ch == '"' && !in_single {
4378 in_double = !in_double;
4379 continue;
4380 }
4381 if ch.is_whitespace() && !in_single && !in_double {
4382 if !current.is_empty() {
4383 words.push(std::mem::take(&mut current));
4384 }
4385 continue;
4386 }
4387 if matches!(ch, ';' | '&' | '|') && !in_single && !in_double {
4388 if !current.is_empty() {
4389 words.push(std::mem::take(&mut current));
4390 }
4391 continue;
4392 }
4393 current.push(ch);
4394 }
4395 if !current.is_empty() {
4396 words.push(current);
4397 }
4398 words
4399}
4400
4401fn is_structured_body(body: &str) -> bool {
4402 let trimmed = body.trim();
4403 if trimmed.is_empty() {
4404 return false;
4405 }
4406 if serde_json::from_str::<serde_json::Value>(trimmed).is_ok() {
4407 return true;
4408 }
4409
4410 let mut saw_line = false;
4411 for line in trimmed
4412 .lines()
4413 .map(str::trim)
4414 .filter(|line| !line.is_empty())
4415 {
4416 saw_line = true;
4417 if serde_json::from_str::<serde_json::Value>(line).is_err() {
4418 return false;
4419 }
4420 }
4421 saw_line
4422}
4423
4424fn stream_starts_like_json(buffer: &BgBuffer, stream: StreamKind) -> bool {
4425 buffer
4426 .read_stream_bounded(stream, 512)
4427 .text
4428 .chars()
4429 .find(|ch| !ch.is_whitespace())
4430 .is_some_and(|ch| matches!(ch, '{' | '[' | '"' | '-' | '0'..='9' | 't' | 'f' | 'n'))
4431}
4432
4433struct CompletionTokenCounts {
4434 original_tokens: Option<u32>,
4435 compressed_tokens: Option<u32>,
4436 original_bytes: Option<i64>,
4437 compressed_bytes: Option<i64>,
4438 tokens_skipped: bool,
4439}
4440
4441impl CompletionTokenCounts {
4442 fn skipped() -> Self {
4443 Self {
4444 original_tokens: None,
4445 compressed_tokens: None,
4446 original_bytes: None,
4447 compressed_bytes: None,
4448 tokens_skipped: true,
4449 }
4450 }
4451}
4452
4453fn completion_status_text(status: &BgTaskStatus, exit_code: Option<i32>) -> String {
4454 match status {
4455 BgTaskStatus::TimedOut => "timed out".to_string(),
4456 BgTaskStatus::Killed => "killed".to_string(),
4457 _ => exit_code
4458 .map(|code| format!("exit {code}"))
4459 .unwrap_or_else(|| format!("{status:?}").to_lowercase()),
4460 }
4461}
4462
4463fn token_count_u32(text: &str) -> u32 {
4464 aft_tokenizer::count_tokens(text)
4465 .try_into()
4466 .unwrap_or(u32::MAX)
4467}
4468
4469impl Default for BgTaskRegistry {
4470 fn default() -> Self {
4471 Self::new(Arc::new(Mutex::new(None)))
4472 }
4473}
4474
4475fn modified_within(path: &Path, grace: Duration) -> bool {
4476 fs::metadata(path)
4477 .and_then(|metadata| metadata.modified())
4478 .ok()
4479 .and_then(|modified| SystemTime::now().duration_since(modified).ok())
4480 .map(|age| age < grace)
4481 .unwrap_or(false)
4482}
4483
4484fn canonicalized_path(path: &Path) -> PathBuf {
4485 fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf())
4486}
4487
4488fn started_instant_from_unix_millis(started_at: u64) -> Instant {
4489 let now_ms = SystemTime::now()
4490 .duration_since(UNIX_EPOCH)
4491 .ok()
4492 .map(|duration| duration.as_millis() as u64)
4493 .unwrap_or(started_at);
4494 let elapsed_ms = now_ms.saturating_sub(started_at);
4495 Instant::now()
4496 .checked_sub(Duration::from_millis(elapsed_ms))
4497 .unwrap_or_else(Instant::now)
4498}
4499
4500fn gc_quarantine(storage_dir: &Path) {
4501 let quarantine_root = storage_dir.join("bash-tasks-quarantine");
4502 let Ok(session_dirs) = fs::read_dir(&quarantine_root) else {
4503 return;
4504 };
4505 for session_entry in session_dirs.flatten() {
4506 let session_quarantine_dir = session_entry.path();
4507 if !session_quarantine_dir.is_dir() {
4508 continue;
4509 }
4510 let entries = match fs::read_dir(&session_quarantine_dir) {
4511 Ok(entries) => entries,
4512 Err(error) => {
4513 crate::slog_warn!(
4514 "failed to read background task quarantine dir {}: {error}",
4515 session_quarantine_dir.display()
4516 );
4517 continue;
4518 }
4519 };
4520 for entry in entries.flatten() {
4521 let path = entry.path();
4522 if modified_within(&path, QUARANTINE_GC_GRACE) {
4523 continue;
4524 }
4525 let result = if path.is_dir() {
4526 fs::remove_dir_all(&path)
4527 } else {
4528 fs::remove_file(&path)
4529 };
4530 match result {
4531 Ok(()) => log::debug!(
4532 "deleted old background task quarantine entry {}",
4533 path.display()
4534 ),
4535 Err(error) => crate::slog_warn!(
4536 "failed to delete old background task quarantine entry {}: {error}",
4537 path.display()
4538 ),
4539 }
4540 }
4541 let _ = fs::remove_dir(&session_quarantine_dir);
4542 }
4543 let _ = fs::remove_dir(&quarantine_root);
4544}
4545
4546fn read_for_token_count_from_disk(
4547 metadata: &PersistedTask,
4548 paths: &TaskPaths,
4549 max_bytes_per_stream: usize,
4550) -> TokenCountInput {
4551 if metadata.mode == BgMode::Pty {
4552 return TokenCountInput::Skipped;
4553 }
4554 let stdout = read_file_tail_capped(paths, TaskArtifact::Stdout, max_bytes_per_stream);
4561 let stderr = read_file_tail_capped(paths, TaskArtifact::Stderr, max_bytes_per_stream);
4562 match (stdout, stderr) {
4563 (Ok(stdout), Ok(stderr)) => TokenCountInput::Text(combine_streams(
4564 String::from_utf8_lossy(&stdout).as_ref(),
4565 String::from_utf8_lossy(&stderr).as_ref(),
4566 )),
4567 (Ok(stdout), Err(_)) => TokenCountInput::Text(combine_streams(
4568 String::from_utf8_lossy(&stdout).as_ref(),
4569 "",
4570 )),
4571 (Err(_), Ok(stderr)) => TokenCountInput::Text(combine_streams(
4572 "",
4573 String::from_utf8_lossy(&stderr).as_ref(),
4574 )),
4575 (Err(_), Err(_)) => TokenCountInput::Skipped,
4576 }
4577}
4578
4579fn read_file_tail_capped(
4580 paths: &TaskPaths,
4581 artifact: TaskArtifact,
4582 max_bytes: usize,
4583) -> std::io::Result<Vec<u8>> {
4584 let mut file = open_task_artifact(paths, artifact)?;
4585 file.tail(max_bytes).map(|(bytes, _)| bytes)
4586}
4587
4588impl BgTask {
4589 fn snapshot(&self, preview_bytes: usize) -> BgTaskSnapshot {
4590 let state = self
4591 .state
4592 .lock()
4593 .unwrap_or_else(|poison| poison.into_inner());
4594 self.snapshot_locked(&state, preview_bytes)
4595 }
4596
4597 fn snapshot_locked(&self, state: &BgTaskState, preview_bytes: usize) -> BgTaskSnapshot {
4598 let metadata = &state.metadata;
4599 let duration_ms = metadata.duration_ms.or_else(|| {
4600 metadata
4601 .status
4602 .is_terminal()
4603 .then(|| self.started.elapsed().as_millis() as u64)
4604 });
4605 let (output_preview, output_truncated) = if metadata.mode == BgMode::Pty {
4606 (String::new(), false)
4607 } else if metadata.status.is_terminal() {
4608 state
4609 .terminal_output_cache
4610 .as_ref()
4611 .map(|cache| (cache.output_preview.clone(), cache.output_truncated))
4612 .unwrap_or_else(|| (String::new(), false))
4613 } else if preview_bytes == 0 {
4614 (String::new(), false)
4615 } else {
4616 state.buffer.read_tail(preview_bytes)
4617 };
4618 BgTaskSnapshot {
4619 info: BgTaskInfo {
4620 task_id: self.task_id.clone(),
4621 status: metadata.status.clone(),
4622 command: metadata.command.clone(),
4623 mode: metadata.mode.clone(),
4624 started_at: metadata.started_at,
4625 duration_ms,
4626 },
4627 exit_code: metadata.exit_code,
4628 child_pid: metadata.child_pid,
4629 workdir: metadata.workdir.display().to_string(),
4630 output_preview,
4631 output_truncated,
4632 output_path: state
4633 .buffer
4634 .output_path()
4635 .map(|path| path.display().to_string()),
4636 stderr_path: state
4637 .buffer
4638 .stderr_path()
4639 .map(|path| path.display().to_string()),
4640 pty_rows: (metadata.mode == BgMode::Pty).then_some(metadata.pty_rows.unwrap_or(24)),
4641 pty_cols: (metadata.mode == BgMode::Pty).then_some(metadata.pty_cols.unwrap_or(80)),
4642 pty_screen: None,
4643 scanner_report: metadata.scanner_report.clone(),
4644 sandbox_native: metadata.sandbox_native,
4645 sandbox_unavailable: metadata.sandbox_native
4646 && open_task_artifact(&self.paths, TaskArtifact::SandboxUnavailable)
4647 .and_then(|mut file| file.read_all())
4648 .is_ok_and(|bytes| bytes == b"sandbox_unavailable"),
4649 }
4650 }
4651
4652 pub(crate) fn is_running(&self) -> bool {
4653 self.state
4654 .lock()
4655 .map(|state| {
4656 state.metadata.status == BgTaskStatus::Running
4657 || (state.metadata.mode == BgMode::Pty
4658 && state.metadata.status == BgTaskStatus::Killing)
4659 })
4660 .unwrap_or(false)
4661 }
4662
4663 fn is_terminal(&self) -> bool {
4664 self.state
4665 .lock()
4666 .map(|state| state.metadata.status.is_terminal())
4667 .unwrap_or(false)
4668 }
4669
4670 fn mark_terminal_now(&self) {
4671 if let Ok(mut terminal_at) = self.terminal_at.lock() {
4672 if terminal_at.is_none() {
4673 *terminal_at = Some(Instant::now());
4674 }
4675 }
4676 }
4677
4678 fn set_completion_delivered(
4679 &self,
4680 delivered: bool,
4681 registry: &BgTaskRegistry,
4682 ) -> Result<(), String> {
4683 let mut state = self
4684 .state
4685 .lock()
4686 .map_err(|_| "background task lock poisoned".to_string())?;
4687 let updated = registry
4688 .update_task_metadata(&self.paths, |metadata| {
4689 metadata.completion_delivered = delivered;
4690 })
4691 .map_err(|e| format!("failed to update completion delivery: {e}"))?;
4692 state.metadata = updated;
4693 Ok(())
4694 }
4695}
4696
4697#[cfg(unix)]
4718fn reap_piped_child(child_slot: &mut Option<Child>) {
4719 if let Some(mut child) = child_slot.take() {
4720 if matches!(child.try_wait(), Ok(None)) {
4721 let _ = child.wait();
4722 }
4723 }
4724}
4725
4726#[cfg(windows)]
4731fn reap_piped_child(child_slot: &mut Option<Child>) {
4732 *child_slot = None;
4733}
4734
4735fn terminal_metadata_from_marker(
4736 mut metadata: PersistedTask,
4737 marker: ExitMarker,
4738 reason: Option<String>,
4739) -> PersistedTask {
4740 match marker {
4741 ExitMarker::Code(code) => {
4742 let status = if code == 0 {
4743 BgTaskStatus::Completed
4744 } else {
4745 BgTaskStatus::Failed
4746 };
4747 metadata.mark_terminal(status, Some(code), reason);
4748 }
4749 ExitMarker::Killed => metadata.mark_terminal(
4750 BgTaskStatus::Killed,
4751 terminal_exit_code_for_status(&BgTaskStatus::Killed),
4752 reason,
4753 ),
4754 }
4755 metadata
4756}
4757
4758fn terminal_exit_code_for_status(status: &BgTaskStatus) -> Option<i32> {
4759 match status {
4760 BgTaskStatus::TimedOut => Some(124),
4761 BgTaskStatus::Killed => Some(137),
4762 _ => None,
4763 }
4764}
4765
4766fn attach_sandbox_metadata(metadata: &mut PersistedTask, spawn_plan: &SpawnPlan) {
4767 metadata.sandbox_native = spawn_plan.is_native_launcher();
4768 metadata.sandbox_temp_dir = spawn_plan.temp_dir().map(Path::to_path_buf);
4769}
4770
4771#[cfg(unix)]
4772pub(crate) fn resolve_posix_shell() -> PathBuf {
4773 static POSIX_SHELL: OnceLock<PathBuf> = OnceLock::new();
4774 POSIX_SHELL
4775 .get_or_init(|| {
4776 std::env::var_os("BASH")
4777 .filter(|value| !value.is_empty())
4778 .map(PathBuf::from)
4779 .filter(|path| path.exists())
4780 .or_else(|| which::which("bash").ok())
4781 .or_else(|| which::which("zsh").ok())
4782 .unwrap_or_else(|| PathBuf::from("/bin/sh"))
4783 })
4784 .clone()
4785}
4786
4787#[cfg(windows)]
4788fn detached_shell_command_for(
4789 shell: crate::windows_shell::WindowsShell,
4790 command: &str,
4791 exit_path: &Path,
4792 paths: &TaskPaths,
4793 creation_flags: u32,
4794) -> Result<Command, String> {
4795 use crate::windows_shell::WindowsShell;
4796 let wrapper_body = shell.wrapper_script_bytes(command, exit_path);
4809 let wrapper_ext = match shell {
4810 WindowsShell::Pwsh | WindowsShell::Powershell => "ps1",
4811 WindowsShell::Cmd => "bat",
4812 WindowsShell::Posix(_) => "sh",
4816 };
4817 let wrapper_path = paths.dir.join(format!(
4818 "{}.{}",
4819 paths
4820 .json
4821 .file_stem()
4822 .and_then(|s| s.to_str())
4823 .unwrap_or("wrapper"),
4824 wrapper_ext
4825 ));
4826 fs::write(&wrapper_path, wrapper_body)
4827 .map_err(|e| format!("failed to write background bash wrapper script: {e}"))?;
4828
4829 let mut cmd = Command::new(shell.binary().as_ref());
4830 match shell {
4831 WindowsShell::Pwsh | WindowsShell::Powershell => {
4832 cmd.args([
4835 "-NoLogo",
4836 "-NoProfile",
4837 "-NonInteractive",
4838 "-ExecutionPolicy",
4839 "Bypass",
4840 "-File",
4841 ]);
4842 cmd.arg(&wrapper_path);
4843 }
4844 WindowsShell::Cmd => {
4845 cmd.args(["/D", "/C"]);
4852 cmd.arg(&wrapper_path);
4853 }
4854 WindowsShell::Posix(_) => {
4855 cmd.arg(&wrapper_path);
4860 }
4861 }
4862
4863 cmd.creation_flags(creation_flags);
4867 Ok(cmd)
4868}
4869
4870fn spawn_detached_child(
4886 spawn_plan: &SpawnPlan,
4887 command: &str,
4888 paths: &TaskPaths,
4889 workdir: &Path,
4890 env: &HashMap<String, String>,
4891 io_handles: &mut TaskIoHandles,
4892) -> Result<std::process::Child, String> {
4893 #[cfg(not(windows))]
4894 let _ = command;
4895 #[cfg(not(windows))]
4896 {
4897 use std::os::fd::AsRawFd;
4898
4899 let stdout = io_handles
4900 .clone_file(TaskArtifact::Stdout)
4901 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
4902 let stderr = io_handles
4903 .clone_file(TaskArtifact::Stderr)
4904 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
4905 let prepared = spawn_plan
4906 .prepared_task()
4907 .ok_or_else(|| "background task payload was not prepared".to_string())?;
4908 let payload = prepared.invocation()?;
4909 let exit = io_handles
4910 .inheritable_file(TaskArtifact::Exit)
4911 .map_err(|e| format!("failed to inherit exit marker handle: {e}"))?;
4912 let failure = io_handles
4913 .inheritable_file(TaskArtifact::SandboxUnavailable)
4914 .map_err(|e| format!("failed to inherit sandbox failure marker handle: {e}"))?;
4915 let shell = spawn_plan
4916 .host_shell_path()
4917 .map(Path::to_path_buf)
4918 .unwrap_or_else(resolve_posix_shell);
4919 let args = vec![
4920 OsString::from("-c"),
4921 payload.wrapper_text.clone(),
4922 OsString::from("aft-payload-wrapper"),
4923 shell.as_os_str().to_os_string(),
4924 payload.command_text.clone(),
4925 OsString::from(crate::sandbox_spawn::CHILD_EXIT_FD.to_string()),
4926 ];
4927 let (mut child_command, profile_handle) = crate::sandbox_spawn::detached_command_for_plan(
4928 spawn_plan,
4929 std::ffi::OsStr::new("/bin/sh"),
4930 &args,
4931 &paths.json,
4932 crate::sandbox_spawn::CHILD_EXIT_FD,
4933 crate::sandbox_spawn::CHILD_FAILURE_FD,
4934 )?;
4935 crate::sandbox_spawn::apply_marker_fd_allowlist(
4936 &mut child_command,
4937 exit.as_raw_fd(),
4938 failure.as_raw_fd(),
4939 )?;
4940 child_command
4941 .current_dir(workdir)
4942 .envs(env)
4943 .stdin(Stdio::null())
4944 .stdout(Stdio::from(stdout))
4945 .stderr(Stdio::from(stderr));
4946 crate::sandbox_spawn::apply_sandbox_environment(spawn_plan, &mut child_command, env);
4947 let child = child_command
4948 .spawn()
4949 .map_err(|e| format!("failed to spawn background bash command: {e}"));
4950 drop((payload, exit, failure, profile_handle));
4951 child
4952 }
4953 #[cfg(windows)]
4954 {
4955 use crate::windows_shell::shell_candidates;
4956 match spawn_plan {
4957 SpawnPlan::Unsandboxed | SpawnPlan::Host { .. } => {}
4958 SpawnPlan::Refused { code, .. } => return Err((*code).to_string()),
4959 SpawnPlan::Launcher { .. } => return Err("sandbox_unavailable".to_string()),
4960 }
4961 let candidates: Vec<crate::windows_shell::WindowsShell> = shell_candidates();
4972 const FLAG_CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200;
4985 const FLAG_CREATE_BREAKAWAY_FROM_JOB: u32 = 0x0100_0000;
4986 const FLAG_CREATE_NO_WINDOW: u32 = 0x0800_0000;
4987 let with_breakaway =
4988 FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP | FLAG_CREATE_BREAKAWAY_FROM_JOB;
4989 let without_breakaway = FLAG_CREATE_NO_WINDOW | FLAG_CREATE_NEW_PROCESS_GROUP;
4990 let mut last_error: Option<String> = None;
4991 for (idx, shell) in candidates.iter().enumerate() {
4992 for &flags in &[with_breakaway, without_breakaway] {
4996 let stdout = io_handles
4999 .clone_file(TaskArtifact::Stdout)
5000 .map_err(|e| format!("failed to clone stdout capture handle: {e}"))?;
5001 let stderr = io_handles
5002 .clone_file(TaskArtifact::Stderr)
5003 .map_err(|e| format!("failed to clone stderr capture handle: {e}"))?;
5004 let mut cmd =
5005 detached_shell_command_for(shell.clone(), command, &paths.exit, paths, flags)?;
5006 cmd.current_dir(workdir)
5007 .envs(env)
5008 .stdin(Stdio::null())
5009 .stdout(Stdio::from(stdout))
5010 .stderr(Stdio::from(stderr));
5011 match cmd.spawn() {
5012 Ok(child) => {
5013 if idx > 0 {
5014 crate::slog_warn!("background bash spawn fell back to {} after {} earlier candidate(s) failed; \
5015 the cached PATH probe disagreed with runtime spawn — likely PATH \
5016 inheritance, antivirus / AppLocker / Defender ASR, or sandbox policy.",
5017 shell.binary(),
5018 idx);
5019 }
5020 if flags == without_breakaway {
5021 crate::slog_warn!(
5022 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected \
5023 (likely a restrictive Job Object — CI sandbox or MDM policy). \
5024 Spawned without breakaway; the bg task will be torn down if the \
5025 AFT process group is killed."
5026 );
5027 }
5028 return Ok(child);
5029 }
5030 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
5031 crate::slog_warn!("background bash spawn: {} returned NotFound at runtime — trying next candidate",
5032 shell.binary());
5033 last_error = Some(format!("{}: {e}", shell.binary()));
5034 break;
5037 }
5038 Err(e) if flags == with_breakaway && e.raw_os_error() == Some(5) => {
5039 crate::slog_warn!(
5041 "background bash spawn: CREATE_BREAKAWAY_FROM_JOB rejected with \
5042 Access Denied — retrying {} without breakaway",
5043 shell.binary()
5044 );
5045 last_error = Some(format!("{}: {e}", shell.binary()));
5046 continue;
5047 }
5048 Err(e) => {
5049 return Err(format!(
5050 "failed to spawn background bash command via {}: {e}",
5051 shell.binary()
5052 ));
5053 }
5054 }
5055 }
5056 }
5057 Err(format!(
5058 "failed to spawn background bash command: no Windows shell could be spawned. \
5059 Last error: {}. PATH-probed candidates: {:?}",
5060 last_error.unwrap_or_else(|| "no candidates were attempted".to_string()),
5061 candidates.iter().map(|s| s.binary()).collect::<Vec<_>>()
5062 ))
5063 }
5064}
5065
5066#[cfg(test)]
5067fn random_slug() -> String {
5068 let mut bytes = [0u8; 8];
5076 getrandom::fill(&mut bytes).unwrap_or_else(|_| {
5078 let t = SystemTime::now()
5080 .duration_since(UNIX_EPOCH)
5081 .map(|d| d.as_nanos() as u64)
5082 .unwrap_or(0);
5083 let p = u64::from(std::process::id());
5084 bytes.copy_from_slice(&(t ^ p.rotate_left(32)).to_le_bytes());
5085 });
5086 let hex: String = bytes.iter().map(|b| format!("{b:02x}")).collect();
5088 format!("bash-{hex}")
5089}
5090
5091#[cfg(test)]
5092mod tests {
5093 use std::collections::HashMap;
5094 use std::fs;
5095 use std::io::Write;
5096 #[cfg(unix)]
5097 use std::os::unix::fs::PermissionsExt;
5098 use std::sync::atomic::{AtomicBool, AtomicUsize};
5099 use std::sync::{Arc, Mutex};
5100 use std::time::{Duration, Instant, SystemTime};
5101
5102 use super::*;
5103 use crate::bash_background::persistence::{read_task, task_paths, write_task};
5104
5105 #[cfg(unix)]
5106 const QUICK_SUCCESS_COMMAND: &str = "true";
5107 #[cfg(windows)]
5108 const QUICK_SUCCESS_COMMAND: &str = "cmd /c exit 0";
5109
5110 #[cfg(unix)]
5111 const LONG_RUNNING_COMMAND: &str = "sleep 5";
5112 #[cfg(windows)]
5113 const LONG_RUNNING_COMMAND: &str = "cmd /c timeout /t 5 /nobreak > nul";
5114
5115 #[test]
5116 fn bash_memory_estimate_is_zero_when_empty_and_nonzero_for_completion_cache() {
5117 let registry = BgTaskRegistry::default();
5118 assert_eq!(registry.estimated_memory().estimated_bytes, Some(0));
5119 registry
5120 .inner
5121 .completions
5122 .lock()
5123 .unwrap()
5124 .push_back(BgCompletion {
5125 task_id: "bash-memory".to_string(),
5126 session_id: "session-memory".to_string(),
5127 status: BgTaskStatus::Completed,
5128 exit_code: Some(0),
5129 command: "printf memory".to_string(),
5130 output_preview: "resident completion output".to_string(),
5131 output_truncated: false,
5132 original_tokens: None,
5133 compressed_tokens: None,
5134 tokens_skipped: false,
5135 });
5136 let estimate = registry.estimated_memory();
5137 assert!(estimate.estimated_bytes.unwrap() > 0);
5138 assert_eq!(estimate.counts["completion_caches"], 1);
5139 assert_eq!(estimate.counts["sessions"], 1);
5140 }
5141
5142 #[test]
5143 fn gh_structured_detection_rejects_piped_commands() {
5144 assert!(is_gh_structured_command(
5145 "gh issue list --json number,title"
5146 ));
5147 assert!(is_gh_structured_command(
5148 "cd repo && gh issue list --json number,title"
5149 ));
5150
5151 assert!(!is_gh_structured_command(
5152 "gh issue list --json number,title | jq '.[]'"
5153 ));
5154 assert!(!is_gh_structured_command(
5155 "gh issue list --json number,title |"
5156 ));
5157 }
5158
5159 fn insert_terminal_piped_task(
5160 registry: &BgTaskRegistry,
5161 dir: &tempfile::TempDir,
5162 command: &str,
5163 stdout: &str,
5164 stderr: &str,
5165 compressed: bool,
5166 ) -> (String, Arc<BgTask>) {
5167 let task_id = random_slug();
5168 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5169 fs::create_dir_all(&paths.dir).unwrap();
5170 fs::write(&paths.stdout, stdout).unwrap();
5171 fs::write(&paths.stderr, stderr).unwrap();
5172 let mut metadata = PersistedTask::starting(
5173 task_id.clone(),
5174 "session".to_string(),
5175 command.to_string(),
5176 dir.path().to_path_buf(),
5177 Some(dir.path().to_path_buf()),
5178 Some(30_000),
5179 true,
5180 compressed,
5181 );
5182 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
5183 write_task(&paths.json, &metadata).unwrap();
5184 registry
5185 .insert_rehydrated_task(metadata, paths, true)
5186 .expect("insert terminal task");
5187 let task = registry.task_for_session(&task_id, "session").unwrap();
5188 (task_id, task)
5189 }
5190
5191 #[test]
5192 fn bash_zero_preview_running_status_skips_output_read_while_explicit_preview_reads() {
5193 let registry = BgTaskRegistry::default();
5194 let dir = tempfile::tempdir().unwrap();
5195 let task_id = random_slug();
5196 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5197 fs::create_dir_all(&paths.dir).unwrap();
5198 fs::write(&paths.stdout, "live output\n").unwrap();
5199 fs::write(&paths.stderr, "").unwrap();
5200 let stdout_path = paths.stdout.clone();
5201 let mut metadata = PersistedTask::starting(
5202 task_id.clone(),
5203 "session".to_string(),
5204 "sleep 60".to_string(),
5205 dir.path().to_path_buf(),
5206 Some(dir.path().to_path_buf()),
5207 Some(30_000),
5208 true,
5209 false,
5210 );
5211 metadata.status = BgTaskStatus::Running;
5212 write_task(&paths.json, &metadata).unwrap();
5213 registry
5214 .insert_rehydrated_task(metadata, paths, false)
5215 .expect("insert running task");
5216
5217 crate::bash_background::buffer::reset_tail_read_count(&stdout_path);
5218 for _ in 0..5 {
5219 let snapshot = registry
5220 .status(&task_id, "session", Some(dir.path()), Some(dir.path()), 0)
5221 .expect("running snapshot");
5222 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
5223 assert!(snapshot.output_preview.is_empty());
5224 }
5225 assert_eq!(
5226 crate::bash_background::buffer::tail_read_count(&stdout_path),
5227 0
5228 );
5229
5230 let snapshot = registry
5231 .status(
5232 &task_id,
5233 "session",
5234 Some(dir.path()),
5235 Some(dir.path()),
5236 RUNNING_OUTPUT_PREVIEW_BYTES,
5237 )
5238 .expect("explicit running snapshot");
5239 assert_eq!(snapshot.output_preview, "live output\n");
5240 assert_eq!(
5241 crate::bash_background::buffer::tail_read_count(&stdout_path),
5242 1
5243 );
5244 }
5245
5246 #[test]
5247 fn artifact_read_capability_requires_exact_canonical_path_and_session() {
5248 let registry = BgTaskRegistry::default();
5249 let dir = tempfile::tempdir().unwrap();
5250 let (_task_id, task) = insert_terminal_piped_task(
5251 ®istry,
5252 &dir,
5253 "printf output",
5254 "stdout\n",
5255 "stderr\n",
5256 true,
5257 );
5258 fs::write(&task.paths.exit, "0\n").unwrap();
5259
5260 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stdout));
5261 assert!(registry.is_session_owned_artifact_path("session", &task.paths.stderr));
5262 assert!(registry.is_session_owned_artifact_path("session", &task.paths.exit));
5263 assert!(!registry.is_session_owned_artifact_path("different-session", &task.paths.stdout));
5264 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.json));
5265
5266 let unregistered = task.paths.dir.join("unregistered-output");
5267 fs::write(&unregistered, "not a task artifact\n").unwrap();
5268 assert!(!registry.is_session_owned_artifact_path("session", &unregistered));
5269 }
5270
5271 #[cfg(unix)]
5272 #[test]
5273 fn artifact_directory_symlink_does_not_create_a_prefix_exception() {
5274 let registry = BgTaskRegistry::default();
5275 let dir = tempfile::tempdir().unwrap();
5276 let project = dir.path().join("project");
5277 fs::create_dir_all(&project).unwrap();
5278 let (_task_id, task) =
5279 insert_terminal_piped_task(®istry, &dir, "printf output", "stdout\n", "", true);
5280 let link = project.join("task-artifacts");
5281 std::os::unix::fs::symlink(&task.paths.dir, &link).unwrap();
5282 let unregistered = task.paths.dir.join("unregistered-output");
5283 fs::write(&unregistered, "not registered\n").unwrap();
5284
5285 assert!(!registry.is_session_owned_artifact_path("session", &link));
5286 assert!(
5287 !registry.is_session_owned_artifact_path("session", &link.join("unregistered-output"))
5288 );
5289 assert!(registry.is_session_owned_artifact_path(
5290 "session",
5291 &link.join(task.paths.stdout.file_name().unwrap())
5292 ));
5293
5294 let outside = dir.path().join("outside-secret");
5295 fs::write(&outside, "must stay private\n").unwrap();
5296 fs::remove_file(&task.paths.stdout).unwrap();
5297 std::os::unix::fs::symlink(&outside, &task.paths.stdout).unwrap();
5298 assert!(!registry.is_session_owned_artifact_path("session", &task.paths.stdout));
5299 }
5300
5301 #[test]
5302 fn recovery_footer_uses_bash_status_when_artifact_is_not_registered() {
5303 let registry = BgTaskRegistry::default();
5304 let dir = tempfile::tempdir().unwrap();
5305 let task_id = "bash-1111111111111111";
5306 let paths = task_paths(dir.path(), "session", task_id).unwrap();
5307 fs::create_dir_all(&paths.dir).unwrap();
5308 fs::write(
5309 &paths.stdout,
5310 format!("{}tail\n", "output-line\n".repeat(2_000)),
5311 )
5312 .unwrap();
5313 fs::write(&paths.stderr, "").unwrap();
5314 let mut metadata = PersistedTask::starting(
5315 task_id.to_string(),
5316 "session".to_string(),
5317 "printf output".to_string(),
5318 dir.path().to_path_buf(),
5319 Some(dir.path().to_path_buf()),
5320 Some(30_000),
5321 true,
5322 true,
5323 );
5324 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
5325 write_task(&paths.json, &metadata).unwrap();
5326
5327 let cache = registry
5328 .render_terminal_output_from_paths(&metadata, &paths)
5329 .expect("terminal render");
5330
5331 assert!(cache
5332 .output_preview
5333 .contains("use bash_status({taskId: \"bash-1111111111111111\"})"));
5334 assert!(!cache.output_preview.contains("full output: read "));
5335 }
5336
5337 fn insert_terminal_pty_task(
5338 registry: &BgTaskRegistry,
5339 dir: &tempfile::TempDir,
5340 pty_output: &str,
5341 ) -> (String, Arc<BgTask>) {
5342 let task_id = random_slug();
5343 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5344 fs::create_dir_all(&paths.dir).unwrap();
5345 fs::write(&paths.pty, pty_output).unwrap();
5346 let mut metadata = PersistedTask::starting(
5347 task_id.clone(),
5348 "session".to_string(),
5349 "python".to_string(),
5350 dir.path().to_path_buf(),
5351 Some(dir.path().to_path_buf()),
5352 Some(30_000),
5353 true,
5354 true,
5355 );
5356 metadata.mode = BgMode::Pty;
5357 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
5358 write_task(&paths.json, &metadata).unwrap();
5359 registry
5360 .insert_rehydrated_task(metadata, paths, true)
5361 .expect("insert terminal pty task");
5362 let task = registry.task_for_session(&task_id, "session").unwrap();
5363 (task_id, task)
5364 }
5365
5366 #[cfg(unix)]
5367 fn wait_for_terminal_snapshot(
5368 registry: &BgTaskRegistry,
5369 task_id: &str,
5370 session_id: &str,
5371 project: &Path,
5372 storage: &Path,
5373 ) -> BgTaskSnapshot {
5374 let started = Instant::now();
5375 loop {
5376 let snapshot = registry
5377 .status(task_id, session_id, Some(project), Some(storage), 4096)
5378 .expect("spawned task should be visible to status");
5379 if snapshot.info.status.is_terminal() {
5380 return snapshot;
5381 }
5382 assert!(
5383 started.elapsed() < Duration::from_secs(10),
5384 "timed out waiting for task {task_id} to finish; last status={:?}",
5385 snapshot.info.status
5386 );
5387 std::thread::sleep(Duration::from_millis(50));
5388 }
5389 }
5390
5391 fn write_running_project_task(storage: &Path, project: &Path, session: &str, task_id: &str) {
5392 let paths = task_paths(storage, session, task_id).unwrap();
5393 let mut metadata = PersistedTask::starting(
5394 task_id.to_string(),
5395 session.to_string(),
5396 "sleep 60".to_string(),
5397 project.to_path_buf(),
5398 Some(project.to_path_buf()),
5399 Some(30_000),
5400 true,
5401 true,
5402 );
5403 metadata.status = BgTaskStatus::Running;
5404 write_task(&paths.json, &metadata).unwrap();
5405 fs::write(&paths.stdout, "still running\n").unwrap();
5406 fs::write(&paths.stderr, "").unwrap();
5407 }
5408
5409 #[test]
5410 fn status_replay_filters_same_session_by_project_root() {
5411 let project_a = tempfile::tempdir().unwrap();
5412 let project_b = tempfile::tempdir().unwrap();
5413 let storage = tempfile::tempdir().unwrap();
5414 let session = "shared-session";
5415 let task_id = "bash-2222222222222222";
5416 write_running_project_task(storage.path(), project_a.path(), session, task_id);
5417
5418 let actor_b = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5419 assert!(actor_b
5420 .status(
5421 task_id,
5422 session,
5423 Some(project_b.path()),
5424 Some(storage.path()),
5425 1024,
5426 )
5427 .is_none());
5428 assert!(actor_b.task_for_session(task_id, session).is_none());
5429
5430 let actor_a = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5431 let snapshot = actor_a
5432 .status(
5433 task_id,
5434 session,
5435 Some(project_a.path()),
5436 Some(storage.path()),
5437 1024,
5438 )
5439 .expect("owning project should replay its task");
5440 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
5441 }
5442
5443 #[cfg(unix)]
5444 #[test]
5445 fn multiline_pipeline_stdout_persists_all_lines_after_terminal_status() {
5446 let cases = [
5447 (
5448 "long-first",
5449 "sleep 0.5; printf 'one\\n' | cat\nprintf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
5450 vec!["one", "1", "three"],
5451 ),
5452 (
5453 "short-first",
5454 "printf 'one\\n' | cat\nsleep 0.2; printf 'two\\n' | grep -c two\nprintf 'three\\n' | cat",
5455 vec!["one", "1", "three"],
5456 ),
5457 (
5458 "failing-middle",
5459 "sleep 0.2; printf 'one\\n' | cat\nfalse; printf 'after-false\\n' | cat\nprintf 'three\\n' | cat",
5460 vec!["one", "after-false", "three"],
5461 ),
5462 ];
5463
5464 for (name, command, expected_lines) in cases {
5465 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
5466 let dir = tempfile::tempdir().unwrap();
5467 let session_id = format!("session-{name}");
5468 let task_id = registry
5469 .spawn(
5470 SpawnPlan::Unsandboxed,
5471 command,
5472 session_id.clone(),
5473 dir.path().to_path_buf(),
5474 HashMap::new(),
5475 Some(Duration::from_secs(30)),
5476 dir.path().to_path_buf(),
5477 10,
5478 true,
5479 true,
5480 Some(dir.path().to_path_buf()),
5481 )
5482 .unwrap();
5483
5484 let snapshot = wait_for_terminal_snapshot(
5485 ®istry,
5486 &task_id,
5487 &session_id,
5488 dir.path(),
5489 dir.path(),
5490 );
5491 assert_eq!(
5492 snapshot.info.status,
5493 BgTaskStatus::Completed,
5494 "{name}: task should complete; snapshot={snapshot:?}"
5495 );
5496 assert_eq!(
5497 snapshot.exit_code,
5498 Some(0),
5499 "{name}: script should use the final command's exit code"
5500 );
5501
5502 let stdout = String::from_utf8(
5503 registry
5504 .read_artifact(&task_id, &session_id, TaskArtifact::Stdout)
5505 .expect("read validated stdout artifact"),
5506 )
5507 .expect("stdout is UTF-8");
5508 let lines: Vec<&str> = stdout.lines().collect();
5509 assert_eq!(
5510 lines, expected_lines,
5511 "{name}: raw stdout artifact must include every newline-separated command's output"
5512 );
5513 }
5514 }
5515
5516 #[test]
5517 fn recognizes_all_recovery_marker_forms() {
5518 assert!(is_recovery_marker(
5519 "[truncated output; full output: read \"/tmp/out\"]"
5520 ));
5521 assert!(is_recovery_marker(
5522 "[omitted output; see remaining: tail -n +42 \"/tmp/out\"]"
5523 ));
5524 assert!(is_recovery_marker(
5525 "[truncated output; full output unavailable]"
5526 ));
5527 assert!(is_recovery_marker(
5528 r#"[truncated 123 bytes from saved output prefix; retained output: read "/tmp/out"]"#
5529 ));
5530 }
5531
5532 #[test]
5533 fn recovery_marker_reports_disk_prefix_truncation_as_retained_output() {
5534 let recovery = RecoveryContext {
5535 dropped_by_class: BTreeMap::new(),
5536 had_inner_drop: false,
5537 offset_hint_eligible: false,
5538 offset_start_line: None,
5539 byte_truncated: false,
5540 disk_truncated_prefix_bytes: 4096,
5541 output_path: Some("/tmp/stdout".to_string()),
5542 stderr_path: None,
5543 include_stderr_path: false,
5544 artifact_access: ArtifactRecoveryAccess {
5545 task_id: "bash-test".to_string(),
5546 readable: true,
5547 },
5548 };
5549
5550 let marker = recovery_marker(&recovery).expect("disk truncation must emit marker");
5551
5552 assert!(marker.contains("truncated 4096 bytes from saved output prefix"));
5553 assert!(marker.contains(r#"retained output: read "/tmp/stdout""#));
5554 assert!(!marker.contains("full output: read"));
5555 }
5556
5557 #[test]
5558 fn killed_exit_marker_sets_nonzero_sentinel_exit_code() {
5559 let metadata = PersistedTask::starting(
5560 "task".to_string(),
5561 "session".to_string(),
5562 "cargo test".to_string(),
5563 PathBuf::from("/tmp"),
5564 None,
5565 None,
5566 true,
5567 true,
5568 );
5569
5570 let terminal = terminal_metadata_from_marker(metadata, ExitMarker::Killed, None);
5571
5572 assert_eq!(terminal.status, BgTaskStatus::Killed);
5573 assert_eq!(terminal.exit_code, Some(137));
5574 }
5575
5576 #[test]
5577 fn terminal_status_polls_use_cached_render_once_and_off_lock() {
5578 let registry = BgTaskRegistry::default();
5579 let dir = tempfile::tempdir().unwrap();
5580 let (_task_id, task) = insert_terminal_piped_task(
5581 ®istry,
5582 &dir,
5583 "custom-tool --verbose",
5584 &"stdout line\n".repeat(200_000),
5585 "",
5586 true,
5587 );
5588 let calls = Arc::new(AtomicUsize::new(0));
5589 let saw_unlocked_state = Arc::new(AtomicBool::new(false));
5590 let task_holder = Arc::new(Mutex::new(Some(Arc::clone(&task))));
5591 let calls_for_closure = Arc::clone(&calls);
5592 let unlocked_for_closure = Arc::clone(&saw_unlocked_state);
5593 let task_for_closure = Arc::clone(&task_holder);
5594 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
5595 calls_for_closure.fetch_add(1, Ordering::SeqCst);
5596 if let Some(task) = task_for_closure.lock().unwrap().as_ref() {
5597 if task.state.try_lock().is_ok() {
5598 unlocked_for_closure.store(true, Ordering::SeqCst);
5599 }
5600 }
5601 CompressionResult::new(format!("compressed {} bytes", output.len()))
5602 });
5603
5604 let first = registry
5605 .status(
5606 &task.task_id,
5607 "session",
5608 None,
5609 Some(dir.path()),
5610 RUNNING_OUTPUT_PREVIEW_BYTES,
5611 )
5612 .unwrap();
5613 let second = registry
5614 .status(
5615 &task.task_id,
5616 "session",
5617 None,
5618 Some(dir.path()),
5619 RUNNING_OUTPUT_PREVIEW_BYTES,
5620 )
5621 .unwrap();
5622 let listed = registry.list(RUNNING_OUTPUT_PREVIEW_BYTES);
5623
5624 assert_eq!(
5625 calls.load(Ordering::SeqCst),
5626 1,
5627 "terminal render must be cached"
5628 );
5629 assert!(
5630 saw_unlocked_state.load(Ordering::SeqCst),
5631 "compressor must run after releasing the task state lock"
5632 );
5633 assert!(first.output_preview.starts_with("compressed "));
5634 assert_eq!(second.output_preview, first.output_preview);
5635 assert_eq!(listed[0].output_preview, first.output_preview);
5636 }
5637
5638 #[test]
5639 fn completion_preview_success_keeps_tail_only() {
5640 let registry = BgTaskRegistry::default();
5645 let dir = tempfile::tempdir().unwrap();
5646 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
5647 let (_task_id, task) =
5648 insert_terminal_piped_task(®istry, &dir, "cat big.log", &output, "", false);
5649
5650 registry.post_terminal_transition(&task, true).unwrap();
5651 let completions = registry.drain_completions_for_session(Some("session"));
5652 assert_eq!(completions.len(), 1);
5653 let preview = &completions[0].output_preview;
5654 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
5655 assert!(!preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
5656 assert!(completions[0].output_truncated);
5657 }
5658
5659 #[test]
5660 fn completion_preview_failure_keeps_head_and_tail() {
5661 let registry = BgTaskRegistry::default();
5664 let dir = tempfile::tempdir().unwrap();
5665 let output = format!("HEAD-SIGNAL\n{}TAIL-SIGNAL\n", "middle\n".repeat(2_000));
5666 let task_id = random_slug();
5667 let paths = task_paths(dir.path(), "session", &task_id).unwrap();
5668 fs::create_dir_all(&paths.dir).unwrap();
5669 fs::write(&paths.stdout, &output).unwrap();
5670 fs::write(&paths.stderr, "").unwrap();
5671 let mut metadata = PersistedTask::starting(
5672 task_id.clone(),
5673 "session".to_string(),
5674 "cat big.log".to_string(),
5675 dir.path().to_path_buf(),
5676 Some(dir.path().to_path_buf()),
5677 Some(30_000),
5678 true,
5679 false,
5680 );
5681 metadata.mark_terminal(BgTaskStatus::Failed, Some(1), None);
5682 write_task(&paths.json, &metadata).unwrap();
5683 registry
5684 .insert_rehydrated_task(metadata, paths, true)
5685 .expect("insert terminal task");
5686 let task = registry.task_for_session(&task_id, "session").unwrap();
5687
5688 registry.post_terminal_transition(&task, true).unwrap();
5689 let completions = registry.drain_completions_for_session(Some("session"));
5690 assert_eq!(completions.len(), 1);
5691 let preview = &completions[0].output_preview;
5692 assert!(preview.contains("HEAD-SIGNAL"), "preview was {preview:?}");
5693 assert!(preview.contains("TAIL-SIGNAL"), "preview was {preview:?}");
5694 }
5695
5696 #[test]
5697 fn has_completions_for_session_matches_pending_delivery() {
5698 let registry = BgTaskRegistry::default();
5699 assert!(!registry.has_completions_for_session(Some("session")));
5700 assert!(!registry.has_completions_for_session(None));
5701
5702 let dir = tempfile::tempdir().unwrap();
5703 let (_task_id, task) =
5704 insert_terminal_piped_task(®istry, &dir, QUICK_SUCCESS_COMMAND, "done\n", "", false);
5705 registry.post_terminal_transition(&task, true).unwrap();
5706
5707 assert!(registry.has_completions_for_session(Some("session")));
5708 assert!(registry.has_completions_for_session(None));
5709 assert!(!registry.has_completions_for_session(Some("other-session")));
5710
5711 let completions = registry.drain_completions_for_session(Some("session"));
5712 assert_eq!(completions.len(), 1);
5713 assert_eq!(completions[0].task_id, task.task_id);
5714 }
5715
5716 #[test]
5717 fn structured_gh_json_survives_intact_and_ignores_stderr() {
5718 let registry = BgTaskRegistry::default();
5719 let dir = tempfile::tempdir().unwrap();
5720 let calls = Arc::new(AtomicUsize::new(0));
5721 let calls_for_closure = Arc::clone(&calls);
5722 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
5723 calls_for_closure.fetch_add(1, Ordering::SeqCst);
5724 CompressionResult::new(output)
5725 });
5726 let (task_id, _task) = insert_terminal_piped_task(
5727 ®istry,
5728 &dir,
5729 "gh pr view 123 --json body",
5730 "{\"body\":\"hello\"}",
5731 "warning: stderr must not join json",
5732 true,
5733 );
5734
5735 let snapshot = registry
5736 .status(
5737 &task_id,
5738 "session",
5739 None,
5740 Some(dir.path()),
5741 RUNNING_OUTPUT_PREVIEW_BYTES,
5742 )
5743 .unwrap();
5744
5745 assert_eq!(snapshot.output_preview, "{\"body\":\"hello\"}");
5746 assert!(!snapshot.output_preview.contains("warning"));
5747 assert!(!snapshot.output_truncated);
5748 assert_eq!(
5749 calls.load(Ordering::SeqCst),
5750 0,
5751 "structured JSON bypasses compression"
5752 );
5753 }
5754
5755 #[test]
5756 fn registry_emits_single_recovery_marker_for_class_drops() {
5757 let registry = BgTaskRegistry::default();
5758 let dir = tempfile::tempdir().unwrap();
5759 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
5760 let mut dropped = BTreeMap::new();
5761 dropped.insert(DropClass::Error, 18);
5762 dropped.insert(DropClass::Warning, 6);
5763 CompressionResult::with_class_drops("kept diagnostic", dropped)
5764 });
5765 let (task_id, task) =
5766 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
5767
5768 let snapshot = registry
5769 .status(
5770 &task_id,
5771 "session",
5772 None,
5773 Some(dir.path()),
5774 RUNNING_OUTPUT_PREVIEW_BYTES,
5775 )
5776 .unwrap();
5777
5778 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
5779 assert!(snapshot.output_preview.contains("+18 more errors"));
5780 assert!(snapshot.output_preview.contains("+6 more warnings"));
5781 assert!(snapshot
5782 .output_preview
5783 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
5784 assert!(!snapshot.output_preview.contains("tail -n +"));
5785 assert!(snapshot.output_truncated);
5786 }
5787
5788 #[test]
5789 fn registry_marker_reports_semantic_and_byte_drops_once() {
5790 let registry = BgTaskRegistry::default();
5791 let dir = tempfile::tempdir().unwrap();
5792 registry.set_compressor_with_exit_code(move |_command, _output, _exit_code| {
5793 let mut dropped = BTreeMap::new();
5794 dropped.insert(DropClass::Error, 1);
5795 CompressionResult::with_class_drops(
5796 format!("HEAD-SIGNAL\n{}TAIL-SIGNAL", "middle\n".repeat(8_000)),
5797 dropped,
5798 )
5799 });
5800 let (task_id, _task) =
5801 insert_terminal_piped_task(®istry, &dir, "custom-tool", "raw", "", true);
5802
5803 let snapshot = registry
5804 .status(
5805 &task_id,
5806 "session",
5807 None,
5808 Some(dir.path()),
5809 RUNNING_OUTPUT_PREVIEW_BYTES,
5810 )
5811 .unwrap();
5812
5813 assert_eq!(snapshot.output_preview.matches("full output:").count(), 1);
5814 assert!(snapshot.output_preview.contains("+1 more error"));
5815 assert!(snapshot.output_preview.contains("truncated output"));
5816 assert!(snapshot.output_preview.contains("HEAD-SIGNAL"));
5817 assert!(snapshot.output_preview.contains("TAIL-SIGNAL"));
5818 assert!(!snapshot.output_preview.contains("...<truncated"));
5819 assert!(snapshot.output_truncated);
5820 }
5821
5822 #[test]
5823 fn cargo_stderr_class_drops_name_both_capture_paths() {
5824 let registry = BgTaskRegistry::default();
5825 let dir = tempfile::tempdir().unwrap();
5826 let filter_registry = crate::compress::toml_filter::FilterRegistry::default();
5827 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
5828 crate::compress::compress_with_registry_exit_code(
5829 command,
5830 &output,
5831 exit_code,
5832 &filter_registry,
5833 )
5834 });
5835 let stderr = (0..22)
5836 .map(|index| {
5837 format!(
5838 "error: cargo failure {index}\n --> src/lib.rs:{}:1\n |\n{} | boom\n",
5839 index + 1,
5840 index + 1
5841 )
5842 })
5843 .collect::<Vec<_>>()
5844 .join("\n");
5845 let (task_id, task) = insert_terminal_piped_task(
5846 ®istry,
5847 &dir,
5848 "cargo check",
5849 "Finished dev [unoptimized] target(s) in 0.01s\n",
5850 &stderr,
5851 true,
5852 );
5853
5854 let snapshot = registry
5855 .status(
5856 &task_id,
5857 "session",
5858 None,
5859 Some(dir.path()),
5860 RUNNING_OUTPUT_PREVIEW_BYTES,
5861 )
5862 .unwrap();
5863
5864 assert!(snapshot.output_preview.contains("+2 more errors"));
5865 assert!(snapshot
5866 .output_preview
5867 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
5868 assert!(snapshot
5869 .output_preview
5870 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
5871 assert!(!snapshot.output_preview.contains("tail -n +"));
5872 }
5873
5874 #[test]
5875 fn over_ceiling_structured_json_uses_pointer_not_partial_json() {
5876 let registry = BgTaskRegistry::default();
5877 let dir = tempfile::tempdir().unwrap();
5878 let body = format!("{{\"body\":\"{}\"}}", "x".repeat(60 * 1024));
5879 let (task_id, task) = insert_terminal_piped_task(
5880 ®istry,
5881 &dir,
5882 "cd /repo && gh pr view 123 --json body",
5883 &body,
5884 "",
5885 true,
5886 );
5887
5888 let snapshot = registry
5889 .status(
5890 &task_id,
5891 "session",
5892 None,
5893 Some(dir.path()),
5894 RUNNING_OUTPUT_PREVIEW_BYTES,
5895 )
5896 .unwrap();
5897
5898 assert!(snapshot.output_preview.starts_with("[JSON output "));
5899 assert!(snapshot
5900 .output_preview
5901 .contains(&task.paths.stdout.display().to_string()));
5902 assert!(!snapshot.output_preview.contains(&"x".repeat(1024)));
5903 assert!(snapshot.output_truncated);
5904 }
5905
5906 #[test]
5907 fn toml_strip_tail_cap_uses_full_output_hint_not_offset_hint() {
5908 let registry = BgTaskRegistry::default();
5909 let dir = tempfile::tempdir().unwrap();
5910 let filter_registry = crate::compress::toml_filter::build_registry(
5911 crate::compress::builtin_filters::ALL,
5912 None,
5913 None,
5914 );
5915 registry.set_compressor_with_exit_code(move |command, output, exit_code| {
5916 crate::compress::compress_with_registry_exit_code(
5917 command,
5918 &output,
5919 exit_code,
5920 &filter_registry,
5921 )
5922 });
5923 let stdout = format!(
5924 "make[1]: Entering directory `/tmp`\n{}",
5925 (0..100)
5926 .map(|index| format!("compile line {index}"))
5927 .collect::<Vec<_>>()
5928 .join("\n")
5929 );
5930 let (task_id, task) =
5931 insert_terminal_piped_task(®istry, &dir, "make all", &stdout, "", true);
5932
5933 let snapshot = registry
5934 .status(
5935 &task_id,
5936 "session",
5937 None,
5938 Some(dir.path()),
5939 RUNNING_OUTPUT_PREVIEW_BYTES,
5940 )
5941 .unwrap();
5942
5943 assert!(snapshot.output_preview.contains("compile line 99"));
5944 assert!(snapshot.output_preview.contains(&format!(
5945 "full output: read \"{}\"",
5946 task.paths.stdout.display()
5947 )));
5948 assert!(!snapshot
5949 .output_preview
5950 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
5951 assert!(!snapshot.output_preview.contains("tail -n +"));
5952 }
5953
5954 #[test]
5955 fn compressed_false_raw_passthrough_uses_wider_head_tail_cap() {
5956 let registry = BgTaskRegistry::default();
5957 let dir = tempfile::tempdir().unwrap();
5958 let output = format!("RAW-HEAD\n{}RAW-TAIL\n", "raw-middle\n".repeat(8_000));
5959 let (task_id, task) =
5960 insert_terminal_piped_task(®istry, &dir, "cat raw.log", &output, "RAW-ERR\n", false);
5961
5962 let snapshot = registry
5963 .status(
5964 &task_id,
5965 "session",
5966 None,
5967 Some(dir.path()),
5968 RUNNING_OUTPUT_PREVIEW_BYTES,
5969 )
5970 .unwrap();
5971
5972 assert!(snapshot.output_preview.contains("RAW-HEAD"));
5973 assert!(snapshot.output_preview.contains("RAW-TAIL"));
5974 assert!(snapshot.output_preview.contains("truncated output"));
5975 assert!(snapshot
5976 .output_preview
5977 .contains(&format!("read \"{}\"", task.paths.stdout.display())));
5978 assert!(snapshot
5979 .output_preview
5980 .contains(&format!("read \"{}\"", task.paths.stderr.display())));
5981 assert!(!snapshot.output_preview.contains("tail -n +"));
5982 assert!(snapshot.output_preview.len() > 16 * 1024);
5983 assert!(snapshot.output_truncated);
5984 }
5985
5986 #[test]
5987 fn pty_terminal_snapshot_bypasses_line_compression() {
5988 let registry = BgTaskRegistry::default();
5989 let dir = tempfile::tempdir().unwrap();
5990 let calls = Arc::new(AtomicUsize::new(0));
5991 let calls_for_closure = Arc::clone(&calls);
5992 registry.set_compressor_with_exit_code(move |_command, output, _exit_code| {
5993 calls_for_closure.fetch_add(1, Ordering::SeqCst);
5994 CompressionResult::new(output)
5995 });
5996 let (task_id, _task) = insert_terminal_pty_task(®istry, &dir, "raw\u{1b}[31m pty bytes");
5997
5998 let snapshot = registry
5999 .status(
6000 &task_id,
6001 "session",
6002 None,
6003 Some(dir.path()),
6004 RUNNING_OUTPUT_PREVIEW_BYTES,
6005 )
6006 .unwrap();
6007
6008 assert_eq!(snapshot.info.mode, BgMode::Pty);
6009 assert_eq!(snapshot.output_preview, "");
6010 assert_eq!(calls.load(Ordering::SeqCst), 0);
6011 }
6012
6013 #[test]
6014 fn pty_dimensions_are_persisted_and_returned_in_snapshot() {
6015 let registry = BgTaskRegistry::default();
6016 let dir = tempfile::tempdir().unwrap();
6017 let task_id = registry
6018 .spawn_pty(
6019 SpawnPlan::Unsandboxed,
6020 QUICK_SUCCESS_COMMAND,
6021 "session".to_string(),
6022 dir.path().to_path_buf(),
6023 HashMap::new(),
6024 Some(Duration::from_secs(30)),
6025 dir.path().to_path_buf(),
6026 10,
6027 true,
6028 false,
6029 Some(dir.path().to_path_buf()),
6030 50,
6031 120,
6032 )
6033 .unwrap();
6034
6035 let resolved =
6036 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
6037 let metadata = read_task_at(&resolved).unwrap();
6038 assert_eq!(
6039 metadata.schema_version,
6040 crate::bash_background::persistence::SCHEMA_VERSION
6041 );
6042 assert_eq!(metadata.mode, BgMode::Pty);
6043 assert_eq!(metadata.pty_rows, Some(50));
6044 assert_eq!(metadata.pty_cols, Some(120));
6045
6046 let snapshot = registry
6047 .status(&task_id, "session", None, Some(dir.path()), 1024)
6048 .unwrap();
6049 assert_eq!(snapshot.pty_rows, Some(50));
6050 assert_eq!(snapshot.pty_cols, Some(120));
6051 }
6052
6053 fn spawn_dead_child() -> std::process::Child {
6058 #[cfg(unix)]
6059 let mut cmd = std::process::Command::new("true");
6060 #[cfg(windows)]
6061 let mut cmd = {
6062 let mut c = std::process::Command::new("cmd");
6063 c.args(["/c", "exit", "0"]);
6064 c
6065 };
6066 cmd.stdin(std::process::Stdio::null());
6067 cmd.stdout(std::process::Stdio::null());
6068 cmd.stderr(std::process::Stdio::null());
6069 let mut child = cmd.spawn().expect("spawn replacement child for reap test");
6070 let started = Instant::now();
6079 loop {
6080 match child.try_wait() {
6081 Ok(Some(_)) => break,
6082 Ok(None) => {
6083 if started.elapsed() > Duration::from_secs(5) {
6084 panic!("dead-child stand-in did not exit within 5s");
6085 }
6086 std::thread::sleep(Duration::from_millis(10));
6087 }
6088 Err(error) => panic!("dead-child try_wait failed: {error}"),
6089 }
6090 }
6091 child
6092 }
6093
6094 #[test]
6095 fn ack_marks_delivered_even_when_completion_was_already_consumed_locally() {
6096 let registry = BgTaskRegistry::default();
6097 let dir = tempfile::tempdir().unwrap();
6098 let task_id = registry
6099 .spawn(
6100 SpawnPlan::Unsandboxed,
6101 LONG_RUNNING_COMMAND,
6102 "session".to_string(),
6103 dir.path().to_path_buf(),
6104 HashMap::new(),
6105 Some(Duration::from_secs(30)),
6106 dir.path().to_path_buf(),
6107 10,
6108 true,
6109 false,
6110 Some(dir.path().to_path_buf()),
6111 )
6112 .unwrap();
6113 registry
6114 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6115 .unwrap();
6116 assert_eq!(
6117 registry
6118 .drain_completions_for_session(Some("session"))
6119 .len(),
6120 1
6121 );
6122
6123 registry.inner.completions.lock().unwrap().clear();
6126
6127 assert_eq!(
6128 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
6129 vec![task_id.clone()]
6130 );
6131 assert!(registry
6132 .drain_completions_for_session(Some("session"))
6133 .is_empty());
6134
6135 let resolved =
6136 resolve_task_layout(&session_tasks_dir(dir.path(), "session"), &task_id).unwrap();
6137 let metadata = read_task_at(&resolved).unwrap();
6138 assert!(metadata.completion_delivered);
6139
6140 let replayed = BgTaskRegistry::default();
6141 replayed
6142 .replay_session_inner(dir.path(), "session", None)
6143 .unwrap();
6144 assert!(replayed
6145 .drain_completions_for_session(Some("session"))
6146 .is_empty());
6147 }
6148
6149 #[test]
6150 fn register_watch_rejects_unknown_task() {
6151 let registry = BgTaskRegistry::default();
6152
6153 let result = registry.register_watch(
6154 "missing-task".to_string(),
6155 WatchPattern::Substring("READY".into()),
6156 true,
6157 );
6158
6159 assert_eq!(result, Err("task_not_found"));
6160 }
6161
6162 #[test]
6163 fn register_watch_on_terminal_task_scans_existing_output() {
6164 let frames = Arc::new(Mutex::new(Vec::new()));
6165 let captured = Arc::clone(&frames);
6166 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
6167 captured.lock().unwrap().push(frame);
6168 })
6169 as Box<dyn Fn(PushFrame) + Send + Sync>);
6170 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
6171 let dir = tempfile::tempdir().unwrap();
6172 let task_id = registry
6173 .spawn(
6174 SpawnPlan::Unsandboxed,
6175 LONG_RUNNING_COMMAND,
6176 "session".to_string(),
6177 dir.path().to_path_buf(),
6178 HashMap::new(),
6179 Some(Duration::from_secs(30)),
6180 dir.path().to_path_buf(),
6181 10,
6182 true,
6183 false,
6184 Some(dir.path().to_path_buf()),
6185 )
6186 .unwrap();
6187 registry
6188 .inner
6189 .shutdown
6190 .store(true, std::sync::atomic::Ordering::SeqCst);
6191 let task = registry.task_for_session(&task_id, "session").unwrap();
6192 std::fs::write(&task.paths.stdout, "READY\n").unwrap();
6193 registry
6194 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6195 .unwrap();
6196 frames.lock().unwrap().clear();
6197 registry.inner.completions.lock().unwrap().clear();
6198
6199 registry
6200 .register_watch(
6201 task_id.clone(),
6202 WatchPattern::Substring("READY".into()),
6203 true,
6204 )
6205 .unwrap();
6206
6207 let frames = frames.lock().unwrap();
6208 let frame = frames
6209 .iter()
6210 .find_map(|frame| match frame {
6211 PushFrame::BashPatternMatch(frame) => Some(frame),
6212 _ => None,
6213 })
6214 .expect("terminal watch registration should emit pattern frame");
6215 assert_eq!(frame.reason, "pattern_match");
6216 assert_eq!(frame.task_id, task_id);
6217 assert_eq!(frame.session_id, "session");
6218 assert_eq!(frame.match_text, "READY");
6219 assert_eq!(frame.match_offset, 0);
6220 assert_eq!(registry.active_watch_count(&frame.task_id), 0);
6221 let metadata = read_task(&task.paths.json).unwrap();
6222 assert!(metadata.completion_delivered);
6223 }
6224
6225 #[test]
6226 fn cleanup_finished_removes_terminal_tasks_older_than_threshold() {
6227 let registry = BgTaskRegistry::default();
6228 let dir = tempfile::tempdir().unwrap();
6229 let task_id = registry
6230 .spawn(
6231 SpawnPlan::Unsandboxed,
6232 QUICK_SUCCESS_COMMAND,
6233 "session".to_string(),
6234 dir.path().to_path_buf(),
6235 HashMap::new(),
6236 Some(Duration::from_secs(30)),
6237 dir.path().to_path_buf(),
6238 10,
6239 true,
6240 false,
6241 Some(dir.path().to_path_buf()),
6242 )
6243 .unwrap();
6244 registry
6245 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6246 .unwrap();
6247 let completions = registry.drain_completions_for_session(Some("session"));
6248 assert_eq!(completions.len(), 1);
6249 assert_eq!(
6250 registry.ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id)),
6251 vec![task_id.clone()]
6252 );
6253
6254 registry.cleanup_finished(Duration::ZERO);
6255
6256 assert!(registry.inner.tasks.lock().unwrap().is_empty());
6257 }
6258
6259 #[test]
6260 fn cleanup_finished_retains_undelivered_terminals() {
6261 let registry = BgTaskRegistry::default();
6262 let dir = tempfile::tempdir().unwrap();
6263 let task_id = registry
6264 .spawn(
6265 SpawnPlan::Unsandboxed,
6266 QUICK_SUCCESS_COMMAND,
6267 "session".to_string(),
6268 dir.path().to_path_buf(),
6269 HashMap::new(),
6270 Some(Duration::from_secs(30)),
6271 dir.path().to_path_buf(),
6272 10,
6273 true,
6274 false,
6275 Some(dir.path().to_path_buf()),
6276 )
6277 .unwrap();
6278 registry
6279 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6280 .unwrap();
6281
6282 registry.cleanup_finished(Duration::ZERO);
6283
6284 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
6285 }
6286
6287 #[test]
6295 fn reap_child_marks_failed_when_child_exits_without_exit_marker() {
6296 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6297 let dir = tempfile::tempdir().unwrap();
6298 let task_id = registry
6299 .spawn(
6300 SpawnPlan::Unsandboxed,
6301 QUICK_SUCCESS_COMMAND,
6302 "session".to_string(),
6303 dir.path().to_path_buf(),
6304 HashMap::new(),
6305 Some(Duration::from_secs(30)),
6306 dir.path().to_path_buf(),
6307 10,
6308 true,
6309 false,
6310 Some(dir.path().to_path_buf()),
6311 )
6312 .unwrap();
6313
6314 let task = registry.task_for_session(&task_id, "session").unwrap();
6315
6316 let started = Instant::now();
6321 loop {
6322 let exited = {
6323 let mut state = task.state.lock().unwrap();
6324 match &mut state.runtime {
6325 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
6326 _ => true,
6327 }
6328 };
6329 if exited {
6330 break;
6331 }
6332 assert!(
6333 started.elapsed() < Duration::from_secs(5),
6334 "child should exit quickly"
6335 );
6336 std::thread::sleep(Duration::from_millis(20));
6337 }
6338
6339 registry
6347 .inner
6348 .shutdown
6349 .store(true, std::sync::atomic::Ordering::SeqCst);
6350 std::thread::sleep(Duration::from_millis(550));
6354
6355 let _ = std::fs::remove_file(&task.paths.exit);
6358
6359 {
6374 let mut state = task.state.lock().unwrap();
6375 state.metadata.status = BgTaskStatus::Running;
6376 state.metadata.status_reason = None;
6377 state.metadata.exit_code = None;
6378 state.metadata.finished_at = None;
6379 state.metadata.duration_ms = None;
6380 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
6383 .expect("persist reset Running metadata for reap_child test");
6384 if matches!(state.runtime, TaskRuntime::Piped(None)) {
6388 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
6389 }
6390 }
6391 *task.terminal_at.lock().unwrap() = None;
6394
6395 assert!(
6398 task.is_running(),
6399 "precondition: metadata.status == Running"
6400 );
6401 assert!(
6402 !task.paths.exit.exists(),
6403 "precondition: exit marker absent"
6404 );
6405
6406 registry.reap_child(&task);
6411
6412 {
6413 let state = task.state.lock().unwrap();
6414 assert_eq!(
6415 state.metadata.status,
6416 BgTaskStatus::Running,
6417 "first reap must leave status Running while waiting one pass for marker"
6418 );
6419 assert_eq!(
6420 state.metadata.status_reason, None,
6421 "first reap must not record a failure reason"
6422 );
6423 assert!(
6424 matches!(state.runtime, TaskRuntime::Piped(None)),
6425 "child handle must be released after first reap"
6426 );
6427 assert!(
6428 state.detached,
6429 "task must be marked detached after first reap"
6430 );
6431 }
6432
6433 registry.reap_child(&task);
6437
6438 let state = task.state.lock().unwrap();
6439 assert!(
6440 state.metadata.status.is_terminal(),
6441 "second reap must transition to terminal when PID dead and no marker. Got status={:?}",
6442 state.metadata.status
6443 );
6444 assert_eq!(
6445 state.metadata.status,
6446 BgTaskStatus::Failed,
6447 "must specifically be Failed (not Killed): status={:?}",
6448 state.metadata.status
6449 );
6450 assert_eq!(
6451 state.metadata.status_reason.as_deref(),
6452 Some("process exited without exit marker"),
6453 "reason must match replay path's wording: {:?}",
6454 state.metadata.status_reason
6455 );
6456 assert!(
6457 matches!(state.runtime, TaskRuntime::Piped(None)),
6458 "child handle must stay released after second reap"
6459 );
6460 assert!(
6461 state.detached,
6462 "task must remain detached after second reap"
6463 );
6464 }
6465
6466 #[test]
6471 fn reap_child_preserves_running_when_exit_marker_exists() {
6472 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6473 let dir = tempfile::tempdir().unwrap();
6474 let task_id = registry
6475 .spawn(
6476 SpawnPlan::Unsandboxed,
6477 QUICK_SUCCESS_COMMAND,
6478 "session".to_string(),
6479 dir.path().to_path_buf(),
6480 HashMap::new(),
6481 Some(Duration::from_secs(30)),
6482 dir.path().to_path_buf(),
6483 10,
6484 true,
6485 false,
6486 Some(dir.path().to_path_buf()),
6487 )
6488 .unwrap();
6489
6490 let task = registry.task_for_session(&task_id, "session").unwrap();
6491
6492 let started = Instant::now();
6495 loop {
6496 let exited = {
6497 let mut state = task.state.lock().unwrap();
6498 match &mut state.runtime {
6499 TaskRuntime::Piped(Some(child)) => matches!(child.try_wait(), Ok(Some(_))),
6500 _ => true,
6501 }
6502 };
6503 if exited && task.paths.exit.exists() {
6504 break;
6505 }
6506 assert!(
6507 started.elapsed() < Duration::from_secs(5),
6508 "child should exit and write marker quickly"
6509 );
6510 std::thread::sleep(Duration::from_millis(20));
6511 }
6512
6513 registry
6519 .inner
6520 .shutdown
6521 .store(true, std::sync::atomic::Ordering::SeqCst);
6522 std::thread::sleep(Duration::from_millis(550));
6523
6524 {
6530 let mut state = task.state.lock().unwrap();
6531 state.metadata.status = BgTaskStatus::Running;
6532 state.metadata.status_reason = None;
6533 if matches!(state.runtime, TaskRuntime::Piped(None)) {
6534 state.runtime = TaskRuntime::Piped(Some(spawn_dead_child()));
6535 }
6536 }
6537 *task.terminal_at.lock().unwrap() = None;
6538 if !task.paths.exit.exists() {
6541 std::fs::write(&task.paths.exit, "0").expect("write replacement exit marker");
6542 }
6543
6544 registry.reap_child(&task);
6548
6549 let state = task.state.lock().unwrap();
6550 assert!(
6551 matches!(state.runtime, TaskRuntime::Piped(None)),
6552 "child handle still released even when marker exists"
6553 );
6554 assert!(
6555 state.detached,
6556 "task still marked detached even when marker exists"
6557 );
6558 assert_eq!(
6563 state.metadata.status,
6564 BgTaskStatus::Running,
6565 "reap_child must defer to poll_task when marker exists"
6566 );
6567 }
6568
6569 #[cfg(unix)]
6573 fn pid_stat(pid: u32) -> Option<String> {
6574 let output = std::process::Command::new("ps")
6575 .args(["-o", "stat=", "-p", &pid.to_string()])
6576 .output()
6577 .ok()?;
6578 if !output.status.success() {
6579 return None;
6580 }
6581 let stat = String::from_utf8_lossy(&output.stdout).trim().to_string();
6582 if stat.is_empty() {
6583 None
6584 } else {
6585 Some(stat)
6586 }
6587 }
6588
6589 #[cfg(unix)]
6591 fn is_zombie(pid: u32) -> bool {
6592 pid_stat(pid).is_some_and(|stat| stat.starts_with('Z'))
6593 }
6594
6595 #[cfg(unix)]
6601 fn spawn_unreaped_zombie() -> std::process::Child {
6602 let child = std::process::Command::new("true")
6603 .stdin(std::process::Stdio::null())
6604 .stdout(std::process::Stdio::null())
6605 .stderr(std::process::Stdio::null())
6606 .spawn()
6607 .expect("spawn zombie stand-in");
6608 let pid = child.id();
6609 let started = Instant::now();
6610 while !is_zombie(pid) {
6611 assert!(
6612 started.elapsed() < Duration::from_secs(5),
6613 "stand-in child should become a zombie within 5s"
6614 );
6615 std::thread::sleep(Duration::from_millis(10));
6616 }
6617 child
6619 }
6620
6621 #[cfg(unix)]
6631 #[test]
6632 fn finalize_from_marker_reaps_child_no_zombie() {
6633 use std::sync::atomic::Ordering;
6634
6635 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6636 let dir = tempfile::tempdir().unwrap();
6637 let task_id = registry
6638 .spawn(
6639 SpawnPlan::Unsandboxed,
6640 QUICK_SUCCESS_COMMAND,
6641 "session".to_string(),
6642 dir.path().to_path_buf(),
6643 HashMap::new(),
6644 Some(Duration::from_secs(30)),
6645 dir.path().to_path_buf(),
6646 10,
6647 true,
6648 false,
6649 Some(dir.path().to_path_buf()),
6650 )
6651 .unwrap();
6652
6653 registry.inner.shutdown.store(true, Ordering::SeqCst);
6657 std::thread::sleep(Duration::from_millis(550));
6658
6659 let task = registry.task_for_session(&task_id, "session").unwrap();
6660
6661 let started = Instant::now();
6665 while !task.paths.exit.exists() {
6666 assert!(
6667 started.elapsed() < Duration::from_secs(5),
6668 "exit marker should land quickly for `true`"
6669 );
6670 std::thread::sleep(Duration::from_millis(20));
6671 }
6672
6673 let zombie_pid;
6679 {
6680 let mut state = task.state.lock().unwrap();
6681 state.metadata.status = BgTaskStatus::Running;
6682 state.metadata.status_reason = None;
6683 state.metadata.exit_code = None;
6684 state.metadata.finished_at = None;
6685 state.metadata.duration_ms = None;
6686 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
6687 .expect("persist reset Running metadata");
6688 let zombie = spawn_unreaped_zombie();
6689 zombie_pid = zombie.id();
6690 state.runtime = TaskRuntime::Piped(Some(zombie));
6691 }
6692 *task.terminal_at.lock().unwrap() = None;
6693
6694 assert!(
6696 is_zombie(zombie_pid),
6697 "precondition: stand-in child {zombie_pid} must be a zombie before finalize"
6698 );
6699
6700 registry.poll_task(&task).unwrap();
6703
6704 {
6705 let state = task.state.lock().unwrap();
6706 assert!(
6707 matches!(state.runtime, TaskRuntime::Piped(None)),
6708 "child handle must be released after marker finalize"
6709 );
6710 assert!(
6711 state.metadata.status.is_terminal(),
6712 "task must be terminal after marker finalize: {:?}",
6713 state.metadata.status
6714 );
6715 }
6716
6717 assert!(
6720 !is_zombie(zombie_pid),
6721 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
6722 after the exit-marker terminal transition"
6723 );
6724 }
6725
6726 #[cfg(unix)]
6730 #[test]
6731 fn kill_with_existing_marker_reaps_child_no_zombie() {
6732 use std::sync::atomic::Ordering;
6733
6734 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6735 let dir = tempfile::tempdir().unwrap();
6736 let task_id = registry
6737 .spawn(
6738 SpawnPlan::Unsandboxed,
6739 QUICK_SUCCESS_COMMAND,
6740 "session".to_string(),
6741 dir.path().to_path_buf(),
6742 HashMap::new(),
6743 Some(Duration::from_secs(30)),
6744 dir.path().to_path_buf(),
6745 10,
6746 true,
6747 false,
6748 Some(dir.path().to_path_buf()),
6749 )
6750 .unwrap();
6751
6752 registry.inner.shutdown.store(true, Ordering::SeqCst);
6753 std::thread::sleep(Duration::from_millis(550));
6754
6755 let task = registry.task_for_session(&task_id, "session").unwrap();
6756
6757 let started = Instant::now();
6758 while !task.paths.exit.exists() {
6759 assert!(
6760 started.elapsed() < Duration::from_secs(5),
6761 "exit marker should land quickly for `true`"
6762 );
6763 std::thread::sleep(Duration::from_millis(20));
6764 }
6765
6766 let zombie_pid;
6767 {
6768 let mut state = task.state.lock().unwrap();
6769 state.metadata.status = BgTaskStatus::Running;
6770 state.metadata.status_reason = None;
6771 state.metadata.exit_code = None;
6772 state.metadata.finished_at = None;
6773 state.metadata.duration_ms = None;
6774 crate::bash_background::persistence::write_task(&task.paths.json, &state.metadata)
6775 .expect("persist reset Running metadata");
6776 let zombie = spawn_unreaped_zombie();
6777 zombie_pid = zombie.id();
6778 state.runtime = TaskRuntime::Piped(Some(zombie));
6779 }
6780 *task.terminal_at.lock().unwrap() = None;
6781
6782 assert!(
6783 is_zombie(zombie_pid),
6784 "precondition: stand-in child {zombie_pid} must be a zombie before kill"
6785 );
6786
6787 registry
6789 .kill_with_status(&task_id, "session", BgTaskStatus::Killed)
6790 .expect("kill should succeed");
6791
6792 {
6793 let state = task.state.lock().unwrap();
6794 assert!(
6795 matches!(state.runtime, TaskRuntime::Piped(None)),
6796 "child handle must be released after marker-aware kill"
6797 );
6798 assert!(state.metadata.status.is_terminal());
6799 }
6800
6801 assert!(
6802 !is_zombie(zombie_pid),
6803 "issue #91 regression: child {zombie_pid} left as <defunct> zombie \
6804 after a marker-aware kill"
6805 );
6806 }
6807
6808 #[test]
6809 fn cleanup_finished_keeps_running_tasks() {
6810 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6811 let dir = tempfile::tempdir().unwrap();
6812 let task_id = registry
6813 .spawn(
6814 SpawnPlan::Unsandboxed,
6815 LONG_RUNNING_COMMAND,
6816 "session".to_string(),
6817 dir.path().to_path_buf(),
6818 HashMap::new(),
6819 Some(Duration::from_secs(30)),
6820 dir.path().to_path_buf(),
6821 10,
6822 true,
6823 false,
6824 Some(dir.path().to_path_buf()),
6825 )
6826 .unwrap();
6827
6828 registry.cleanup_finished(Duration::ZERO);
6829
6830 assert!(registry.inner.tasks.lock().unwrap().contains_key(&task_id));
6831 let _ = registry.kill(&task_id, "session");
6832 }
6833
6834 #[cfg(unix)]
6835 #[test]
6836 fn rehydrating_sandboxed_task_never_respawns_persisted_command() {
6837 let project = tempfile::tempdir().unwrap();
6838 let storage = tempfile::tempdir().unwrap();
6839 let sandbox_temp = storage.path().join("sandbox-temp");
6840 fs::create_dir(&sandbox_temp).unwrap();
6841 let launcher_script = project.path().join("sandbox-launch");
6842 let launcher = PathBuf::from("/bin/sh");
6843 fs::write(
6844 &launcher_script,
6845 "while [ \"$#\" -gt 0 ]; do\n if [ \"$1\" = -- ]; then\n shift\n exec \"$@\"\n fi\n shift\ndone\nexit 78\n",
6846 )
6847 .unwrap();
6848 let mut permissions = fs::metadata(&launcher_script).unwrap().permissions();
6849 permissions.set_mode(0o700);
6850 fs::set_permissions(&launcher_script, permissions).unwrap();
6851
6852 let profile = crate::sandbox_profile::SandboxProfile::build(
6853 vec![project.path().to_path_buf()],
6854 Vec::new(),
6855 Vec::new(),
6856 Vec::new(),
6857 Vec::new(),
6858 Vec::new(),
6859 Vec::new(),
6860 sandbox_temp,
6861 )
6862 .unwrap();
6863 let plan = SpawnPlan::launcher_for_test(profile, launcher);
6864 let spawn_marker = project.path().join("spawn-count");
6865 let stop_marker = project.path().join("stop-command");
6866 let quote =
6867 |path: &Path| format!("'{}'", path.display().to_string().replace('\'', "'\\''"));
6868 let command = format!(
6869 "printf 'spawn\\n' >> {}; while [ ! -e {} ]; do sleep 0.05; done",
6870 quote(&spawn_marker),
6871 quote(&stop_marker)
6872 );
6873
6874 let original = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6875 let task_id = original
6876 .spawn(
6877 plan,
6878 &command,
6879 "sandbox-rehydrate".to_string(),
6880 project.path().to_path_buf(),
6881 HashMap::new(),
6882 Some(Duration::from_secs(30)),
6883 storage.path().to_path_buf(),
6884 10,
6885 true,
6886 false,
6887 Some(project.path().to_path_buf()),
6888 )
6889 .unwrap();
6890 let started = Instant::now();
6891 while !spawn_marker.exists() {
6892 assert!(
6893 started.elapsed() < Duration::from_secs(20),
6894 "original sandboxed task did not start"
6895 );
6896 std::thread::sleep(Duration::from_millis(10));
6897 }
6898 original.detach();
6899
6900 let restarted = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6901 restarted
6902 .replay_session(storage.path(), "sandbox-rehydrate")
6903 .unwrap();
6904 let replayed = restarted
6905 .status(
6906 &task_id,
6907 "sandbox-rehydrate",
6908 Some(project.path()),
6909 Some(storage.path()),
6910 4096,
6911 )
6912 .expect("rehydrated sandbox task");
6913 assert_eq!(replayed.info.status, BgTaskStatus::Running);
6914 assert!(replayed.sandbox_native);
6915
6916 std::thread::sleep(Duration::from_millis(650));
6917 assert_eq!(
6918 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
6919 1,
6920 "registry replay must observe the persisted process without spawning its command"
6921 );
6922
6923 fs::write(&stop_marker, "stop").unwrap();
6924 let terminal = wait_for_terminal_snapshot(
6925 &restarted,
6926 &task_id,
6927 "sandbox-rehydrate",
6928 project.path(),
6929 storage.path(),
6930 );
6931 assert_eq!(terminal.info.status, BgTaskStatus::Completed);
6932 assert_eq!(
6933 fs::read_to_string(&spawn_marker).unwrap().lines().count(),
6934 1
6935 );
6936 restarted.detach();
6937 }
6938
6939 #[cfg(windows)]
6940 fn wait_for_file(path: &Path) -> String {
6941 let started = Instant::now();
6948 loop {
6949 if let Ok(content) = fs::read_to_string(path) {
6950 if !content.trim().is_empty() {
6951 return content;
6952 }
6953 }
6954 assert!(
6955 started.elapsed() < Duration::from_secs(30),
6956 "timed out waiting for non-empty {}",
6957 path.display()
6958 );
6959 std::thread::sleep(Duration::from_millis(100));
6960 }
6961 }
6962
6963 #[cfg(windows)]
6964 fn spawn_windows_registry_command(
6965 command: &str,
6966 ) -> (BgTaskRegistry, tempfile::TempDir, String) {
6967 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(None)));
6968 let dir = tempfile::tempdir().unwrap();
6969 let task_id = registry
6970 .spawn(
6971 SpawnPlan::Unsandboxed,
6972 command,
6973 "session".to_string(),
6974 dir.path().to_path_buf(),
6975 HashMap::new(),
6976 Some(Duration::from_secs(30)),
6977 dir.path().to_path_buf(),
6978 10,
6979 false,
6980 false,
6981 Some(dir.path().to_path_buf()),
6982 )
6983 .unwrap();
6984 (registry, dir, task_id)
6985 }
6986
6987 #[cfg(windows)]
6988 #[test]
6989 fn windows_spawn_writes_exit_marker_for_zero_exit() {
6990 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 0");
6991 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
6992
6993 let content = wait_for_file(&exit_path);
6994
6995 assert_eq!(content.trim(), "0");
6996 }
6997
6998 #[cfg(windows)]
6999 #[test]
7000 fn windows_spawn_writes_exit_marker_for_nonzero_exit() {
7001 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c exit 42");
7002 let exit_path = registry.task_exit_path(&task_id, "session").unwrap();
7003
7004 let content = wait_for_file(&exit_path);
7005
7006 assert_eq!(content.trim(), "42");
7007 }
7008
7009 #[cfg(windows)]
7010 #[test]
7011 fn windows_spawn_captures_stdout_to_disk() {
7012 let (registry, _dir, task_id) = spawn_windows_registry_command("cmd /c echo hello");
7013 let task = registry.task_for_session(&task_id, "session").unwrap();
7014 let stdout_path = task.paths.stdout.clone();
7015 let exit_path = task.paths.exit.clone();
7016
7017 let _ = wait_for_file(&exit_path);
7018 let stdout = fs::read_to_string(stdout_path).expect("read stdout");
7019
7020 assert!(stdout.contains("hello"), "stdout was {stdout:?}");
7021 }
7022
7023 #[cfg(windows)]
7024 #[test]
7025 fn windows_spawn_uses_pwsh_when_available() {
7026 let candidates = crate::windows_shell::shell_candidates_with(
7030 |binary| match binary {
7031 "pwsh.exe" => Some(std::path::PathBuf::from(r"C:\pwsh\pwsh.exe")),
7032 "powershell.exe" => Some(std::path::PathBuf::from(r"C:\ps\powershell.exe")),
7033 _ => None,
7034 },
7035 || None,
7036 );
7037 let shell = candidates.first().expect("at least one candidate").clone();
7038 assert_eq!(shell, crate::windows_shell::WindowsShell::Pwsh);
7039 assert_eq!(shell.binary().as_ref(), "pwsh.exe");
7040 }
7041
7042 #[cfg(windows)]
7045 #[test]
7046 fn windows_shell_cmd_wrapper_writes_marker_via_temp_rename() {
7047 let exit_path = Path::new(r"C:\Temp\bash-test.exit");
7048 let script =
7049 crate::windows_shell::WindowsShell::Cmd.wrapper_script("cmd /c exit 42", exit_path);
7050
7051 assert!(
7052 script.contains("set CODE=%ERRORLEVEL%"),
7053 "wrapper must capture the child exit code: {script}"
7054 );
7055 assert!(
7056 script.contains("exit /B %CODE%"),
7057 "wrapper must propagate the child exit code: {script}"
7058 );
7059 assert!(
7064 script.contains("bash-test.exit"),
7065 "wrapper must target the exit marker path: {script}"
7066 );
7067 assert!(
7068 script.contains("move /Y"),
7069 "wrapper must write the marker atomically via temp-file + rename: {script}"
7070 );
7071 }
7072
7073 #[cfg(windows)]
7079 #[test]
7080 fn windows_shell_cmd_bg_command_uses_minimal_cmd_flags() {
7081 use crate::windows_shell::WindowsShell;
7082 let cmd = WindowsShell::Cmd.bg_command("echo wrapped");
7083 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
7084 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
7085 assert_eq!(
7086 args_strs,
7087 vec!["/D", "/S", "/C", "echo wrapped"],
7088 "Cmd::bg_command must prepend /D /S /C"
7089 );
7090 }
7091
7092 #[cfg(windows)]
7095 #[test]
7096 fn windows_shell_pwsh_bg_command_uses_standard_args() {
7097 use crate::windows_shell::WindowsShell;
7098 let cmd = WindowsShell::Pwsh.bg_command("Get-Date");
7099 let args: Vec<&std::ffi::OsStr> = cmd.get_args().collect();
7100 let args_strs: Vec<&str> = args.iter().filter_map(|a| a.to_str()).collect();
7101 assert!(
7102 args_strs.contains(&"-Command"),
7103 "Pwsh::bg_command must use -Command: {args_strs:?}"
7104 );
7105 assert!(
7106 args_strs.contains(&"Get-Date"),
7107 "Pwsh::bg_command must include the user command body"
7108 );
7109 }
7110
7111 fn registry_with_db_and_frames(
7112 storage: &Path,
7113 ) -> (
7114 BgTaskRegistry,
7115 Arc<Mutex<Connection>>,
7116 Arc<Mutex<Vec<PushFrame>>>,
7117 ) {
7118 let frames = Arc::new(Mutex::new(Vec::new()));
7119 let captured = Arc::clone(&frames);
7120 let sender: crate::context::ProgressSender = Arc::new(Box::new(move |frame| {
7121 captured.lock().unwrap().push(frame);
7122 })
7123 as Box<dyn Fn(PushFrame) + Send + Sync>);
7124 let registry = BgTaskRegistry::new(Arc::new(Mutex::new(Some(sender))));
7125 registry.set_harness(Harness::Opencode);
7126 let conn = crate::db::open(&storage.join("aft.db")).expect("open test DB");
7127 let shared = Arc::new(Mutex::new(conn));
7128 registry.set_db_pool(shared.clone());
7129 (registry, shared, frames)
7130 }
7131
7132 fn pattern_match_frames(frames: &Mutex<Vec<PushFrame>>) -> Vec<BashPatternMatchFrame> {
7133 frames
7134 .lock()
7135 .unwrap()
7136 .iter()
7137 .filter_map(|frame| match frame {
7138 PushFrame::BashPatternMatch(frame) => Some(frame.clone()),
7139 _ => None,
7140 })
7141 .collect()
7142 }
7143
7144 #[test]
7145 fn pattern_watch_survives_registry_teardown_and_rehydrate() {
7146 let dir = tempfile::tempdir().unwrap();
7147 let storage = dir.path();
7148 let (registry, _db, frames) = registry_with_db_and_frames(storage);
7149 let task_id = registry
7150 .spawn(
7151 SpawnPlan::Unsandboxed,
7152 LONG_RUNNING_COMMAND,
7153 "session".to_string(),
7154 storage.to_path_buf(),
7155 HashMap::new(),
7156 Some(Duration::from_secs(30)),
7157 storage.to_path_buf(),
7158 10,
7159 true,
7160 false,
7161 Some(storage.to_path_buf()),
7162 )
7163 .unwrap();
7164 registry
7165 .register_watch(
7166 task_id.clone(),
7167 WatchPattern::Substring("READY".into()),
7168 true,
7169 )
7170 .unwrap();
7171 let task = registry.task_for_session(&task_id, "session").unwrap();
7172 registry.clear_task_watch_state(&task_id);
7174 assert_eq!(registry.active_watch_count(&task_id), 0);
7175
7176 std::fs::OpenOptions::new()
7177 .append(true)
7178 .open(&task.paths.stdout)
7179 .unwrap()
7180 .write_all(b"READY\n")
7181 .unwrap();
7182 frames.lock().unwrap().clear();
7183
7184 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
7185 registry
7187 .inner
7188 .shutdown
7189 .store(true, std::sync::atomic::Ordering::SeqCst);
7190 replayed
7191 .replay_session_inner(storage, "session", None)
7192 .unwrap();
7193
7194 let matches = pattern_match_frames(&replay_frames);
7195 assert!(
7196 matches.iter().any(|frame| {
7197 frame.task_id == task_id
7198 && frame.reason == "pattern_match"
7199 && frame.match_text == "READY"
7200 }),
7201 "rehydrate should deliver gap match: {matches:?}"
7202 );
7203 }
7204
7205 #[test]
7206 fn pattern_watch_gap_match_between_teardown_and_rehydrate_delivers_once() {
7207 let dir = tempfile::tempdir().unwrap();
7208 let storage = dir.path();
7209 let (registry, _db, frames) = registry_with_db_and_frames(storage);
7210 let task_id = registry
7211 .spawn(
7212 SpawnPlan::Unsandboxed,
7213 LONG_RUNNING_COMMAND,
7214 "session".to_string(),
7215 storage.to_path_buf(),
7216 HashMap::new(),
7217 Some(Duration::from_secs(30)),
7218 storage.to_path_buf(),
7219 10,
7220 true,
7221 false,
7222 Some(storage.to_path_buf()),
7223 )
7224 .unwrap();
7225 registry
7226 .register_watch(
7227 task_id.clone(),
7228 WatchPattern::Substring("GAP-HIT".into()),
7229 true,
7230 )
7231 .unwrap();
7232 let task = registry.task_for_session(&task_id, "session").unwrap();
7233 let cursor_before = registry.watch_stream_cursors(&task_id).0;
7234 registry.clear_task_watch_state(&task_id);
7235
7236 std::fs::OpenOptions::new()
7238 .append(true)
7239 .open(&task.paths.stdout)
7240 .unwrap()
7241 .write_all(b"prefix GAP-HIT suffix\n")
7242 .unwrap();
7243 frames.lock().unwrap().clear();
7244
7245 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
7246 registry
7247 .inner
7248 .shutdown
7249 .store(true, std::sync::atomic::Ordering::SeqCst);
7250 replayed
7251 .replay_session_inner(storage, "session", None)
7252 .unwrap();
7253
7254 let matches: Vec<_> = pattern_match_frames(&replay_frames)
7255 .into_iter()
7256 .filter(|frame| frame.task_id == task_id && frame.match_text.contains("GAP-HIT"))
7257 .collect();
7258 assert_eq!(
7259 matches.len(),
7260 1,
7261 "gap match must deliver exactly once: {matches:?}"
7262 );
7263 assert!(
7264 matches[0].match_offset >= cursor_before,
7265 "match offset should be at/after the persisted cursor ({cursor_before}), got {}",
7266 matches[0].match_offset
7267 );
7268 }
7269
7270 #[test]
7271 fn pattern_watch_acked_match_does_not_redeliver_after_restart() {
7272 let dir = tempfile::tempdir().unwrap();
7273 let storage = dir.path();
7274 let (registry, db, frames) = registry_with_db_and_frames(storage);
7275 let task_id = registry
7276 .spawn(
7277 SpawnPlan::Unsandboxed,
7278 LONG_RUNNING_COMMAND,
7279 "session".to_string(),
7280 storage.to_path_buf(),
7281 HashMap::new(),
7282 Some(Duration::from_secs(30)),
7283 storage.to_path_buf(),
7284 10,
7285 true,
7286 false,
7287 Some(storage.to_path_buf()),
7288 )
7289 .unwrap();
7290 registry
7291 .register_watch(
7292 task_id.clone(),
7293 WatchPattern::Substring("READY".into()),
7294 true,
7295 )
7296 .unwrap();
7297 let task = registry.task_for_session(&task_id, "session").unwrap();
7298 std::fs::OpenOptions::new()
7299 .append(true)
7300 .open(&task.paths.stdout)
7301 .unwrap()
7302 .write_all(b"READY\n")
7303 .unwrap();
7304 registry.scan_task_watch_output(&task);
7305 let delivered = pattern_match_frames(&frames);
7306 assert!(
7307 delivered
7308 .iter()
7309 .any(|frame| frame.task_id == task_id && frame.match_text == "READY"),
7310 "live path should deliver match: {delivered:?}"
7311 );
7312 assert!(registry
7314 .ack_completions_for_session(Some("session"), std::slice::from_ref(&task_id))
7315 .contains(&task_id));
7316 {
7317 let conn = db.lock().unwrap();
7318 let rows = crate::db::bash_watches::list_bash_pattern_watches_for_task(
7319 &conn, "opencode", "session", &task_id,
7320 )
7321 .unwrap();
7322 assert!(
7323 rows.is_empty(),
7324 "acked once-watch rows must be deleted: {rows:?}"
7325 );
7326 }
7327
7328 frames.lock().unwrap().clear();
7329 let (replayed, _db2, replay_frames) = registry_with_db_and_frames(storage);
7330 registry
7331 .inner
7332 .shutdown
7333 .store(true, std::sync::atomic::Ordering::SeqCst);
7334 replayed
7335 .replay_session_inner(storage, "session", None)
7336 .unwrap();
7337 let matches = pattern_match_frames(&replay_frames)
7338 .into_iter()
7339 .filter(|frame| frame.task_id == task_id)
7340 .collect::<Vec<_>>();
7341 assert!(
7342 matches.is_empty(),
7343 "acked match must not re-deliver after restart: {matches:?}"
7344 );
7345 }
7346
7347 #[test]
7348 fn pattern_watch_rows_are_removed_when_task_is_gc_deleted() {
7349 let dir = tempfile::tempdir().unwrap();
7350 let storage = dir.path();
7351 let (registry, db, _frames) = registry_with_db_and_frames(storage);
7352 let task_id = "bash-aaaaaaaaaaaaaaaa";
7353 let paths = task_paths(storage, "session", task_id).unwrap();
7354 let mut metadata = PersistedTask::starting(
7355 task_id.to_string(),
7356 "session".to_string(),
7357 "true".to_string(),
7358 storage.to_path_buf(),
7359 Some(storage.to_path_buf()),
7360 None,
7361 true,
7362 true,
7363 );
7364 metadata.mark_terminal(BgTaskStatus::Completed, Some(0), None);
7365 metadata.completion_delivered = true;
7366 write_task(&paths.json, &metadata).unwrap();
7367 {
7368 let conn = db.lock().unwrap();
7369 crate::db::bash_tasks::upsert_bash_task(
7370 &conn,
7371 &metadata.to_bash_task_row("opencode", &paths).unwrap(),
7372 )
7373 .unwrap();
7374 crate::db::bash_watches::upsert_bash_pattern_watch(
7375 &conn,
7376 &BashPatternWatchRow {
7377 harness: "opencode".into(),
7378 session_id: "session".into(),
7379 task_id: task_id.into(),
7380 watch_id: "watch-00000001".into(),
7381 pattern_kind: "substring".into(),
7382 pattern: "x".into(),
7383 once: true,
7384 created_at: 1,
7385 stdout_offset: 0,
7386 stderr_offset: 0,
7387 pty_offset: 0,
7388 scanning: true,
7389 pending_match: false,
7390 match_text: None,
7391 match_offset: None,
7392 match_context: None,
7393 },
7394 )
7395 .unwrap();
7396 }
7397 let old = SystemTime::now()
7398 .checked_sub(Duration::from_secs(25 * 60 * 60))
7399 .unwrap();
7400 filetime::set_file_mtime(&paths.json, filetime::FileTime::from_system_time(old)).unwrap();
7401
7402 let deleted = registry.maybe_gc_persisted(storage).unwrap();
7403 assert!(
7404 deleted >= 1,
7405 "expected GC to delete the terminal task bundle"
7406 );
7407 let conn = db.lock().unwrap();
7408 let watches = crate::db::bash_watches::list_bash_pattern_watches_for_task(
7409 &conn, "opencode", "session", task_id,
7410 )
7411 .unwrap();
7412 assert!(
7413 watches.is_empty(),
7414 "task GC must remove watch rows: {watches:?}"
7415 );
7416 }
7417}