Skip to main content

aft/
logging.rs

1//! Durable process logging and low-cost periodic performance summaries.
2//!
3//! Rust module processes use one file per PID. That avoids cross-process rename
4//! races while preserving a single greppable directory for all AFT activity.
5
6use crate::executor::Executor;
7use crate::run_tool_call::ToolCallPhaseDurations;
8use std::collections::{BTreeMap, VecDeque};
9use std::fs::{self, File, OpenOptions};
10use std::io::{self, BufWriter, Write};
11use std::path::{Path, PathBuf};
12use std::sync::atomic::{AtomicU64, Ordering};
13use std::sync::mpsc::{self, SyncSender, TrySendError};
14use std::sync::{LazyLock, Mutex, OnceLock};
15use std::thread;
16use std::time::{Duration, Instant, SystemTime};
17
18const LOG_FILE_BYTES: u64 = 20 * 1024 * 1024;
19const LOG_GENERATIONS: usize = 5;
20const ROTATION_CHECK_EVERY: u64 = 64;
21const LOG_CHANNEL_CAPACITY: usize = 4096;
22const DEAD_PROCESS_LOG_MAX_AGE: Duration = Duration::from_secs(7 * 24 * 60 * 60);
23const DEFAULT_PERF_TICK_INTERVAL: Duration = Duration::from_secs(60);
24const PERF_SAMPLE_INTERVAL: Duration = Duration::from_millis(250);
25const SLOW_TOOL_CALL_THRESHOLD: Duration = Duration::from_millis(50);
26const TOOL_CALL_SAMPLE_CAPACITY: usize = 256;
27
28/// Initialize the `RUST_LOG`-filtered stderr logger and its additive file sink.
29pub fn init() {
30    let storage_root = crate::bash_background::storage_dir(None);
31    let logs_dir = storage_root.join("logs");
32    let file_name = format!("aft-{}.log", std::process::id());
33    let file_path = logs_dir.join(file_name);
34
35    let file_tx = match prepare_file_sink(&logs_dir, &file_path) {
36        Ok(sink) => {
37            let (tx, rx) = mpsc::sync_channel(LOG_CHANNEL_CAPACITY);
38            thread::Builder::new()
39                .name("aft-log-writer".to_string())
40                .spawn(move || run_file_writer(sink, rx))
41                .map(|_| {
42                    if let Ok(mut control) = FILE_CONTROL.lock() {
43                        control.tx = Some(tx.clone());
44                        control.storage_root = Some(storage_root.clone());
45                    }
46                    Some(tx)
47                })
48                .unwrap_or_else(|error| {
49                    write_stderr_once(&format!(
50                        "[aft] durable log disabled: cannot start writer thread: {error}\n"
51                    ));
52                    None
53                })
54        }
55        Err(error) => {
56            write_stderr_once(&format!(
57                "[aft] durable log disabled for {}: {error}\n",
58                file_path.display()
59            ));
60            None
61        }
62    };
63
64    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info"))
65        .target(env_logger::Target::Pipe(Box::new(TeeWriter { file_tx })))
66        .format(|buf, record| {
67            let prefix = if record.target().starts_with("aft::lsp")
68                || record.target().starts_with("aft_lsp")
69            {
70                "[aft-lsp]"
71            } else {
72                "[aft]"
73            };
74            writeln!(buf, "{} {}", prefix, record.args())
75        })
76        .init();
77}
78
79fn prepare_file_sink(logs_dir: &Path, file_path: &Path) -> io::Result<RotatingFile> {
80    fs::create_dir_all(logs_dir)?;
81    sweep_dead_process_logs(logs_dir, SystemTime::now(), DEAD_PROCESS_LOG_MAX_AGE)?;
82    RotatingFile::open(
83        file_path.to_path_buf(),
84        LOG_FILE_BYTES,
85        LOG_GENERATIONS,
86        ROTATION_CHECK_EVERY,
87    )
88}
89
90enum LogMessage {
91    Write(Vec<u8>),
92    Reconfigure(PathBuf),
93}
94
95#[derive(Default)]
96struct FileControl {
97    tx: Option<SyncSender<LogMessage>>,
98    storage_root: Option<PathBuf>,
99}
100
101static FILE_CONTROL: LazyLock<Mutex<FileControl>> =
102    LazyLock::new(|| Mutex::new(FileControl::default()));
103
104struct TeeWriter {
105    file_tx: Option<SyncSender<LogMessage>>,
106}
107
108impl Write for TeeWriter {
109    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
110        io::stderr().write_all(buf)?;
111        if let Some(tx) = self.file_tx.as_ref() {
112            match tx.try_send(LogMessage::Write(buf.to_vec())) {
113                Ok(()) => {}
114                Err(TrySendError::Full(_)) => {
115                    PERF.file_lines_dropped.fetch_add(1, Ordering::Relaxed);
116                }
117                Err(TrySendError::Disconnected(_)) => self.file_tx = None,
118            }
119        }
120        Ok(buf.len())
121    }
122
123    fn flush(&mut self) -> io::Result<()> {
124        io::stderr().flush()
125    }
126}
127
128fn run_file_writer(mut sink: RotatingFile, rx: mpsc::Receiver<LogMessage>) {
129    while let Ok(message) = rx.recv() {
130        let mut lines = Vec::new();
131        let mut reconfigure = None;
132        match message {
133            LogMessage::Write(line) => {
134                lines.push(line);
135                while lines.len() < 256 {
136                    match rx.try_recv() {
137                        Ok(LogMessage::Write(line)) => lines.push(line),
138                        Ok(LogMessage::Reconfigure(storage_root)) => {
139                            reconfigure = Some(storage_root);
140                            break;
141                        }
142                        Err(_) => break,
143                    }
144                }
145            }
146            LogMessage::Reconfigure(storage_root) => reconfigure = Some(storage_root),
147        }
148        if !lines.is_empty() {
149            if let Err(error) = sink.write_batch(&lines) {
150                write_stderr_once(&format!(
151                    "[aft] durable log disabled after write failure for {}: {error}\n",
152                    sink.path.display()
153                ));
154                break;
155            }
156        }
157        if let Some(storage_root) = reconfigure {
158            let logs_dir = storage_root.join("logs");
159            let path = logs_dir.join(format!("aft-{}.log", std::process::id()));
160            match prepare_file_sink(&logs_dir, &path) {
161                Ok(new_sink) => sink = new_sink,
162                Err(error) => write_stderr_once(&format!(
163                    "[aft] durable log could not switch to {}: {error}\n",
164                    path.display()
165                )),
166            }
167        }
168    }
169}
170
171fn write_stderr_once(message: &str) {
172    let _ = io::stderr().write_all(message.as_bytes());
173}
174
175struct RotatingFile {
176    path: PathBuf,
177    writer: Option<BufWriter<File>>,
178    size: u64,
179    threshold: u64,
180    generations: usize,
181    check_every: u64,
182    writes_since_check: u64,
183}
184
185impl RotatingFile {
186    fn open(
187        path: PathBuf,
188        threshold: u64,
189        generations: usize,
190        check_every: u64,
191    ) -> io::Result<Self> {
192        let file = OpenOptions::new().create(true).append(true).open(&path)?;
193        let size = file.metadata()?.len();
194        Ok(Self {
195            path,
196            writer: Some(BufWriter::new(file)),
197            size,
198            threshold,
199            generations,
200            check_every: check_every.max(1),
201            writes_since_check: 0,
202        })
203    }
204
205    fn write_batch(&mut self, lines: &[Vec<u8>]) -> io::Result<()> {
206        let batch_bytes = lines.iter().map(Vec::len).sum::<usize>() as u64;
207        self.writes_since_check = self.writes_since_check.saturating_add(lines.len() as u64);
208        if self.writes_since_check >= self.check_every
209            && self.size > 0
210            && self.size.saturating_add(batch_bytes) > self.threshold
211        {
212            self.rotate()?;
213        }
214        let writer = self
215            .writer
216            .as_mut()
217            .ok_or_else(|| io::Error::other("log writer unavailable"))?;
218        for line in lines {
219            writer.write_all(line)?;
220        }
221        // The worker batches channel messages before this flush. File I/O never
222        // runs on request, watcher, executor, or transport threads.
223        writer.flush()?;
224        self.size = self.size.saturating_add(batch_bytes);
225        if self.writes_since_check >= self.check_every {
226            self.writes_since_check = 0;
227        }
228        Ok(())
229    }
230
231    fn rotate(&mut self) -> io::Result<()> {
232        if let Some(mut writer) = self.writer.take() {
233            writer.flush()?;
234        }
235        if self.generations > 0 {
236            let oldest = rotated_path(&self.path, self.generations);
237            remove_file_if_present(&oldest)?;
238            for generation in (1..self.generations).rev() {
239                let from = rotated_path(&self.path, generation);
240                let to = rotated_path(&self.path, generation + 1);
241                rename_if_present(&from, &to)?;
242            }
243            rename_if_present(&self.path, &rotated_path(&self.path, 1))?;
244        } else {
245            remove_file_if_present(&self.path)?;
246        }
247        let file = OpenOptions::new()
248            .create(true)
249            .write(true)
250            .truncate(true)
251            .open(&self.path)?;
252        self.writer = Some(BufWriter::new(file));
253        self.size = 0;
254        self.writes_since_check = 0;
255        Ok(())
256    }
257}
258
259fn rotated_path(base: &Path, generation: usize) -> PathBuf {
260    let mut path = base.as_os_str().to_os_string();
261    path.push(format!(".{generation}"));
262    PathBuf::from(path)
263}
264
265fn remove_file_if_present(path: &Path) -> io::Result<()> {
266    match fs::remove_file(path) {
267        Ok(()) => Ok(()),
268        Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
269        Err(error) => Err(error),
270    }
271}
272
273fn rename_if_present(from: &Path, to: &Path) -> io::Result<()> {
274    match fs::rename(from, to) {
275        Ok(()) => Ok(()),
276        Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
277        Err(error) => Err(error),
278    }
279}
280
281fn sweep_dead_process_logs(dir: &Path, now: SystemTime, max_age: Duration) -> io::Result<()> {
282    for entry in fs::read_dir(dir)? {
283        let entry = match entry {
284            Ok(entry) => entry,
285            Err(_) => continue,
286        };
287        let name = entry.file_name();
288        let Some(pid) = process_log_pid(name.to_string_lossy().as_ref()) else {
289            continue;
290        };
291        let metadata = match entry.metadata() {
292            Ok(metadata) => metadata,
293            Err(_) => continue,
294        };
295        let age = metadata
296            .modified()
297            .ok()
298            .and_then(|modified| now.duration_since(modified).ok());
299        if age.is_some_and(|age| age >= max_age) && !process_is_alive(pid) {
300            let _ = fs::remove_file(entry.path());
301        }
302    }
303    Ok(())
304}
305
306fn process_log_pid(name: &str) -> Option<u32> {
307    let rest = name.strip_prefix("aft-")?;
308    let (pid, suffix) = rest.split_once(".log")?;
309    if !suffix.is_empty()
310        && !(suffix.starts_with('.') && suffix[1..].chars().all(|ch| ch.is_ascii_digit()))
311    {
312        return None;
313    }
314    pid.parse().ok()
315}
316
317#[cfg(unix)]
318fn process_is_alive(pid: u32) -> bool {
319    if pid == std::process::id() {
320        return true;
321    }
322    let result = unsafe { libc::kill(pid as libc::pid_t, 0) };
323    result == 0 || io::Error::last_os_error().raw_os_error() == Some(libc::EPERM)
324}
325
326#[cfg(windows)]
327fn process_is_alive(pid: u32) -> bool {
328    use std::ffi::c_void;
329
330    const PROCESS_QUERY_LIMITED_INFORMATION: u32 = 0x1000;
331    const STILL_ACTIVE: u32 = 259;
332    const ERROR_INVALID_PARAMETER: u32 = 87;
333    #[link(name = "kernel32")]
334    unsafe extern "system" {
335        #[link_name = "OpenProcess"]
336        fn open_process(access: u32, inherit_handle: i32, process_id: u32) -> *mut c_void;
337        #[link_name = "GetExitCodeProcess"]
338        fn get_exit_code_process(process: *mut c_void, exit_code: *mut u32) -> i32;
339        #[link_name = "CloseHandle"]
340        fn close_handle(object: *mut c_void) -> i32;
341        #[link_name = "GetLastError"]
342        fn get_last_error() -> u32;
343    }
344
345    if pid == std::process::id() {
346        return true;
347    }
348    let handle = unsafe { open_process(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) };
349    if handle.is_null() {
350        // ERROR_INVALID_PARAMETER means no process owns this PID. For access
351        // errors, retain the file rather than risk deleting a live process log.
352        return unsafe { get_last_error() } != ERROR_INVALID_PARAMETER;
353    }
354    let mut exit_code = 0;
355    let queried = unsafe { get_exit_code_process(handle, &mut exit_code) } != 0;
356    unsafe { close_handle(handle) };
357    queried && exit_code == STILL_ACTIVE
358}
359
360#[cfg(not(any(unix, windows)))]
361fn process_is_alive(pid: u32) -> bool {
362    pid == std::process::id()
363}
364
365#[derive(Default)]
366struct PerfMetrics {
367    watcher_ingested: AtomicU64,
368    watcher_paths: AtomicU64,
369    watcher_dropped: AtomicU64,
370    drain_slices: AtomicU64,
371    semantic_collects: AtomicU64,
372    semantic_files: AtomicU64,
373    semantic_chunks: AtomicU64,
374    semantic_ms: AtomicU64,
375    callgraph_invalidations: AtomicU64,
376    file_lines_dropped: AtomicU64,
377    tool_call_count: AtomicU64,
378    tool_calls: Mutex<VecDeque<ToolCallPerfSample>>,
379    tier2: Mutex<BTreeMap<String, (u64, u64)>>,
380    next_sample_ns: AtomicU64,
381    reporter: Mutex<PerfReporter>,
382}
383
384struct PerfReporter {
385    last_report: Instant,
386    last_completed_interactive: u64,
387    last_completed_maintenance: u64,
388    last_tool_call_count: u64,
389}
390
391impl Default for PerfReporter {
392    fn default() -> Self {
393        Self {
394            last_report: Instant::now(),
395            last_completed_interactive: 0,
396            last_completed_maintenance: 0,
397            last_tool_call_count: 0,
398        }
399    }
400}
401
402#[derive(Clone, Copy)]
403struct ToolCallPerfSample {
404    total_ms: u64,
405    queue_ms: u64,
406}
407
408#[derive(Clone, Copy, Default)]
409struct ToolCallPerfSummary {
410    count: usize,
411    p50_total_ms: u64,
412    max_total_ms: u64,
413    p50_queue_ms: u64,
414    max_queue_ms: u64,
415}
416
417#[derive(Clone, Copy, Default)]
418struct ExecutorSample {
419    interactive_running: usize,
420    maintenance_running: usize,
421    interactive_queued: usize,
422    maintenance_queued: usize,
423    interactive_oldest_ms: Option<u64>,
424    maintenance_oldest_ms: Option<u64>,
425}
426
427static PERF: LazyLock<PerfMetrics> = LazyLock::new(PerfMetrics::default);
428
429/// Move subsequent file log writes to a newly configured storage root.
430///
431/// Reconfiguration is queued behind existing writes and is a no-op when the
432/// root has not changed. Initialization and explicit configure changes call
433/// this directly, avoiding storage-root polling on transport drain turns.
434pub fn sync_storage_root(storage_root: PathBuf) {
435    let Ok(mut control) = FILE_CONTROL.lock() else {
436        return;
437    };
438    if control.storage_root.as_ref() == Some(&storage_root) {
439        return;
440    }
441    let Some(tx) = control.tx.as_ref() else {
442        return;
443    };
444    if tx
445        .try_send(LogMessage::Reconfigure(storage_root.clone()))
446        .is_ok()
447    {
448        control.storage_root = Some(storage_root);
449    }
450}
451
452/// Called by `drain_watcher_events_bounded` for dispatch events actually received.
453pub fn note_watcher_events(count: usize) {
454    PERF.watcher_ingested
455        .fetch_add(count as u64, Ordering::Relaxed);
456}
457
458/// Called when a watcher drain slice takes paths from dispatch continuation state.
459pub fn note_drain_paths(count: usize) {
460    PERF.watcher_paths
461        .fetch_add(count as u64, Ordering::Relaxed);
462}
463
464/// Called when `drain_watcher_events_bounded` receives a rescan-required overflow signal.
465pub fn note_watcher_overflow() {
466    PERF.watcher_dropped.fetch_add(1, Ordering::Relaxed);
467}
468
469/// Called by the standalone request loop before a request-triggered runtime drain.
470pub fn note_drain_slice() {
471    PERF.drain_slices.fetch_add(1, Ordering::Relaxed);
472}
473
474/// Called after `SemanticIndex::collect_chunks` has collected one real file batch.
475pub fn note_semantic_collect(chunks: usize, files: usize, elapsed_ms: u64) {
476    PERF.semantic_collects.fetch_add(1, Ordering::Relaxed);
477    PERF.semantic_chunks
478        .fetch_add(chunks as u64, Ordering::Relaxed);
479    PERF.semantic_files
480        .fetch_add(files as u64, Ordering::Relaxed);
481    PERF.semantic_ms.fetch_add(elapsed_ms, Ordering::Relaxed);
482}
483
484/// Called by `Tier2PhaseTimings::log` after a Tier-2 scan performs measurable work.
485pub fn note_tier2_scan(category: String, elapsed_ms: u64) {
486    if let Ok(mut tier2) = PERF.tier2.lock() {
487        let entry = tier2.entry(category).or_default();
488        entry.0 = entry.0.saturating_add(1);
489        entry.1 = entry.1.saturating_add(elapsed_ms);
490    }
491}
492
493/// Called after watcher-driven callgraph `refresh_files` succeeds for concrete paths.
494pub fn note_callgraph_invalidations(files: usize) {
495    PERF.callgraph_invalidations
496        .fetch_add(files as u64, Ordering::Relaxed);
497}
498
499/// Record a completed subc tool call for slow-call diagnostics and the standing
500/// perf-tick window. The writer calls this only after `write_all` has handed the
501/// complete response frame to the transport.
502pub fn note_tool_call_trace(
503    name: &str,
504    root: &Path,
505    channel: u16,
506    corr: u64,
507    phases: ToolCallPhaseDurations,
508) {
509    let sample = ToolCallPerfSample {
510        total_ms: duration_millis_u64(phases.total),
511        queue_ms: duration_millis_u64(phases.queue),
512    };
513    if let Ok(mut samples) = PERF.tool_calls.lock() {
514        if samples.len() == TOOL_CALL_SAMPLE_CAPACITY {
515            samples.pop_front();
516        }
517        samples.push_back(sample);
518        PERF.tool_call_count.fetch_add(1, Ordering::Relaxed);
519    }
520
521    crate::slog_debug!(
522        "tool_call phase name={} channel={} corr={} total_ms={:.3} queue_ms={:.3} translate_ms={:.3} exec_ms={:.3} format_ms={:.3} finalize_ms={:.3} egress_ms={:.3} egress_enqueue_ms={:.3} egress_queue_ms={:.3} egress_prepare_ms={:.3} egress_write_ms={:.3} frame_bytes={} writer_queue_depth={} writer_active={} writer_queue_full={} reserve_timeouts={} root={}",
523        name,
524        channel,
525        corr,
526        duration_millis_f64(phases.total),
527        duration_millis_f64(phases.queue),
528        duration_millis_f64(phases.translate),
529        duration_millis_f64(phases.execute),
530        duration_millis_f64(phases.format),
531        duration_millis_f64(phases.finalize),
532        duration_millis_f64(phases.egress),
533        duration_millis_f64(phases.egress_enqueue),
534        duration_millis_f64(phases.egress_queue),
535        duration_millis_f64(phases.egress_prepare),
536        duration_millis_f64(phases.egress_write),
537        phases.frame_bytes,
538        phases.writer_queue_depth,
539        phases.writer_active_at_enqueue,
540        phases.writer_queue_was_full,
541        phases.writer_reserve_timeouts,
542        root.display(),
543    );
544
545    if phases.total > SLOW_TOOL_CALL_THRESHOLD {
546        crate::slog_warn!(
547            "slow tool_call name={} channel={} corr={} total={}ms queue={} translate={} exec={} format={} finalize={} egress={} egress_enqueue={} egress_queue={} egress_prepare={} egress_write={} frame_bytes={} writer_queue_depth={} writer_active={} writer_queue_full={} reserve_timeouts={} root={}",
548            name,
549            channel,
550            corr,
551            duration_millis_u64(phases.total),
552            duration_millis_u64(phases.queue),
553            duration_millis_u64(phases.translate),
554            duration_millis_u64(phases.execute),
555            duration_millis_u64(phases.format),
556            duration_millis_u64(phases.finalize),
557            duration_millis_u64(phases.egress),
558            duration_millis_u64(phases.egress_enqueue),
559            duration_millis_u64(phases.egress_queue),
560            duration_millis_u64(phases.egress_prepare),
561            duration_millis_u64(phases.egress_write),
562            phases.frame_bytes,
563            phases.writer_queue_depth,
564            phases.writer_active_at_enqueue,
565            phases.writer_queue_was_full,
566            phases.writer_reserve_timeouts,
567            root.display(),
568        );
569    }
570}
571
572/// Sample executor liveness and emit one busy-only aggregate at the configured cadence.
573///
574/// The transport may call this every loop turn; an atomic deadline keeps all
575/// executor sampling and reporter locking off that path between drain ticks.
576pub fn perf_tick(executor: Option<&Executor>) {
577    if !perf_sample_due() {
578        return;
579    }
580
581    let sample = executor.and_then(|executor| {
582        executor
583            .try_dispatch_liveness_snapshot()
584            .map(|snapshot| ExecutorSample {
585                interactive_running: snapshot.running.interactive,
586                maintenance_running: snapshot.running.maintenance,
587                interactive_queued: snapshot.interactive.queued,
588                maintenance_queued: snapshot.maintenance.queued,
589                interactive_oldest_ms: snapshot.interactive.oldest_age_ms,
590                maintenance_oldest_ms: snapshot.maintenance.oldest_age_ms,
591            })
592    });
593
594    let completion_counts = executor.map_or((0, 0), Executor::completion_counts);
595    let tool_call_count = PERF.tool_call_count.load(Ordering::Relaxed);
596    let (completed_interactive, completed_maintenance, new_tool_calls) = {
597        let Ok(mut reporter) = PERF.reporter.lock() else {
598            return;
599        };
600        if reporter.last_report.elapsed() < perf_tick_interval() {
601            return;
602        }
603        reporter.last_report = Instant::now();
604        let completed = (
605            completion_counts
606                .0
607                .saturating_sub(reporter.last_completed_interactive),
608            completion_counts
609                .1
610                .saturating_sub(reporter.last_completed_maintenance),
611            tool_call_count.saturating_sub(reporter.last_tool_call_count),
612        );
613        reporter.last_completed_interactive = completion_counts.0;
614        reporter.last_completed_maintenance = completion_counts.1;
615        reporter.last_tool_call_count = tool_call_count;
616        completed
617    };
618
619    let watcher_ingested = PERF.watcher_ingested.swap(0, Ordering::Relaxed);
620    let watcher_paths = PERF.watcher_paths.swap(0, Ordering::Relaxed);
621    let watcher_dropped = PERF.watcher_dropped.swap(0, Ordering::Relaxed);
622    let drain_slices = PERF.drain_slices.swap(0, Ordering::Relaxed);
623    let semantic_collects = PERF.semantic_collects.swap(0, Ordering::Relaxed);
624    let semantic_files = PERF.semantic_files.swap(0, Ordering::Relaxed);
625    let semantic_chunks = PERF.semantic_chunks.swap(0, Ordering::Relaxed);
626    let semantic_ms = PERF.semantic_ms.swap(0, Ordering::Relaxed);
627    let callgraph_invalidations = PERF.callgraph_invalidations.swap(0, Ordering::Relaxed);
628    let file_lines_dropped = PERF.file_lines_dropped.swap(0, Ordering::Relaxed);
629    let tier2 = PERF
630        .tier2
631        .lock()
632        .map(|mut tier2| std::mem::take(&mut *tier2))
633        .unwrap_or_default();
634    let tool_calls = PERF
635        .tool_calls
636        .lock()
637        .map(|samples| summarize_tool_calls(&samples))
638        .unwrap_or_default();
639
640    let executor_busy = sample.is_some_and(|sample| {
641        sample.interactive_running > 0
642            || sample.maintenance_running > 0
643            || sample.interactive_queued > 0
644            || sample.maintenance_queued > 0
645    });
646    let active = watcher_ingested > 0
647        || watcher_paths > 0
648        || watcher_dropped > 0
649        || drain_slices > 0
650        || semantic_collects > 0
651        || callgraph_invalidations > 0
652        || completed_interactive > 0
653        || completed_maintenance > 0
654        || new_tool_calls > 0
655        || file_lines_dropped > 0
656        || !tier2.is_empty()
657        || executor_busy;
658    if !active {
659        return;
660    }
661
662    let tier2_summary = if tier2.is_empty() {
663        "none".to_string()
664    } else {
665        tier2
666            .into_iter()
667            .map(|(category, (count, ms))| format!("{category}:{count}/{ms}ms"))
668            .collect::<Vec<_>>()
669            .join(",")
670    };
671    let sample = sample.unwrap_or_default();
672    crate::slog_info!(
673        "perf tick: watcher={{ingested:{},paths:{},dropped:{}}} drains={} tier2=[{}] semantic={{collects:{},files:{},chunks:{},ms:{}}} callgraph_invalidations={} executor_completed={{interactive:{},maintenance:{}}} oldest_queued_ms={{interactive:{},maintenance:{}}} toolcall={{count:{},p50_total_ms:{},max_total_ms:{},p50_queue_ms:{},max_queue_ms:{}}} file_log_dropped={}",
674        watcher_ingested,
675        watcher_paths,
676        watcher_dropped,
677        drain_slices,
678        tier2_summary,
679        semantic_collects,
680        semantic_files,
681        semantic_chunks,
682        semantic_ms,
683        callgraph_invalidations,
684        completed_interactive,
685        completed_maintenance,
686        format_optional_ms(sample.interactive_oldest_ms),
687        format_optional_ms(sample.maintenance_oldest_ms),
688        tool_calls.count,
689        tool_calls.p50_total_ms,
690        tool_calls.max_total_ms,
691        tool_calls.p50_queue_ms,
692        tool_calls.max_queue_ms,
693        file_lines_dropped,
694    );
695}
696
697fn duration_millis_f64(duration: Duration) -> f64 {
698    duration.as_secs_f64() * 1_000.0
699}
700
701fn duration_millis_u64(duration: Duration) -> u64 {
702    duration.as_millis().min(u64::MAX as u128) as u64
703}
704
705fn summarize_tool_calls(samples: &VecDeque<ToolCallPerfSample>) -> ToolCallPerfSummary {
706    if samples.is_empty() {
707        return ToolCallPerfSummary::default();
708    }
709    let mut totals = samples
710        .iter()
711        .map(|sample| sample.total_ms)
712        .collect::<Vec<_>>();
713    let mut queues = samples
714        .iter()
715        .map(|sample| sample.queue_ms)
716        .collect::<Vec<_>>();
717    totals.sort_unstable();
718    queues.sort_unstable();
719    let median_index = (samples.len() - 1) / 2;
720    ToolCallPerfSummary {
721        count: samples.len(),
722        p50_total_ms: totals[median_index],
723        max_total_ms: totals[totals.len() - 1],
724        p50_queue_ms: queues[median_index],
725        max_queue_ms: queues[queues.len() - 1],
726    }
727}
728
729fn format_optional_ms(value: Option<u64>) -> String {
730    value
731        .map(|value| value.to_string())
732        .unwrap_or_else(|| "none".to_string())
733}
734
735fn perf_sample_due() -> bool {
736    static ORIGIN: LazyLock<Instant> = LazyLock::new(Instant::now);
737    let now_ns = ORIGIN.elapsed().as_nanos().min(u64::MAX as u128) as u64;
738    let mut deadline = PERF.next_sample_ns.load(Ordering::Relaxed);
739    loop {
740        if now_ns < deadline {
741            return false;
742        }
743        let next = now_ns.saturating_add(PERF_SAMPLE_INTERVAL.as_nanos() as u64);
744        match PERF.next_sample_ns.compare_exchange_weak(
745            deadline,
746            next,
747            Ordering::Relaxed,
748            Ordering::Relaxed,
749        ) {
750            Ok(_) => return true,
751            Err(observed) => deadline = observed,
752        }
753    }
754}
755
756fn perf_tick_interval() -> Duration {
757    static INTERVAL: OnceLock<Duration> = OnceLock::new();
758    *INTERVAL.get_or_init(|| {
759        std::env::var("AFT_PERF_TICK_INTERVAL_MS")
760            .ok()
761            .and_then(|value| value.parse::<u64>().ok())
762            .filter(|value| *value > 0)
763            .map(Duration::from_millis)
764            .unwrap_or(DEFAULT_PERF_TICK_INTERVAL)
765    })
766}
767
768#[cfg(test)]
769mod tests {
770    use super::*;
771    use filetime::{set_file_mtime, FileTime};
772    use tempfile::TempDir;
773
774    fn line(value: &str) -> Vec<Vec<u8>> {
775        vec![format!("{value}\n").into_bytes()]
776    }
777
778    #[test]
779    fn rotation_rolls_at_threshold_and_preserves_newest_generations() {
780        let temp = TempDir::new().unwrap();
781        let path = temp.path().join("aft-123.log");
782        let mut sink = RotatingFile::open(path.clone(), 10, 2, 1).unwrap();
783        sink.write_batch(&line("aaaa")).unwrap();
784        sink.write_batch(&line("bbbb")).unwrap();
785        sink.write_batch(&line("cccc")).unwrap();
786        sink.write_batch(&line("dddd")).unwrap();
787        sink.write_batch(&line("eeee")).unwrap();
788
789        assert_eq!(fs::read_to_string(&path).unwrap(), "eeee\n");
790        assert_eq!(
791            fs::read_to_string(rotated_path(&path, 1)).unwrap(),
792            "cccc\ndddd\n"
793        );
794        assert_eq!(
795            fs::read_to_string(rotated_path(&path, 2)).unwrap(),
796            "aaaa\nbbbb\n"
797        );
798        assert!(!rotated_path(&path, 3).exists());
799    }
800
801    #[test]
802    fn dead_pid_sweep_removes_only_old_process_logs() {
803        let temp = TempDir::new().unwrap();
804        let dead = temp.path().join("aft-4294967294.log");
805        let dead_rotated = temp.path().join("aft-4294967294.log.1");
806        let live = temp.path().join(format!("aft-{}.log", std::process::id()));
807        let unrelated = temp.path().join("aft-plugin.log");
808        for path in [&dead, &dead_rotated, &live, &unrelated] {
809            fs::write(path, "log").unwrap();
810            set_file_mtime(path, FileTime::from_unix_time(1, 0)).unwrap();
811        }
812
813        sweep_dead_process_logs(
814            temp.path(),
815            SystemTime::UNIX_EPOCH + Duration::from_secs(10 * 24 * 60 * 60),
816            DEAD_PROCESS_LOG_MAX_AGE,
817        )
818        .unwrap();
819
820        assert!(!dead.exists());
821        assert!(!dead_rotated.exists());
822        assert!(live.exists());
823        assert!(unrelated.exists());
824    }
825
826    #[test]
827    fn tool_call_summary_uses_bounded_window_median_and_maxima() {
828        let samples = VecDeque::from([
829            ToolCallPerfSample {
830                total_ms: 9,
831                queue_ms: 5,
832            },
833            ToolCallPerfSample {
834                total_ms: 3,
835                queue_ms: 1,
836            },
837            ToolCallPerfSample {
838                total_ms: 7,
839                queue_ms: 2,
840            },
841            ToolCallPerfSample {
842                total_ms: 5,
843                queue_ms: 4,
844            },
845        ]);
846
847        let summary = summarize_tool_calls(&samples);
848
849        assert_eq!(summary.count, 4);
850        assert_eq!(summary.p50_total_ms, 5);
851        assert_eq!(summary.max_total_ms, 9);
852        assert_eq!(summary.p50_queue_ms, 2);
853        assert_eq!(summary.max_queue_ms, 5);
854    }
855}