1use crate::executor::Executor;
7use crate::run_tool_call::ToolCallPhaseDurations;
8use std::collections::{BTreeMap, VecDeque};
9use std::fs::{self, File, OpenOptions};
10use std::io::{self, BufWriter, Write};
11use std::path::{Path, PathBuf};
12use std::sync::atomic::{AtomicU64, Ordering};
13use std::sync::mpsc::{self, SyncSender, TrySendError};
14use std::sync::{LazyLock, Mutex, OnceLock};
15use std::thread;
16use std::time::{Duration, Instant, SystemTime};
17
18const LOG_FILE_BYTES: u64 = 20 * 1024 * 1024;
19const LOG_GENERATIONS: usize = 5;
20const ROTATION_CHECK_EVERY: u64 = 64;
21const LOG_CHANNEL_CAPACITY: usize = 4096;
22const DEAD_PROCESS_LOG_MAX_AGE: Duration = Duration::from_secs(7 * 24 * 60 * 60);
23const DEFAULT_PERF_TICK_INTERVAL: Duration = Duration::from_secs(60);
24const PERF_SAMPLE_INTERVAL: Duration = Duration::from_millis(250);
25const SLOW_TOOL_CALL_THRESHOLD: Duration = Duration::from_millis(50);
26const TOOL_CALL_SAMPLE_CAPACITY: usize = 256;
27
28pub fn init() {
30 let storage_root = crate::bash_background::storage_dir(None);
31 let logs_dir = storage_root.join("logs");
32 let file_name = format!("aft-{}.log", std::process::id());
33 let file_path = logs_dir.join(file_name);
34
35 let file_tx = match prepare_file_sink(&logs_dir, &file_path) {
36 Ok(sink) => {
37 let (tx, rx) = mpsc::sync_channel(LOG_CHANNEL_CAPACITY);
38 thread::Builder::new()
39 .name("aft-log-writer".to_string())
40 .spawn(move || run_file_writer(sink, rx))
41 .map(|_| {
42 if let Ok(mut control) = FILE_CONTROL.lock() {
43 control.tx = Some(tx.clone());
44 control.storage_root = Some(storage_root.clone());
45 }
46 Some(tx)
47 })
48 .unwrap_or_else(|error| {
49 write_stderr_once(&format!(
50 "[aft] durable log disabled: cannot start writer thread: {error}\n"
51 ));
52 None
53 })
54 }
55 Err(error) => {
56 write_stderr_once(&format!(
57 "[aft] durable log disabled for {}: {error}\n",
58 file_path.display()
59 ));
60 None
61 }
62 };
63
64 env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info"))
65 .target(env_logger::Target::Pipe(Box::new(TeeWriter { file_tx })))
66 .format(|buf, record| {
67 let prefix = if record.target().starts_with("aft::lsp")
68 || record.target().starts_with("aft_lsp")
69 {
70 "[aft-lsp]"
71 } else {
72 "[aft]"
73 };
74 writeln!(buf, "{} {}", prefix, record.args())
75 })
76 .init();
77}
78
79fn prepare_file_sink(logs_dir: &Path, file_path: &Path) -> io::Result<RotatingFile> {
80 fs::create_dir_all(logs_dir)?;
81 sweep_dead_process_logs(logs_dir, SystemTime::now(), DEAD_PROCESS_LOG_MAX_AGE)?;
82 RotatingFile::open(
83 file_path.to_path_buf(),
84 LOG_FILE_BYTES,
85 LOG_GENERATIONS,
86 ROTATION_CHECK_EVERY,
87 )
88}
89
90enum LogMessage {
91 Write(Vec<u8>),
92 Reconfigure(PathBuf),
93}
94
95#[derive(Default)]
96struct FileControl {
97 tx: Option<SyncSender<LogMessage>>,
98 storage_root: Option<PathBuf>,
99}
100
101static FILE_CONTROL: LazyLock<Mutex<FileControl>> =
102 LazyLock::new(|| Mutex::new(FileControl::default()));
103
104struct TeeWriter {
105 file_tx: Option<SyncSender<LogMessage>>,
106}
107
108impl Write for TeeWriter {
109 fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
110 io::stderr().write_all(buf)?;
111 if let Some(tx) = self.file_tx.as_ref() {
112 match tx.try_send(LogMessage::Write(buf.to_vec())) {
113 Ok(()) => {}
114 Err(TrySendError::Full(_)) => {
115 PERF.file_lines_dropped.fetch_add(1, Ordering::Relaxed);
116 }
117 Err(TrySendError::Disconnected(_)) => self.file_tx = None,
118 }
119 }
120 Ok(buf.len())
121 }
122
123 fn flush(&mut self) -> io::Result<()> {
124 io::stderr().flush()
125 }
126}
127
128fn run_file_writer(mut sink: RotatingFile, rx: mpsc::Receiver<LogMessage>) {
129 while let Ok(message) = rx.recv() {
130 let mut lines = Vec::new();
131 let mut reconfigure = None;
132 match message {
133 LogMessage::Write(line) => {
134 lines.push(line);
135 while lines.len() < 256 {
136 match rx.try_recv() {
137 Ok(LogMessage::Write(line)) => lines.push(line),
138 Ok(LogMessage::Reconfigure(storage_root)) => {
139 reconfigure = Some(storage_root);
140 break;
141 }
142 Err(_) => break,
143 }
144 }
145 }
146 LogMessage::Reconfigure(storage_root) => reconfigure = Some(storage_root),
147 }
148 if !lines.is_empty() {
149 if let Err(error) = sink.write_batch(&lines) {
150 write_stderr_once(&format!(
151 "[aft] durable log disabled after write failure for {}: {error}\n",
152 sink.path.display()
153 ));
154 break;
155 }
156 }
157 if let Some(storage_root) = reconfigure {
158 let logs_dir = storage_root.join("logs");
159 let path = logs_dir.join(format!("aft-{}.log", std::process::id()));
160 match prepare_file_sink(&logs_dir, &path) {
161 Ok(new_sink) => sink = new_sink,
162 Err(error) => write_stderr_once(&format!(
163 "[aft] durable log could not switch to {}: {error}\n",
164 path.display()
165 )),
166 }
167 }
168 }
169}
170
171fn write_stderr_once(message: &str) {
172 let _ = io::stderr().write_all(message.as_bytes());
173}
174
175struct RotatingFile {
176 path: PathBuf,
177 writer: Option<BufWriter<File>>,
178 size: u64,
179 threshold: u64,
180 generations: usize,
181 check_every: u64,
182 writes_since_check: u64,
183}
184
185impl RotatingFile {
186 fn open(
187 path: PathBuf,
188 threshold: u64,
189 generations: usize,
190 check_every: u64,
191 ) -> io::Result<Self> {
192 let file = OpenOptions::new().create(true).append(true).open(&path)?;
193 let size = file.metadata()?.len();
194 Ok(Self {
195 path,
196 writer: Some(BufWriter::new(file)),
197 size,
198 threshold,
199 generations,
200 check_every: check_every.max(1),
201 writes_since_check: 0,
202 })
203 }
204
205 fn write_batch(&mut self, lines: &[Vec<u8>]) -> io::Result<()> {
206 let batch_bytes = lines.iter().map(Vec::len).sum::<usize>() as u64;
207 self.writes_since_check = self.writes_since_check.saturating_add(lines.len() as u64);
208 if self.writes_since_check >= self.check_every
209 && self.size > 0
210 && self.size.saturating_add(batch_bytes) > self.threshold
211 {
212 self.rotate()?;
213 }
214 let writer = self
215 .writer
216 .as_mut()
217 .ok_or_else(|| io::Error::other("log writer unavailable"))?;
218 for line in lines {
219 writer.write_all(line)?;
220 }
221 writer.flush()?;
224 self.size = self.size.saturating_add(batch_bytes);
225 if self.writes_since_check >= self.check_every {
226 self.writes_since_check = 0;
227 }
228 Ok(())
229 }
230
231 fn rotate(&mut self) -> io::Result<()> {
232 if let Some(mut writer) = self.writer.take() {
233 writer.flush()?;
234 }
235 if self.generations > 0 {
236 let oldest = rotated_path(&self.path, self.generations);
237 remove_file_if_present(&oldest)?;
238 for generation in (1..self.generations).rev() {
239 let from = rotated_path(&self.path, generation);
240 let to = rotated_path(&self.path, generation + 1);
241 rename_if_present(&from, &to)?;
242 }
243 rename_if_present(&self.path, &rotated_path(&self.path, 1))?;
244 } else {
245 remove_file_if_present(&self.path)?;
246 }
247 let file = OpenOptions::new()
248 .create(true)
249 .write(true)
250 .truncate(true)
251 .open(&self.path)?;
252 self.writer = Some(BufWriter::new(file));
253 self.size = 0;
254 self.writes_since_check = 0;
255 Ok(())
256 }
257}
258
259fn rotated_path(base: &Path, generation: usize) -> PathBuf {
260 let mut path = base.as_os_str().to_os_string();
261 path.push(format!(".{generation}"));
262 PathBuf::from(path)
263}
264
265fn remove_file_if_present(path: &Path) -> io::Result<()> {
266 match fs::remove_file(path) {
267 Ok(()) => Ok(()),
268 Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
269 Err(error) => Err(error),
270 }
271}
272
273fn rename_if_present(from: &Path, to: &Path) -> io::Result<()> {
274 match fs::rename(from, to) {
275 Ok(()) => Ok(()),
276 Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
277 Err(error) => Err(error),
278 }
279}
280
281fn sweep_dead_process_logs(dir: &Path, now: SystemTime, max_age: Duration) -> io::Result<()> {
282 for entry in fs::read_dir(dir)? {
283 let entry = match entry {
284 Ok(entry) => entry,
285 Err(_) => continue,
286 };
287 let name = entry.file_name();
288 let Some(pid) = process_log_pid(name.to_string_lossy().as_ref()) else {
289 continue;
290 };
291 let metadata = match entry.metadata() {
292 Ok(metadata) => metadata,
293 Err(_) => continue,
294 };
295 let age = metadata
296 .modified()
297 .ok()
298 .and_then(|modified| now.duration_since(modified).ok());
299 if age.is_some_and(|age| age >= max_age) && !process_is_alive(pid) {
300 let _ = fs::remove_file(entry.path());
301 }
302 }
303 Ok(())
304}
305
306fn process_log_pid(name: &str) -> Option<u32> {
307 let rest = name.strip_prefix("aft-")?;
308 let (pid, suffix) = rest.split_once(".log")?;
309 if !suffix.is_empty()
310 && !(suffix.starts_with('.') && suffix[1..].chars().all(|ch| ch.is_ascii_digit()))
311 {
312 return None;
313 }
314 pid.parse().ok()
315}
316
317#[cfg(unix)]
318fn process_is_alive(pid: u32) -> bool {
319 if pid == std::process::id() {
320 return true;
321 }
322 let result = unsafe { libc::kill(pid as libc::pid_t, 0) };
323 result == 0 || io::Error::last_os_error().raw_os_error() == Some(libc::EPERM)
324}
325
326#[cfg(windows)]
327fn process_is_alive(pid: u32) -> bool {
328 use std::ffi::c_void;
329
330 const PROCESS_QUERY_LIMITED_INFORMATION: u32 = 0x1000;
331 const STILL_ACTIVE: u32 = 259;
332 const ERROR_INVALID_PARAMETER: u32 = 87;
333 #[link(name = "kernel32")]
334 unsafe extern "system" {
335 #[link_name = "OpenProcess"]
336 fn open_process(access: u32, inherit_handle: i32, process_id: u32) -> *mut c_void;
337 #[link_name = "GetExitCodeProcess"]
338 fn get_exit_code_process(process: *mut c_void, exit_code: *mut u32) -> i32;
339 #[link_name = "CloseHandle"]
340 fn close_handle(object: *mut c_void) -> i32;
341 #[link_name = "GetLastError"]
342 fn get_last_error() -> u32;
343 }
344
345 if pid == std::process::id() {
346 return true;
347 }
348 let handle = unsafe { open_process(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) };
349 if handle.is_null() {
350 return unsafe { get_last_error() } != ERROR_INVALID_PARAMETER;
353 }
354 let mut exit_code = 0;
355 let queried = unsafe { get_exit_code_process(handle, &mut exit_code) } != 0;
356 unsafe { close_handle(handle) };
357 queried && exit_code == STILL_ACTIVE
358}
359
360#[cfg(not(any(unix, windows)))]
361fn process_is_alive(pid: u32) -> bool {
362 pid == std::process::id()
363}
364
365#[derive(Default)]
366struct PerfMetrics {
367 watcher_ingested: AtomicU64,
368 watcher_paths: AtomicU64,
369 watcher_dropped: AtomicU64,
370 drain_slices: AtomicU64,
371 semantic_collects: AtomicU64,
372 semantic_files: AtomicU64,
373 semantic_chunks: AtomicU64,
374 semantic_ms: AtomicU64,
375 callgraph_invalidations: AtomicU64,
376 file_lines_dropped: AtomicU64,
377 tool_call_count: AtomicU64,
378 tool_calls: Mutex<VecDeque<ToolCallPerfSample>>,
379 tier2: Mutex<BTreeMap<String, (u64, u64)>>,
380 next_sample_ns: AtomicU64,
381 reporter: Mutex<PerfReporter>,
382}
383
384struct PerfReporter {
385 last_report: Instant,
386 last_completed_interactive: u64,
387 last_completed_maintenance: u64,
388 last_tool_call_count: u64,
389}
390
391impl Default for PerfReporter {
392 fn default() -> Self {
393 Self {
394 last_report: Instant::now(),
395 last_completed_interactive: 0,
396 last_completed_maintenance: 0,
397 last_tool_call_count: 0,
398 }
399 }
400}
401
402#[derive(Clone, Copy)]
403struct ToolCallPerfSample {
404 total_ms: u64,
405 queue_ms: u64,
406}
407
408#[derive(Clone, Copy, Default)]
409struct ToolCallPerfSummary {
410 count: usize,
411 p50_total_ms: u64,
412 max_total_ms: u64,
413 p50_queue_ms: u64,
414 max_queue_ms: u64,
415}
416
417#[derive(Clone, Copy, Default)]
418struct ExecutorSample {
419 interactive_running: usize,
420 maintenance_running: usize,
421 interactive_queued: usize,
422 maintenance_queued: usize,
423 interactive_oldest_ms: Option<u64>,
424 maintenance_oldest_ms: Option<u64>,
425}
426
427static PERF: LazyLock<PerfMetrics> = LazyLock::new(PerfMetrics::default);
428
429pub fn sync_storage_root(storage_root: PathBuf) {
435 let Ok(mut control) = FILE_CONTROL.lock() else {
436 return;
437 };
438 if control.storage_root.as_ref() == Some(&storage_root) {
439 return;
440 }
441 let Some(tx) = control.tx.as_ref() else {
442 return;
443 };
444 if tx
445 .try_send(LogMessage::Reconfigure(storage_root.clone()))
446 .is_ok()
447 {
448 control.storage_root = Some(storage_root);
449 }
450}
451
452pub fn note_watcher_events(count: usize) {
454 PERF.watcher_ingested
455 .fetch_add(count as u64, Ordering::Relaxed);
456}
457
458pub fn note_drain_paths(count: usize) {
460 PERF.watcher_paths
461 .fetch_add(count as u64, Ordering::Relaxed);
462}
463
464pub fn note_watcher_overflow() {
466 PERF.watcher_dropped.fetch_add(1, Ordering::Relaxed);
467}
468
469pub fn note_drain_slice() {
471 PERF.drain_slices.fetch_add(1, Ordering::Relaxed);
472}
473
474pub fn note_semantic_collect(chunks: usize, files: usize, elapsed_ms: u64) {
476 PERF.semantic_collects.fetch_add(1, Ordering::Relaxed);
477 PERF.semantic_chunks
478 .fetch_add(chunks as u64, Ordering::Relaxed);
479 PERF.semantic_files
480 .fetch_add(files as u64, Ordering::Relaxed);
481 PERF.semantic_ms.fetch_add(elapsed_ms, Ordering::Relaxed);
482}
483
484pub fn note_tier2_scan(category: String, elapsed_ms: u64) {
486 if let Ok(mut tier2) = PERF.tier2.lock() {
487 let entry = tier2.entry(category).or_default();
488 entry.0 = entry.0.saturating_add(1);
489 entry.1 = entry.1.saturating_add(elapsed_ms);
490 }
491}
492
493pub fn note_callgraph_invalidations(files: usize) {
495 PERF.callgraph_invalidations
496 .fetch_add(files as u64, Ordering::Relaxed);
497}
498
499pub fn note_tool_call_trace(
503 name: &str,
504 root: &Path,
505 channel: u16,
506 corr: u64,
507 phases: ToolCallPhaseDurations,
508) {
509 let sample = ToolCallPerfSample {
510 total_ms: duration_millis_u64(phases.total),
511 queue_ms: duration_millis_u64(phases.queue),
512 };
513 if let Ok(mut samples) = PERF.tool_calls.lock() {
514 if samples.len() == TOOL_CALL_SAMPLE_CAPACITY {
515 samples.pop_front();
516 }
517 samples.push_back(sample);
518 PERF.tool_call_count.fetch_add(1, Ordering::Relaxed);
519 }
520
521 crate::slog_debug!(
522 "tool_call phase name={} channel={} corr={} total_ms={:.3} queue_ms={:.3} translate_ms={:.3} exec_ms={:.3} format_ms={:.3} finalize_ms={:.3} egress_ms={:.3} egress_enqueue_ms={:.3} egress_queue_ms={:.3} egress_prepare_ms={:.3} egress_write_ms={:.3} frame_bytes={} writer_queue_depth={} writer_active={} writer_queue_full={} reserve_timeouts={} root={}",
523 name,
524 channel,
525 corr,
526 duration_millis_f64(phases.total),
527 duration_millis_f64(phases.queue),
528 duration_millis_f64(phases.translate),
529 duration_millis_f64(phases.execute),
530 duration_millis_f64(phases.format),
531 duration_millis_f64(phases.finalize),
532 duration_millis_f64(phases.egress),
533 duration_millis_f64(phases.egress_enqueue),
534 duration_millis_f64(phases.egress_queue),
535 duration_millis_f64(phases.egress_prepare),
536 duration_millis_f64(phases.egress_write),
537 phases.frame_bytes,
538 phases.writer_queue_depth,
539 phases.writer_active_at_enqueue,
540 phases.writer_queue_was_full,
541 phases.writer_reserve_timeouts,
542 root.display(),
543 );
544
545 if phases.total > SLOW_TOOL_CALL_THRESHOLD {
546 crate::slog_warn!(
547 "slow tool_call name={} channel={} corr={} total={}ms queue={} translate={} exec={} format={} finalize={} egress={} egress_enqueue={} egress_queue={} egress_prepare={} egress_write={} frame_bytes={} writer_queue_depth={} writer_active={} writer_queue_full={} reserve_timeouts={} root={}",
548 name,
549 channel,
550 corr,
551 duration_millis_u64(phases.total),
552 duration_millis_u64(phases.queue),
553 duration_millis_u64(phases.translate),
554 duration_millis_u64(phases.execute),
555 duration_millis_u64(phases.format),
556 duration_millis_u64(phases.finalize),
557 duration_millis_u64(phases.egress),
558 duration_millis_u64(phases.egress_enqueue),
559 duration_millis_u64(phases.egress_queue),
560 duration_millis_u64(phases.egress_prepare),
561 duration_millis_u64(phases.egress_write),
562 phases.frame_bytes,
563 phases.writer_queue_depth,
564 phases.writer_active_at_enqueue,
565 phases.writer_queue_was_full,
566 phases.writer_reserve_timeouts,
567 root.display(),
568 );
569 }
570}
571
572pub fn perf_tick(executor: Option<&Executor>) {
577 if !perf_sample_due() {
578 return;
579 }
580
581 let sample = executor.and_then(|executor| {
582 executor
583 .try_dispatch_liveness_snapshot()
584 .map(|snapshot| ExecutorSample {
585 interactive_running: snapshot.running.interactive,
586 maintenance_running: snapshot.running.maintenance,
587 interactive_queued: snapshot.interactive.queued,
588 maintenance_queued: snapshot.maintenance.queued,
589 interactive_oldest_ms: snapshot.interactive.oldest_age_ms,
590 maintenance_oldest_ms: snapshot.maintenance.oldest_age_ms,
591 })
592 });
593
594 let completion_counts = executor.map_or((0, 0), Executor::completion_counts);
595 let tool_call_count = PERF.tool_call_count.load(Ordering::Relaxed);
596 let (completed_interactive, completed_maintenance, new_tool_calls) = {
597 let Ok(mut reporter) = PERF.reporter.lock() else {
598 return;
599 };
600 if reporter.last_report.elapsed() < perf_tick_interval() {
601 return;
602 }
603 reporter.last_report = Instant::now();
604 let completed = (
605 completion_counts
606 .0
607 .saturating_sub(reporter.last_completed_interactive),
608 completion_counts
609 .1
610 .saturating_sub(reporter.last_completed_maintenance),
611 tool_call_count.saturating_sub(reporter.last_tool_call_count),
612 );
613 reporter.last_completed_interactive = completion_counts.0;
614 reporter.last_completed_maintenance = completion_counts.1;
615 reporter.last_tool_call_count = tool_call_count;
616 completed
617 };
618
619 let watcher_ingested = PERF.watcher_ingested.swap(0, Ordering::Relaxed);
620 let watcher_paths = PERF.watcher_paths.swap(0, Ordering::Relaxed);
621 let watcher_dropped = PERF.watcher_dropped.swap(0, Ordering::Relaxed);
622 let drain_slices = PERF.drain_slices.swap(0, Ordering::Relaxed);
623 let semantic_collects = PERF.semantic_collects.swap(0, Ordering::Relaxed);
624 let semantic_files = PERF.semantic_files.swap(0, Ordering::Relaxed);
625 let semantic_chunks = PERF.semantic_chunks.swap(0, Ordering::Relaxed);
626 let semantic_ms = PERF.semantic_ms.swap(0, Ordering::Relaxed);
627 let callgraph_invalidations = PERF.callgraph_invalidations.swap(0, Ordering::Relaxed);
628 let file_lines_dropped = PERF.file_lines_dropped.swap(0, Ordering::Relaxed);
629 let tier2 = PERF
630 .tier2
631 .lock()
632 .map(|mut tier2| std::mem::take(&mut *tier2))
633 .unwrap_or_default();
634 let tool_calls = PERF
635 .tool_calls
636 .lock()
637 .map(|samples| summarize_tool_calls(&samples))
638 .unwrap_or_default();
639
640 let executor_busy = sample.is_some_and(|sample| {
641 sample.interactive_running > 0
642 || sample.maintenance_running > 0
643 || sample.interactive_queued > 0
644 || sample.maintenance_queued > 0
645 });
646 let active = watcher_ingested > 0
647 || watcher_paths > 0
648 || watcher_dropped > 0
649 || drain_slices > 0
650 || semantic_collects > 0
651 || callgraph_invalidations > 0
652 || completed_interactive > 0
653 || completed_maintenance > 0
654 || new_tool_calls > 0
655 || file_lines_dropped > 0
656 || !tier2.is_empty()
657 || executor_busy;
658 if !active {
659 return;
660 }
661
662 let tier2_summary = if tier2.is_empty() {
663 "none".to_string()
664 } else {
665 tier2
666 .into_iter()
667 .map(|(category, (count, ms))| format!("{category}:{count}/{ms}ms"))
668 .collect::<Vec<_>>()
669 .join(",")
670 };
671 let sample = sample.unwrap_or_default();
672 crate::slog_info!(
673 "perf tick: watcher={{ingested:{},paths:{},dropped:{}}} drains={} tier2=[{}] semantic={{collects:{},files:{},chunks:{},ms:{}}} callgraph_invalidations={} executor_completed={{interactive:{},maintenance:{}}} oldest_queued_ms={{interactive:{},maintenance:{}}} toolcall={{count:{},p50_total_ms:{},max_total_ms:{},p50_queue_ms:{},max_queue_ms:{}}} file_log_dropped={}",
674 watcher_ingested,
675 watcher_paths,
676 watcher_dropped,
677 drain_slices,
678 tier2_summary,
679 semantic_collects,
680 semantic_files,
681 semantic_chunks,
682 semantic_ms,
683 callgraph_invalidations,
684 completed_interactive,
685 completed_maintenance,
686 format_optional_ms(sample.interactive_oldest_ms),
687 format_optional_ms(sample.maintenance_oldest_ms),
688 tool_calls.count,
689 tool_calls.p50_total_ms,
690 tool_calls.max_total_ms,
691 tool_calls.p50_queue_ms,
692 tool_calls.max_queue_ms,
693 file_lines_dropped,
694 );
695}
696
697fn duration_millis_f64(duration: Duration) -> f64 {
698 duration.as_secs_f64() * 1_000.0
699}
700
701fn duration_millis_u64(duration: Duration) -> u64 {
702 duration.as_millis().min(u64::MAX as u128) as u64
703}
704
705fn summarize_tool_calls(samples: &VecDeque<ToolCallPerfSample>) -> ToolCallPerfSummary {
706 if samples.is_empty() {
707 return ToolCallPerfSummary::default();
708 }
709 let mut totals = samples
710 .iter()
711 .map(|sample| sample.total_ms)
712 .collect::<Vec<_>>();
713 let mut queues = samples
714 .iter()
715 .map(|sample| sample.queue_ms)
716 .collect::<Vec<_>>();
717 totals.sort_unstable();
718 queues.sort_unstable();
719 let median_index = (samples.len() - 1) / 2;
720 ToolCallPerfSummary {
721 count: samples.len(),
722 p50_total_ms: totals[median_index],
723 max_total_ms: totals[totals.len() - 1],
724 p50_queue_ms: queues[median_index],
725 max_queue_ms: queues[queues.len() - 1],
726 }
727}
728
729fn format_optional_ms(value: Option<u64>) -> String {
730 value
731 .map(|value| value.to_string())
732 .unwrap_or_else(|| "none".to_string())
733}
734
735fn perf_sample_due() -> bool {
736 static ORIGIN: LazyLock<Instant> = LazyLock::new(Instant::now);
737 let now_ns = ORIGIN.elapsed().as_nanos().min(u64::MAX as u128) as u64;
738 let mut deadline = PERF.next_sample_ns.load(Ordering::Relaxed);
739 loop {
740 if now_ns < deadline {
741 return false;
742 }
743 let next = now_ns.saturating_add(PERF_SAMPLE_INTERVAL.as_nanos() as u64);
744 match PERF.next_sample_ns.compare_exchange_weak(
745 deadline,
746 next,
747 Ordering::Relaxed,
748 Ordering::Relaxed,
749 ) {
750 Ok(_) => return true,
751 Err(observed) => deadline = observed,
752 }
753 }
754}
755
756fn perf_tick_interval() -> Duration {
757 static INTERVAL: OnceLock<Duration> = OnceLock::new();
758 *INTERVAL.get_or_init(|| {
759 std::env::var("AFT_PERF_TICK_INTERVAL_MS")
760 .ok()
761 .and_then(|value| value.parse::<u64>().ok())
762 .filter(|value| *value > 0)
763 .map(Duration::from_millis)
764 .unwrap_or(DEFAULT_PERF_TICK_INTERVAL)
765 })
766}
767
768#[cfg(test)]
769mod tests {
770 use super::*;
771 use filetime::{set_file_mtime, FileTime};
772 use tempfile::TempDir;
773
774 fn line(value: &str) -> Vec<Vec<u8>> {
775 vec![format!("{value}\n").into_bytes()]
776 }
777
778 #[test]
779 fn rotation_rolls_at_threshold_and_preserves_newest_generations() {
780 let temp = TempDir::new().unwrap();
781 let path = temp.path().join("aft-123.log");
782 let mut sink = RotatingFile::open(path.clone(), 10, 2, 1).unwrap();
783 sink.write_batch(&line("aaaa")).unwrap();
784 sink.write_batch(&line("bbbb")).unwrap();
785 sink.write_batch(&line("cccc")).unwrap();
786 sink.write_batch(&line("dddd")).unwrap();
787 sink.write_batch(&line("eeee")).unwrap();
788
789 assert_eq!(fs::read_to_string(&path).unwrap(), "eeee\n");
790 assert_eq!(
791 fs::read_to_string(rotated_path(&path, 1)).unwrap(),
792 "cccc\ndddd\n"
793 );
794 assert_eq!(
795 fs::read_to_string(rotated_path(&path, 2)).unwrap(),
796 "aaaa\nbbbb\n"
797 );
798 assert!(!rotated_path(&path, 3).exists());
799 }
800
801 #[test]
802 fn dead_pid_sweep_removes_only_old_process_logs() {
803 let temp = TempDir::new().unwrap();
804 let dead = temp.path().join("aft-4294967294.log");
805 let dead_rotated = temp.path().join("aft-4294967294.log.1");
806 let live = temp.path().join(format!("aft-{}.log", std::process::id()));
807 let unrelated = temp.path().join("aft-plugin.log");
808 for path in [&dead, &dead_rotated, &live, &unrelated] {
809 fs::write(path, "log").unwrap();
810 set_file_mtime(path, FileTime::from_unix_time(1, 0)).unwrap();
811 }
812
813 sweep_dead_process_logs(
814 temp.path(),
815 SystemTime::UNIX_EPOCH + Duration::from_secs(10 * 24 * 60 * 60),
816 DEAD_PROCESS_LOG_MAX_AGE,
817 )
818 .unwrap();
819
820 assert!(!dead.exists());
821 assert!(!dead_rotated.exists());
822 assert!(live.exists());
823 assert!(unrelated.exists());
824 }
825
826 #[test]
827 fn tool_call_summary_uses_bounded_window_median_and_maxima() {
828 let samples = VecDeque::from([
829 ToolCallPerfSample {
830 total_ms: 9,
831 queue_ms: 5,
832 },
833 ToolCallPerfSample {
834 total_ms: 3,
835 queue_ms: 1,
836 },
837 ToolCallPerfSample {
838 total_ms: 7,
839 queue_ms: 2,
840 },
841 ToolCallPerfSample {
842 total_ms: 5,
843 queue_ms: 4,
844 },
845 ]);
846
847 let summary = summarize_tool_calls(&samples);
848
849 assert_eq!(summary.count, 4);
850 assert_eq!(summary.p50_total_ms, 5);
851 assert_eq!(summary.max_total_ms, 9);
852 assert_eq!(summary.p50_queue_ms, 2);
853 assert_eq!(summary.max_queue_ms, 5);
854 }
855}