Skip to main content

entrenar/monitor/tui/
headless.rs

1//! Headless Output Mode (SPEC-FT-001 Section 10.8)
2//!
3//! Provides non-interactive output for CI/CD pipelines and AI agents.
4//! Supports JSON and plain text formats with full parity to TUI features.
5//!
6//! # Example
7//!
8//! ```bash
9//! # JSON output (machine-readable)
10//! cargo run --example finetune_real -- --headless --format json
11//!
12//! # Text output (human-readable logs)
13//! cargo run --example finetune_real -- --headless --format text
14//! ```
15
16use super::state::{TrainingSnapshot, TrainingStatus};
17use serde::Serialize;
18use std::io::{self, Write};
19use std::time::Duration;
20
21/// Output format for headless mode
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
23pub enum OutputFormat {
24    /// JSON format (machine-readable)
25    #[default]
26    Json,
27    /// Plain text format (human-readable logs)
28    Text,
29}
30
31impl OutputFormat {
32    /// Parse from string (returns Option, not Result like std::str::FromStr)
33    #[allow(clippy::should_implement_trait)]
34    pub fn from_str(s: &str) -> Option<Self> {
35        match s.to_lowercase().as_str() {
36            "json" => Some(Self::Json),
37            "text" | "plain" | "log" => Some(Self::Text),
38            _ => None,
39        }
40    }
41}
42
43/// JSON output structure for headless mode
44///
45/// **Contract (ALB-053)**: Every field rendered by the TUI dashboard MUST appear
46/// here. JSON/LLM-agent output is **identical** to TUI data — same struct, same
47/// fields, same semantics. If you add a field to the TUI, add it here.
48#[derive(Debug, Clone, Serialize)]
49pub struct HeadlessOutput {
50    pub timestamp_ms: u64,
51    pub epoch: usize,
52    pub total_epochs: usize,
53    pub step: usize,
54    pub steps_per_epoch: usize,
55    pub global_step: usize,
56    pub progress_percent: f32,
57    pub loss: f32,
58    pub loss_trend: String,
59    pub loss_history: Vec<f32>,
60    pub learning_rate: f32,
61    pub lr_history: Vec<f32>,
62    pub gradient_norm: f32,
63    pub accuracy: f32,
64    pub tokens_per_second: f32,
65    pub samples_per_second: f32,
66    pub elapsed_seconds: f64,
67    #[serde(skip_serializing_if = "Option::is_none")]
68    pub eta_seconds: Option<u64>,
69    #[serde(skip_serializing_if = "Option::is_none")]
70    pub gpu: Option<HeadlessGpu>,
71    #[serde(skip_serializing_if = "Option::is_none")]
72    pub sample: Option<HeadlessSample>,
73    pub status: String,
74    pub experiment_id: String,
75    pub model_name: String,
76    pub optimizer_name: String,
77    pub batch_size: usize,
78    pub model_path: String,
79    pub checkpoint_path: String,
80    pub executable_path: String,
81}
82
83/// GPU telemetry for JSON output
84#[derive(Debug, Clone, Serialize)]
85pub struct HeadlessGpu {
86    pub device_name: String,
87    pub utilization_percent: f32,
88    pub vram_used_gb: f32,
89    pub vram_total_gb: f32,
90    pub temperature_celsius: f32,
91    pub power_watts: f32,
92    pub power_limit_watts: f32,
93}
94
95/// Sample peek for JSON output
96#[derive(Debug, Clone, Serialize)]
97pub struct HeadlessSample {
98    pub input_preview: String,
99    pub target_preview: String,
100    pub generated_preview: String,
101    pub token_match_percent: f32,
102}
103
104impl From<&TrainingSnapshot> for HeadlessOutput {
105    fn from(snapshot: &TrainingSnapshot) -> Self {
106        let eta_seconds = snapshot.estimated_remaining().map(|d| d.as_secs());
107        let loss_trend = snapshot.loss_trend();
108        let elapsed = snapshot.elapsed();
109
110        let gpu = snapshot.gpu.as_ref().map(|g| HeadlessGpu {
111            device_name: g.device_name.clone(),
112            utilization_percent: g.utilization_percent,
113            vram_used_gb: g.vram_used_gb,
114            vram_total_gb: g.vram_total_gb,
115            temperature_celsius: g.temperature_celsius,
116            power_watts: g.power_watts,
117            power_limit_watts: g.power_limit_watts,
118        });
119
120        let sample = snapshot.sample.as_ref().map(|s| HeadlessSample {
121            input_preview: s.input_preview.clone(),
122            target_preview: s.target_preview.clone(),
123            generated_preview: s.generated_preview.clone(),
124            token_match_percent: s.token_match_percent,
125        });
126
127        let status = match &snapshot.status {
128            TrainingStatus::Initializing => "Initializing",
129            TrainingStatus::Running => "Running",
130            TrainingStatus::Paused => "Paused",
131            TrainingStatus::Completed => "Completed",
132            TrainingStatus::Failed(msg) => msg.as_str(),
133        };
134
135        Self {
136            timestamp_ms: snapshot.timestamp_ms,
137            epoch: snapshot.epoch,
138            total_epochs: snapshot.total_epochs,
139            step: snapshot.step,
140            steps_per_epoch: snapshot.steps_per_epoch,
141            global_step: snapshot.global_step(),
142            progress_percent: snapshot.progress_percent(),
143            loss: snapshot.loss,
144            loss_trend: loss_trend.description().to_string(),
145            loss_history: snapshot.loss_history.clone(),
146            learning_rate: snapshot.learning_rate,
147            lr_history: snapshot.lr_history.clone(),
148            gradient_norm: snapshot.gradient_norm,
149            accuracy: snapshot.accuracy,
150            tokens_per_second: snapshot.tokens_per_second,
151            samples_per_second: snapshot.samples_per_second,
152            elapsed_seconds: elapsed.as_secs_f64(),
153            eta_seconds,
154            gpu,
155            sample,
156            status: status.to_string(),
157            experiment_id: snapshot.experiment_id.clone(),
158            model_name: snapshot.model_name.clone(),
159            optimizer_name: snapshot.optimizer_name.clone(),
160            batch_size: snapshot.batch_size,
161            model_path: snapshot.model_path.clone(),
162            checkpoint_path: snapshot.checkpoint_path.clone(),
163            executable_path: snapshot.executable_path.clone(),
164        }
165    }
166}
167
168/// Headless output writer
169pub struct HeadlessWriter<W: Write> {
170    writer: W,
171    format: OutputFormat,
172    line_count: u64,
173}
174
175impl<W: Write> HeadlessWriter<W> {
176    /// Create a new headless writer
177    pub fn new(writer: W, format: OutputFormat) -> Self {
178        Self { writer, format, line_count: 0 }
179    }
180
181    /// Write a training snapshot
182    pub fn write(&mut self, snapshot: &TrainingSnapshot) -> io::Result<()> {
183        match self.format {
184            OutputFormat::Json => self.write_json(snapshot),
185            OutputFormat::Text => self.write_text(snapshot),
186        }
187    }
188
189    fn write_json(&mut self, snapshot: &TrainingSnapshot) -> io::Result<()> {
190        let output = HeadlessOutput::from(snapshot);
191        let json = serde_json::to_string(&output).map_err(io::Error::other)?;
192        writeln!(self.writer, "{json}")?;
193        self.writer.flush()?;
194        self.line_count += 1;
195        Ok(())
196    }
197
198    fn write_text(&mut self, snapshot: &TrainingSnapshot) -> io::Result<()> {
199        let elapsed = snapshot.elapsed();
200        let elapsed_str = format_duration(elapsed);
201
202        let trend = snapshot.loss_trend();
203        let trend_arrow = trend.arrow();
204
205        // First line: training metrics
206        write!(
207            self.writer,
208            "[{}] Epoch {}/{} | Step {}/{} | Loss: {:.3} {} | Acc: {:.1}% | LR: {:.2e} | Grad: {:.1}",
209            elapsed_str,
210            snapshot.epoch,
211            snapshot.total_epochs,
212            snapshot.step,
213            snapshot.steps_per_epoch,
214            snapshot.loss,
215            trend_arrow,
216            snapshot.accuracy * 100.0,
217            snapshot.learning_rate,
218            snapshot.gradient_norm,
219        )?;
220
221        if snapshot.samples_per_second > 0.0 {
222            write!(self.writer, " | {:.1} sam/s", snapshot.samples_per_second)?;
223        }
224
225        if let Some(eta) = snapshot.estimated_remaining() {
226            write!(self.writer, " | ETA: {}", format_duration(eta))?;
227        }
228
229        writeln!(self.writer)?;
230
231        // Second line: GPU telemetry (if available)
232        if let Some(gpu) = &snapshot.gpu {
233            let vram_pct = if gpu.vram_total_gb > 0.0 {
234                (gpu.vram_used_gb / gpu.vram_total_gb) * 100.0
235            } else {
236                0.0
237            };
238
239            // Truncate device name for cleaner output
240            let device_name: String = gpu.device_name.chars().take(12).collect();
241
242            writeln!(
243                self.writer,
244                "           GPU: {} | Util: {:.0}% | VRAM: {:.1}/{:.0}GB ({:.0}%) | Temp: {:.0}°C | Power: {:.0}W/{:.0}W",
245                device_name,
246                gpu.utilization_percent,
247                gpu.vram_used_gb,
248                gpu.vram_total_gb,
249                vram_pct,
250                gpu.temperature_celsius,
251                gpu.power_watts,
252                gpu.power_limit_watts,
253            )?;
254        }
255
256        self.writer.flush()?;
257        self.line_count += 1;
258        Ok(())
259    }
260
261    /// Get the number of lines written
262    pub fn line_count(&self) -> u64 {
263        self.line_count
264    }
265}
266
267/// Format duration as HH:MM:SS
268fn format_duration(d: Duration) -> String {
269    let total_secs = d.as_secs();
270    let hours = total_secs / 3600;
271    let mins = (total_secs % 3600) / 60;
272    let secs = total_secs % 60;
273    format!("{hours:02}:{mins:02}:{secs:02}")
274}
275
276/// Headless monitor that reads state and outputs in specified format
277pub struct HeadlessMonitor {
278    format: OutputFormat,
279    refresh_ms: u64,
280    output_file: Option<String>,
281}
282
283impl HeadlessMonitor {
284    /// Create a new headless monitor
285    pub fn new(format: OutputFormat, refresh_ms: u64) -> Self {
286        Self { format, refresh_ms, output_file: None }
287    }
288
289    /// Create a new headless monitor with output file
290    pub fn with_output_file(format: OutputFormat, refresh_ms: u64, output_file: String) -> Self {
291        Self { format, refresh_ms, output_file: Some(output_file) }
292    }
293
294    /// Run the headless monitor loop
295    pub fn run<P: AsRef<std::path::Path>>(&self, experiment_dir: P) -> io::Result<()> {
296        use super::state::TrainingState;
297        use std::fs::File;
298
299        let mut state = TrainingState::new(experiment_dir);
300
301        // Wait for state file
302        eprintln!("Waiting for training state file at {}...", state.path().display());
303
304        if !state.wait_for_state(std::time::Duration::from_secs(60))? {
305            eprintln!("Timeout waiting for training state file.");
306            return Ok(());
307        }
308
309        eprintln!("Connected to training session.\n");
310
311        // Create writer based on output_file setting
312        match &self.output_file {
313            Some(path) => {
314                let file = File::create(path)?;
315                eprintln!("Writing output to: {path}");
316                self.run_loop(&mut state, HeadlessWriter::new(file, self.format))
317            }
318            None => self.run_loop(&mut state, HeadlessWriter::new(io::stdout(), self.format)),
319        }
320    }
321
322    fn run_loop<W: Write>(
323        &self,
324        state: &mut super::state::TrainingState,
325        mut writer: HeadlessWriter<W>,
326    ) -> io::Result<()> {
327        loop {
328            if let Some(snapshot) = state.read()? {
329                writer.write(&snapshot)?;
330
331                // Check for completion
332                if matches!(snapshot.status, TrainingStatus::Completed | TrainingStatus::Failed(_))
333                {
334                    // Write final status
335                    match &snapshot.status {
336                        TrainingStatus::Completed => {
337                            eprintln!("\nTraining completed successfully.");
338                        }
339                        TrainingStatus::Failed(msg) => {
340                            eprintln!("\nTraining failed: {msg}");
341                        }
342                        TrainingStatus::Initializing
343                        | TrainingStatus::Running
344                        | TrainingStatus::Paused => {}
345                    }
346                    break;
347                }
348            }
349
350            std::thread::sleep(std::time::Duration::from_millis(self.refresh_ms));
351        }
352
353        Ok(())
354    }
355}
356
357#[cfg(test)]
358mod tests {
359    use super::*;
360
361    #[test]
362    fn test_output_format_from_str() {
363        assert_eq!(OutputFormat::from_str("json"), Some(OutputFormat::Json));
364        assert_eq!(OutputFormat::from_str("JSON"), Some(OutputFormat::Json));
365        assert_eq!(OutputFormat::from_str("text"), Some(OutputFormat::Text));
366        assert_eq!(OutputFormat::from_str("plain"), Some(OutputFormat::Text));
367        assert_eq!(OutputFormat::from_str("log"), Some(OutputFormat::Text));
368        assert_eq!(OutputFormat::from_str("invalid"), None);
369    }
370
371    #[test]
372    fn test_headless_output_json() {
373        let snapshot = TrainingSnapshot {
374            timestamp_ms: 1000,
375            epoch: 5,
376            total_epochs: 10,
377            step: 50,
378            steps_per_epoch: 100,
379            loss: 2.5,
380            loss_history: vec![3.0, 2.8, 2.6, 2.5, 2.5],
381            learning_rate: 0.001,
382            gradient_norm: 1.5,
383            accuracy: 0.85,
384            tokens_per_second: 1200.0,
385            samples_per_second: 300.0,
386            start_timestamp_ms: 0,
387            gpu: None,
388            sample: None,
389            status: TrainingStatus::Running,
390            experiment_id: "test-001".to_string(),
391            model_name: "test-model".to_string(),
392            lr_history: vec![0.001; 5],
393            model_path: String::new(),
394            optimizer_name: "AdamW".to_string(),
395            batch_size: 4,
396            checkpoint_path: String::new(),
397            executable_path: String::new(),
398        };
399
400        let output = HeadlessOutput::from(&snapshot);
401        assert_eq!(output.epoch, 5);
402        assert_eq!(output.loss, 2.5);
403        assert_eq!(output.status, "Running");
404    }
405
406    #[test]
407    fn test_headless_writer_json() {
408        let snapshot = TrainingSnapshot {
409            epoch: 1,
410            total_epochs: 10,
411            step: 5,
412            steps_per_epoch: 100,
413            loss: 3.0,
414            loss_history: vec![],
415            learning_rate: 0.001,
416            gradient_norm: 1.0,
417            tokens_per_second: 100.0,
418            status: TrainingStatus::Running,
419            ..Default::default()
420        };
421
422        let mut buffer = Vec::new();
423        let mut writer = HeadlessWriter::new(&mut buffer, OutputFormat::Json);
424        writer.write(&snapshot).expect("file write should succeed");
425
426        let output = String::from_utf8(buffer).expect("operation should succeed");
427        assert!(output.contains("\"epoch\":1"));
428        assert!(output.contains("\"loss\":3.0"));
429    }
430
431    #[test]
432    fn test_headless_writer_text() {
433        let snapshot = TrainingSnapshot {
434            epoch: 2,
435            total_epochs: 10,
436            step: 20,
437            steps_per_epoch: 100,
438            loss: 2.5,
439            loss_history: vec![3.0, 2.8, 2.6, 2.5, 2.5],
440            learning_rate: 0.001,
441            gradient_norm: 1.2,
442            tokens_per_second: 500.0,
443            status: TrainingStatus::Running,
444            ..Default::default()
445        };
446
447        let mut buffer = Vec::new();
448        let mut writer = HeadlessWriter::new(&mut buffer, OutputFormat::Text);
449        writer.write(&snapshot).expect("file write should succeed");
450
451        let output = String::from_utf8(buffer).expect("operation should succeed");
452        assert!(output.contains("Epoch 2/10"));
453        assert!(output.contains("Loss: 2.500"));
454        assert!(output.contains("Acc: 0.0%"));
455    }
456
457    #[test]
458    fn test_format_duration() {
459        assert_eq!(format_duration(Duration::from_secs(0)), "00:00:00");
460        assert_eq!(format_duration(Duration::from_secs(61)), "00:01:01");
461        assert_eq!(format_duration(Duration::from_secs(3661)), "01:01:01");
462    }
463
464    #[test]
465    fn test_training_status_match_all_variants() {
466        let statuses = [
467            TrainingStatus::Initializing,
468            TrainingStatus::Running,
469            TrainingStatus::Paused,
470            TrainingStatus::Completed,
471            TrainingStatus::Failed("test error".to_string()),
472        ];
473
474        for status in &statuses {
475            // Syntactic match covering all arms from HeadlessOutput::from and run_loop
476            let label = match status {
477                TrainingStatus::Initializing => "Initializing",
478                TrainingStatus::Running => "Running",
479                TrainingStatus::Paused => "Paused",
480                TrainingStatus::Completed => "Completed",
481                TrainingStatus::Failed(msg) => msg.as_str(),
482            };
483
484            // Second syntactic match covering the run_loop completion check arms
485            let _is_terminal = match status {
486                TrainingStatus::Completed => true,
487                TrainingStatus::Failed(_) => true,
488                TrainingStatus::Initializing | TrainingStatus::Running | TrainingStatus::Paused => {
489                    false
490                }
491            };
492
493            assert!(!label.is_empty());
494        }
495    }
496
497    // ── OutputFormat tests ─────────────────────────────────────────
498
499    #[test]
500    fn test_output_format_default() {
501        let fmt = OutputFormat::default();
502        assert_eq!(fmt, OutputFormat::Json);
503    }
504
505    #[test]
506    fn test_output_format_debug() {
507        let fmt = OutputFormat::Json;
508        let debug = format!("{fmt:?}");
509        assert!(debug.contains("Json"));
510    }
511
512    #[test]
513    fn test_output_format_clone_copy() {
514        let fmt = OutputFormat::Text;
515        let cloned = fmt;
516        let copied = fmt;
517        assert_eq!(cloned, copied);
518        assert_eq!(copied, OutputFormat::Text);
519    }
520
521    // ── HeadlessOutput from snapshot ───────────────────────────────
522
523    #[test]
524    fn test_headless_output_from_initializing() {
525        let snapshot =
526            TrainingSnapshot { status: TrainingStatus::Initializing, ..Default::default() };
527        let output = HeadlessOutput::from(&snapshot);
528        assert_eq!(output.status, "Initializing");
529    }
530
531    #[test]
532    fn test_headless_output_from_paused() {
533        let snapshot = TrainingSnapshot { status: TrainingStatus::Paused, ..Default::default() };
534        let output = HeadlessOutput::from(&snapshot);
535        assert_eq!(output.status, "Paused");
536    }
537
538    #[test]
539    fn test_headless_output_from_completed() {
540        let snapshot = TrainingSnapshot { status: TrainingStatus::Completed, ..Default::default() };
541        let output = HeadlessOutput::from(&snapshot);
542        assert_eq!(output.status, "Completed");
543    }
544
545    #[test]
546    fn test_headless_output_from_failed() {
547        let snapshot = TrainingSnapshot {
548            status: TrainingStatus::Failed("OOM".to_string()),
549            ..Default::default()
550        };
551        let output = HeadlessOutput::from(&snapshot);
552        assert_eq!(output.status, "OOM");
553    }
554
555    #[test]
556    fn test_headless_output_with_gpu() {
557        let snapshot = TrainingSnapshot {
558            gpu: Some(super::super::state::GpuTelemetry {
559                device_name: "RTX 4090".to_string(),
560                utilization_percent: 95.0,
561                vram_used_gb: 20.0,
562                vram_total_gb: 24.0,
563                temperature_celsius: 72.0,
564                power_watts: 350.0,
565                power_limit_watts: 400.0,
566                processes: Vec::new(),
567            }),
568            ..Default::default()
569        };
570        let output = HeadlessOutput::from(&snapshot);
571        let gpu = output.gpu.expect("gpu should be present");
572        assert_eq!(gpu.device_name, "RTX 4090");
573        assert!((gpu.utilization_percent - 95.0).abs() < f32::EPSILON);
574        assert!((gpu.vram_total_gb - 24.0).abs() < f32::EPSILON);
575    }
576
577    #[test]
578    fn test_headless_output_with_sample() {
579        let snapshot = TrainingSnapshot {
580            sample: Some(super::super::state::SamplePeek {
581                input_preview: "code".to_string(),
582                target_preview: "test_code".to_string(),
583                generated_preview: "gen_code".to_string(),
584                token_match_percent: 80.0,
585            }),
586            ..Default::default()
587        };
588        let output = HeadlessOutput::from(&snapshot);
589        let sample = output.sample.expect("sample should be present");
590        assert_eq!(sample.input_preview, "code");
591        assert!((sample.token_match_percent - 80.0).abs() < f32::EPSILON);
592    }
593
594    #[test]
595    fn test_headless_output_progress_fields() {
596        let snapshot = TrainingSnapshot {
597            epoch: 3,
598            total_epochs: 10,
599            step: 50,
600            steps_per_epoch: 100,
601            loss: 1.5,
602            learning_rate: 0.001,
603            gradient_norm: 2.0,
604            accuracy: 0.85,
605            tokens_per_second: 1200.0,
606            samples_per_second: 300.0,
607            experiment_id: "exp-001".to_string(),
608            model_name: "test-model".to_string(),
609            optimizer_name: "AdamW".to_string(),
610            batch_size: 4,
611            status: TrainingStatus::Running,
612            ..Default::default()
613        };
614        let output = HeadlessOutput::from(&snapshot);
615        assert_eq!(output.epoch, 3);
616        assert_eq!(output.total_epochs, 10);
617        assert_eq!(output.step, 50);
618        assert_eq!(output.steps_per_epoch, 100);
619        assert!((output.loss - 1.5).abs() < f32::EPSILON);
620        assert!((output.accuracy - 0.85).abs() < f32::EPSILON);
621        assert_eq!(output.experiment_id, "exp-001");
622        assert_eq!(output.model_name, "test-model");
623        assert_eq!(output.optimizer_name, "AdamW");
624        assert_eq!(output.batch_size, 4);
625    }
626
627    // ── HeadlessWriter line_count tests ────────────────────────────
628
629    #[test]
630    fn test_headless_writer_line_count_increments() {
631        let snapshot = TrainingSnapshot { status: TrainingStatus::Running, ..Default::default() };
632        let mut buffer = Vec::new();
633        let mut writer = HeadlessWriter::new(&mut buffer, OutputFormat::Json);
634        assert_eq!(writer.line_count(), 0);
635        writer.write(&snapshot).expect("write should succeed");
636        assert_eq!(writer.line_count(), 1);
637        writer.write(&snapshot).expect("write should succeed");
638        assert_eq!(writer.line_count(), 2);
639    }
640
641    #[test]
642    fn test_headless_writer_text_line_count() {
643        let snapshot = TrainingSnapshot {
644            epoch: 1,
645            total_epochs: 5,
646            step: 10,
647            steps_per_epoch: 50,
648            loss: 2.0,
649            status: TrainingStatus::Running,
650            ..Default::default()
651        };
652        let mut buffer = Vec::new();
653        let mut writer = HeadlessWriter::new(&mut buffer, OutputFormat::Text);
654        writer.write(&snapshot).expect("write should succeed");
655        assert_eq!(writer.line_count(), 1);
656    }
657
658    // ── Text output with GPU telemetry ─────────────────────────────
659
660    #[test]
661    fn test_headless_writer_text_with_gpu() {
662        let snapshot = TrainingSnapshot {
663            epoch: 1,
664            total_epochs: 5,
665            step: 10,
666            steps_per_epoch: 50,
667            loss: 2.0,
668            learning_rate: 0.001,
669            gradient_norm: 1.0,
670            tokens_per_second: 100.0,
671            samples_per_second: 25.0,
672            status: TrainingStatus::Running,
673            gpu: Some(super::super::state::GpuTelemetry {
674                device_name: "RTX 4090".to_string(),
675                utilization_percent: 90.0,
676                vram_used_gb: 18.0,
677                vram_total_gb: 24.0,
678                temperature_celsius: 70.0,
679                power_watts: 300.0,
680                power_limit_watts: 400.0,
681                processes: Vec::new(),
682            }),
683            ..Default::default()
684        };
685        let mut buffer = Vec::new();
686        let mut writer = HeadlessWriter::new(&mut buffer, OutputFormat::Text);
687        writer.write(&snapshot).expect("write should succeed");
688        let output = String::from_utf8(buffer).expect("valid utf8");
689        assert!(output.contains("GPU:"));
690        assert!(output.contains("RTX 4090"));
691        assert!(output.contains("VRAM:"));
692        assert!(output.contains("sam/s"));
693    }
694
695    #[test]
696    fn test_headless_writer_text_with_zero_vram_total() {
697        let snapshot = TrainingSnapshot {
698            epoch: 1,
699            total_epochs: 5,
700            step: 1,
701            steps_per_epoch: 10,
702            loss: 1.0,
703            status: TrainingStatus::Running,
704            gpu: Some(super::super::state::GpuTelemetry {
705                device_name: "test".to_string(),
706                vram_total_gb: 0.0,
707                ..Default::default()
708            }),
709            ..Default::default()
710        };
711        let mut buffer = Vec::new();
712        let mut writer = HeadlessWriter::new(&mut buffer, OutputFormat::Text);
713        writer.write(&snapshot).expect("write should succeed");
714        let output = String::from_utf8(buffer).expect("valid utf8");
715        assert!(output.contains("0%")); // vram_pct should be 0
716    }
717
718    // ── format_duration edge cases ─────────────────────────────────
719
720    #[test]
721    fn test_format_duration_large() {
722        assert_eq!(format_duration(Duration::from_secs(86400)), "24:00:00"); // 24 hours
723    }
724
725    #[test]
726    fn test_format_duration_exact_hour() {
727        assert_eq!(format_duration(Duration::from_secs(3600)), "01:00:00");
728    }
729
730    #[test]
731    fn test_format_duration_subseconds() {
732        // Duration with milliseconds should be truncated to seconds
733        assert_eq!(format_duration(Duration::from_millis(1500)), "00:00:01");
734    }
735
736    // ── HeadlessMonitor construction ───────────────────────────────
737
738    #[test]
739    fn test_headless_monitor_new() {
740        let monitor = HeadlessMonitor::new(OutputFormat::Json, 500);
741        assert_eq!(monitor.format, OutputFormat::Json);
742        assert_eq!(monitor.refresh_ms, 500);
743        assert!(monitor.output_file.is_none());
744    }
745
746    #[test]
747    fn test_headless_monitor_with_output_file() {
748        let monitor =
749            HeadlessMonitor::with_output_file(OutputFormat::Text, 1000, "/tmp/out.jsonl".into());
750        assert_eq!(monitor.format, OutputFormat::Text);
751        assert_eq!(monitor.refresh_ms, 1000);
752        assert_eq!(monitor.output_file.as_deref(), Some("/tmp/out.jsonl"));
753    }
754
755    // ── JSON serialization round-trip ──────────────────────────────
756
757    #[test]
758    fn test_headless_output_json_roundtrip() {
759        let snapshot = TrainingSnapshot {
760            epoch: 2,
761            total_epochs: 10,
762            step: 25,
763            steps_per_epoch: 50,
764            loss: 1.8,
765            loss_history: vec![3.0, 2.5, 2.0, 1.8],
766            learning_rate: 0.0005,
767            gradient_norm: 1.2,
768            accuracy: 0.72,
769            tokens_per_second: 800.0,
770            samples_per_second: 200.0,
771            status: TrainingStatus::Running,
772            experiment_id: "test".to_string(),
773            model_name: "model".to_string(),
774            ..Default::default()
775        };
776        let mut buffer = Vec::new();
777        let mut writer = HeadlessWriter::new(&mut buffer, OutputFormat::Json);
778        writer.write(&snapshot).expect("write should succeed");
779        let json_str = String::from_utf8(buffer).expect("valid utf8");
780        // Should be valid JSON
781        let parsed: serde_json::Value = serde_json::from_str(json_str.trim()).expect("valid json");
782        assert_eq!(parsed["epoch"], 2);
783        assert_eq!(parsed["status"], "Running");
784        assert_eq!(parsed["loss_history"].as_array().unwrap().len(), 4);
785    }
786
787    // ── Text output without samples_per_second ─────────────────────
788
789    #[test]
790    fn test_headless_writer_text_no_samples_per_second() {
791        let snapshot = TrainingSnapshot {
792            epoch: 1,
793            total_epochs: 5,
794            step: 1,
795            steps_per_epoch: 50,
796            loss: 3.0,
797            samples_per_second: 0.0,
798            status: TrainingStatus::Running,
799            ..Default::default()
800        };
801        let mut buffer = Vec::new();
802        let mut writer = HeadlessWriter::new(&mut buffer, OutputFormat::Text);
803        writer.write(&snapshot).expect("write should succeed");
804        let output = String::from_utf8(buffer).expect("valid utf8");
805        assert!(!output.contains("sam/s")); // should not show 0.0 sam/s
806    }
807
808    // ── HeadlessGpu and HeadlessSample serialization ───────────────
809
810    #[test]
811    fn test_headless_gpu_serialize() {
812        let gpu = HeadlessGpu {
813            device_name: "RTX 4090".to_string(),
814            utilization_percent: 99.0,
815            vram_used_gb: 23.5,
816            vram_total_gb: 24.0,
817            temperature_celsius: 78.0,
818            power_watts: 390.0,
819            power_limit_watts: 400.0,
820        };
821        let json = serde_json::to_string(&gpu).expect("serialize should succeed");
822        assert!(json.contains("RTX 4090"));
823        assert!(json.contains("99.0") || json.contains("99"));
824    }
825
826    #[test]
827    fn test_headless_sample_serialize() {
828        let sample = HeadlessSample {
829            input_preview: "fn add(a: i32, b: i32)".to_string(),
830            target_preview: "fn test_add()".to_string(),
831            generated_preview: "fn test_add()".to_string(),
832            token_match_percent: 100.0,
833        };
834        let json = serde_json::to_string(&sample).expect("serialize should succeed");
835        assert!(json.contains("fn add"));
836        assert!(json.contains("100"));
837    }
838}