1use yscv_eval::{
2 CameraDiagnosticsThresholds, DetectionEvalConfig, TrackingEvalConfig,
3 evaluate_detections_from_dataset, evaluate_tracking_from_dataset,
4 load_camera_diagnostics_report_json_file, load_detection_dataset_coco_files,
5 load_detection_dataset_jsonl_file, load_detection_dataset_kitti_label_dirs,
6 load_detection_dataset_openimages_csv_files, load_detection_dataset_voc_xml_dirs,
7 load_detection_dataset_widerface_files, load_detection_dataset_yolo_label_dirs,
8 load_tracking_dataset_jsonl_file, load_tracking_dataset_mot_txt_files,
9 validate_camera_diagnostics_report,
10};
11
12use crate::config::{CliConfig, CliError};
13use crate::error::AppError;
14
15pub fn run_dataset_evaluation(cli: &CliConfig) -> Result<(), AppError> {
16 println!("yscv-cli eval: starting dataset evaluation");
17 if let Some(path) = cli.eval_detection_dataset_path.as_deref() {
18 let frames = load_detection_dataset_jsonl_file(path)?;
19 let metrics = evaluate_detections_from_dataset(
20 &frames,
21 DetectionEvalConfig {
22 iou_threshold: cli.eval_iou_threshold,
23 score_threshold: cli.eval_score_threshold,
24 },
25 )?;
26 println!(
27 "detection_eval dataset={} frames={}",
28 path.display(),
29 frames.len()
30 );
31 println!(
32 " tp={} fp={} fn={} precision={:.4} recall={:.4} f1={:.4} ap={:.4}",
33 metrics.true_positives,
34 metrics.false_positives,
35 metrics.false_negatives,
36 metrics.precision,
37 metrics.recall,
38 metrics.f1,
39 metrics.average_precision,
40 );
41 }
42
43 if let (Some(gt_path), Some(pred_path)) = (
44 cli.eval_detection_coco_gt_path.as_deref(),
45 cli.eval_detection_coco_pred_path.as_deref(),
46 ) {
47 let frames = load_detection_dataset_coco_files(gt_path, pred_path)?;
48 let metrics = evaluate_detections_from_dataset(
49 &frames,
50 DetectionEvalConfig {
51 iou_threshold: cli.eval_iou_threshold,
52 score_threshold: cli.eval_score_threshold,
53 },
54 )?;
55 println!(
56 "detection_eval_coco ground_truth={} predictions={} frames={}",
57 gt_path.display(),
58 pred_path.display(),
59 frames.len()
60 );
61 println!(
62 " tp={} fp={} fn={} precision={:.4} recall={:.4} f1={:.4} ap={:.4}",
63 metrics.true_positives,
64 metrics.false_positives,
65 metrics.false_negatives,
66 metrics.precision,
67 metrics.recall,
68 metrics.f1,
69 metrics.average_precision,
70 );
71 }
72
73 if let (Some(gt_path), Some(pred_path)) = (
74 cli.eval_detection_openimages_gt_path.as_deref(),
75 cli.eval_detection_openimages_pred_path.as_deref(),
76 ) {
77 let frames = load_detection_dataset_openimages_csv_files(gt_path, pred_path)?;
78 let metrics = evaluate_detections_from_dataset(
79 &frames,
80 DetectionEvalConfig {
81 iou_threshold: cli.eval_iou_threshold,
82 score_threshold: cli.eval_score_threshold,
83 },
84 )?;
85 println!(
86 "detection_eval_openimages ground_truth={} predictions={} frames={}",
87 gt_path.display(),
88 pred_path.display(),
89 frames.len()
90 );
91 println!(
92 " tp={} fp={} fn={} precision={:.4} recall={:.4} f1={:.4} ap={:.4}",
93 metrics.true_positives,
94 metrics.false_positives,
95 metrics.false_negatives,
96 metrics.precision,
97 metrics.recall,
98 metrics.f1,
99 metrics.average_precision,
100 );
101 }
102
103 if let (Some(manifest_path), Some(gt_dir), Some(pred_dir)) = (
104 cli.eval_detection_yolo_manifest_path.as_deref(),
105 cli.eval_detection_yolo_gt_dir_path.as_deref(),
106 cli.eval_detection_yolo_pred_dir_path.as_deref(),
107 ) {
108 let frames = load_detection_dataset_yolo_label_dirs(manifest_path, gt_dir, pred_dir)?;
109 let metrics = evaluate_detections_from_dataset(
110 &frames,
111 DetectionEvalConfig {
112 iou_threshold: cli.eval_iou_threshold,
113 score_threshold: cli.eval_score_threshold,
114 },
115 )?;
116 println!(
117 "detection_eval_yolo manifest={} gt_dir={} pred_dir={} frames={}",
118 manifest_path.display(),
119 gt_dir.display(),
120 pred_dir.display(),
121 frames.len()
122 );
123 println!(
124 " tp={} fp={} fn={} precision={:.4} recall={:.4} f1={:.4} ap={:.4}",
125 metrics.true_positives,
126 metrics.false_positives,
127 metrics.false_negatives,
128 metrics.precision,
129 metrics.recall,
130 metrics.f1,
131 metrics.average_precision,
132 );
133 }
134
135 if let (Some(manifest_path), Some(gt_dir), Some(pred_dir)) = (
136 cli.eval_detection_voc_manifest_path.as_deref(),
137 cli.eval_detection_voc_gt_dir_path.as_deref(),
138 cli.eval_detection_voc_pred_dir_path.as_deref(),
139 ) {
140 let frames = load_detection_dataset_voc_xml_dirs(manifest_path, gt_dir, pred_dir)?;
141 let metrics = evaluate_detections_from_dataset(
142 &frames,
143 DetectionEvalConfig {
144 iou_threshold: cli.eval_iou_threshold,
145 score_threshold: cli.eval_score_threshold,
146 },
147 )?;
148 println!(
149 "detection_eval_voc manifest={} gt_dir={} pred_dir={} frames={}",
150 manifest_path.display(),
151 gt_dir.display(),
152 pred_dir.display(),
153 frames.len()
154 );
155 println!(
156 " tp={} fp={} fn={} precision={:.4} recall={:.4} f1={:.4} ap={:.4}",
157 metrics.true_positives,
158 metrics.false_positives,
159 metrics.false_negatives,
160 metrics.precision,
161 metrics.recall,
162 metrics.f1,
163 metrics.average_precision,
164 );
165 }
166
167 if let (Some(manifest_path), Some(gt_dir), Some(pred_dir)) = (
168 cli.eval_detection_kitti_manifest_path.as_deref(),
169 cli.eval_detection_kitti_gt_dir_path.as_deref(),
170 cli.eval_detection_kitti_pred_dir_path.as_deref(),
171 ) {
172 let frames = load_detection_dataset_kitti_label_dirs(manifest_path, gt_dir, pred_dir)?;
173 let metrics = evaluate_detections_from_dataset(
174 &frames,
175 DetectionEvalConfig {
176 iou_threshold: cli.eval_iou_threshold,
177 score_threshold: cli.eval_score_threshold,
178 },
179 )?;
180 println!(
181 "detection_eval_kitti manifest={} gt_dir={} pred_dir={} frames={}",
182 manifest_path.display(),
183 gt_dir.display(),
184 pred_dir.display(),
185 frames.len()
186 );
187 println!(
188 " tp={} fp={} fn={} precision={:.4} recall={:.4} f1={:.4} ap={:.4}",
189 metrics.true_positives,
190 metrics.false_positives,
191 metrics.false_negatives,
192 metrics.precision,
193 metrics.recall,
194 metrics.f1,
195 metrics.average_precision,
196 );
197 }
198
199 if let (Some(gt_path), Some(pred_path)) = (
200 cli.eval_detection_widerface_gt_path.as_deref(),
201 cli.eval_detection_widerface_pred_path.as_deref(),
202 ) {
203 let frames = load_detection_dataset_widerface_files(gt_path, pred_path)?;
204 let metrics = evaluate_detections_from_dataset(
205 &frames,
206 DetectionEvalConfig {
207 iou_threshold: cli.eval_iou_threshold,
208 score_threshold: cli.eval_score_threshold,
209 },
210 )?;
211 println!(
212 "detection_eval_widerface ground_truth={} predictions={} frames={}",
213 gt_path.display(),
214 pred_path.display(),
215 frames.len()
216 );
217 println!(
218 " tp={} fp={} fn={} precision={:.4} recall={:.4} f1={:.4} ap={:.4}",
219 metrics.true_positives,
220 metrics.false_positives,
221 metrics.false_negatives,
222 metrics.precision,
223 metrics.recall,
224 metrics.f1,
225 metrics.average_precision,
226 );
227 }
228
229 if let Some(path) = cli.eval_tracking_dataset_path.as_deref() {
230 let frames = load_tracking_dataset_jsonl_file(path)?;
231 let metrics = evaluate_tracking_from_dataset(
232 &frames,
233 TrackingEvalConfig {
234 iou_threshold: cli.eval_iou_threshold,
235 },
236 )?;
237 println!(
238 "tracking_eval dataset={} frames={}",
239 path.display(),
240 frames.len()
241 );
242 println!(
243 " gt={} matches={} fp={} fn={} idsw={} precision={:.4} recall={:.4} f1={:.4} mota={:.4} motp={:.4}",
244 metrics.total_ground_truth,
245 metrics.matches,
246 metrics.false_positives,
247 metrics.false_negatives,
248 metrics.id_switches,
249 metrics.precision,
250 metrics.recall,
251 metrics.f1,
252 metrics.mota,
253 metrics.motp,
254 );
255 }
256
257 if let (Some(gt_path), Some(pred_path)) = (
258 cli.eval_tracking_mot_gt_path.as_deref(),
259 cli.eval_tracking_mot_pred_path.as_deref(),
260 ) {
261 let frames = load_tracking_dataset_mot_txt_files(gt_path, pred_path)?;
262 let metrics = evaluate_tracking_from_dataset(
263 &frames,
264 TrackingEvalConfig {
265 iou_threshold: cli.eval_iou_threshold,
266 },
267 )?;
268 println!(
269 "tracking_eval_mot ground_truth={} predictions={} frames={}",
270 gt_path.display(),
271 pred_path.display(),
272 frames.len()
273 );
274 println!(
275 " gt={} matches={} fp={} fn={} idsw={} precision={:.4} recall={:.4} f1={:.4} mota={:.4} motp={:.4}",
276 metrics.total_ground_truth,
277 metrics.matches,
278 metrics.false_positives,
279 metrics.false_negatives,
280 metrics.id_switches,
281 metrics.precision,
282 metrics.recall,
283 metrics.f1,
284 metrics.mota,
285 metrics.motp,
286 );
287 }
288 println!("yscv-cli eval: completed");
289 Ok(())
290}
291
292pub fn run_diagnostics_report_validation(cli: &CliConfig) -> Result<(), AppError> {
293 let Some(path) = cli.validate_diagnostics_report_path.as_deref() else {
294 return Err(CliError::Message(
295 "missing diagnostics report path for validation mode".to_string(),
296 )
297 .into());
298 };
299 println!(
300 "yscv-cli eval: validating diagnostics report {}",
301 path.display()
302 );
303 let report = load_camera_diagnostics_report_json_file(path)?;
304 let thresholds = CameraDiagnosticsThresholds {
305 min_collected_frames: cli.validate_diagnostics_min_frames,
306 max_abs_wall_drift_pct: cli.validate_diagnostics_max_drift_pct,
307 max_abs_sensor_drift_pct: cli.validate_diagnostics_max_drift_pct,
308 max_dropped_frames: cli.validate_diagnostics_max_dropped_frames,
309 };
310 let violations = validate_camera_diagnostics_report(&report, thresholds);
311 if violations.is_empty() {
312 println!(
313 "diagnostics_report_validation passed: status={} collected_frames={} max_abs_drift_pct={} max_dropped_frames={}",
314 report.status,
315 report
316 .capture
317 .as_ref()
318 .map(|capture| capture.collected_frames)
319 .unwrap_or(0),
320 cli.validate_diagnostics_max_drift_pct,
321 cli.validate_diagnostics_max_dropped_frames,
322 );
323 return Ok(());
324 }
325
326 println!("diagnostics_report_validation failed:");
327 for violation in &violations {
328 println!(" - {}: {}", violation.field, violation.message);
329 }
330 Err(CliError::Message("diagnostics report validation failed".to_string()).into())
331}