yscv-eval 0.1.8

Evaluation metrics (mAP, MOTA, HOTA) and dataset adapters
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

use super::*;

#[test]
fn parse_pipeline_thresholds_parses_supported_keys() {
    let text = r#"
            # comment
            detect.max_mean_ms=10
            detect.max_p95_ms=15
            end_to_end.min_fps=25
        "#;
    let thresholds = parse_pipeline_benchmark_thresholds(text).unwrap();
    assert_eq!(
        thresholds,
        PipelineBenchmarkThresholds {
            detect: StageThresholds {
                min_fps: None,
                max_mean_ms: Some(10.0),
                max_p95_ms: Some(15.0),
            },
            track: StageThresholds::default(),
            recognize: StageThresholds::default(),
            end_to_end: StageThresholds {
                min_fps: Some(25.0),
                max_mean_ms: None,
                max_p95_ms: None,
            },
        }
    );
}

#[test]
fn parse_pipeline_thresholds_rejects_bad_entries() {
    let err = parse_pipeline_benchmark_thresholds("bad").unwrap_err();
    assert_eq!(
        err,
        EvalError::InvalidThresholdEntry {
            line: 1,
            message: "expected `stage.metric=value` entry".to_string(),
        }
    );

    let err = parse_pipeline_benchmark_thresholds("unknown.max_mean_ms=1").unwrap_err();
    assert_eq!(
        err,
        EvalError::InvalidThresholdEntry {
            line: 1,
            message: "unsupported stage `unknown`".to_string(),
        }
    );
}

#[test]
fn validate_pipeline_thresholds_reports_violations() {
    let report = PipelineBenchmarkReport {
        frames: 4,
        detect: TimingStats {
            runs: 4,
            total_ms: 40.0,
            mean_ms: 10.0,
            min_ms: 8.0,
            max_ms: 12.0,
            p50_ms: 10.0,
            p95_ms: 11.8,
            throughput_fps: 100.0,
        },
        track: TimingStats {
            runs: 4,
            total_ms: 4.0,
            mean_ms: 1.0,
            min_ms: 1.0,
            max_ms: 1.0,
            p50_ms: 1.0,
            p95_ms: 1.0,
            throughput_fps: 1000.0,
        },
        recognize: TimingStats {
            runs: 4,
            total_ms: 8.0,
            mean_ms: 2.0,
            min_ms: 2.0,
            max_ms: 2.0,
            p50_ms: 2.0,
            p95_ms: 2.0,
            throughput_fps: 500.0,
        },
        end_to_end: TimingStats {
            runs: 4,
            total_ms: 80.0,
            mean_ms: 20.0,
            min_ms: 18.0,
            max_ms: 22.0,
            p50_ms: 20.0,
            p95_ms: 21.8,
            throughput_fps: 50.0,
        },
    };
    let thresholds = PipelineBenchmarkThresholds {
        detect: StageThresholds {
            min_fps: Some(110.0),
            max_mean_ms: Some(9.0),
            max_p95_ms: None,
        },
        track: StageThresholds::default(),
        recognize: StageThresholds::default(),
        end_to_end: StageThresholds {
            min_fps: Some(40.0),
            max_mean_ms: None,
            max_p95_ms: Some(21.0),
        },
    };
    let violations = validate_pipeline_benchmark_thresholds(&report, &thresholds);
    assert_eq!(violations.len(), 3);
    assert_eq!(violations[0].stage, "detect");
    assert_eq!(violations[1].stage, "detect");
    assert_eq!(violations[2].stage, "end_to_end");
}