kermit-bench 0.0.5

Benchmarking utilities and benchmarks
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

//! Benchmark infrastructure for Kermit.
//!
//! Provides dataset download, conversion (to Parquet), and benchmark
//! task/subtask definitions. The
//! [`BenchmarkManager`](manager::BenchmarkManager) coordinates downloading and
//! loading benchmark datasets, while
//! [`BenchmarkConfig`](benchmark::BenchmarkConfig) defines the interface each
//! benchmark must implement.

pub mod benchmark;
pub mod benchmarks;
pub mod downloader;
pub mod generation;
pub mod manager;
mod utils;

#[cfg(test)]
mod tests {
    use {
        crate::{benchmarks::Benchmark, manager::BenchmarkManager},
        std::path::PathBuf,
    };

    #[test]
    fn oxford_benchmark() {
        let tmp_dir = PathBuf::from("data"); // env::temp_dir().join("kermit-bench-testing");
        if tmp_dir.exists() {
            std::fs::remove_dir_all(&tmp_dir).expect("Failed to remove temporary directory");
        }
        std::fs::create_dir_all(&tmp_dir).expect("Failed to create temporary directory");
        let mut man = BenchmarkManager::new(&tmp_dir);
        assert!(man.add_benchmark(Benchmark::Oxford).is_ok());

        let ds_dir = tmp_dir.join(Benchmark::Oxford.config().metadata().download_spec.name);
        assert!(ds_dir.exists());

        // Test validation - should succeed when dataset is properly loaded
        assert!(Benchmark::Oxford.config().validate(&tmp_dir).is_ok());

        assert!(man.rm_benchmark(Benchmark::Oxford).is_ok());
    }
}