#[cfg(feature = "inference")]
#[test]
fn test_calculate_stats_single_iteration() {
let times = vec![Duration::from_millis(500)];
let config = BenchConfig {
warmup: 0,
iterations: 1,
max_tokens: 32,
prompt: "test".to_string(),
quiet: false,
};
let result =
calculate_benchmark_stats(times, 10, Duration::from_millis(50), &config).expect("&config)");
assert_eq!(result.total_time, Duration::from_millis(500));
assert_eq!(result.mean_time, Duration::from_millis(500));
assert_eq!(result.median_time, Duration::from_millis(500));
assert_eq!(result.std_dev, Duration::ZERO);
assert!((result.tokens_per_second - 20.0).abs() < 0.01);
}
#[cfg(feature = "inference")]
#[test]
fn test_calculate_stats_passed_threshold_high() {
let times = vec![Duration::from_millis(100); 5];
let config = BenchConfig {
warmup: 1,
iterations: 5,
max_tokens: 32,
prompt: "test".to_string(),
quiet: false,
};
let result =
calculate_benchmark_stats(times, 500, Duration::from_millis(5), &config).expect("&config)");
assert!(result.passed); assert!(result.tokens_per_second >= 60.0);
}
#[cfg(feature = "inference")]
#[test]
fn test_calculate_stats_passed_threshold_low() {
let times = vec![Duration::from_secs(2); 5];
let config = BenchConfig {
warmup: 1,
iterations: 5,
max_tokens: 32,
prompt: "test".to_string(),
quiet: false,
};
let result =
calculate_benchmark_stats(times, 10, Duration::from_millis(200), &config).expect("&config)");
assert!(!result.passed); assert!(result.tokens_per_second < 60.0);
}
#[cfg(feature = "inference")]
#[test]
fn test_calculate_stats_passed_threshold_exactly_60() {
let times = vec![Duration::from_secs(1); 5];
let config = BenchConfig {
warmup: 1,
iterations: 5,
max_tokens: 32,
prompt: "test".to_string(),
quiet: false,
};
let result =
calculate_benchmark_stats(times, 300, Duration::from_millis(10), &config).expect("&config)");
assert!(result.passed); assert!((result.tokens_per_second - 60.0).abs() < 0.01);
}
#[cfg(feature = "inference")]
#[test]
fn test_calculate_stats_preserves_iteration_times() {
let original_times = vec![
Duration::from_millis(100),
Duration::from_millis(300),
Duration::from_millis(200),
];
let config = BenchConfig {
warmup: 0,
iterations: 3,
max_tokens: 32,
prompt: "test".to_string(),
quiet: false,
};
let result = calculate_benchmark_stats(
original_times.clone(),
30,
Duration::from_millis(10),
&config,
)
.expect("expected value");
assert_eq!(result.iteration_times, original_times);
}
#[cfg(feature = "inference")]
#[test]
fn test_calculate_stats_median_even_count() {
let times = vec![
Duration::from_millis(100),
Duration::from_millis(200),
Duration::from_millis(300),
Duration::from_millis(400),
];
let config = BenchConfig {
warmup: 0,
iterations: 4,
max_tokens: 32,
prompt: "test".to_string(),
quiet: false,
};
let result =
calculate_benchmark_stats(times, 40, Duration::from_millis(25), &config).expect("&config)");
assert_eq!(result.median_time, Duration::from_millis(250));
}
#[cfg(feature = "inference")]
#[test]
fn test_calculate_stats_unsorted_input() {
let times = vec![
Duration::from_millis(500),
Duration::from_millis(100),
Duration::from_millis(300),
];
let config = BenchConfig {
warmup: 0,
iterations: 3,
max_tokens: 32,
prompt: "test".to_string(),
quiet: false,
};
let result =
calculate_benchmark_stats(times, 30, Duration::from_millis(10), &config).expect("&config)");
assert_eq!(result.median_time, Duration::from_millis(300));
}
#[cfg(feature = "inference")]
#[test]
fn test_calculate_stats_very_fast_iterations() {
let times = vec![Duration::from_nanos(100); 10];
let config = BenchConfig {
warmup: 0,
iterations: 10,
max_tokens: 32,
prompt: "test".to_string(),
quiet: false,
};
let result =
calculate_benchmark_stats(times, 1000, Duration::from_nanos(10), &config).expect("&config)");
assert!(result.tokens_per_second > 1e6);
assert!(result.passed);
}
#[cfg(feature = "inference")]
#[test]
fn test_calculate_stats_large_outlier() {
let times = vec![
Duration::from_millis(100),
Duration::from_millis(100),
Duration::from_millis(100),
Duration::from_millis(100),
Duration::from_secs(10), ];
let config = BenchConfig {
warmup: 1,
iterations: 5,
max_tokens: 32,
prompt: "test".to_string(),
quiet: false,
};
let result =
calculate_benchmark_stats(times, 50, Duration::from_millis(10), &config).expect("&config)");
assert_eq!(result.median_time, Duration::from_millis(100));
assert!(result.mean_time > result.median_time);
assert!(result.std_dev > Duration::from_secs(1));
}
#[cfg(feature = "inference")]
#[test]
fn test_calculate_stats_two_iterations() {
let times = vec![Duration::from_millis(200), Duration::from_millis(800)];
let config = BenchConfig {
warmup: 0,
iterations: 2,
max_tokens: 32,
prompt: "test".to_string(),
quiet: false,
};
let result =
calculate_benchmark_stats(times, 20, Duration::from_millis(50), &config).expect("&config)");
assert_eq!(result.mean_time, Duration::from_millis(500));
assert_eq!(result.median_time, Duration::from_millis(500));
let std_ms = result.std_dev.as_secs_f64() * 1000.0;
assert!((std_ms - 300.0).abs() < 1.0);
}
#[cfg(feature = "inference")]
#[test]
fn test_calculate_stats_zero_tokens() {
let times = vec![Duration::from_millis(100); 3];
let config = BenchConfig {
warmup: 0,
iterations: 3,
max_tokens: 32,
prompt: "test".to_string(),
quiet: false,
};
let result = calculate_benchmark_stats(times, 0, Duration::ZERO, &config).expect("&config)");
assert_eq!(result.total_tokens, 0);
assert_eq!(result.tokens_per_second, 0.0);
assert!(!result.passed);
}
#[cfg(feature = "inference")]
#[test]
fn test_calculate_stats_first_token_time_preserved() {
let times = vec![Duration::from_millis(100); 3];
let config = BenchConfig {
warmup: 0,
iterations: 3,
max_tokens: 32,
prompt: "test".to_string(),
quiet: false,
};
let ttft = Duration::from_millis(42);
let result = calculate_benchmark_stats(times, 30, ttft, &config).expect("&config)");
assert_eq!(result.time_to_first_token, ttft);
}
#[test]
fn test_run_prompt_none_uses_default() {
let mut file = NamedTempFile::with_suffix(".gguf").expect("create temp file");
file.write_all(b"fake gguf data").expect("write");
let result = run(file.path(), 1, 1, 16, None, false, None, false);
assert!(result.is_err());
}
#[test]
fn test_run_prompt_some_uses_custom() {
let mut file = NamedTempFile::with_suffix(".gguf").expect("create temp file");
file.write_all(b"fake gguf data").expect("write");
let result = run(
file.path(),
1,
1,
16,
Some("Hello world"),
false,
None,
false,
);
assert!(result.is_err());
}
#[test]
fn test_run_fast_flag_deprecated() {
let mut file = NamedTempFile::with_suffix(".gguf").expect("create temp file");
file.write_all(b"fake gguf data").expect("write");
let result = run(file.path(), 1, 1, 16, None, true, None, false);
assert!(result.is_err());
}
#[test]
fn test_run_zero_warmup_iterations() {
let mut file = NamedTempFile::with_suffix(".gguf").expect("create temp file");
file.write_all(b"fake gguf data").expect("write");
let result = run(file.path(), 0, 0, 0, None, false, None, false);
assert!(result.is_err());
}
#[test]
fn test_run_large_max_tokens() {
let mut file = NamedTempFile::with_suffix(".gguf").expect("create temp file");
file.write_all(b"fake gguf data").expect("write");
let result = run(file.path(), 1, 1, 100_000, None, false, None, false);
assert!(result.is_err());
}
#[cfg(feature = "inference")]
#[test]
fn test_brick_name_rms_norm_valid() {
let file = NamedTempFile::with_suffix(".gguf").expect("create temp file");
let result = run(file.path(), 1, 3, 16, None, false, Some("rms_norm"), false);
if let Err(e) = &result {
let msg = format!("{e}");
assert!(!msg.contains("Unknown brick type"));
}
}
#[cfg(feature = "inference")]
#[test]
fn test_brick_name_qkv_valid() {
let file = NamedTempFile::with_suffix(".gguf").expect("create temp file");
let result = run(file.path(), 1, 3, 16, None, false, Some("qkv"), false);
if let Err(e) = &result {
let msg = format!("{e}");
assert!(!msg.contains("Unknown brick type"));
}
}
#[cfg(feature = "inference")]
#[test]
fn test_brick_name_rope_valid() {
let file = NamedTempFile::with_suffix(".gguf").expect("create temp file");
let result = run(file.path(), 1, 3, 16, None, false, Some("rope"), false);
if let Err(e) = &result {
let msg = format!("{e}");
assert!(!msg.contains("Unknown brick type"));
}
}
#[cfg(feature = "inference")]
#[test]
fn test_brick_name_attn_valid() {
let file = NamedTempFile::with_suffix(".gguf").expect("create temp file");
let result = run(file.path(), 1, 3, 16, None, false, Some("attn"), false);
if let Err(e) = &result {
let msg = format!("{e}");
assert!(!msg.contains("Unknown brick type"));
}
}
#[cfg(feature = "inference")]
#[test]
fn test_brick_name_attention_alias_valid() {
let file = NamedTempFile::with_suffix(".gguf").expect("create temp file");
let result = run(file.path(), 1, 3, 16, None, false, Some("attention"), false);
if let Err(e) = &result {
let msg = format!("{e}");
assert!(!msg.contains("Unknown brick type"));
}
}
#[cfg(feature = "inference")]
#[test]
fn test_brick_name_o_proj_valid() {
let file = NamedTempFile::with_suffix(".gguf").expect("create temp file");
let result = run(file.path(), 1, 3, 16, None, false, Some("o_proj"), false);
if let Err(e) = &result {
let msg = format!("{e}");
assert!(!msg.contains("Unknown brick type"));
}
}
#[cfg(feature = "inference")]
#[test]
fn test_brick_name_ffn_valid() {
let file = NamedTempFile::with_suffix(".gguf").expect("create temp file");
let result = run(file.path(), 1, 3, 16, None, false, Some("ffn"), false);
if let Err(e) = &result {
let msg = format!("{e}");
assert!(!msg.contains("Unknown brick type"));
}
}
#[cfg(feature = "inference")]
#[test]
fn test_brick_name_layer_valid() {
let file = NamedTempFile::with_suffix(".gguf").expect("create temp file");
let result = run(file.path(), 1, 3, 16, None, false, Some("layer"), false);
if let Err(e) = &result {
let msg = format!("{e}");
assert!(!msg.contains("Unknown brick type"));
}
}
#[cfg(feature = "inference")]
#[test]
fn test_brick_name_unknown_returns_error() {
let file = NamedTempFile::with_suffix(".gguf").expect("create temp file");
let result = run(
file.path(),
1,
3,
16,
None,
false,
Some("unknown_thing"),
false,
);
assert!(result.is_err());
let msg = format!("{}", result.unwrap_err());
assert!(msg.contains("Unknown brick type"));
assert!(msg.contains("unknown_thing"));
}
#[cfg(feature = "inference")]
#[test]
fn test_brick_name_empty_string_returns_error() {
let file = NamedTempFile::with_suffix(".gguf").expect("create temp file");
let result = run(file.path(), 1, 3, 16, None, false, Some(""), false);
assert!(result.is_err());
let msg = format!("{}", result.unwrap_err());
assert!(msg.contains("Unknown brick type"));
}