sipp-rs 0.1.4

Unified Rust library for extensible Sipp inference
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86

//! Tests the `runtime::config::inference_config::placement` module in `sipp`.
//!
//! Covers runtime configuration normalization, serialization, and boundary choices through pure value assertions.

use super::super::arg_value;
use super::{GpuLayerConfig, ModelLoadMode, ModelPlacementConfig, SplitMode};
use crate::defaults::BYTES_PER_MIB_U64;

#[test]
fn gpu_layer_count_matches_llama_negative_all_convention() {
    assert_eq!(GpuLayerConfig::from_layer_count(-1), GpuLayerConfig::All);
    assert_eq!(
        GpuLayerConfig::from_layer_count(0),
        GpuLayerConfig::Count(0)
    );
    assert_eq!(
        GpuLayerConfig::from_layer_count(1),
        GpuLayerConfig::Count(1)
    );
}

#[test]
fn placement_arg_len_matches_emitted_args() {
    let placement = ModelPlacementConfig {
        devices: vec!["gpu0".to_string(), "gpu1".to_string()],
        gpu_layers: GpuLayerConfig::Count(99),
        split_mode: SplitMode::Tensor,
        main_gpu: Some(1),
        tensor_split: vec![0.5, 0.5],
        load_mode: ModelLoadMode::Mlock,
        fit_params: true,
        fit_params_min_ctx: Some(2048),
        fit_params_target_bytes: vec![BYTES_PER_MIB_U64],
        check_tensors: true,
        no_extra_bufts: true,
        no_host: true,
    };
    let mut args = Vec::with_capacity(placement.arg_len());

    placement.push_args(&mut args);

    assert_eq!(args.capacity(), args.len());
    assert_eq!(arg_value(&args, "--device"), Some("gpu0,gpu1"));
    assert_eq!(arg_value(&args, "--gpu-layers"), Some("99"));
    assert_eq!(arg_value(&args, "--split-mode"), Some("tensor"));
    assert_eq!(arg_value(&args, "--load-mode"), Some("mlock"));
    assert!(args.iter().any(|arg| arg == "--no-host"));
}

#[test]
fn model_load_modes_match_llama_arguments() {
    for (mode, expected) in [
        (ModelLoadMode::Auto, "auto"),
        (ModelLoadMode::None, "none"),
        (ModelLoadMode::Mmap, "mmap"),
        (ModelLoadMode::Mlock, "mlock"),
        (ModelLoadMode::MmapMlock, "mmap+mlock"),
        (ModelLoadMode::DirectIo, "dio"),
    ] {
        let placement = ModelPlacementConfig {
            load_mode: mode,
            ..ModelPlacementConfig::default()
        };
        let mut args = Vec::with_capacity(placement.arg_len());

        placement.push_args(&mut args);

        assert_eq!(args.capacity(), args.len());
        assert_eq!(arg_value(&args, "--load-mode"), Some(expected));
    }
}

#[test]
fn cpu_placement_omits_gpu_arguments() {
    let placement = ModelPlacementConfig {
        gpu_layers: GpuLayerConfig::Count(0),
        ..ModelPlacementConfig::default()
    };
    let mut args = Vec::with_capacity(placement.arg_len());

    placement.push_args(&mut args);

    assert_eq!(args.capacity(), args.len());
    assert_eq!(arg_value(&args, "--gpu-layers"), None);
    assert_eq!(arg_value(&args, "--split-mode"), None);
}