skippy-runtime 0.77.0

Rust runtime layer for Skippy staged model execution
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

//! Real-projector lifecycle regressions, opt-in through the workload fixture.

use super::*;
use crate::{ModelInfo, RuntimeConfig, TensorRole};

/// Load the opt-in speech model and projector without a synthetic runtime fallback.
fn speech_fixture() -> Result<Option<StageModel>> {
    if std::env::var("SKIPPY_WORKLOAD_CLASS").as_deref() != Ok("speech_synthesis") {
        return Ok(None);
    }
    let path = std::env::var("SKIPPY_WORKLOAD_MODEL").context("speech fixture model")?;
    let projector =
        std::env::var("SKIPPY_WORKLOAD_PROJECTOR").context("speech fixture projector")?;
    let layer_end = ModelInfo::open(&path)?
        .tensors()?
        .into_iter()
        .filter(|tensor| tensor.role == TensorRole::Layer)
        .filter_map(|tensor| tensor.layer_index)
        .max()
        .context("speech fixture layers")?
        + 1;
    let config = RuntimeConfig {
        layer_end,
        ctx_size: 2048,
        n_batch: Some(2048),
        n_ubatch: Some(2048),
        projector_path: Some(projector),
        projector_use_gpu: Some(false),
        kv_offload: Some(false),
        op_offload: Some(false),
        ..RuntimeConfig::default()
    };
    StageModel::open(path, &config).map(Some)
}

/// Prove a speech session can resume ordinary decoding with a real output row.
fn assert_generation_reusable(model: &StageModel, session: &mut StageSession) -> Result<()> {
    session.reset()?;
    let tokens = model.tokenize("The mesh is ready.", true)?;
    let (last, prefix) = tokens.split_last().context("generation fixture tokens")?;
    session.prefill_chunked(prefix)?;
    // Plain prefill deliberately produces no logits. Decode the final token
    // with an output row so this checks real generation, not a sampler fallback.
    let token = session.decode_step(*last)?;
    assert!(token >= 0, "normal generation must produce a valid token");
    assert_eq!(session.sample_current(None)?, token);
    Ok(())
}

#[test]
/// Verify all speech exit paths restore ordinary session decoding.
fn speech_success_cancellation_and_native_failure_leave_session_reusable() -> Result<()> {
    let Some(model) = speech_fixture()? else {
        return Ok(());
    };
    let mut session = model.create_session()?;
    let mut config = SpeechSynthesisConfig {
        prompt: "Hello.".into(),
        language: None,
        top_k: 1,
        top_p: 1.0,
        seed: 42,
        output_format: SpeechOutputFormat::Wav,
        max_frames: 2,
    };
    let cancelled = model
        .synthesize_speech(&mut session, &config, || true)
        .unwrap_err();
    assert!(cancelled.to_string().contains("cancelled"), "{cancelled:#}");
    assert_generation_reusable(&model, &mut session)?;
    config.language = Some("not-a-supported-language".into());
    let rejected = model
        .synthesize_speech(&mut session, &config, || false)
        .unwrap_err();
    assert!(
        rejected.to_string().contains("rejected the input"),
        "{rejected:#}"
    );
    assert_generation_reusable(&model, &mut session)?;
    config.language = None;
    let capped = model
        .synthesize_speech(&mut session, &config, || false)
        .unwrap_err();
    assert!(capped.to_string().contains("frame limit"), "{capped:#}");
    assert_generation_reusable(&model, &mut session)?;
    config.max_frames = 512;
    let audio = model.synthesize_speech(&mut session, &config, || false)?;
    assert!(audio.sample_count > 0 && audio.bytes.len() > 44);
    assert_generation_reusable(&model, &mut session)?;
    Ok(())
}