rlx-flow 0.2.1

Block assembly-line API for RLX model builders — fusion-first, config-driven
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

// RLX — versatile ML compiler + runtime.
// Copyright (C) 2026 Eugene Hauptmann, Nataliya Kosmyna.

use anyhow::Result;
use rlx_ir::op::MaskKind;

use super::BlockStage;
use crate::context::FlowCtx;
use crate::value::FlowValue;
#[derive(Debug, Clone)]
pub struct LlamaDecoderStage {
    pub layer_prefix: String,
    pub num_heads: usize,
    pub head_dim: usize,
    pub num_kv_heads: usize,
    pub eps: f32,
    pub mask: MaskKind,
    pub hidden_shape: rlx_ir::Shape,
}

impl LlamaDecoderStage {
    pub fn layer(layer_idx: usize, spec: LlamaDecoderSpec) -> Self {
        Self {
            layer_prefix: format!("model.layers.{layer_idx}"),
            num_heads: spec.num_heads,
            head_dim: spec.head_dim,
            num_kv_heads: spec.num_kv_heads,
            eps: spec.eps,
            mask: spec.mask,
            hidden_shape: spec.hidden_shape,
        }
    }
}

#[derive(Debug, Clone)]
pub struct LlamaDecoderSpec {
    pub num_heads: usize,
    pub head_dim: usize,
    pub num_kv_heads: usize,
    pub eps: f32,
    pub mask: MaskKind,
    pub hidden_shape: rlx_ir::Shape,
}

impl BlockStage for LlamaDecoderStage {
    fn emit(&self, ctx: &mut FlowCtx<'_>, input: FlowValue) -> Result<Option<FlowValue>> {
        let lp = &self.layer_prefix;
        let zero_beta = ctx
            .state
            .zero_beta
            .ok_or_else(|| anyhow::anyhow!("LlamaDecoder requires ZeroBeta stage"))?;
        let cos = ctx
            .state
            .rope_cos
            .ok_or_else(|| anyhow::anyhow!("LlamaDecoder requires RopeTables stage"))?;
        let sin = ctx
            .state
            .rope_sin
            .ok_or_else(|| anyhow::anyhow!("LlamaDecoder requires RopeTables stage"))?;

        let in_ln_g = ctx.load_param(&format!("{lp}.input_layernorm.weight"), false)?;
        let q_w = ctx.load_param(&format!("{lp}.self_attn.q_proj.weight"), true)?;
        let k_w = ctx.load_param(&format!("{lp}.self_attn.k_proj.weight"), true)?;
        let v_w = ctx.load_param(&format!("{lp}.self_attn.v_proj.weight"), true)?;
        let o_w = ctx.load_param(&format!("{lp}.self_attn.o_proj.weight"), true)?;
        let post_ln_g = ctx.load_param(&format!("{lp}.post_attention_layernorm.weight"), false)?;
        let gate_w = ctx.load_param(&format!("{lp}.mlp.gate_proj.weight"), true)?;
        let up_w = ctx.load_param(&format!("{lp}.mlp.up_proj.weight"), true)?;
        let down_w = ctx.load_param(&format!("{lp}.mlp.down_proj.weight"), true)?;

        let id = ctx.hir().llama_decoder_block(
            input.id,
            in_ln_g,
            zero_beta,
            q_w,
            k_w,
            v_w,
            o_w,
            post_ln_g,
            zero_beta,
            gate_w,
            up_w,
            down_w,
            cos,
            sin,
            None,
            self.num_heads,
            self.head_dim,
            self.num_kv_heads,
            self.eps,
            self.mask,
            self.hidden_shape.clone(),
        );

        Ok(Some(ctx.wrap(id, self.hidden_shape.clone())))
    }
}