rlx-flow 0.2.1

Block assembly-line API for RLX model builders — fusion-first, config-driven
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

// RLX — versatile ML compiler + runtime.
// Copyright (C) 2026 Eugene Hauptmann, Nataliya Kosmyna.

use std::sync::Arc;

use super::{GeluFfnStage, LayerScaleStage, VitSelfAttnStage};
use crate::layer::LayerStack;
use crate::stage::FlowStage;

/// Fused DINOv2 ViT encoder block (pre-norm + LayerScale + tanh-approx GELU FFN).
pub fn dinov2_layer_fused(
    layer_idx: usize,
    hidden_size: usize,
    num_heads: usize,
    eps: f32,
) -> FlowStage {
    let lp = format!("blocks.{layer_idx}");
    FlowStage::Named {
        name: format!("layer{layer_idx}"),
        inner: Arc::new(
            LayerStack::named(lp.clone())
                .residual_save()
                .layer_norm(
                    format!("{lp}.norm1.weight"),
                    format!("{lp}.norm1.bias"),
                    eps,
                )
                .stage(FlowStage::VitSelfAttn(VitSelfAttnStage::dinov2(
                    &lp,
                    hidden_size,
                    num_heads,
                )))
                .stage(FlowStage::LayerScale(LayerScaleStage::new(format!(
                    "{lp}.ls1.gamma"
                ))))
                .residual_add()
                .residual_save()
                .layer_norm(
                    format!("{lp}.norm2.weight"),
                    format!("{lp}.norm2.bias"),
                    eps,
                )
                .stage(FlowStage::GeluFfn(GeluFfnStage::dinov2(&lp)))
                .stage(FlowStage::LayerScale(LayerScaleStage::new(format!(
                    "{lp}.ls2.gamma"
                ))))
                .residual_add()
                .build()
                .unwrap_sequence(),
        ),
    }
}

trait UnwrapSequence {
    fn unwrap_sequence(self) -> FlowStage;
}

impl UnwrapSequence for FlowStage {
    fn unwrap_sequence(self) -> FlowStage {
        match self {
            FlowStage::Named { inner, .. } => (*inner).clone(),
            other => other,
        }
    }
}