qwen_image_encoder_fixture/
qwen_image_encoder_fixture.rs

1//! Full tiny Qwen2.5-VL text+vision parity gate.
2//!
3//! Generate the fixture with `qwen_image_encoder_oracle.py` in the bundled
4//! reference environment, then this example writes a real CMF with the
5//! official tensor names/assets and compares native `Conditioning` against
6//! the reference model's final hidden state after the 64-token drop:
7//!
8//! ```text
9//! /private/tmp/cmf-qwen-image-ref-venv/bin/python \
10//!   crates/cortiq-engine/examples/qwen_image_encoder_oracle.py /tmp/qwen-encoder.json
11//! cargo run -p cortiq-engine --example qwen_image_encoder_fixture -- \
12//!   /tmp/qwen-encoder.json /tmp/qwen-encoder.cmf
13//! ```
14
15use cortiq_core::{CmfHeader, CmfModel, TensorDtype, TensorSpec, CMF_VERSION};
16use cortiq_engine::qwen_image_encoder::QwenImageEncoder;
17use serde::Deserialize;
18use serde_json::json;
19use std::collections::HashMap;
20use std::error::Error;
21use std::path::{Path, PathBuf};
22
23#[derive(Deserialize)]
24struct Fixture {
25    config: serde_json::Value,
26    processor: serde_json::Value,
27    tokenizer: serde_json::Value,
28    prompt: String,
29    image_width: u32,
30    image_height: u32,
31    image_rgb: Vec<u8>,
32    ids: Vec<u32>,
33    grid: Vec<usize>,
34    shapes: HashMap<String, Vec<usize>>,
35    weights: HashMap<String, Vec<f32>>,
36    expected: Vec<f32>,
37}
38
39fn f32_bytes(values: &[f32]) -> Vec<u8> {
40    values.iter().flat_map(|v| v.to_le_bytes()).collect()
41}
42
43fn blob(name: &str, value: &serde_json::Value) -> Result<TensorSpec, Box<dyn Error>> {
44    let data = serde_json::to_vec(value)?;
45    Ok(TensorSpec {
46        name: name.into(),
47        dtype: TensorDtype::U8,
48        shape: vec![data.len()],
49        data,
50    })
51}
52
53fn tiny_header() -> Result<CmfHeader, Box<dyn Error>> {
54    Ok(serde_json::from_value(json!({
55        "version": CMF_VERSION,
56        "arch": {
57            "arch_name": "qwen-image-encoder-tiny-fixture",
58            "hidden_size": 12,
59            "intermediate_size": 24,
60            "num_layers": 1,
61            "num_attention_heads": 2,
62            "num_kv_heads": 1,
63            "head_dim": 6,
64            "vocab_size": 305,
65            "layer_types": ["FullAttention"],
66            "rms_norm_eps": 1e-6,
67            "max_position_embeddings": 512
68        },
69        "quant_type": "F32"
70    }))?)
71}
72
73fn pack(fixture: &Fixture, output: &Path) -> Result<(), Box<dyn Error>> {
74    let mut tensors = vec![
75        blob("image.config_json", &fixture.config)?,
76        blob("image.processor_config_json", &fixture.processor)?,
77        blob("image.tokenizer_json", &fixture.tokenizer)?,
78    ];
79    let mut names: Vec<&String> = fixture.weights.keys().collect();
80    names.sort();
81    for name in names {
82        let values = fixture.weights.get(name).expect("weight key disappeared");
83        let shape = fixture
84            .shapes
85            .get(name)
86            .ok_or_else(|| format!("missing shape for weight '{name}'"))?;
87        if shape.iter().product::<usize>() != values.len() {
88            return Err(format!(
89                "weight '{name}' shape {:?} has {} elements, fixture stores {}",
90                shape,
91                shape.iter().product::<usize>(),
92                values.len()
93            )
94            .into());
95        }
96        tensors.push(TensorSpec {
97            name: name.clone(),
98            dtype: TensorDtype::F32,
99            shape: shape.clone(),
100            data: f32_bytes(values),
101        });
102    }
103    CmfModel::write(output, &tiny_header()?, &tensors, None, None)?;
104    Ok(())
105}
106
107fn check(got: &[f32], expected: &[f32]) -> Result<(), Box<dyn Error>> {
108    if got.len() != expected.len() {
109        return Err(format!(
110            "native hidden length {} != oracle {}",
111            got.len(),
112            expected.len()
113        )
114        .into());
115    }
116    let mut max_abs = 0.0f32;
117    let mut sum_sq = 0.0f64;
118    let mut ref_sq = 0.0f64;
119    for (&a, &b) in got.iter().zip(expected) {
120        if !a.is_finite() {
121            return Err("native hidden contains non-finite values".into());
122        }
123        max_abs = max_abs.max((a - b).abs());
124        let d = a as f64 - b as f64;
125        sum_sq += d * d;
126        ref_sq += (b as f64) * (b as f64);
127    }
128    let rel_rms = (sum_sq / ref_sq.max(1e-30)).sqrt();
129    println!("encoder tiny parity: max_abs={max_abs:.6e} rel_rms={rel_rms:.6e}");
130    if max_abs > 5e-4 || rel_rms > 2e-4 {
131        return Err(
132            format!("native/oracle mismatch max_abs={max_abs:.6e} rel_rms={rel_rms:.6e}").into(),
133        );
134    }
135    Ok(())
136}
137
138fn main() -> Result<(), Box<dyn Error>> {
139    let mut args = std::env::args_os().skip(1);
140    let fixture_path = PathBuf::from(args.next().ok_or("missing fixture JSON")?);
141    let cmf_path = PathBuf::from(args.next().ok_or("missing output CMF")?);
142    let fixture: Fixture = serde_json::from_slice(&std::fs::read(&fixture_path)?)?;
143    if fixture.grid != vec![1usize, 6, 10] || fixture.ids.len() <= 64 {
144        return Err(format!(
145            "unexpected fixture geometry grid={:?} ids={}",
146            fixture.grid,
147            fixture.ids.len()
148        )
149        .into());
150    }
151    if fixture.image_rgb.len() != fixture.image_width as usize * fixture.image_height as usize * 3 {
152        return Err("fixture RGB byte count does not match image dimensions".into());
153    }
154    pack(&fixture, &cmf_path)?;
155    let model = CmfModel::open(&cmf_path)?;
156    if !model.verify().is_empty() {
157        return Err(format!("fixture CMF failed verification: {:?}", model.verify()).into());
158    }
159    let encoder = QwenImageEncoder::open(&cmf_path)?;
160    let image = image::RgbImage::from_raw(
161        fixture.image_width,
162        fixture.image_height,
163        fixture.image_rgb.clone(),
164    )
165    .ok_or("fixture RGB image could not be constructed")?;
166    let output = encoder.encode(&fixture.prompt, &[image])?;
167    if output.hidden_size != 12 || output.seq_len * output.hidden_size != fixture.expected.len() {
168        return Err(format!(
169            "native conditioning shape {}×{} does not match oracle {} values",
170            output.seq_len,
171            output.hidden_size,
172            fixture.expected.len()
173        )
174        .into());
175    }
176    check(&output.hidden, &fixture.expected)?;
177    println!(
178        "encoder tiny fixture PASS: tokens={} rows={} grid={:?}",
179        fixture.ids.len(),
180        output.seq_len,
181        fixture.grid
182    );
183    Ok(())
184}
qwen_image_encoder_fixture/qwen_image_encoder_fixture.rs

qwen_image_encoder_fixture/
qwen_image_encoder_fixture.rs
