1use std::path::PathBuf;
28
29use crate::png::encode_rgb8;
30use crate::sample;
31use crate::te::{Qwen3Tokenizer, TeError, TeWeights, TextEncoder};
32use crate::vae::{VaeDecoder, VaeError, VaeWeights};
33use crate::{DitError, DitForward, DitWeights, PngError, StepTap};
34
35#[derive(Debug, Clone)]
37pub enum TeSource {
38 Mlx4bit(PathBuf),
41 NpyDir(PathBuf),
44}
45
46#[derive(Debug, Clone)]
54pub struct GoldenOverride {
55 pub golden_dir: PathBuf,
59 pub use_golden_cond: bool,
61 pub use_golden_latent: bool,
63 pub vae_golden_dir: Option<PathBuf>,
65}
66
67#[derive(Debug, Clone)]
69pub struct TextToImageCfg {
70 pub prompt: String,
72 pub seed: u64,
74 pub steps: usize,
76 pub width: usize,
78 pub height: usize,
80 pub guidance: f32,
83 pub dit_gguf: PathBuf,
85 pub vae_weights_dir: PathBuf,
89 pub te_source: TeSource,
91 pub tokenizer_dir: PathBuf,
93 pub golden_override: Option<GoldenOverride>,
95}
96
97pub struct TextToImageOut {
99 pub png: Vec<u8>,
101 pub width: usize,
103 pub height: usize,
105 pub stage_cosines: Vec<(String, f32)>,
109}
110
111#[derive(Debug, thiserror::Error)]
113pub enum PipelineError {
114 #[error("DiT error: {0}")]
116 Dit(#[from] DitError),
117 #[error("text-encoder error: {0}")]
119 Te(#[from] TeError),
120 #[error("VAE error: {0}")]
122 Vae(#[from] VaeError),
123 #[error("PNG error: {0}")]
125 Png(#[from] PngError),
126 #[error("I/O error for {path}: {source}")]
128 Io {
129 path: String,
131 source: std::io::Error,
133 },
134 #[error("pipeline shape error: {0}")]
136 Shape(String),
137 #[error("missing input: {0}")]
139 MissingInput(String),
140}
141
142struct Npy {
144 data: Vec<f32>,
145 shape: Vec<usize>,
146}
147
148impl Npy {
149 fn numel(&self) -> usize {
150 self.shape.iter().product()
151 }
152}
153
154fn read_npy(path: &std::path::Path) -> Result<Npy, PipelineError> {
158 let bytes = std::fs::read(path).map_err(|e| PipelineError::Io {
159 path: path.display().to_string(),
160 source: e,
161 })?;
162 if bytes.len() < 10 || &bytes[..6] != b"\x93NUMPY" {
163 return Err(PipelineError::Shape(format!(
164 "{}: bad npy magic",
165 path.display()
166 )));
167 }
168 let major = bytes[6];
169 let (header_start, header_len) = if major >= 2 {
170 let len = u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]) as usize;
171 (12usize, len)
172 } else {
173 let len = u16::from_le_bytes([bytes[8], bytes[9]]) as usize;
174 (10usize, len)
175 };
176 let header_end = header_start + header_len;
177 if bytes.len() < header_end {
178 return Err(PipelineError::Shape(format!(
179 "{}: truncated npy header",
180 path.display()
181 )));
182 }
183 let header = std::str::from_utf8(&bytes[header_start..header_end])
184 .map_err(|e| PipelineError::Shape(format!("{}: header utf8: {e}", path.display())))?;
185 if !header.contains("'<f4'") {
186 return Err(PipelineError::Shape(format!(
187 "{}: descr is not '<f4': {header}",
188 path.display()
189 )));
190 }
191 let fortran = header.contains("'fortran_order': True");
192 let s_idx = header
193 .find("'shape':")
194 .ok_or_else(|| PipelineError::Shape(format!("{}: no shape key", path.display())))?;
195 let open = header[s_idx..]
196 .find('(')
197 .map(|o| s_idx + o + 1)
198 .ok_or_else(|| PipelineError::Shape(format!("{}: no shape open paren", path.display())))?;
199 let close = header[open..]
200 .find(')')
201 .map(|c| open + c)
202 .ok_or_else(|| PipelineError::Shape(format!("{}: no shape close paren", path.display())))?;
203 let shape: Vec<usize> = header[open..close]
204 .split(',')
205 .map(str::trim)
206 .filter(|s| !s.is_empty())
207 .map(|s| {
208 s.parse::<usize>()
209 .map_err(|e| PipelineError::Shape(format!("{}: shape parse: {e}", path.display())))
210 })
211 .collect::<Result<_, _>>()?;
212 let payload = &bytes[header_end..];
213 if payload.len() % 4 != 0 {
214 return Err(PipelineError::Shape(format!(
215 "{}: payload not f32-aligned",
216 path.display()
217 )));
218 }
219 let raw: Vec<f32> = payload
220 .chunks_exact(4)
221 .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
222 .collect();
223 let numel: usize = shape.iter().product();
224 if raw.len() < numel {
225 return Err(PipelineError::Shape(format!(
226 "{}: payload short ({} < {numel})",
227 path.display(),
228 raw.len()
229 )));
230 }
231 let data = if fortran && shape.len() > 1 {
232 fortran_to_c(&raw[..numel], &shape)
233 } else {
234 raw
235 };
236 Ok(Npy { data, shape })
237}
238
239fn fortran_to_c(src: &[f32], shape: &[usize]) -> Vec<f32> {
241 let ndim = shape.len();
242 let numel: usize = shape.iter().product();
243 let mut f_stride = vec![1usize; ndim];
244 for d in 1..ndim {
245 f_stride[d] = f_stride[d - 1] * shape[d - 1];
246 }
247 let mut out = vec![0.0f32; numel];
248 for (c_pos, slot) in out.iter_mut().enumerate() {
249 let mut rem = c_pos;
250 let mut f_off = 0usize;
251 for d in 0..ndim {
252 let stride_c: usize = shape[d + 1..].iter().product();
253 let idx = rem / stride_c;
254 rem %= stride_c;
255 f_off += idx * f_stride[d];
256 }
257 *slot = src[f_off];
258 }
259 out
260}
261
262fn load_golden(dir: &std::path::Path, name: &str) -> Result<Npy, PipelineError> {
264 read_npy(&dir.join(format!("{name}.npy")))
265}
266
267fn cosine(a: &[f32], b: &[f32]) -> f64 {
269 let mut dot = 0.0f64;
270 let mut na = 0.0f64;
271 let mut nb = 0.0f64;
272 for (&x, &y) in a.iter().zip(b.iter()) {
273 let (x, y) = (x as f64, y as f64);
274 dot += x * y;
275 na += x * x;
276 nb += y * y;
277 }
278 if na > 0.0 && nb > 0.0 {
279 dot / (na.sqrt() * nb.sqrt())
280 } else {
281 0.0
282 }
283}
284
285pub fn latent_seq_to_packed_nchw(
295 latent: &[f32],
296 seq_img: usize,
297 in_channels: usize,
298 ph: usize,
299 pw: usize,
300) -> Result<Vec<f32>, PipelineError> {
301 if seq_img != ph * pw {
302 return Err(PipelineError::Shape(format!(
303 "seq_img {seq_img} != ph*pw ({ph}*{pw})"
304 )));
305 }
306 if latent.len() != seq_img * in_channels {
307 return Err(PipelineError::Shape(format!(
308 "latent len {} != seq_img*in_channels ({seq_img}*{in_channels})",
309 latent.len()
310 )));
311 }
312 let plane = ph * pw;
313 let mut packed = vec![0.0f32; in_channels * plane];
314 for hh in 0..ph {
315 for ww in 0..pw {
316 let s = hh * pw + ww;
317 let src_base = s * in_channels;
318 for ch in 0..in_channels {
319 packed[ch * plane + hh * pw + ww] = latent[src_base + ch];
320 }
321 }
322 }
323 Ok(packed)
324}
325
326fn compute_cond(
328 cfg: &TextToImageCfg,
329 seq_txt: usize,
330 joint_dim: usize,
331) -> Result<Vec<f32>, PipelineError> {
332 let tok = Qwen3Tokenizer::open(&cfg.tokenizer_dir)?;
333 let toks = tok.tokenize(&cfg.prompt, seq_txt)?;
334
335 let weights = match &cfg.te_source {
336 TeSource::Mlx4bit(path) => TeWeights::open_mlx_4bit(path)?,
337 TeSource::NpyDir(dir) => TeWeights::open(dir)?,
338 };
339 let encoder = TextEncoder::new(&weights);
340 let out = encoder.forward(&toks.input_ids, &toks.attention_mask)?;
341 let cond = out.cond_7680()?;
342 if cond.len() != seq_txt * joint_dim {
343 return Err(PipelineError::Shape(format!(
344 "TE cond len {} != seq_txt*joint_dim ({seq_txt}*{joint_dim})",
345 cond.len()
346 )));
347 }
348 Ok(cond)
349}
350
351fn vae_path_present(path: &std::path::Path) -> bool {
360 path.is_file() || path.is_dir()
361}
362
363pub(crate) fn decoded_chw_to_rgb8(
372 c: usize,
373 h: usize,
374 w: usize,
375 data: &[f32],
376) -> Result<(usize, usize, Vec<u8>), PipelineError> {
377 if c != 3 {
378 return Err(PipelineError::Shape(format!(
379 "expected 3 output channels, got {c}"
380 )));
381 }
382 let plane = h * w;
383 let mut rgb = vec![0u8; data.len()];
384 for y in 0..h {
385 for x in 0..w {
386 let hw = y * w + x;
387 let dst = (y * w + x) * 3;
388 for ch in 0..3 {
389 let v = (data[ch * plane + hw] / 2.0 + 0.5).clamp(0.0, 1.0);
390 rgb[dst + ch] = (v * 255.0).round() as u8;
391 }
392 }
393 }
394 Ok((w, h, rgb))
395}
396
397pub fn text_to_image(cfg: &TextToImageCfg) -> Result<TextToImageOut, PipelineError> {
404 let mut stage_cosines: Vec<(String, f32)> = Vec::new();
405 let timing = std::env::var("PICTOR_IMAGE_TIMING").is_ok();
406
407 if !cfg.dit_gguf.exists() {
409 return Err(PipelineError::MissingInput(format!(
410 "DiT GGUF not found: {}",
411 cfg.dit_gguf.display()
412 )));
413 }
414 if !vae_path_present(&cfg.vae_weights_dir) {
415 return Err(PipelineError::MissingInput(format!(
416 "VAE weights not found: {}",
417 cfg.vae_weights_dir.display()
418 )));
419 }
420 let t_load_dit = std::time::Instant::now();
421 let dit_weights = DitWeights::open(&cfg.dit_gguf)?;
422 let dcfg = dit_weights.config();
423 let in_channels = dcfg.in_channels as usize;
424 let joint_dim = dcfg.joint_attention_dim as usize;
425 let num_axes = dcfg.axes_dims_rope.len();
426 if timing {
427 eprintln!(
428 "[timing] load DiT: {:.2}s",
429 t_load_dit.elapsed().as_secs_f64()
430 );
431 }
432
433 let t_load_vae = std::time::Instant::now();
434 let vae_weights = VaeWeights::open(&cfg.vae_weights_dir)?;
435 let vae = VaeDecoder::from_weights(&vae_weights)?;
436 if timing {
437 eprintln!(
438 "[timing] load VAE: {:.2}s",
439 t_load_vae.elapsed().as_secs_f64()
440 );
441 }
442
443 let (lat_h, lat_w) = sample::latent_grid(cfg.height, cfg.width);
445 let seq_img = lat_h * lat_w;
446 let seq_txt = 512usize;
448
449 let golden = cfg.golden_override.as_ref();
450
451 let cond_vec: Vec<f32> = match golden {
453 Some(g) if g.use_golden_cond => {
454 let c = load_golden(&g.golden_dir, "cond")?;
455 if c.numel() < seq_txt * joint_dim {
456 return Err(PipelineError::Shape(format!(
457 "golden cond numel {} < {seq_txt}*{joint_dim}",
458 c.numel()
459 )));
460 }
461 c.data[..seq_txt * joint_dim].to_vec()
462 }
463 _ => {
464 let t_te = std::time::Instant::now();
465 let cond = compute_cond(cfg, seq_txt, joint_dim)?;
466 if timing {
467 eprintln!("[timing] TE encode: {:.2}s", t_te.elapsed().as_secs_f64());
468 }
469 if let Some(g) = golden {
471 if let Ok(gc) = load_golden(&g.golden_dir, "cond") {
472 if gc.numel() >= cond.len() {
473 let cos = cosine(&cond, &gc.data[..cond.len()]) as f32;
474 stage_cosines.push(("cond_vs_golden".to_string(), cos));
475 }
476 }
477 }
478 cond
479 }
480 };
481
482 let final_latent: Vec<f32> = match golden {
484 Some(g) if g.use_golden_latent => {
485 let glat = load_golden(&g.golden_dir, "latent_after_step3")?;
486 if glat.numel() != seq_img * in_channels {
487 return Err(PipelineError::Shape(format!(
488 "latent_after_step3 numel {} != {seq_img}*{in_channels}",
489 glat.numel()
490 )));
491 }
492 glat.data
493 }
494 _ => {
495 let fwd = DitForward::new(&dit_weights);
496
497 let (init, img_ids, txt_ids, timesteps, sigmas) = match golden {
499 Some(g) => {
500 let init = load_golden(&g.golden_dir, "tf_in_hidden_states")?;
501 let img_ids = load_golden(&g.golden_dir, "img_ids")?;
502 let txt_ids = load_golden(&g.golden_dir, "txt_ids")?;
503 let timesteps = load_golden(&g.golden_dir, "timesteps")?;
504 let sigmas = load_golden(&g.golden_dir, "sigmas")?;
505 (
506 init.data[..seq_img * in_channels].to_vec(),
507 img_ids.data[..seq_img * num_axes].to_vec(),
508 txt_ids.data[..seq_txt * num_axes].to_vec(),
509 timesteps.data,
510 sigmas.data,
511 )
512 }
513 None => {
514 let init = sample::create_noise(cfg.seed, cfg.height, cfg.width);
515 if init.len() != seq_img * in_channels {
516 return Err(PipelineError::Shape(format!(
517 "native noise len {} != {seq_img}*{in_channels} \
518 (in_channels {in_channels} vs PACKED_CHANNELS {})",
519 init.len(),
520 sample::PACKED_CHANNELS
521 )));
522 }
523 let img_ids = sample::img_ids(lat_h, lat_w);
524 let txt_ids = sample::txt_ids(seq_txt);
525 let (timesteps, sigmas) = sample::flow_match_schedule(seq_img, cfg.steps);
526 (init, img_ids, txt_ids, timesteps, sigmas)
527 }
528 };
529
530 let mut steps: Vec<StepTap> = Vec::new();
531 let t_sample = std::time::Instant::now();
532 let latent = fwd.sample(
533 &init,
534 &cond_vec,
535 &img_ids,
536 &txt_ids,
537 seq_img,
538 seq_txt,
539 ×teps,
540 &sigmas,
541 Some(&mut steps),
542 )?;
543 if timing {
544 eprintln!(
545 "[timing] DiT sample ({} steps): {:.2}s",
546 timesteps.len(),
547 t_sample.elapsed().as_secs_f64()
548 );
549 }
550
551 if let Some(g) = golden {
553 if let Ok(glat) = load_golden(&g.golden_dir, "latent_after_step3") {
554 if glat.numel() == latent.len() {
555 let cos = cosine(&latent, &glat.data) as f32;
556 stage_cosines.push(("latent_vs_golden".to_string(), cos));
557 }
558 }
559 }
560 latent
561 }
562 };
563
564 let (ph, pw) = (lat_h, lat_w);
566 let packed = latent_seq_to_packed_nchw(&final_latent, seq_img, in_channels, ph, pw)?;
567 let t_vae = std::time::Instant::now();
568 let decoded = vae.decode_packed_latents(&packed, ph, pw, None)?;
569 if timing {
570 eprintln!("[timing] VAE decode: {:.2}s", t_vae.elapsed().as_secs_f64());
571 }
572
573 if let Some(g) = golden {
574 if let Some(vae_golden_dir) = &g.vae_golden_dir {
575 if let Ok(gd) = load_golden(vae_golden_dir, "vae_decoded") {
576 if gd.numel() == decoded.data.len() {
577 let cos = cosine(&decoded.data, &gd.data) as f32;
578 stage_cosines.push(("vae_vs_golden".to_string(), cos));
579 }
580 }
581 }
582 }
583
584 let (w, h, rgb) = decoded_chw_to_rgb8(decoded.c, decoded.h, decoded.w, &decoded.data)?;
586
587 let t_png = std::time::Instant::now();
589 let png = encode_rgb8(w, h, &rgb)?;
590 if timing {
591 eprintln!("[timing] PNG encode: {:.2}s", t_png.elapsed().as_secs_f64());
592 }
593
594 Ok(TextToImageOut {
595 png,
596 width: w,
597 height: h,
598 stage_cosines,
599 })
600}
601
602#[cfg(test)]
603mod tests {
604 use super::vae_path_present;
605 use std::path::PathBuf;
606
607 fn scratch(label: &str) -> PathBuf {
611 let nanos = std::time::SystemTime::now()
612 .duration_since(std::time::UNIX_EPOCH)
613 .map(|d| d.as_nanos())
614 .unwrap_or(0);
615 std::env::temp_dir().join(format!(
616 "pictor_issue9_{label}_{}_{nanos}",
617 std::process::id()
618 ))
619 }
620
621 #[test]
630 fn test_issue_9_vae_safetensors_file_accepted() -> std::io::Result<()> {
631 let file = scratch("vae").with_extension("safetensors");
632 std::fs::write(&file, b"not-a-real-safetensors-but-a-real-file")?;
635
636 let present = vae_path_present(&file);
637 std::fs::remove_file(&file)?;
638
639 assert!(
640 present,
641 "issue #9: a .safetensors FILE must pass the VAE precheck (got false), \
642 path = {}",
643 file.display()
644 );
645 Ok(())
646 }
647
648 #[test]
652 fn test_issue_9_vae_directory_still_accepted() -> std::io::Result<()> {
653 let dir = scratch("vae_dir");
654 std::fs::create_dir_all(&dir)?;
655
656 let present = vae_path_present(&dir);
657 std::fs::remove_dir_all(&dir)?;
658
659 assert!(
660 present,
661 "a VAE weights directory must still pass the precheck, path = {}",
662 dir.display()
663 );
664 Ok(())
665 }
666
667 #[test]
670 fn test_issue_9_vae_nonexistent_rejected() {
671 let missing = scratch("vae_missing").with_extension("safetensors");
672 assert!(
674 !vae_path_present(&missing),
675 "a nonexistent path must be rejected by the precheck, path = {}",
676 missing.display()
677 );
678 }
679}