1use std::{collections::HashMap, sync::Arc};
2
3use crate::{
4 composition::Composition,
5 error::RenderError,
6 gpu::{
7 BoundFrame, CompileContext, CompiledComposition, FrameBindContext, MediaTextureKey,
8 RasterHandle,
9 },
10 media::{CpuMediaFrame, MediaFrame, MediaStore},
11 node::{NodeId, NodeKind, NodeParamEvalContext, NodeParams, PortRef},
12};
13
14#[cfg(feature = "ffmpeg")]
15use super::media::GpuMediaFrameImporter;
16
17#[derive(Debug, Clone, PartialEq, Eq, Hash)]
18pub struct CompiledPlanKey(Vec<(NodeId, Option<usize>)>);
19
20struct PreparedComposition {
21 renderer: lumen_gpu::Renderer,
22 compiled: CompiledComposition,
23 current_media_textures: HashMap<lumen_gpu::TextureId, MediaTextureKey>,
24}
25
26pub struct GpuCompositionRenderer {
27 renderer: lumen_gpu::Renderer,
28 compiled_plans: HashMap<CompiledPlanKey, PreparedComposition>,
29 active_key: Option<CompiledPlanKey>,
30 output_format: lumen_gpu::wgpu::TextureFormat,
31 media_texture_cache: HashMap<MediaTextureKey, Arc<lumen_gpu::wgpu::Texture>>,
32 #[cfg(feature = "ffmpeg")]
33 gpu_media_importer: GpuMediaFrameImporter,
34}
35
36impl GpuCompositionRenderer {
37 pub async fn new() -> crate::Result<Self> {
38 let renderer = lumen_gpu::Renderer::new()
39 .await
40 .map_err(|error| RenderError::Gpu {
41 details: error.to_string(),
42 })?;
43 tracing::debug!(target: "lumen_render", "created gpu composition renderer");
44 Ok(Self {
45 renderer,
46 compiled_plans: HashMap::new(),
47 active_key: None,
48 output_format: lumen_gpu::wgpu::TextureFormat::Rgba8Unorm,
49 media_texture_cache: HashMap::new(),
50 #[cfg(feature = "ffmpeg")]
51 gpu_media_importer: GpuMediaFrameImporter::default(),
52 })
53 }
54
55 pub fn from_device(device: lumen_gpu::wgpu::Device, queue: lumen_gpu::wgpu::Queue) -> Self {
56 Self {
57 renderer: lumen_gpu::Renderer::from_device(device, queue),
58 compiled_plans: HashMap::new(),
59 active_key: None,
60 output_format: lumen_gpu::wgpu::TextureFormat::Rgba8Unorm,
61 media_texture_cache: HashMap::new(),
62 #[cfg(feature = "ffmpeg")]
63 gpu_media_importer: GpuMediaFrameImporter::default(),
64 }
65 }
66
67 pub fn compile(&mut self, composition: &Composition) -> crate::Result<()> {
68 self.compile_with_output_format(composition, lumen_gpu::wgpu::TextureFormat::Rgba8Unorm)
69 }
70
71 pub fn compile_with_output_format(
72 &mut self,
73 composition: &Composition,
74 output_format: lumen_gpu::wgpu::TextureFormat,
75 ) -> crate::Result<()> {
76 self.reset_compiled(output_format);
77 let key = self.compiled_plan_key(composition, 0)?;
78 let compiled = CompileContext::with_frame(composition, 0, output_format).compile()?;
79 self.prepare_compiled(key, compiled)
80 }
81
82 pub fn compile_with_media<M: MediaStore>(
83 &mut self,
84 composition: &Composition,
85 media: &M,
86 output_format: lumen_gpu::wgpu::TextureFormat,
87 ) -> crate::Result<()> {
88 self.reset_compiled(output_format);
89 self.ensure_compiled_for_frame(composition, 0, Some(media))
90 }
91
92 fn reset_compiled(&mut self, output_format: lumen_gpu::wgpu::TextureFormat) {
93 tracing::debug!(
94 target: "lumen_render",
95 ?output_format,
96 cached_plans = self.compiled_plans.len(),
97 media_textures = self.media_texture_cache.len(),
98 "reset compiled composition"
99 );
100 self.output_format = output_format;
101 self.compiled_plans.clear();
102 self.active_key = None;
103 }
104
105 fn prepare_compiled(
106 &mut self,
107 key: CompiledPlanKey,
108 compiled: CompiledComposition,
109 ) -> crate::Result<()> {
110 tracing::debug!(
111 target: "lumen_render",
112 key_entries = key.0.len(),
113 textures = compiled.plan.textures().len(),
114 buffers = compiled.plan.buffers().len(),
115 programs = compiled.plan.programs().len(),
116 passes = compiled.plan.passes().len(),
117 compiled_nodes = compiled.compiled_nodes.len(),
118 "prepare compiled render plan"
119 );
120 let mut renderer = lumen_gpu::Renderer::from_device_with_adapter_info(
121 self.renderer.device.clone(),
122 self.renderer.queue.clone(),
123 self.renderer.adapter_info().clone(),
124 );
125 renderer
126 .prepare_plan(&compiled.plan)
127 .map_err(|error| RenderError::Gpu {
128 details: error.to_string(),
129 })?;
130 self.compiled_plans.insert(
131 key.clone(),
132 PreparedComposition {
133 renderer,
134 compiled,
135 current_media_textures: HashMap::new(),
136 },
137 );
138 self.active_key = Some(key);
139 Ok(())
140 }
141
142 pub fn precompile_frame<M: MediaStore>(
143 &mut self,
144 composition: &Composition,
145 frame: u32,
146 media: &M,
147 ) -> crate::Result<()> {
148 self.ensure_compiled_for_frame(composition, frame, Some(media))
149 }
150
151 pub fn precompile_frame_window<M: MediaStore>(
152 &mut self,
153 composition: &Composition,
154 start_frame: u32,
155 frame_count: u32,
156 media: &M,
157 ) -> crate::Result<()> {
158 tracing::trace!(
159 target: "lumen_render",
160 start_frame,
161 frame_count,
162 "precompile frame window"
163 );
164 for offset in 0..frame_count {
165 self.precompile_frame(composition, start_frame.saturating_add(offset), media)?;
166 }
167 Ok(())
168 }
169
170 pub fn render_frame<M: MediaStore>(
171 &mut self,
172 composition: &Composition,
173 frame: u32,
174 media: &M,
175 ) -> crate::Result<RasterHandle> {
176 self.render_frame_submitted(composition, frame, media)
177 .map(|(raster, _submission)| raster)
178 }
179
180 pub fn render_frame_submitted<M: MediaStore>(
181 &mut self,
182 composition: &Composition,
183 frame: u32,
184 media: &M,
185 ) -> crate::Result<(RasterHandle, lumen_gpu::wgpu::SubmissionIndex)> {
186 let span = tracing::trace_span!(target: "lumen_render", "render frame", frame);
187 let _entered = span.enter();
188 self.ensure_compiled_for_frame(composition, frame, Some(media))?;
189 let bound = self.bind_frame(composition, frame, media)?;
190 tracing::trace!(
191 target: "lumen_render",
192 frame,
193 buffer_uploads = bound.buffer_upload_count(),
194 texture_uploads = bound.texture_upload_count(),
195 media_textures = bound.media_textures().len(),
196 "bound frame"
197 );
198 self.submit_bound_frame(&bound)
199 }
200
201 pub fn render_frame_into_external<M: MediaStore>(
202 &mut self,
203 composition: &Composition,
204 frame: u32,
205 media: &M,
206 external: lumen_gpu::ExternalTexture,
207 ) -> crate::Result<lumen_gpu::SubmittedExternalTexture> {
208 let span = tracing::trace_span!(target: "lumen_render", "render frame into external texture", frame);
209 let _entered = span.enter();
210 self.ensure_compiled_for_frame(composition, frame, Some(media))?;
211 let bound = self.bind_frame(composition, frame, media)?;
212 self.upload_bound_frame(&bound)?;
213 let prepared = self.active_prepared_mut()?;
214 prepared
215 .renderer
216 .submit_plan_with_external_texture(
217 &prepared.compiled.plan,
218 prepared.compiled.output.texture,
219 external,
220 )
221 .map_err(|error| RenderError::Gpu {
222 details: error.to_string(),
223 })
224 .map_err(Into::into)
225 }
226
227 pub fn bind_frame<M: MediaStore>(
228 &mut self,
229 composition: &Composition,
230 frame: u32,
231 media: &M,
232 ) -> crate::Result<BoundFrame> {
233 self.ensure_compiled_for_frame(composition, frame, Some(media))?;
234 let compiled = self.active_compiled()?;
235 FrameBindContext::with_media(composition, frame, media).bind(compiled)
236 }
237
238 pub fn submit_bound_frame(
239 &mut self,
240 bound: &BoundFrame,
241 ) -> crate::Result<(RasterHandle, lumen_gpu::wgpu::SubmissionIndex)> {
242 self.upload_bound_frame(bound)?;
243 self.submit_render()
244 }
245
246 pub fn upload_bound_frame(&mut self, bound: &BoundFrame) -> crate::Result<()> {
247 tracing::trace!(
248 target: "lumen_render",
249 buffer_uploads = bound.buffer_upload_count(),
250 texture_uploads = bound.texture_upload_count(),
251 media_textures = bound.media_textures().len(),
252 "upload bound frame"
253 );
254 self.upload_media_textures(bound)?;
255 let update = bound.frame_update();
256 let prepared = self.active_prepared_mut()?;
257 prepared
258 .renderer
259 .apply_frame_update(&prepared.compiled.plan, &update)
260 .map_err(|error| RenderError::Gpu {
261 details: error.to_string(),
262 })?;
263 Ok(())
264 }
265
266 fn upload_media_textures(&mut self, bound: &BoundFrame) -> crate::Result<()> {
267 let key = self.active_key.clone().ok_or_else(|| RenderError::Gpu {
268 details: "composition has not been compiled".to_string(),
269 })?;
270 let prepared = self
271 .compiled_plans
272 .get_mut(&key)
273 .ok_or_else(|| RenderError::Gpu {
274 details: "composition has not been compiled".to_string(),
275 })?;
276 let media_texture_cache = &mut self.media_texture_cache;
277 #[cfg(feature = "ffmpeg")]
278 let gpu_media_importer = &mut self.gpu_media_importer;
279 for upload in bound.media_textures() {
280 tracing::trace!(
281 target: "lumen_render",
282 texture = ?upload.texture,
283 source = %upload.key.source,
284 frame = ?upload.key.frame,
285 width = upload.size.width,
286 height = upload.size.height,
287 "resolve media texture upload"
288 );
289 #[cfg(feature = "ffmpeg")]
290 if let MediaFrame::GpuVideo(frame) = &upload.frame {
291 let texture = gpu_media_importer
292 .import(&prepared.renderer, upload, frame)
293 .map_err(|details| RenderError::Gpu {
294 details: format!("failed to import GPU media frame: {details}"),
295 })?;
296 let desc = lumen_gpu::TextureDesc::sampled(
297 upload.size,
298 match frame.frame.backend() {
299 #[cfg(all(target_os = "macos", feature = "metal"))]
300 lumen_ffmpeg::GpuBackend::Metal => {
301 lumen_gpu::wgpu::TextureFormat::Bgra8Unorm
302 }
303 _ => lumen_gpu::wgpu::TextureFormat::Rgba8Unorm,
304 },
305 );
306 prepared
307 .renderer
308 .replace_texture_arc(upload.texture, texture, desc)
309 .map_err(|error| RenderError::Gpu {
310 details: error.to_string(),
311 })?;
312 prepared.current_media_textures.remove(&upload.texture);
313 continue;
314 }
315
316 let MediaFrame::CpuRgba(frame) = &upload.frame else {
317 return Err(RenderError::Gpu {
318 details: "media frame backend is not supported by this GPU renderer build"
319 .to_string(),
320 }
321 .into());
322 };
323
324 if upload.key.frame.is_some() {
325 tracing::trace!(
326 target: "lumen_render",
327 texture = ?upload.texture,
328 source = %upload.key.source,
329 frame = ?upload.key.frame,
330 "upload uncached video frame"
331 );
332 let rgba = fit_frame_to_rgba8(frame, upload.size.width, upload.size.height);
333 prepared.renderer.queue.write_texture(
334 lumen_gpu::wgpu::TexelCopyTextureInfo {
335 texture: prepared.renderer.texture(upload.texture).ok_or_else(|| {
336 RenderError::Gpu {
337 details: format!("unknown media texture {:?}", upload.texture),
338 }
339 })?,
340 mip_level: 0,
341 origin: lumen_gpu::wgpu::Origin3d::ZERO,
342 aspect: lumen_gpu::wgpu::TextureAspect::All,
343 },
344 &rgba,
345 lumen_gpu::wgpu::TexelCopyBufferLayout {
346 offset: 0,
347 bytes_per_row: Some(upload.size.width * 4),
348 rows_per_image: Some(upload.size.height),
349 },
350 upload.size.as_extent(),
351 );
352 prepared.current_media_textures.remove(&upload.texture);
353 continue;
354 }
355
356 if prepared
357 .current_media_textures
358 .get(&upload.texture)
359 .is_some_and(|current| current == &upload.key)
360 {
361 tracing::trace!(
362 target: "lumen_render",
363 texture = ?upload.texture,
364 source = %upload.key.source,
365 frame = ?upload.key.frame,
366 "reuse current media texture binding"
367 );
368 continue;
369 }
370
371 let texture = if let Some(texture) = media_texture_cache.get(&upload.key) {
372 tracing::trace!(
373 target: "lumen_render",
374 source = %upload.key.source,
375 frame = ?upload.key.frame,
376 "reuse cached media texture"
377 );
378 Arc::clone(texture)
379 } else {
380 tracing::debug!(
381 target: "lumen_render",
382 source = %upload.key.source,
383 frame = ?upload.key.frame,
384 width = upload.size.width,
385 height = upload.size.height,
386 "cache media texture"
387 );
388 let texture = Arc::new(prepared.renderer.device.create_texture(
389 &lumen_gpu::wgpu::TextureDescriptor {
390 label: Some("lumen media cached frame"),
391 size: upload.size.as_extent(),
392 mip_level_count: 1,
393 sample_count: 1,
394 dimension: lumen_gpu::wgpu::TextureDimension::D2,
395 format: lumen_gpu::wgpu::TextureFormat::Rgba8Unorm,
396 usage: lumen_gpu::wgpu::TextureUsages::TEXTURE_BINDING
397 | lumen_gpu::wgpu::TextureUsages::COPY_DST
398 | lumen_gpu::wgpu::TextureUsages::COPY_SRC,
399 view_formats: &[],
400 },
401 ));
402 let rgba = fit_frame_to_rgba8(frame, upload.size.width, upload.size.height);
403 prepared.renderer.queue.write_texture(
404 texture.as_image_copy(),
405 &rgba,
406 lumen_gpu::wgpu::TexelCopyBufferLayout {
407 offset: 0,
408 bytes_per_row: Some(upload.size.width * 4),
409 rows_per_image: Some(upload.size.height),
410 },
411 upload.size.as_extent(),
412 );
413 media_texture_cache.insert(upload.key.clone(), Arc::clone(&texture));
414 texture
415 };
416
417 let desc = lumen_gpu::TextureDesc::sampled(
418 upload.size,
419 lumen_gpu::wgpu::TextureFormat::Rgba8Unorm,
420 );
421 prepared
422 .renderer
423 .replace_texture_arc(upload.texture, texture, desc)
424 .map_err(|error| RenderError::Gpu {
425 details: error.to_string(),
426 })?;
427 prepared
428 .current_media_textures
429 .insert(upload.texture, upload.key.clone());
430 }
431 Ok(())
432 }
433
434 pub fn submit_render(
435 &mut self,
436 ) -> crate::Result<(RasterHandle, lumen_gpu::wgpu::SubmissionIndex)> {
437 let prepared = self.active_prepared_mut()?;
438 tracing::trace!(
439 target: "lumen_render",
440 passes = prepared.compiled.plan.passes().len(),
441 "submit render"
442 );
443 let submission = prepared
444 .renderer
445 .submit_plan(&prepared.compiled.plan)
446 .map_err(|error| RenderError::Gpu {
447 details: error.to_string(),
448 })?;
449 Ok((prepared.compiled.output, submission))
450 }
451
452 pub fn gpu_renderer(&self) -> &lumen_gpu::Renderer {
453 self.active_key
454 .as_ref()
455 .and_then(|key| self.compiled_plans.get(key))
456 .map(|prepared| &prepared.renderer)
457 .unwrap_or(&self.renderer)
458 }
459
460 pub fn gpu_renderer_mut(&mut self) -> &mut lumen_gpu::Renderer {
461 let Some(key) = self.active_key.clone() else {
462 return &mut self.renderer;
463 };
464 self.compiled_plans
465 .get_mut(&key)
466 .map(|prepared| &mut prepared.renderer)
467 .unwrap_or(&mut self.renderer)
468 }
469
470 pub fn compiled(&self) -> Option<&CompiledComposition> {
471 self.active_key
472 .as_ref()
473 .and_then(|key| self.compiled_plans.get(key))
474 .map(|prepared| &prepared.compiled)
475 }
476
477 fn active_compiled(&self) -> crate::Result<&CompiledComposition> {
478 self.compiled().ok_or_else(|| {
479 RenderError::Gpu {
480 details: "composition has not been compiled".to_string(),
481 }
482 .into()
483 })
484 }
485
486 fn active_prepared_mut(&mut self) -> crate::Result<&mut PreparedComposition> {
487 let key = self.active_key.clone().ok_or_else(|| RenderError::Gpu {
488 details: "composition has not been compiled".to_string(),
489 })?;
490 self.compiled_plans.get_mut(&key).ok_or_else(|| {
491 RenderError::Gpu {
492 details: "composition has not been compiled".to_string(),
493 }
494 .into()
495 })
496 }
497
498 fn ensure_compiled_for_frame<M: MediaStore>(
499 &mut self,
500 composition: &Composition,
501 frame: u32,
502 media: Option<&M>,
503 ) -> crate::Result<()> {
504 let key = self.compiled_plan_key(composition, frame)?;
505 if self.active_key.as_ref() == Some(&key) {
506 tracing::trace!(target: "lumen_render", frame, "reuse active compiled plan");
507 return Ok(());
508 }
509 if self.compiled_plans.contains_key(&key) {
510 tracing::debug!(
511 target: "lumen_render",
512 frame,
513 key_entries = key.0.len(),
514 "switch to cached compiled plan"
515 );
516 self.active_key = Some(key);
517 return Ok(());
518 }
519
520 tracing::debug!(
521 target: "lumen_render",
522 frame,
523 key_entries = key.0.len(),
524 "compile render plan for frame"
525 );
526 let compiled = match media {
527 Some(media) => {
528 CompileContext::with_media_for_frame(composition, frame, media, self.output_format)
529 .compile()?
530 }
531 None => CompileContext::with_frame(composition, frame, self.output_format).compile()?,
532 };
533 self.prepare_compiled(key, compiled)
534 }
535
536 fn compiled_plan_key(
537 &self,
538 composition: &Composition,
539 frame: u32,
540 ) -> crate::Result<CompiledPlanKey> {
541 let mut selections = Vec::new();
542 let mut visited = std::collections::HashSet::new();
543 let output = self.media_output_port(composition)?;
544 self.collect_plan_key(composition, &output, frame, &mut visited, &mut selections)?;
545 selections.sort_by_key(|(node_id, _)| node_id.0);
546 selections.dedup();
547 Ok(CompiledPlanKey(selections))
548 }
549
550 fn collect_plan_key(
551 &self,
552 composition: &Composition,
553 port: &PortRef,
554 frame: u32,
555 visited: &mut std::collections::HashSet<(NodeId, u32)>,
556 selections: &mut Vec<(NodeId, Option<usize>)>,
557 ) -> crate::Result<()> {
558 if port.is_empty() || !visited.insert((port.id, frame)) {
559 return Ok(());
560 }
561
562 let Some(node) = composition.graph.nodes.get(&port.id) else {
563 return Ok(());
564 };
565 match node {
566 NodeKind::MediaOutput(media_output) => self.collect_plan_key(
567 composition,
568 &media_output.source,
569 frame,
570 visited,
571 selections,
572 ),
573 NodeKind::TimeRemap(time_remap) => {
574 let ctx = self.expression_context(composition, frame, time_remap.id, "params");
575 let params = time_remap.params.eval(&NodeParamEvalContext {
576 node_id: time_remap.id,
577 expr: &ctx,
578 })?;
579 let target_frame = crate::node::processing::time_remap::remap_frame(
580 crate::node::processing::time_remap::TimeRemapSettings {
581 frame: params.frame,
582 loop_enabled: params.loop_enabled,
583 loop_start: params.loop_start,
584 loop_end: params.loop_end,
585 },
586 );
587 self.collect_plan_key(
588 composition,
589 &time_remap.source,
590 target_frame,
591 visited,
592 selections,
593 )
594 }
595 NodeKind::Switch(switch) => {
596 let ctx = self.expression_context(composition, frame, switch.id, "selected_layer");
597 let selection =
598 crate::node::compositing::switch::selected_layer_for_frame(switch, &ctx)?;
599 selections.push((switch.id, selection));
600 if let Some(layer) = selection.and_then(|index| switch.layers.get(index)) {
601 self.collect_plan_key(composition, layer, frame, visited, selections)?;
602 }
603 Ok(())
604 }
605 _ => {
606 for input in composition
607 .graph
608 .connections
609 .iter()
610 .filter(|connection| connection.to_node == port.id)
611 .map(|connection| {
612 PortRef::new(connection.from_node, connection.from_port.clone())
613 })
614 {
615 self.collect_plan_key(composition, &input, frame, visited, selections)?;
616 }
617 Ok(())
618 }
619 }
620 }
621
622 fn media_output_port(&self, composition: &Composition) -> crate::Result<PortRef> {
623 let mut outputs = composition
624 .graph
625 .nodes
626 .iter()
627 .filter_map(|(node_id, node)| {
628 matches!(node, NodeKind::MediaOutput(_)).then_some(*node_id)
629 });
630 let Some(output) = outputs.next() else {
631 return Err(crate::error::GraphValidationError::MissingMediaOutput.into());
632 };
633 if outputs.next().is_some() {
634 return Err(
635 crate::error::GraphValidationError::MultipleMediaOutputs { count: 2 }.into(),
636 );
637 }
638 Ok(PortRef::new(output, "output".to_string()))
639 }
640
641 fn expression_context<'a>(
642 &self,
643 composition: &'a Composition,
644 frame: u32,
645 node_id: NodeId,
646 property_path: &str,
647 ) -> crate::expr::ExpressionContext<'a> {
648 crate::expr::ExpressionContext {
649 frame,
650 fps: composition.timeline.fps,
651 width: composition.render_settings.width,
652 height: composition.render_settings.height,
653 duration_frames: composition.timeline.duration_frames,
654 path: Some(format!("{node_id}.{property_path}")),
655 graph: Some(&composition.graph),
656 }
657 }
658}
659
660fn fit_frame_to_rgba8(frame: &CpuMediaFrame, width: u32, height: u32) -> Vec<u8> {
661 if frame.width == width && frame.height == height && frame.row_bytes == width as usize * 4 {
662 return frame.rgba.as_ref().clone();
663 }
664
665 let mut out = vec![0; width as usize * height as usize * 4];
666 for y in 0..height {
667 let src_y = ((u64::from(y) * u64::from(frame.height)) / u64::from(height)) as usize;
668 for x in 0..width {
669 let src_x = ((u64::from(x) * u64::from(frame.width)) / u64::from(width)) as usize;
670 let src = src_y
671 .saturating_mul(frame.row_bytes)
672 .saturating_add(src_x.saturating_mul(4));
673 let dst = (y as usize)
674 .saturating_mul(width as usize * 4)
675 .saturating_add(x as usize * 4);
676 out[dst..dst + 4].copy_from_slice(&frame.rgba[src..src + 4]);
677 }
678 }
679 out
680}