1use std::{collections::HashMap, sync::Arc};
2
3#[cfg(all(
4 target_os = "linux",
5 feature = "ffmpeg",
6 feature = "cuda",
7 feature = "vulkan"
8))]
9use std::sync::OnceLock;
10
11use crate::{
12 composition::Composition,
13 error::RenderError,
14 gpu::{
15 BoundFrame, CompileContext, CompiledComposition, FrameBindContext, MediaTextureKey,
16 RasterHandle,
17 },
18 media::{CpuMediaFrame, MediaFrame, MediaStore},
19 node::{NodeId, NodeKind, PortRef},
20};
21
22#[cfg(all(
23 target_os = "linux",
24 feature = "ffmpeg",
25 feature = "cuda",
26 feature = "vulkan"
27))]
28use super::types::MediaTextureUpload;
29
30#[derive(Debug, Clone, PartialEq, Eq, Hash)]
31pub struct CompiledPlanKey(Vec<(NodeId, Option<usize>)>);
32
33struct PreparedComposition {
34 renderer: lumen_gpu::Renderer,
35 compiled: CompiledComposition,
36 current_media_textures: HashMap<lumen_gpu::TextureId, MediaTextureKey>,
37}
38
39pub struct GpuCompositionRenderer {
40 renderer: lumen_gpu::Renderer,
41 compiled_plans: HashMap<CompiledPlanKey, PreparedComposition>,
42 active_key: Option<CompiledPlanKey>,
43 output_format: lumen_gpu::wgpu::TextureFormat,
44 media_texture_cache: HashMap<MediaTextureKey, Arc<lumen_gpu::wgpu::Texture>>,
45 #[cfg(all(
46 target_os = "linux",
47 feature = "ffmpeg",
48 feature = "cuda",
49 feature = "vulkan"
50 ))]
51 cuda_media: Option<CudaMediaInterop>,
52}
53
54impl GpuCompositionRenderer {
55 pub async fn new() -> crate::Result<Self> {
56 let renderer = lumen_gpu::Renderer::new()
57 .await
58 .map_err(|error| RenderError::Gpu {
59 details: error.to_string(),
60 })?;
61 tracing::debug!(target: "lumen_render", "created gpu composition renderer");
62 Ok(Self {
63 renderer,
64 compiled_plans: HashMap::new(),
65 active_key: None,
66 output_format: lumen_gpu::wgpu::TextureFormat::Rgba8Unorm,
67 media_texture_cache: HashMap::new(),
68 #[cfg(all(
69 target_os = "linux",
70 feature = "ffmpeg",
71 feature = "cuda",
72 feature = "vulkan"
73 ))]
74 cuda_media: None,
75 })
76 }
77
78 pub fn from_device(device: lumen_gpu::wgpu::Device, queue: lumen_gpu::wgpu::Queue) -> Self {
79 Self {
80 renderer: lumen_gpu::Renderer::from_device(device, queue),
81 compiled_plans: HashMap::new(),
82 active_key: None,
83 output_format: lumen_gpu::wgpu::TextureFormat::Rgba8Unorm,
84 media_texture_cache: HashMap::new(),
85 #[cfg(all(
86 target_os = "linux",
87 feature = "ffmpeg",
88 feature = "cuda",
89 feature = "vulkan"
90 ))]
91 cuda_media: None,
92 }
93 }
94
95 pub fn compile(&mut self, composition: &Composition) -> crate::Result<()> {
96 self.compile_with_output_format(composition, lumen_gpu::wgpu::TextureFormat::Rgba8Unorm)
97 }
98
99 pub fn compile_with_output_format(
100 &mut self,
101 composition: &Composition,
102 output_format: lumen_gpu::wgpu::TextureFormat,
103 ) -> crate::Result<()> {
104 self.reset_compiled(output_format);
105 let key = self.compiled_plan_key(composition, 0)?;
106 let compiled = CompileContext::with_frame(composition, 0, output_format).compile()?;
107 self.prepare_compiled(key, compiled)
108 }
109
110 pub fn compile_with_media<M: MediaStore>(
111 &mut self,
112 composition: &Composition,
113 media: &M,
114 output_format: lumen_gpu::wgpu::TextureFormat,
115 ) -> crate::Result<()> {
116 self.reset_compiled(output_format);
117 self.ensure_compiled_for_frame(composition, 0, Some(media))
118 }
119
120 fn reset_compiled(&mut self, output_format: lumen_gpu::wgpu::TextureFormat) {
121 tracing::debug!(
122 target: "lumen_render",
123 ?output_format,
124 cached_plans = self.compiled_plans.len(),
125 media_textures = self.media_texture_cache.len(),
126 "reset compiled composition"
127 );
128 self.output_format = output_format;
129 self.compiled_plans.clear();
130 self.active_key = None;
131 }
132
133 fn prepare_compiled(
134 &mut self,
135 key: CompiledPlanKey,
136 compiled: CompiledComposition,
137 ) -> crate::Result<()> {
138 tracing::debug!(
139 target: "lumen_render",
140 key_entries = key.0.len(),
141 textures = compiled.plan.textures().len(),
142 buffers = compiled.plan.buffers().len(),
143 programs = compiled.plan.programs().len(),
144 passes = compiled.plan.passes().len(),
145 frame_bindings = compiled.frame_bindings.len(),
146 "prepare compiled render plan"
147 );
148 let mut renderer = lumen_gpu::Renderer::from_device(
149 self.renderer.device.clone(),
150 self.renderer.queue.clone(),
151 );
152 renderer
153 .prepare_plan(&compiled.plan)
154 .map_err(|error| RenderError::Gpu {
155 details: error.to_string(),
156 })?;
157 self.compiled_plans.insert(
158 key.clone(),
159 PreparedComposition {
160 renderer,
161 compiled,
162 current_media_textures: HashMap::new(),
163 },
164 );
165 self.active_key = Some(key);
166 Ok(())
167 }
168
169 pub fn precompile_frame<M: MediaStore>(
170 &mut self,
171 composition: &Composition,
172 frame: u32,
173 media: &M,
174 ) -> crate::Result<()> {
175 self.ensure_compiled_for_frame(composition, frame, Some(media))
176 }
177
178 pub fn precompile_frame_window<M: MediaStore>(
179 &mut self,
180 composition: &Composition,
181 start_frame: u32,
182 frame_count: u32,
183 media: &M,
184 ) -> crate::Result<()> {
185 tracing::trace!(
186 target: "lumen_render",
187 start_frame,
188 frame_count,
189 "precompile frame window"
190 );
191 for offset in 0..frame_count {
192 self.precompile_frame(composition, start_frame.saturating_add(offset), media)?;
193 }
194 Ok(())
195 }
196
197 pub fn render_frame<M: MediaStore>(
198 &mut self,
199 composition: &Composition,
200 frame: u32,
201 media: &M,
202 ) -> crate::Result<RasterHandle> {
203 self.render_frame_submitted(composition, frame, media)
204 .map(|(raster, _submission)| raster)
205 }
206
207 pub fn render_frame_submitted<M: MediaStore>(
208 &mut self,
209 composition: &Composition,
210 frame: u32,
211 media: &M,
212 ) -> crate::Result<(RasterHandle, lumen_gpu::wgpu::SubmissionIndex)> {
213 let span = tracing::trace_span!(target: "lumen_render", "render frame", frame);
214 let _entered = span.enter();
215 self.ensure_compiled_for_frame(composition, frame, Some(media))?;
216 let bound = self.bind_frame(composition, frame, media)?;
217 tracing::trace!(
218 target: "lumen_render",
219 frame,
220 buffer_uploads = bound.buffer_upload_count(),
221 texture_uploads = bound.texture_upload_count(),
222 media_textures = bound.media_textures().len(),
223 "bound frame"
224 );
225 self.submit_bound_frame(&bound)
226 }
227
228 pub fn bind_frame<M: MediaStore>(
229 &mut self,
230 composition: &Composition,
231 frame: u32,
232 media: &M,
233 ) -> crate::Result<BoundFrame> {
234 self.ensure_compiled_for_frame(composition, frame, Some(media))?;
235 let compiled = self.active_compiled()?;
236 FrameBindContext::with_media(composition, frame, media).bind(compiled)
237 }
238
239 pub fn submit_bound_frame(
240 &mut self,
241 bound: &BoundFrame,
242 ) -> crate::Result<(RasterHandle, lumen_gpu::wgpu::SubmissionIndex)> {
243 self.upload_bound_frame(bound)?;
244 self.submit_render()
245 }
246
247 pub fn upload_bound_frame(&mut self, bound: &BoundFrame) -> crate::Result<()> {
248 tracing::trace!(
249 target: "lumen_render",
250 buffer_uploads = bound.buffer_upload_count(),
251 texture_uploads = bound.texture_upload_count(),
252 media_textures = bound.media_textures().len(),
253 "upload bound frame"
254 );
255 self.upload_media_textures(bound)?;
256 let update = bound.frame_update();
257 let prepared = self.active_prepared_mut()?;
258 prepared
259 .renderer
260 .apply_frame_update(&prepared.compiled.plan, &update)
261 .map_err(|error| RenderError::Gpu {
262 details: error.to_string(),
263 })?;
264 Ok(())
265 }
266
267 fn upload_media_textures(&mut self, bound: &BoundFrame) -> crate::Result<()> {
268 let key = self.active_key.clone().ok_or_else(|| RenderError::Gpu {
269 details: "composition has not been compiled".to_string(),
270 })?;
271 let prepared = self
272 .compiled_plans
273 .get_mut(&key)
274 .ok_or_else(|| RenderError::Gpu {
275 details: "composition has not been compiled".to_string(),
276 })?;
277 let media_texture_cache = &mut self.media_texture_cache;
278 #[cfg(all(
279 target_os = "linux",
280 feature = "ffmpeg",
281 feature = "cuda",
282 feature = "vulkan"
283 ))]
284 let cuda_media = &mut self.cuda_media;
285 for upload in bound.media_textures() {
286 tracing::trace!(
287 target: "lumen_render",
288 texture = ?upload.texture,
289 source = %upload.key.source,
290 frame = ?upload.key.frame,
291 width = upload.size.width,
292 height = upload.size.height,
293 "resolve media texture upload"
294 );
295 #[cfg(all(
296 target_os = "linux",
297 feature = "ffmpeg",
298 feature = "cuda",
299 feature = "vulkan"
300 ))]
301 if let MediaFrame::GpuVideo(frame) = &upload.frame {
302 let texture =
303 cuda_video_frame_to_texture(cuda_media, &prepared.renderer, upload, frame)
304 .map_err(|details| RenderError::Gpu {
305 details: format!("failed to import CUDA media frame: {details}"),
306 })?;
307 let desc = lumen_gpu::TextureDesc::sampled(
308 upload.size,
309 lumen_gpu::wgpu::TextureFormat::Rgba8Unorm,
310 );
311 prepared
312 .renderer
313 .replace_texture_arc(upload.texture, texture, desc)
314 .map_err(|error| RenderError::Gpu {
315 details: error.to_string(),
316 })?;
317 prepared.current_media_textures.remove(&upload.texture);
318 continue;
319 }
320
321 let MediaFrame::CpuRgba(frame) = &upload.frame else {
322 return Err(RenderError::Gpu {
323 details: "media frame backend is not supported by this GPU renderer build"
324 .to_string(),
325 }
326 .into());
327 };
328
329 if upload.key.frame.is_some() {
330 tracing::trace!(
331 target: "lumen_render",
332 texture = ?upload.texture,
333 source = %upload.key.source,
334 frame = ?upload.key.frame,
335 "upload uncached video frame"
336 );
337 let rgba = fit_frame_to_rgba8(frame, upload.size.width, upload.size.height);
338 prepared.renderer.queue.write_texture(
339 lumen_gpu::wgpu::TexelCopyTextureInfo {
340 texture: prepared.renderer.texture(upload.texture).ok_or_else(|| {
341 RenderError::Gpu {
342 details: format!("unknown media texture {:?}", upload.texture),
343 }
344 })?,
345 mip_level: 0,
346 origin: lumen_gpu::wgpu::Origin3d::ZERO,
347 aspect: lumen_gpu::wgpu::TextureAspect::All,
348 },
349 &rgba,
350 lumen_gpu::wgpu::TexelCopyBufferLayout {
351 offset: 0,
352 bytes_per_row: Some(upload.size.width * 4),
353 rows_per_image: Some(upload.size.height),
354 },
355 upload.size.as_extent(),
356 );
357 prepared.current_media_textures.remove(&upload.texture);
358 continue;
359 }
360
361 if prepared
362 .current_media_textures
363 .get(&upload.texture)
364 .is_some_and(|current| current == &upload.key)
365 {
366 tracing::trace!(
367 target: "lumen_render",
368 texture = ?upload.texture,
369 source = %upload.key.source,
370 frame = ?upload.key.frame,
371 "reuse current media texture binding"
372 );
373 continue;
374 }
375
376 let texture = if let Some(texture) = media_texture_cache.get(&upload.key) {
377 tracing::trace!(
378 target: "lumen_render",
379 source = %upload.key.source,
380 frame = ?upload.key.frame,
381 "reuse cached media texture"
382 );
383 Arc::clone(texture)
384 } else {
385 tracing::debug!(
386 target: "lumen_render",
387 source = %upload.key.source,
388 frame = ?upload.key.frame,
389 width = upload.size.width,
390 height = upload.size.height,
391 "cache media texture"
392 );
393 let texture = Arc::new(prepared.renderer.device.create_texture(
394 &lumen_gpu::wgpu::TextureDescriptor {
395 label: Some("lumen media cached frame"),
396 size: upload.size.as_extent(),
397 mip_level_count: 1,
398 sample_count: 1,
399 dimension: lumen_gpu::wgpu::TextureDimension::D2,
400 format: lumen_gpu::wgpu::TextureFormat::Rgba8Unorm,
401 usage: lumen_gpu::wgpu::TextureUsages::TEXTURE_BINDING
402 | lumen_gpu::wgpu::TextureUsages::COPY_DST
403 | lumen_gpu::wgpu::TextureUsages::COPY_SRC,
404 view_formats: &[],
405 },
406 ));
407 let rgba = fit_frame_to_rgba8(frame, upload.size.width, upload.size.height);
408 prepared.renderer.queue.write_texture(
409 texture.as_image_copy(),
410 &rgba,
411 lumen_gpu::wgpu::TexelCopyBufferLayout {
412 offset: 0,
413 bytes_per_row: Some(upload.size.width * 4),
414 rows_per_image: Some(upload.size.height),
415 },
416 upload.size.as_extent(),
417 );
418 media_texture_cache.insert(upload.key.clone(), Arc::clone(&texture));
419 texture
420 };
421
422 let desc = lumen_gpu::TextureDesc::sampled(
423 upload.size,
424 lumen_gpu::wgpu::TextureFormat::Rgba8Unorm,
425 );
426 prepared
427 .renderer
428 .replace_texture_arc(upload.texture, texture, desc)
429 .map_err(|error| RenderError::Gpu {
430 details: error.to_string(),
431 })?;
432 prepared
433 .current_media_textures
434 .insert(upload.texture, upload.key.clone());
435 }
436 Ok(())
437 }
438
439 pub fn submit_render(
440 &mut self,
441 ) -> crate::Result<(RasterHandle, lumen_gpu::wgpu::SubmissionIndex)> {
442 let prepared = self.active_prepared_mut()?;
443 tracing::trace!(
444 target: "lumen_render",
445 passes = prepared.compiled.plan.passes().len(),
446 "submit render"
447 );
448 let submission = prepared
449 .renderer
450 .submit_plan(&prepared.compiled.plan)
451 .map_err(|error| RenderError::Gpu {
452 details: error.to_string(),
453 })?;
454 Ok((prepared.compiled.output, submission))
455 }
456
457 pub fn gpu_renderer(&self) -> &lumen_gpu::Renderer {
458 self.active_key
459 .as_ref()
460 .and_then(|key| self.compiled_plans.get(key))
461 .map(|prepared| &prepared.renderer)
462 .unwrap_or(&self.renderer)
463 }
464
465 pub fn gpu_renderer_mut(&mut self) -> &mut lumen_gpu::Renderer {
466 let Some(key) = self.active_key.clone() else {
467 return &mut self.renderer;
468 };
469 self.compiled_plans
470 .get_mut(&key)
471 .map(|prepared| &mut prepared.renderer)
472 .unwrap_or(&mut self.renderer)
473 }
474
475 pub fn compiled(&self) -> Option<&CompiledComposition> {
476 self.active_key
477 .as_ref()
478 .and_then(|key| self.compiled_plans.get(key))
479 .map(|prepared| &prepared.compiled)
480 }
481
482 fn active_compiled(&self) -> crate::Result<&CompiledComposition> {
483 self.compiled().ok_or_else(|| {
484 RenderError::Gpu {
485 details: "composition has not been compiled".to_string(),
486 }
487 .into()
488 })
489 }
490
491 fn active_prepared_mut(&mut self) -> crate::Result<&mut PreparedComposition> {
492 let key = self.active_key.clone().ok_or_else(|| RenderError::Gpu {
493 details: "composition has not been compiled".to_string(),
494 })?;
495 self.compiled_plans.get_mut(&key).ok_or_else(|| {
496 RenderError::Gpu {
497 details: "composition has not been compiled".to_string(),
498 }
499 .into()
500 })
501 }
502
503 fn ensure_compiled_for_frame<M: MediaStore>(
504 &mut self,
505 composition: &Composition,
506 frame: u32,
507 media: Option<&M>,
508 ) -> crate::Result<()> {
509 let key = self.compiled_plan_key(composition, frame)?;
510 if self.active_key.as_ref() == Some(&key) {
511 tracing::trace!(target: "lumen_render", frame, "reuse active compiled plan");
512 return Ok(());
513 }
514 if self.compiled_plans.contains_key(&key) {
515 tracing::debug!(
516 target: "lumen_render",
517 frame,
518 key_entries = key.0.len(),
519 "switch to cached compiled plan"
520 );
521 self.active_key = Some(key);
522 return Ok(());
523 }
524
525 tracing::debug!(
526 target: "lumen_render",
527 frame,
528 key_entries = key.0.len(),
529 "compile render plan for frame"
530 );
531 let compiled = match media {
532 Some(media) => {
533 CompileContext::with_media_for_frame(composition, frame, media, self.output_format)
534 .compile()?
535 }
536 None => CompileContext::with_frame(composition, frame, self.output_format).compile()?,
537 };
538 self.prepare_compiled(key, compiled)
539 }
540
541 fn compiled_plan_key(
542 &self,
543 composition: &Composition,
544 frame: u32,
545 ) -> crate::Result<CompiledPlanKey> {
546 let mut selections = Vec::new();
547 let mut visited = std::collections::HashSet::new();
548 let output = self.media_output_port(composition)?;
549 self.collect_plan_key(composition, &output, frame, &mut visited, &mut selections)?;
550 selections.sort_by_key(|(node_id, _)| node_id.0);
551 selections.dedup();
552 Ok(CompiledPlanKey(selections))
553 }
554
555 fn collect_plan_key(
556 &self,
557 composition: &Composition,
558 port: &PortRef,
559 frame: u32,
560 visited: &mut std::collections::HashSet<(NodeId, u32)>,
561 selections: &mut Vec<(NodeId, Option<usize>)>,
562 ) -> crate::Result<()> {
563 if port.is_empty() || !visited.insert((port.id, frame)) {
564 return Ok(());
565 }
566
567 let Some(node) = composition.graph.nodes.get(&port.id) else {
568 return Ok(());
569 };
570 match node {
571 NodeKind::MediaOutput(media_output) => self.collect_plan_key(
572 composition,
573 &media_output.source,
574 frame,
575 visited,
576 selections,
577 ),
578 NodeKind::TimeRemap(time_remap) => {
579 let ctx = self.expression_context(composition, frame, time_remap.id, "frame");
580 let target_frame = crate::node::processing::time_remap::remap_frame(
581 crate::node::processing::time_remap::resolve_settings(
582 time_remap.id,
583 &time_remap.frame,
584 &time_remap.loop_enabled,
585 &time_remap.loop_start,
586 &time_remap.loop_end,
587 &ctx,
588 )?,
589 );
590 self.collect_plan_key(
591 composition,
592 &time_remap.source,
593 target_frame,
594 visited,
595 selections,
596 )
597 }
598 NodeKind::Switch(switch) => {
599 let ctx = self.expression_context(composition, frame, switch.id, "selected_layer");
600 let selection =
601 crate::node::compositing::switch::selected_layer_for_frame(switch, &ctx)?;
602 selections.push((switch.id, selection));
603 if let Some(layer) = selection.and_then(|index| switch.layers.get(index)) {
604 self.collect_plan_key(composition, layer, frame, visited, selections)?;
605 }
606 Ok(())
607 }
608 _ => {
609 for input in composition
610 .graph
611 .connections
612 .iter()
613 .filter(|connection| connection.to_node == port.id)
614 .map(|connection| {
615 PortRef::new(connection.from_node, connection.from_port.clone())
616 })
617 {
618 self.collect_plan_key(composition, &input, frame, visited, selections)?;
619 }
620 Ok(())
621 }
622 }
623 }
624
625 fn media_output_port(&self, composition: &Composition) -> crate::Result<PortRef> {
626 let mut outputs = composition
627 .graph
628 .nodes
629 .iter()
630 .filter_map(|(node_id, node)| {
631 matches!(node, NodeKind::MediaOutput(_)).then_some(*node_id)
632 });
633 let Some(output) = outputs.next() else {
634 return Err(crate::error::GraphValidationError::MissingMediaOutput.into());
635 };
636 if outputs.next().is_some() {
637 return Err(
638 crate::error::GraphValidationError::MultipleMediaOutputs { count: 2 }.into(),
639 );
640 }
641 Ok(PortRef::new(output, "output".to_string()))
642 }
643
644 fn expression_context<'a>(
645 &self,
646 composition: &'a Composition,
647 frame: u32,
648 node_id: NodeId,
649 property_path: &str,
650 ) -> crate::expr::ExpressionContext<'a> {
651 crate::expr::ExpressionContext {
652 frame,
653 fps: composition.timeline.fps,
654 width: composition.render_settings.width,
655 height: composition.render_settings.height,
656 duration_frames: composition.timeline.duration_frames,
657 path: Some(format!("{node_id}.{property_path}")),
658 graph: Some(&composition.graph),
659 }
660 }
661}
662
663fn fit_frame_to_rgba8(frame: &CpuMediaFrame, width: u32, height: u32) -> Vec<u8> {
664 if frame.width == width && frame.height == height && frame.row_bytes == width as usize * 4 {
665 return frame.rgba.as_ref().clone();
666 }
667
668 let mut out = vec![0; width as usize * height as usize * 4];
669 for y in 0..height {
670 let src_y = ((u64::from(y) * u64::from(frame.height)) / u64::from(height)) as usize;
671 for x in 0..width {
672 let src_x = ((u64::from(x) * u64::from(frame.width)) / u64::from(width)) as usize;
673 let src = src_y
674 .saturating_mul(frame.row_bytes)
675 .saturating_add(src_x.saturating_mul(4));
676 let dst = (y as usize)
677 .saturating_mul(width as usize * 4)
678 .saturating_add(x as usize * 4);
679 out[dst..dst + 4].copy_from_slice(&frame.rgba[src..src + 4]);
680 }
681 }
682 out
683}
684
685#[cfg(all(
686 target_os = "linux",
687 feature = "ffmpeg",
688 feature = "cuda",
689 feature = "vulkan"
690))]
691struct CudaMediaInterop {
692 driver: &'static lumen_ffmpeg::CudaDriver,
693 context: lumen_ffmpeg::CudaContext<'static>,
694 converter: lumen_ffmpeg::CudaNv12ToRgbaConverter<'static>,
695 textures: HashMap<lumen_gpu::TextureId, CudaMediaTexture>,
696}
697
698#[cfg(all(
699 target_os = "linux",
700 feature = "ffmpeg",
701 feature = "cuda",
702 feature = "vulkan"
703))]
704struct CudaMediaTexture {
705 size: lumen_gpu::Size,
706 imported: lumen_ffmpeg::ImportedCudaExternalImage<'static>,
707 rgba: lumen_ffmpeg::CudaDeviceAllocation<'static>,
708 exportable: lumen_gpu::ExportableVulkanTexture,
709}
710
711#[cfg(all(
712 target_os = "linux",
713 feature = "ffmpeg",
714 feature = "cuda",
715 feature = "vulkan"
716))]
717impl CudaMediaInterop {
718 fn new(exportable: &lumen_gpu::ExportableVulkanTexture) -> Result<Self, String> {
719 let driver = cuda_driver()?;
720 let vulkan_uuid = exportable.device_info().device_uuid;
721 let cuda_ordinal = driver
722 .devices()?
723 .into_iter()
724 .find(|device| device.uuid == vulkan_uuid)
725 .map(|device| device.ordinal)
726 .unwrap_or(0);
727 let context = driver.create_primary_context_for_ordinal(cuda_ordinal)?;
728 let converter = driver.create_nv12_to_rgba_converter()?;
729 Ok(Self {
730 driver,
731 context,
732 converter,
733 textures: HashMap::new(),
734 })
735 }
736
737 fn import_frame(
738 &mut self,
739 renderer: &lumen_gpu::Renderer,
740 upload: &MediaTextureUpload,
741 frame: &crate::media::GpuVideoMediaFrame,
742 ) -> Result<Arc<lumen_gpu::wgpu::Texture>, String> {
743 use lumen_ffmpeg::GpuVideoFrame;
744
745 let GpuVideoFrame::Cuda(decoded) = &frame.frame else {
746 return Err(format!(
747 "unsupported GPU media backend {:?}; only CUDA media frames can be imported here",
748 frame.frame.backend()
749 ));
750 };
751 let (decoded_width, decoded_height) = decoded.dimensions();
752 let decoded_size = lumen_gpu::Size::new(decoded_width.max(1), decoded_height.max(1));
753
754 let needs_texture = self
755 .textures
756 .get(&upload.texture)
757 .is_none_or(|texture| texture.size != decoded_size);
758 if needs_texture {
759 let texture = create_cuda_media_texture(renderer, self.driver, decoded_size)?;
760 self.textures.insert(upload.texture, texture);
761 }
762 let texture = self
763 .textures
764 .get_mut(&upload.texture)
765 .ok_or_else(|| "CUDA media texture cache entry was not initialized".to_string())?;
766
767 self.context.set_current()?;
768 self.converter.convert(decoded, &texture.rgba)?;
769 self.driver
770 .copy_rgba_frame_to_image(&texture.rgba, &texture.imported)?;
771 self.driver.synchronize_context()?;
772 Ok(texture.exportable.texture_arc())
773 }
774}
775
776#[cfg(all(
777 target_os = "linux",
778 feature = "ffmpeg",
779 feature = "cuda",
780 feature = "vulkan"
781))]
782fn cuda_driver() -> Result<&'static lumen_ffmpeg::CudaDriver, String> {
783 static CUDA_DRIVER: OnceLock<lumen_ffmpeg::CudaDriver> = OnceLock::new();
784 if let Some(driver) = CUDA_DRIVER.get() {
785 return Ok(driver);
786 }
787 let _ = CUDA_DRIVER.set(lumen_ffmpeg::CudaDriver::load()?);
788 CUDA_DRIVER
789 .get()
790 .ok_or_else(|| "CUDA driver was not initialized".to_string())
791}
792
793#[cfg(all(
794 target_os = "linux",
795 feature = "ffmpeg",
796 feature = "cuda",
797 feature = "vulkan"
798))]
799impl Drop for CudaMediaInterop {
800 fn drop(&mut self) {
801 let _ = self.context.set_current();
802 self.textures.clear();
803 }
804}
805
806#[cfg(all(
807 target_os = "linux",
808 feature = "ffmpeg",
809 feature = "cuda",
810 feature = "vulkan"
811))]
812fn cuda_video_frame_to_texture(
813 interop: &mut Option<CudaMediaInterop>,
814 renderer: &lumen_gpu::Renderer,
815 upload: &MediaTextureUpload,
816 frame: &crate::media::GpuVideoMediaFrame,
817) -> Result<Arc<lumen_gpu::wgpu::Texture>, String> {
818 let (width, height) = frame.dimensions();
819 let decoded_size = lumen_gpu::Size::new(width.max(1), height.max(1));
820 if interop.is_none() {
821 let exportable = renderer
822 .create_exportable_vulkan_texture(
823 Some("lumen cuda media texture bootstrap"),
824 decoded_size,
825 lumen_gpu::wgpu::TextureFormat::Rgba8Unorm,
826 lumen_gpu::wgpu::TextureUsages::TEXTURE_BINDING
827 | lumen_gpu::wgpu::TextureUsages::COPY_DST
828 | lumen_gpu::wgpu::TextureUsages::COPY_SRC,
829 )
830 .map_err(|error| error.to_string())?;
831 *interop = Some(CudaMediaInterop::new(&exportable)?);
832 }
833 interop
834 .as_mut()
835 .ok_or_else(|| "CUDA media interop was not initialized".to_string())?
836 .import_frame(renderer, upload, frame)
837}
838
839#[cfg(all(
840 target_os = "linux",
841 feature = "ffmpeg",
842 feature = "cuda",
843 feature = "vulkan"
844))]
845fn create_cuda_media_texture(
846 renderer: &lumen_gpu::Renderer,
847 driver: &'static lumen_ffmpeg::CudaDriver,
848 size: lumen_gpu::Size,
849) -> Result<CudaMediaTexture, String> {
850 use lumen_ffmpeg::import_owned_vulkan_opaque_fd_image;
851 let exportable = renderer
852 .create_exportable_vulkan_texture(
853 Some("lumen cuda media texture"),
854 size,
855 lumen_gpu::wgpu::TextureFormat::Rgba8Unorm,
856 lumen_gpu::wgpu::TextureUsages::TEXTURE_BINDING
857 | lumen_gpu::wgpu::TextureUsages::COPY_DST
858 | lumen_gpu::wgpu::TextureUsages::COPY_SRC,
859 )
860 .map_err(|error| error.to_string())?;
861
862 let rgba = driver.allocate_rgba_frame(size.width, size.height)?;
863 let imported = import_owned_vulkan_opaque_fd_image(
864 driver,
865 exportable
866 .memory_fd()
867 .try_clone()
868 .map_err(|error| format!("failed to duplicate Vulkan memory fd: {error}"))?,
869 exportable.allocation_size(),
870 size.width,
871 size.height,
872 )?;
873 Ok(CudaMediaTexture {
874 size,
875 imported,
876 rgba,
877 exportable,
878 })
879}