1use std::{env, fs, sync::mpsc};
2
3use anyhow::Context;
4use lumen_engine::{composition::Composition, gpu::GpuCompositionRenderer, media::MediaStore};
5use lumen_ffmpeg::VideoCodec;
6
7use super::{
8 ProjectBundle, RenderError, RenderOptions, RenderProgress,
9 encoder::{
10 ENCODER_FRAME_QUEUE_CAPACITY, EncoderFrame, LumenFfmpegEncoder, has_audio,
11 write_composited_audio,
12 },
13 media::{LocalMediaStore, media_root},
14};
15
16#[cfg(all(target_os = "linux", feature = "cuda", feature = "vulkan"))]
17mod linux_cuda;
18#[cfg(not(all(target_os = "linux", feature = "cuda", feature = "vulkan")))]
19mod unsupported_cuda;
20
21#[cfg(all(target_os = "linux", feature = "cuda", feature = "vulkan"))]
22use linux_cuda::render_project_mp4_cuda;
23#[cfg(not(all(target_os = "linux", feature = "cuda", feature = "vulkan")))]
24use unsupported_cuda::render_project_mp4_cuda;
25
26pub(super) const MEDIA_PREFETCH_LOOKAHEAD_FRAMES: u32 = 30;
27
28pub fn render_project_mp4(
29 bundle: &ProjectBundle,
30 options: &RenderOptions,
31 on_progress: &mut dyn FnMut(RenderProgress),
32) -> Result<Vec<u8>, RenderError> {
33 let composition = &bundle.composition;
34 let width = composition.render_settings.width;
35 let height = composition.render_settings.height;
36 let fps = composition.timeline.fps;
37 let total_frames = composition.timeline.duration_frames;
38
39 if fps <= 0.0 {
40 return Err(RenderError {
41 code: "invalid_project_payload",
42 message: format!("invalid timeline fps: {fps}"),
43 retryable: false,
44 });
45 }
46
47 if total_frames == 0 {
48 return Err(RenderError {
49 code: "invalid_project_payload",
50 message: "composition duration_frames must be greater than zero".to_string(),
51 retryable: false,
52 });
53 }
54
55 let media_root = media_root(options.media_root.as_deref()).map_err(|err| RenderError {
56 code: "media_root_error",
57 message: err.to_string(),
58 retryable: false,
59 })?;
60 let media_store = LocalMediaStore::new(media_root);
61 let mut renderer = create_gpu_renderer().map_err(|err| RenderError {
62 code: "render_failed",
63 message: format!("failed to create GPU renderer: {err:#}"),
64 retryable: true,
65 })?;
66 let adapter = renderer.gpu_renderer().adapter_info();
67 tracing::info!(
68 adapter = %adapter.name,
69 backend = ?adapter.backend,
70 device_type = ?adapter.device_type,
71 vendor = adapter.vendor,
72 device = adapter.device,
73 driver = %adapter.driver,
74 driver_info = %adapter.driver_info,
75 "rendering with GPU adapter"
76 );
77 renderer
78 .compile_with_media(
79 composition,
80 &media_store,
81 lumen_gpu::wgpu::TextureFormat::Rgba8Unorm,
82 )
83 .map_err(|err| RenderError {
84 code: "render_failed",
85 message: format!("failed to compile composition: {err}"),
86 retryable: false,
87 })?;
88
89 let encoder =
90 choose_video_encoder(options.video_encoder.as_deref()).map_err(|err| RenderError {
91 code: "invalid_render_profile",
92 message: err.to_string(),
93 retryable: false,
94 })?;
95 tracing::info!(
96 encoder = encoder.name,
97 codec = ?encoder.codec,
98 cuda_fast_path = cfg!(all(target_os = "linux", feature = "cuda", feature = "vulkan"))
99 && encoder.cuda_fast_path,
100 "selected video encoder"
101 );
102 if encoder.cuda_fast_path {
103 return render_project_mp4_cuda(
104 composition,
105 &media_store,
106 width,
107 height,
108 fps,
109 total_frames,
110 &encoder.name,
111 encoder.codec,
112 options.verbose_debug,
113 on_progress,
114 );
115 }
116
117 let tmp = tempfile::tempdir().map_err(|err| RenderError {
118 code: "encode_failed",
119 message: format!("failed to create temp dir: {err}"),
120 retryable: true,
121 })?;
122 let output_path = tmp.path().join("output.mp4");
123 let include_audio = has_audio(composition);
124 let encoder = LumenFfmpegEncoder::create(
125 &output_path,
126 width,
127 height,
128 fps,
129 &encoder.name,
130 encoder.codec,
131 include_audio,
132 )
133 .map_err(|err| RenderError {
134 code: "encode_failed",
135 message: err.to_string(),
136 retryable: true,
137 })?;
138 let (pixel_recycle_tx, _pixel_recycle_rx) =
139 mpsc::sync_channel::<Vec<u8>>(ENCODER_FRAME_QUEUE_CAPACITY + 2);
140
141 for frame in 0..total_frames {
142 prefetch_media_frames(composition, &media_store, frame);
143 let (raster, _submission) = renderer
144 .render_frame_submitted(composition, frame, &media_store)
145 .map_err(|err| RenderError {
146 code: "render_failed",
147 message: format!("render failed at frame {frame}: {err}"),
148 retryable: true,
149 })?;
150 let _ = renderer.precompile_frame_window(
151 composition,
152 frame.saturating_add(1),
153 MEDIA_PREFETCH_LOOKAHEAD_FRAMES,
154 &media_store,
155 );
156 renderer
157 .gpu_renderer()
158 .device
159 .poll(lumen_gpu::wgpu::PollType::Poll)
160 .map_err(|err| RenderError {
161 code: "render_failed",
162 message: format!("GPU poll failed at frame {frame}: {err}"),
163 retryable: true,
164 })?;
165 let size = raster.domain.storage_size;
166 if size.width != width || size.height != height {
167 return Err(RenderError {
168 code: "render_failed",
169 message: format!(
170 "frame {frame} dimensions {}x{} do not match composition {}x{}",
171 size.width, size.height, width, height
172 ),
173 retryable: true,
174 });
175 }
176 let pixels =
177 read_texture_rgba8(renderer.gpu_renderer(), raster.texture, size).map_err(|err| {
178 RenderError {
179 code: "render_failed",
180 message: format!("failed reading rendered pixels for frame {frame}: {err}"),
181 retryable: true,
182 }
183 })?;
184 encoder
185 .send(EncoderFrame {
186 frame,
187 pixels,
188 recycle_tx: pixel_recycle_tx.clone(),
189 })
190 .map_err(|err| RenderError {
191 code: "encode_failed",
192 message: err.to_string(),
193 retryable: true,
194 })?;
195
196 let completed = frame.saturating_add(1);
197 let ratio = (completed as f32 / total_frames as f32).clamp(0.0, 1.0);
198 on_progress(RenderProgress {
199 stage: "rendering",
200 frame: completed,
201 total_frames,
202 ratio,
203 });
204 }
205
206 if include_audio {
207 write_composited_audio(composition, &media_store, &encoder).map_err(|err| RenderError {
208 code: "audio_render_failed",
209 message: err.to_string(),
210 retryable: true,
211 })?;
212 }
213
214 encoder.finish().map_err(|err| RenderError {
215 code: "encode_failed",
216 message: err.to_string(),
217 retryable: true,
218 })?;
219
220 fs::read(&output_path).map_err(|err| RenderError {
221 code: "encode_failed",
222 message: format!("failed to read encoded output: {err}"),
223 retryable: true,
224 })
225}
226
227pub(super) fn prefetch_media_frames(
228 composition: &Composition,
229 media_store: &LocalMediaStore,
230 frame: u32,
231) {
232 let Ok(requirements) =
233 lumen_engine::media::collect_frame_requirements(composition, media_store, frame)
234 else {
235 return;
236 };
237 for video in requirements.videos {
238 let Some(resolver) = media_store.get_video_resolver(&video.stream_id) else {
239 continue;
240 };
241 let mut frames = video.frames;
242 for required_frame in frames.clone() {
243 for offset in 1..=MEDIA_PREFETCH_LOOKAHEAD_FRAMES {
244 frames.push(required_frame.saturating_add(offset));
245 }
246 }
247 frames.sort_unstable();
248 frames.dedup();
249 for frame in &frames {
250 let _ = resolver.enqueue_frame(*frame);
251 }
252 resolver.retain_frames(&frames);
253 }
254}
255
256fn read_texture_rgba8(
257 renderer: &lumen_gpu::Renderer,
258 id: lumen_gpu::TextureId,
259 size: lumen_gpu::Size,
260) -> anyhow::Result<Vec<u8>> {
261 let bytes_per_pixel = 4;
262 let unpadded_bytes_per_row = size.width.saturating_mul(bytes_per_pixel);
263 let padded_bytes_per_row = align_to(
264 unpadded_bytes_per_row,
265 lumen_gpu::wgpu::COPY_BYTES_PER_ROW_ALIGNMENT,
266 );
267 let output_size = u64::from(padded_bytes_per_row).saturating_mul(u64::from(size.height));
268 let output = renderer
269 .device
270 .create_buffer(&lumen_gpu::wgpu::BufferDescriptor {
271 label: Some("lumen-server readback"),
272 size: output_size.max(1),
273 usage: lumen_gpu::wgpu::BufferUsages::COPY_DST
274 | lumen_gpu::wgpu::BufferUsages::MAP_READ,
275 mapped_at_creation: false,
276 });
277 let mut encoder =
278 renderer
279 .device
280 .create_command_encoder(&lumen_gpu::wgpu::CommandEncoderDescriptor {
281 label: Some("lumen-server readback encoder"),
282 });
283 let texture = renderer
284 .texture(id)
285 .ok_or_else(|| anyhow::anyhow!("render output texture {id:?} is unavailable"))?;
286 encoder.copy_texture_to_buffer(
287 texture.as_image_copy(),
288 lumen_gpu::wgpu::TexelCopyBufferInfo {
289 buffer: &output,
290 layout: lumen_gpu::wgpu::TexelCopyBufferLayout {
291 offset: 0,
292 bytes_per_row: Some(padded_bytes_per_row),
293 rows_per_image: Some(size.height),
294 },
295 },
296 lumen_gpu::wgpu::Extent3d {
297 width: size.width,
298 height: size.height,
299 depth_or_array_layers: 1,
300 },
301 );
302 renderer.queue.submit([encoder.finish()]);
303
304 let slice = output.slice(..);
305 let (tx, rx) = mpsc::channel();
306 slice.map_async(lumen_gpu::wgpu::MapMode::Read, move |result| {
307 let _ = tx.send(result);
308 });
309 renderer
310 .device
311 .poll(lumen_gpu::wgpu::PollType::wait_indefinitely())
312 .map_err(|error| anyhow::anyhow!("GPU readback poll failed: {error}"))?;
313 rx.recv()
314 .map_err(|_| anyhow::anyhow!("GPU readback channel closed"))?
315 .map_err(|error| anyhow::anyhow!("GPU readback map failed: {error}"))?;
316
317 let mapped = slice.get_mapped_range();
318 let mut pixels = vec![
319 0;
320 (size.width as usize)
321 .saturating_mul(size.height as usize)
322 .saturating_mul(bytes_per_pixel as usize)
323 ];
324 for row in 0..size.height as usize {
325 let src_start = row.saturating_mul(padded_bytes_per_row as usize);
326 let src_end = src_start.saturating_add(unpadded_bytes_per_row as usize);
327 let dst_start = row.saturating_mul(unpadded_bytes_per_row as usize);
328 let dst_end = dst_start.saturating_add(unpadded_bytes_per_row as usize);
329 pixels[dst_start..dst_end].copy_from_slice(&mapped[src_start..src_end]);
330 }
331 drop(mapped);
332 output.unmap();
333 Ok(pixels)
334}
335
336fn align_to(value: u32, alignment: u32) -> u32 {
337 value.div_ceil(alignment) * alignment
338}
339
340pub(super) fn create_gpu_renderer() -> anyhow::Result<GpuCompositionRenderer> {
341 let handle = tokio::runtime::Handle::try_current()
342 .context("GPU renderer creation requires a Tokio runtime")?;
343 handle
344 .block_on(GpuCompositionRenderer::new())
345 .context("failed to initialize wgpu renderer")
346}
347
348#[derive(Debug)]
349struct VideoEncoderSelection {
350 name: String,
351 codec: VideoCodec,
352 cuda_fast_path: bool,
353}
354
355fn choose_video_encoder(override_encoder: Option<&str>) -> anyhow::Result<VideoEncoderSelection> {
356 if let Some(encoder) = override_encoder {
357 let encoder = encoder.trim();
358 if !encoder.is_empty() {
359 return video_encoder_selection(encoder);
360 }
361 }
362
363 if let Ok(encoder) = env::var("LUMEN_VIDEO_ENCODER") {
364 let encoder = encoder.trim();
365 if !encoder.is_empty() {
366 return video_encoder_selection(encoder);
367 }
368 }
369
370 let name = if cfg!(target_os = "macos") {
371 "h264_videotoolbox"
372 } else {
373 "libx264"
374 };
375 video_encoder_selection(name)
376}
377
378fn video_encoder_selection(name: &str) -> anyhow::Result<VideoEncoderSelection> {
379 let codec = match name {
380 "h264" | "libx264" | "h264_videotoolbox" | "h264_vulkan" | "h264_nvenc" => VideoCodec::H264,
381 "hevc" | "h265" | "libx265" | "hevc_videotoolbox" | "hevc_vulkan" | "hevc_nvenc" => {
382 VideoCodec::Hevc
383 }
384 "av1" | "libaom-av1" | "av1_nvenc" => VideoCodec::Av1,
385 _ => anyhow::bail!(
386 "unsupported video encoder `{name}`; expected a known H.264, HEVC, or AV1 encoder"
387 ),
388 };
389 Ok(VideoEncoderSelection {
390 name: name.to_string(),
391 codec,
392 cuda_fast_path: matches!(name, "h264_nvenc" | "hevc_nvenc" | "av1_nvenc"),
393 })
394}
395
396#[cfg(test)]
397mod tests {
398 use super::*;
399
400 #[test]
401 fn nvenc_encoders_select_cuda_capable_codecs() {
402 let h264 = video_encoder_selection("h264_nvenc").unwrap();
403 assert_eq!(h264.name, "h264_nvenc");
404 assert_eq!(h264.codec, VideoCodec::H264);
405
406 let hevc = video_encoder_selection("hevc_nvenc").unwrap();
407 assert_eq!(hevc.name, "hevc_nvenc");
408 assert_eq!(hevc.codec, VideoCodec::Hevc);
409
410 let av1 = video_encoder_selection("av1_nvenc").unwrap();
411 assert_eq!(av1.name, "av1_nvenc");
412 assert_eq!(av1.codec, VideoCodec::Av1);
413 }
414
415 #[test]
416 fn unknown_encoder_is_rejected_before_rendering() {
417 let error = video_encoder_selection("definitely_not_an_encoder").unwrap_err();
418 assert!(
419 error
420 .to_string()
421 .contains("expected a known H.264, HEVC, or AV1 encoder")
422 );
423 }
424}