1use alpha2_eval::{
9 ActiveClip, ActiveComposition, ActiveEffect, ActiveTransition, EvaluatedVisual, FrameDesc,
10 ShapesPlan, StepOp, sample_scalar, sample_visual,
11};
12use alpha2_media::{SourceId, YuvEncoding};
13use alpha2_model::{
14 ClipBlendMode, EffectValue, Generator, MaskMode, MatteMode, Rgb, ShapeMask, TextStyle,
15};
16use alpha2_render::{
17 BlendMode, CacheIntent, Color, EffectId, EffectInputs, EffectParams, EffectResources,
18 EffectRuntime, FrameSubmitTimings, GENERATOR_EFFECT_ID, GraphError, GraphLayer, LayerTransform,
19 MASK_COMBINE_EFFECT_IDS, MASK_COMBINE_NAMES, MAX_SHAPE_MASKS, Nv12PlaneTargets, ParamSpec,
20 ParamValue, PendingSourceWork, RenderCacheKey, RenderError, RenderGraph, RenderNodeId,
21 RenderNodeKind, RenderSize, RenderedTexture, Renderer, SourceView, TRACK_MATTE_EFFECT_ID,
22 Yuv420pReadback,
23};
24use alpha2_time::MediaTime;
25
26use crate::TextTexture;
27use std::collections::{HashMap, HashSet};
28use std::time::Duration;
29use web_time::Instant;
30
31pub trait FrameProvider {
39 fn texture_for(&self, source: &SourceId, time: MediaTime) -> Option<SourceView<'_>>;
52
53 fn source_key(&self, _source: &SourceId, _time: MediaTime) -> Option<RenderCacheKey> {
60 None
61 }
62
63 fn pending_source_work(
66 &self,
67 _source: &SourceId,
68 _time: MediaTime,
69 ) -> Option<&PendingSourceWork> {
70 None
71 }
72
73 fn text(&self, _key: u64) -> Option<TextTexture<'_>> {
78 None
79 }
80}
81
82#[derive(Clone, Debug, Default, PartialEq, Eq)]
84pub struct RenderReport {
85 pub missing_sources: Vec<SourceId>,
87}
88
89impl RenderReport {
90 #[must_use]
92 pub fn is_complete(&self) -> bool {
93 self.missing_sources.is_empty()
94 }
95}
96
97#[derive(Debug)]
99pub struct FrameRender {
100 pub report: RenderReport,
102 pub texture: Option<RenderedTexture>,
104}
105
106impl FrameRender {
107 #[must_use]
109 pub fn is_complete(&self) -> bool {
110 self.texture.is_some()
111 }
112}
113
114#[derive(Debug)]
116pub struct FrameYuv420pRender {
117 pub report: RenderReport,
119 pub readback: Option<Yuv420pReadback>,
121}
122
123impl FrameYuv420pRender {
124 #[must_use]
126 pub fn is_complete(&self) -> bool {
127 self.readback.is_some()
128 }
129}
130
131#[derive(Debug)]
133#[cfg_attr(not(feature = "wgpu-interop"), doc(hidden))]
134pub struct FrameNv12Render {
135 pub report: RenderReport,
137 pub submitted: bool,
139}
140
141#[derive(Debug)]
143pub struct FrameGraph<'a> {
144 pub report: RenderReport,
146 pub graph: Option<RenderGraph<'a>>,
148 pub pending: Vec<&'a PendingSourceWork>,
151}
152
153pub trait FrameRenderer {
157 #[must_use]
163 fn inspect_frame(&self, frame: &FrameDesc, frames: &dyn FrameProvider) -> RenderReport;
164
165 fn build_frame_graph<'a>(
176 &self,
177 size: RenderSize,
178 frame: &'a FrameDesc<'a>,
179 frames: &'a dyn FrameProvider,
180 ) -> Result<FrameGraph<'a>, RenderError>;
181
182 fn render_frame(
192 &self,
193 size: RenderSize,
194 frame: &FrameDesc,
195 frames: &dyn FrameProvider,
196 ) -> Result<FrameRender, RenderError>;
197
198 fn render_frame_yuv420p(
207 &self,
208 size: RenderSize,
209 frame: &FrameDesc,
210 frames: &dyn FrameProvider,
211 encoding: YuvEncoding,
212 ) -> Result<FrameYuv420pRender, RenderError>;
213
214 fn render_frame_yuv420p_timed(
224 &self,
225 size: RenderSize,
226 frame: &FrameDesc,
227 frames: &dyn FrameProvider,
228 encoding: YuvEncoding,
229 timings: &mut FrameSubmitTimings,
230 ) -> Result<FrameYuv420pRender, RenderError>;
231
232 #[cfg_attr(not(feature = "wgpu-interop"), doc(hidden))]
243 fn render_frame_nv12_timed(
244 &self,
245 size: RenderSize,
246 frame: &FrameDesc,
247 frames: &dyn FrameProvider,
248 planes: Nv12PlaneTargets<'_>,
249 encoding: YuvEncoding,
250 timings: &mut FrameSubmitTimings,
251 ) -> Result<FrameNv12Render, RenderError>;
252}
253
254impl FrameRenderer for Renderer {
255 fn inspect_frame(&self, frame: &FrameDesc, frames: &dyn FrameProvider) -> RenderReport {
256 available_textures(frame, frames).0
257 }
258
259 fn build_frame_graph<'a>(
260 &self,
261 size: RenderSize,
262 frame: &'a FrameDesc<'a>,
263 frames: &'a dyn FrameProvider,
264 ) -> Result<FrameGraph<'a>, RenderError> {
265 validate_project_effects(self, frame)?;
266 let (report, mut available) = available_textures(frame, frames);
267 if !report.is_complete() {
268 return Ok(FrameGraph {
269 report,
270 graph: None,
271 pending: Vec::new(),
272 });
273 }
274 attach_texts(&mut available, frames, size);
275 let captions = caption_layers(frame, frames, size)?;
276 let graph = frame_graph(self, size, frame, &available, &captions)?;
277 Ok(FrameGraph {
278 report,
279 graph: Some(graph),
280 pending: pending_source_work(&available, frames),
281 })
282 }
283
284 fn render_frame(
285 &self,
286 size: RenderSize,
287 frame: &FrameDesc,
288 frames: &dyn FrameProvider,
289 ) -> Result<FrameRender, RenderError> {
290 let FrameGraph {
291 report,
292 graph,
293 pending,
294 } = self.build_frame_graph(size, frame, frames)?;
295 let texture = match graph {
296 Some(graph) => Some(self.render_graph(&graph, &pending)?),
297 None => None,
298 };
299 Ok(FrameRender { report, texture })
300 }
301
302 fn render_frame_yuv420p(
303 &self,
304 size: RenderSize,
305 frame: &FrameDesc,
306 frames: &dyn FrameProvider,
307 encoding: YuvEncoding,
308 ) -> Result<FrameYuv420pRender, RenderError> {
309 render_frame_yuv420p_measured(self, size, frame, frames, encoding, None)
310 }
311
312 fn render_frame_yuv420p_timed(
313 &self,
314 size: RenderSize,
315 frame: &FrameDesc,
316 frames: &dyn FrameProvider,
317 encoding: YuvEncoding,
318 timings: &mut FrameSubmitTimings,
319 ) -> Result<FrameYuv420pRender, RenderError> {
320 render_frame_yuv420p_measured(self, size, frame, frames, encoding, Some(timings))
321 }
322
323 #[cfg_attr(not(feature = "wgpu-interop"), doc(hidden))]
324 fn render_frame_nv12_timed(
325 &self,
326 size: RenderSize,
327 frame: &FrameDesc,
328 frames: &dyn FrameProvider,
329 planes: Nv12PlaneTargets<'_>,
330 encoding: YuvEncoding,
331 timings: &mut FrameSubmitTimings,
332 ) -> Result<FrameNv12Render, RenderError> {
333 let mut clock = StageClock::start(true);
334 let mut collected = FrameSubmitTimings::default();
335 let FrameGraph {
336 report,
337 graph,
338 pending,
339 } = self.build_frame_graph(size, frame, frames)?;
340 collected.graph_build = clock.lap();
341 let submitted = graph.is_some();
342 if let Some(graph) = graph {
343 self.render_graph_nv12_timed(&graph, &pending, planes, encoding, &mut collected)?;
344 }
345 *timings = collected;
346 Ok(FrameNv12Render { report, submitted })
347 }
348}
349
350fn render_frame_yuv420p_measured(
351 renderer: &Renderer,
352 size: RenderSize,
353 frame: &FrameDesc,
354 frames: &dyn FrameProvider,
355 encoding: YuvEncoding,
356 timings: Option<&mut FrameSubmitTimings>,
357) -> Result<FrameYuv420pRender, RenderError> {
358 let measure = timings.is_some();
359 let mut clock = StageClock::start(measure);
360 let mut collected = FrameSubmitTimings::default();
361 let FrameGraph {
362 report,
363 graph,
364 pending,
365 } = renderer.build_frame_graph(size, frame, frames)?;
366 collected.graph_build = clock.lap();
367 let readback = match graph {
368 Some(graph) if measure => {
369 Some(renderer.render_graph_yuv420p_timed(&graph, &pending, encoding, &mut collected)?)
370 }
371 Some(graph) => Some(renderer.render_graph_yuv420p(&graph, &pending, encoding)?),
372 None => None,
373 };
374 if let Some(timings) = timings {
375 *timings = collected;
376 }
377 Ok(FrameYuv420pRender { report, readback })
378}
379
380#[derive(Copy, Clone)]
382struct StageClock(Option<Instant>);
383
384impl StageClock {
385 fn start(enabled: bool) -> Self {
386 Self(enabled.then(Instant::now))
387 }
388
389 fn lap(&mut self) -> Duration {
391 let Some(mark) = self.0 else {
392 return Duration::ZERO;
393 };
394 let now = Instant::now();
395 self.0 = Some(now);
396 now.duration_since(mark)
397 }
398}
399
400fn validate_project_effects(renderer: &Renderer, frame: &FrameDesc) -> Result<(), RenderError> {
401 require_effects(renderer, frame_effects(frame)).map_err(|_| RenderError::UnknownEffect)
402}
403
404fn frame_effects<'a>(frame: &'a FrameDesc) -> impl Iterator<Item = (&'a str, u32)> {
407 let composition_effects = frame
408 .visual
409 .iter()
410 .flat_map(|clip| clip.composition.into_iter().chain(clip.source_composition))
411 .flat_map(|composition| composition.plan().effects())
412 .map(|effect| (effect.id.as_str(), effect.version));
413 let transition_effects = frame
414 .transitions
415 .iter()
416 .map(|transition| (transition.effect.id(), transition.effect.version()));
417 composition_effects.chain(transition_effects)
418}
419
420pub(crate) fn check_frame_effects(
427 renderer: &Renderer,
428 frame: &FrameDesc,
429) -> Result<(), crate::EngineError> {
430 require_effects(renderer, frame_effects(frame))
431}
432
433pub(crate) fn require_effects<'a>(
435 renderer: &Renderer,
436 effects: impl IntoIterator<Item = (&'a str, u32)>,
437) -> Result<(), crate::EngineError> {
438 for (effect, version) in effects {
439 if renderer.project_effect_id(effect, version).is_none() {
440 return Err(crate::EngineError::UnregisteredEffect {
441 effect: effect.to_owned(),
442 version,
443 });
444 }
445 }
446 Ok(())
447}
448
449fn pending_source_work<'a>(
453 available: &[(&ActiveClip, ClipFrames<'_>)],
454 frames: &'a dyn FrameProvider,
455) -> Vec<&'a PendingSourceWork> {
456 let mut work = Vec::new();
457 for (clip, textures) in available {
458 if textures.source.is_some() {
459 work.extend(frames.pending_source_work(clip.source_id, clip.source_time));
460 }
461 for composition in clip.composition.iter().chain(&clip.source_composition) {
462 for (_, source, time) in composition.media() {
463 work.extend(frames.pending_source_work(source, time));
464 }
465 }
466 }
467 work
468}
469
470#[derive(Copy, Clone, Debug)]
472struct Frame<'t> {
473 view: SourceView<'t>,
474 key: Option<RenderCacheKey>,
475}
476
477impl<'t> Frame<'t> {
478 fn add_to(self, graph: &mut RenderGraph<'t>) -> RenderNodeId {
480 match self.key {
481 Some(key) => graph.add_source_with_key(self.view, key),
482 None => graph.add_source(self.view),
483 }
484 }
485}
486
487#[derive(Copy, Clone, Debug)]
490struct PlacedText<'t> {
491 texture: TextTexture<'t>,
492 visual: LayerTransform,
493}
494
495impl<'t> PlacedText<'t> {
496 fn add_to(self, graph: &mut RenderGraph<'t>) -> Result<RenderNodeId, RenderError> {
499 let source = match self.texture.cache_key {
500 Some(key) => graph.add_source_with_key(self.texture.texture, key),
501 None => graph.add_source(self.texture.texture),
502 };
503 graph
504 .add_composite(vec![GraphLayer {
505 source,
506 visual: self.visual,
507 blend_mode: BlendMode::Normal,
508 }])
509 .map_err(GraphError::into_render_error)
510 }
511}
512
513#[derive(Copy, Clone, Debug)]
515enum StepFrame<'t> {
516 Media(Frame<'t>),
518 Text(PlacedText<'t>),
520}
521
522#[derive(Clone, Debug, Default)]
524struct ClipFrames<'t> {
525 layer: usize,
530 source: Option<Frame<'t>>,
533 text: Option<PlacedText<'t>>,
535 media: Vec<Option<StepFrame<'t>>>,
538 source_media: Vec<Option<StepFrame<'t>>>,
540}
541
542fn available_textures<'frame, 'eval, 'texture>(
545 frame: &'frame FrameDesc<'eval>,
546 frames: &'texture dyn FrameProvider,
547) -> (
548 RenderReport,
549 Vec<(&'frame ActiveClip<'eval>, ClipFrames<'texture>)>,
550) {
551 let mut report = RenderReport::default();
552 let mut available = Vec::with_capacity(frame.visual.len());
553 let mut lookup = |source: &SourceId, time| {
554 let texture = frames.texture_for(source, time);
555 if texture.is_none() && !report.missing_sources.contains(source) {
556 report.missing_sources.push(source.clone());
557 }
558 texture.map(|view| Frame {
559 view,
560 key: frames.source_key(source, time),
561 })
562 };
563 for (layer, clip) in frame.visual.iter().enumerate() {
565 if !frame.draws(clip) {
566 continue;
567 }
568 let mut textures = ClipFrames {
569 layer,
570 ..ClipFrames::default()
571 };
572 let mut complete = true;
573 if clip.needs_source() {
574 textures.source = lookup(clip.source_id, clip.source_time);
575 complete &= textures.source.is_some();
576 }
577 for (composition, media) in [
578 (clip.composition, &mut textures.media),
579 (clip.source_composition, &mut textures.source_media),
580 ] {
581 let Some(composition) = composition else {
582 continue;
583 };
584 *media = vec![None; composition.plan().steps.len()];
585 for (step, source, time) in composition.media() {
586 media[step] = lookup(source, time).map(StepFrame::Media);
587 complete &= media[step].is_some();
588 }
589 }
590 if complete {
591 available.push((clip, textures));
592 }
593 }
594 (report, available)
595}
596
597fn attach_texts<'t>(
601 available: &mut [(&ActiveClip<'_>, ClipFrames<'t>)],
602 frames: &'t dyn FrameProvider,
603 size: RenderSize,
604) {
605 let lookup = |generator: &Generator| match generator {
606 Generator::Text { text, style } => {
607 frames
608 .text(crate::text_key(text, style, size))
609 .map(|texture| PlacedText {
610 texture,
611 visual: text_placement(texture, style, size),
612 })
613 }
614 _ => None,
615 };
616 for (clip, textures) in available {
617 textures.text = clip.generator.and_then(lookup);
618 for (composition, media) in [
619 (clip.composition, &mut textures.media),
620 (clip.source_composition, &mut textures.source_media),
621 ] {
622 let Some(composition) = composition else {
623 continue;
624 };
625 for (step, slot) in composition.plan().steps.iter().zip(media.iter_mut()) {
626 if let StepOp::Generator { generator, .. } = &step.op {
627 *slot = lookup(generator).map(StepFrame::Text);
628 }
629 }
630 }
631 }
632}
633
634fn text_placement(texture: TextTexture<'_>, style: &TextStyle, size: RenderSize) -> LayerTransform {
638 #[allow(clippy::cast_precision_loss)]
639 let scale = [
640 texture.width as f32 / size.width as f32,
641 texture.height as f32 / size.height as f32,
642 ];
643 LayerTransform {
644 position: [style.position.x, style.position.y],
645 anchor: [
646 box_anchor(style.anchor.x, texture.width, texture.margin[0]),
647 box_anchor(style.anchor.y, texture.height, texture.margin[1]),
648 ],
649 scale,
650 rotation_degrees: 0.0,
651 opacity: 1.0,
652 }
653}
654
655struct CaptionLayer<'a> {
657 layers_below: usize,
659 texture: TextTexture<'a>,
660 visual: LayerTransform,
661}
662
663fn caption_graph_layer<'a>(graph: &mut RenderGraph<'a>, caption: &CaptionLayer<'a>) -> GraphLayer {
666 let source = match caption.texture.cache_key {
667 Some(key) => graph.add_source_with_key(caption.texture.texture, key),
668 None => graph.add_source(caption.texture.texture),
669 };
670 GraphLayer {
671 source,
672 visual: caption.visual,
673 blend_mode: BlendMode::Normal,
674 }
675}
676
677fn caption_layers<'a>(
680 frame: &FrameDesc,
681 frames: &'a dyn FrameProvider,
682 size: RenderSize,
683) -> Result<Vec<CaptionLayer<'a>>, RenderError> {
684 frame
685 .captions
686 .iter()
687 .map(|caption| {
688 let texture = frames
689 .text(crate::text_key(caption.text, caption.style, size))
690 .ok_or(RenderError::MissingText)?;
691 Ok(CaptionLayer {
692 layers_below: caption.layers_below,
693 visual: text_placement(texture, caption.style, size),
694 texture,
695 })
696 })
697 .collect()
698}
699
700#[allow(clippy::cast_precision_loss)]
704fn box_anchor(anchor: f32, extent: u32, margin: u32) -> f32 {
705 if margin == 0 {
706 return anchor;
707 }
708 let inner = extent.saturating_sub(margin * 2) as f32;
709 (margin as f32 + anchor * inner) / extent as f32
710}
711
712fn frame_runtime(frame: &FrameDesc, size: RenderSize) -> EffectRuntime {
714 #[allow(clippy::cast_precision_loss)]
715 let ratio = size.width as f32 / size.height as f32;
716 EffectRuntime {
717 time: frame.time,
718 ratio,
719 ..EffectRuntime::default()
720 }
721}
722
723fn frame_graph<'a>(
724 renderer: &Renderer,
725 size: RenderSize,
726 frame: &'a FrameDesc<'a>,
727 available: &[(&'a ActiveClip<'a>, ClipFrames<'a>)],
728 captions: &[CaptionLayer<'a>],
729) -> Result<RenderGraph<'a>, RenderError> {
730 let mut graph = RenderGraph::new(size);
731 graph.set_blend_space(match frame.blend_space {
732 alpha2_model::BlendSpace::LinearLight => alpha2_render::BlendSpace::LinearLight,
733 alpha2_model::BlendSpace::SrgbEncoded => alpha2_render::BlendSpace::SrgbEncoded,
734 });
735 let mut nodes = HashMap::with_capacity(available.len());
736 let mut inputs = Vec::with_capacity(available.len() + frame.transitions.len());
737 let mut coverages = HashMap::new();
739 let runtime = frame_runtime(frame, size);
740 for (clip, textures) in available {
741 if clip.is_matte {
742 continue;
743 }
744 let mut layer = clip_layer(renderer, &mut graph, clip, textures, frame.time, runtime)?;
745 if let Some(matte) = clip.matte {
746 layer = apply_matte(
747 renderer,
748 &mut graph,
749 layer,
750 matte,
751 available,
752 (frame.time, runtime),
753 &mut coverages,
754 )?;
755 }
756 nodes.insert(clip.clip_id, inputs.len());
757 inputs.push(FrameInput {
758 kind: FrameInputKind::Layer(layer),
759 uses_backdrop: clip.blend_mode != ClipBlendMode::Normal,
760 });
761 }
762 let background = background_color(frame.background);
763 let mut suppressed = HashSet::new();
764 for transition in &frame.transitions {
765 let from = *nodes
766 .get(transition.from_clip)
767 .ok_or(RenderError::InvalidGraph)?;
768 let to = *nodes
769 .get(transition.to_clip)
770 .ok_or(RenderError::InvalidGraph)?;
771 let output = inputs.len();
772 inputs.push(FrameInput {
773 kind: FrameInputKind::Transition {
774 from,
775 to,
776 transition,
777 },
778 uses_backdrop: inputs[from].uses_backdrop || inputs[to].uses_backdrop,
779 });
780 nodes.insert(transition.to_clip, output);
781 suppressed.insert(transition.from_clip);
782 }
783 let mut layers = Vec::with_capacity(available.len() + captions.len());
784 let mut captions = captions.iter().peekable();
785 for (clip, textures) in available {
786 while let Some(caption) = captions.next_if(|caption| caption.layers_below <= textures.layer)
789 {
790 layers.push(caption_graph_layer(&mut graph, caption));
791 }
792 if clip.is_matte || suppressed.contains(clip.clip_id) {
793 continue;
794 }
795 let index = *nodes.get(clip.clip_id).ok_or(RenderError::InvalidGraph)?;
796 if let FrameInputKind::Layer(layer) = inputs[index].kind {
797 layers.push(layer);
798 continue;
799 }
800 let uses_backdrop = inputs[index].uses_backdrop;
804 let backdrop = if uses_backdrop {
805 layers.as_slice()
806 } else {
807 &[]
808 };
809 let source = compile_frame_input(
812 &mut graph,
813 &inputs,
814 index,
815 backdrop,
816 uses_backdrop.then_some(background),
817 runtime,
818 renderer,
819 )?;
820 if uses_backdrop {
821 layers.clear();
822 }
823 layers.push(identity_layer(source));
824 }
825 for caption in captions {
826 layers.push(caption_graph_layer(&mut graph, caption));
827 }
828 let output = graph
829 .add_composite_over(layers, background)
830 .map_err(GraphError::into_render_error)?;
831 graph
832 .set_output(output)
833 .map_err(GraphError::into_render_error)?;
834 graph
835 .set_cache_intent(output, CacheIntent::Auto)
836 .map_err(GraphError::into_render_error)?;
837 Ok(graph)
838}
839
840const fn cache_intent(intent: alpha2_model::CacheIntent) -> CacheIntent {
841 match intent {
842 alpha2_model::CacheIntent::Auto => CacheIntent::Auto,
843 alpha2_model::CacheIntent::Pin => CacheIntent::Pin,
844 alpha2_model::CacheIntent::Bypass => CacheIntent::Bypass,
845 }
846}
847
848fn apply_matte<'a, 'c>(
850 renderer: &Renderer,
851 graph: &mut RenderGraph<'a>,
852 mut layer: GraphLayer,
853 matte: &'c alpha2_model::Matte,
854 available: &[(&'a ActiveClip<'a>, ClipFrames<'a>)],
855 (time, runtime): (MediaTime, EffectRuntime),
856 coverages: &mut HashMap<&'c alpha2_model::ClipId, RenderNodeId>,
857) -> Result<GraphLayer, RenderError> {
858 let source = available
859 .iter()
860 .find(|(candidate, _)| candidate.clip_id == &matte.source);
861 match source {
862 Some((source, textures)) => {
863 let coverage = if let Some(coverage) = coverages.get(&matte.source) {
864 *coverage
865 } else {
866 let matte_layer = clip_layer(renderer, graph, source, textures, time, runtime)?;
867 let coverage = alone(graph, matte_layer)?;
868 coverages.insert(&matte.source, coverage);
869 coverage
870 };
871 layer = matted_layer(graph, layer, coverage, matte.mode, runtime)?;
872 }
873 None if is_inverted(matte.mode) => {}
876 None => layer.visual.opacity = 0.0,
877 }
878 Ok(layer)
879}
880
881fn background_color(background: Rgb) -> Color {
883 Color::rgba(
884 f64::from(background.red),
885 f64::from(background.green),
886 f64::from(background.blue),
887 1.0,
888 )
889}
890
891struct FrameInput<'a> {
892 kind: FrameInputKind<'a>,
893 uses_backdrop: bool,
894}
895
896fn clip_layer<'a>(
897 renderer: &Renderer,
898 graph: &mut RenderGraph<'a>,
899 clip: &'a ActiveClip<'a>,
900 textures: &ClipFrames<'a>,
901 time: MediaTime,
902 runtime: EffectRuntime,
903) -> Result<GraphLayer, RenderError> {
904 let mut source = match clip.composition {
905 None => clip_picture(renderer, graph, clip, textures, runtime)?,
906 Some(composition) => plan_node(
907 renderer,
908 graph,
909 composition,
910 &textures.media,
911 Some((clip, textures)),
912 runtime,
913 )?,
914 };
915 if !clip.masks.is_empty() {
916 source = mask_node(graph, source, clip.masks, time, runtime)?;
917 }
918 let intent = cache_intent(clip.cache);
922 if intent == CacheIntent::Bypass || graph.node_kind(source) != Some(RenderNodeKind::Source) {
923 graph
924 .set_cache_intent(source, intent)
925 .map_err(GraphError::into_render_error)?;
926 }
927 Ok(GraphLayer {
928 source,
929 visual: layer_transform(clip.visual.ok_or(RenderError::InvalidVisual)?),
930 blend_mode: blend_mode(clip.blend_mode),
931 })
932}
933
934fn clip_picture<'a>(
937 renderer: &Renderer,
938 graph: &mut RenderGraph<'a>,
939 clip: &'a ActiveClip<'a>,
940 textures: &ClipFrames<'a>,
941 runtime: EffectRuntime,
942) -> Result<RenderNodeId, RenderError> {
943 if let Some(source) = clip.source_composition {
944 return plan_node(
946 renderer,
947 graph,
948 source,
949 &textures.source_media,
950 None,
951 runtime,
952 );
953 }
954 match (clip.generator, textures.source) {
955 (Some(Generator::Text { .. }), _) => {
956 textures.text.ok_or(RenderError::MissingText)?.add_to(graph)
957 }
958 (Some(Generator::Shapes { .. }), _) => {
959 let plan = clip.shapes.ok_or(RenderError::InvalidGraph)?;
960 Ok(shapes_node(graph, plan, clip.source_time))
961 }
962 (Some(generator), _) => Ok(generator_node(graph, generator, runtime)),
963 (None, Some(frame)) => Ok(frame.add_to(graph)),
964 (None, None) => Err(RenderError::InvalidGraph),
965 }
966}
967
968fn shapes_node(graph: &mut RenderGraph<'_>, plan: &ShapesPlan, time: MediaTime) -> RenderNodeId {
972 let size = graph.size();
973 let canvas = alpha2_geometry::CanvasSize::new(size.width, size.height);
974 let aspect = f64::from(size.width) / f64::from(size.height);
975 let items = plan.evaluate(time, aspect);
976 graph.add_vector(std::sync::Arc::new(alpha2_geometry::build_scene(
977 &items, canvas,
978 )))
979}
980
981fn generator_node(
984 graph: &mut RenderGraph<'_>,
985 generator: &alpha2_model::Generator,
986 runtime: EffectRuntime,
987) -> RenderNodeId {
988 graph.add_generator(
989 EffectId::from_static(GENERATOR_EFFECT_ID, 1),
990 crate::generated::generator_params(generator),
991 timeless(runtime),
992 )
993}
994
995fn plan_node<'a>(
999 renderer: &Renderer,
1000 graph: &mut RenderGraph<'a>,
1001 composition: ActiveComposition<'a>,
1002 media: &[Option<StepFrame<'a>>],
1003 clip: Option<(&'a ActiveClip<'a>, &ClipFrames<'a>)>,
1004 runtime: EffectRuntime,
1005) -> Result<RenderNodeId, RenderError> {
1006 let plan = composition.plan();
1007 let time = composition.time();
1008 let mut built: Vec<RenderNodeId> = Vec::with_capacity(plan.steps.len());
1009 let mut picture = None;
1010 for (index, step) in plan.steps.iter().enumerate() {
1011 let at = step.time(time);
1012 let step_runtime = EffectRuntime {
1013 time: at,
1014 ..runtime
1015 };
1016 let node = match &step.op {
1017 StepOp::Clip => {
1019 if let Some(node) = picture {
1020 node
1021 } else {
1022 let (clip, textures) = clip.ok_or(RenderError::InvalidGraph)?;
1023 *picture.insert(clip_picture(renderer, graph, clip, textures, runtime)?)
1024 }
1025 }
1026 StepOp::Media { .. } => match media.get(index).copied().flatten() {
1027 Some(StepFrame::Media(frame)) => frame.add_to(graph),
1028 _ => return Err(RenderError::InvalidGraph),
1029 },
1030 StepOp::Generator {
1031 generator: Generator::Text { .. },
1032 ..
1033 } => match media.get(index).copied().flatten() {
1034 Some(StepFrame::Text(text)) => text.add_to(graph)?,
1035 _ => return Err(RenderError::MissingText),
1036 },
1037 op => step_node(renderer, graph, op, &built, at, step_runtime)?,
1038 };
1039 built.push(node);
1040 }
1041 step_input(&built, plan.output)
1042}
1043
1044const _: () = assert!(alpha2_model::MAX_EFFECT_INPUTS == alpha2_render::MAX_EFFECT_INPUTS);
1046
1047fn step_input(built: &[RenderNodeId], step: usize) -> Result<RenderNodeId, RenderError> {
1049 built.get(step).copied().ok_or(RenderError::InvalidGraph)
1050}
1051
1052fn step_node<'a>(
1054 renderer: &Renderer,
1055 graph: &mut RenderGraph<'a>,
1056 op: &'a StepOp,
1057 built: &[RenderNodeId],
1058 time: MediaTime,
1059 runtime: EffectRuntime,
1060) -> Result<RenderNodeId, RenderError> {
1061 let input = |step: usize| step_input(built, step);
1062 let node = match op {
1063 StepOp::Clip | StepOp::Media { .. } => return Err(RenderError::InvalidGraph),
1064 StepOp::Generator {
1065 generator: Generator::Shapes { .. },
1066 start,
1067 shapes,
1068 } => {
1069 let plan = shapes.as_deref().ok_or(RenderError::InvalidGraph)?;
1070 let at = MediaTime(time.ticks().saturating_add(start.ticks()));
1071 shapes_node(graph, plan, at)
1072 }
1073 StepOp::Generator { generator, .. } => generator_node(graph, generator, runtime),
1074 StepOp::Effect {
1075 input: from,
1076 inputs,
1077 effect,
1078 } => {
1079 let id = project_effect_id(renderer, &effect.id, effect.version)?;
1080 let active = ActiveEffect::new(effect, time);
1081 let params = effect_params(renderer, &id, active);
1082 let runtime = effect_runtime(renderer, &id, runtime);
1083 let mut named = EffectInputs::new();
1084 for (name, step) in inputs {
1085 named.insert(name.as_str(), input(*step)?);
1086 }
1087 graph.add_effect_with_inputs(
1088 id,
1089 input(*from)?,
1090 named,
1091 params,
1092 effect_resources(active),
1093 runtime,
1094 )
1095 }
1096 .map_err(GraphError::into_render_error)?,
1097 StepOp::Transition {
1098 from,
1099 to,
1100 effect,
1101 progress,
1102 } => {
1103 let id = project_effect_id(renderer, &effect.id, effect.version)?;
1104 let active = ActiveEffect::new(effect, time);
1105 let params = effect_params(renderer, &id, active);
1106 let runtime = EffectRuntime {
1107 progress: sample_scalar(progress, time).clamp(0.0, 1.0),
1108 ..effect_runtime(renderer, &id, runtime)
1109 };
1110 graph.add_transition_with_resources(
1111 id,
1112 input(*from)?,
1113 input(*to)?,
1114 params,
1115 effect_resources(active),
1116 runtime,
1117 )
1118 }
1119 .map_err(GraphError::into_render_error)?,
1120 StepOp::Transform {
1121 input: from,
1122 transform,
1123 } => graph
1124 .add_composite(vec![GraphLayer {
1125 source: input(*from)?,
1126 visual: layer_transform(sample_visual(transform, time, graph_aspect(graph))),
1127 blend_mode: BlendMode::Normal,
1128 }])
1129 .map_err(GraphError::into_render_error)?,
1130 StepOp::Merge { layers } => {
1131 let layers = layers
1132 .iter()
1133 .map(|layer| {
1134 let mut placed = identity_layer(input(layer.input)?);
1135 placed.visual.opacity = sample_scalar(&layer.opacity, time).clamp(0.0, 1.0);
1136 placed.blend_mode = blend_mode(layer.blend_mode);
1137 Ok(placed)
1138 })
1139 .collect::<Result<Vec<_>, RenderError>>()?;
1140 graph
1141 .add_composite(layers)
1142 .map_err(GraphError::into_render_error)?
1143 }
1144 StepOp::Matte {
1145 input: from,
1146 matte,
1147 mode,
1148 } => matte_node(graph, input(*from)?, input(*matte)?, *mode, runtime)?,
1149 StepOp::ShapeMask { input: from, masks } => {
1150 mask_node(graph, input(*from)?, masks, time, runtime)?
1151 }
1152 };
1153 Ok(node)
1154}
1155
1156fn graph_aspect(graph: &RenderGraph<'_>) -> f64 {
1159 let size = graph.size();
1160 f64::from(size.width) / f64::from(size.height)
1161}
1162
1163fn mask_node(
1170 graph: &mut RenderGraph<'_>,
1171 source: RenderNodeId,
1172 masks: &[ShapeMask],
1173 time: MediaTime,
1174 runtime: EffectRuntime,
1175) -> Result<RenderNodeId, RenderError> {
1176 let size = graph.size();
1177 let canvas = alpha2_geometry::CanvasSize::new(size.width, size.height);
1178 let aspect = graph_aspect(graph);
1179 let unit = size.width.min(size.height);
1180 let white = alpha2_model::Paint::Solid {
1181 color: alpha2_model::Rgba::new(1.0, 1.0, 1.0, 1.0),
1182 };
1183 let shown = &masks[..masks.len().min(MAX_SHAPE_MASKS)];
1184 if shown.is_empty() {
1185 return Ok(source);
1186 }
1187 let mut inputs = EffectInputs::new();
1188 let mut params = EffectParams::with_capacity(shown.len() * 2);
1189 for (mask, names) in shown.iter().zip(MASK_COMBINE_NAMES) {
1190 let outline = alpha2_eval::evaluate_geometry(&mask.shape, time, aspect);
1191 let items = [
1192 alpha2_geometry::EvaluatedItem::Geometry(outline),
1193 alpha2_geometry::EvaluatedItem::Fill {
1194 paint: white.clone(),
1195 opacity: 1.0,
1196 rule: mask.fill_rule,
1197 },
1198 ];
1199 let coverage = graph.add_vector(std::sync::Arc::new(alpha2_geometry::build_scene(
1200 &items, canvas,
1201 )));
1202 #[allow(clippy::cast_precision_loss)]
1203 let sigma = 0.3125 * sample_scalar(&mask.feather, time).max(0.0) * unit as f32;
1204 let coverage = graph
1205 .add_gaussian_pixels(coverage, sigma, timeless(runtime))
1206 .map_err(GraphError::into_render_error)?;
1207 inputs.insert(names.input, coverage);
1208 params.insert(names.invert, ParamValue::Bool(mask.invert));
1209 params.insert(
1210 names.mode,
1211 ParamValue::Enum(match mask.mode {
1212 MaskMode::Add => 0,
1213 MaskMode::Subtract => 1,
1214 MaskMode::Intersect => 2,
1215 }),
1216 );
1217 }
1218 graph
1219 .add_effect_with_inputs(
1220 EffectId::from_static(MASK_COMBINE_EFFECT_IDS[shown.len() - 1], 1),
1221 source,
1222 inputs,
1223 params,
1224 EffectResources::new(),
1225 timeless(runtime),
1226 )
1227 .map_err(GraphError::into_render_error)
1228}
1229
1230const fn is_inverted(mode: MatteMode) -> bool {
1231 matches!(mode, MatteMode::AlphaInverted | MatteMode::LumaInverted)
1232}
1233
1234fn matted_layer(
1238 graph: &mut RenderGraph<'_>,
1239 layer: GraphLayer,
1240 coverage: RenderNodeId,
1241 mode: MatteMode,
1242 runtime: EffectRuntime,
1243) -> Result<GraphLayer, RenderError> {
1244 let picture = alone(graph, layer)?;
1245 let matted = matte_node(graph, picture, coverage, mode, runtime)?;
1246 Ok(GraphLayer {
1247 blend_mode: layer.blend_mode,
1248 ..identity_layer(matted)
1249 })
1250}
1251
1252fn alone(graph: &mut RenderGraph<'_>, layer: GraphLayer) -> Result<RenderNodeId, RenderError> {
1254 graph
1255 .add_composite(vec![GraphLayer {
1256 blend_mode: BlendMode::Normal,
1257 ..layer
1258 }])
1259 .map_err(GraphError::into_render_error)
1260}
1261
1262fn effect_runtime(renderer: &Renderer, effect: &EffectId, runtime: EffectRuntime) -> EffectRuntime {
1268 if renderer.effect_reads_time(effect) {
1269 runtime
1270 } else {
1271 EffectRuntime {
1272 time: MediaTime::ZERO,
1273 ..runtime
1274 }
1275 }
1276}
1277
1278fn timeless(runtime: EffectRuntime) -> EffectRuntime {
1282 EffectRuntime {
1283 ratio: runtime.ratio,
1284 ..EffectRuntime::default()
1285 }
1286}
1287
1288fn matte_node(
1290 graph: &mut RenderGraph<'_>,
1291 picture: RenderNodeId,
1292 coverage: RenderNodeId,
1293 mode: MatteMode,
1294 runtime: EffectRuntime,
1295) -> Result<RenderNodeId, RenderError> {
1296 graph
1297 .add_transition_with_resources(
1298 EffectId::from_static(TRACK_MATTE_EFFECT_ID, 1),
1299 picture,
1300 coverage,
1301 EffectParams::new()
1302 .with(
1303 "luma",
1304 ParamValue::Bool(matches!(mode, MatteMode::Luma | MatteMode::LumaInverted)),
1305 )
1306 .with("inverted", ParamValue::Bool(is_inverted(mode))),
1307 EffectResources::new(),
1308 EffectRuntime {
1310 progress: 0.5,
1311 ..timeless(runtime)
1312 },
1313 )
1314 .map_err(GraphError::into_render_error)
1315}
1316
1317#[derive(Copy, Clone)]
1318enum FrameInputKind<'a> {
1319 Layer(GraphLayer),
1320 Transition {
1321 from: usize,
1322 to: usize,
1323 transition: &'a ActiveTransition<'a>,
1324 },
1325}
1326
1327fn compile_frame_input<'a>(
1328 graph: &mut RenderGraph<'a>,
1329 inputs: &[FrameInput<'a>],
1330 index: usize,
1331 backdrop: &[GraphLayer],
1332 background: Option<Color>,
1333 runtime: EffectRuntime,
1334 renderer: &Renderer,
1335) -> Result<RenderNodeId, RenderError> {
1336 if index >= inputs.len() {
1337 return Err(RenderError::InvalidGraph);
1338 }
1339 let mut compiled = vec![None; inputs.len()];
1340 let mut pending = vec![(index, false)];
1341 while let Some((current, expanded)) = pending.pop() {
1342 if compiled[current].is_some() {
1343 continue;
1344 }
1345 match inputs[current].kind {
1346 FrameInputKind::Layer(layer) => {
1347 let mut layers = Vec::with_capacity(backdrop.len() + 1);
1348 layers.extend_from_slice(backdrop);
1349 layers.push(layer);
1350 let composite = match background {
1351 Some(background) => graph.add_composite_over(layers, background),
1352 None => graph.add_composite(layers),
1353 };
1354 compiled[current] = Some(composite.map_err(GraphError::into_render_error)?);
1355 }
1356 FrameInputKind::Transition {
1357 from,
1358 to,
1359 transition,
1360 } => {
1361 if from >= inputs.len() || to >= inputs.len() {
1362 return Err(RenderError::InvalidGraph);
1363 }
1364 if !expanded {
1365 pending.push((current, true));
1366 if compiled[to].is_none() {
1367 pending.push((to, false));
1368 }
1369 if compiled[from].is_none() {
1370 pending.push((from, false));
1371 }
1372 continue;
1373 }
1374 let from = compiled[from].ok_or(RenderError::InvalidGraph)?;
1375 let to = compiled[to].ok_or(RenderError::InvalidGraph)?;
1376 let id = project_effect_id(
1377 renderer,
1378 transition.effect.id(),
1379 transition.effect.version(),
1380 )?;
1381 let params = effect_params(renderer, &id, transition.effect);
1382 let runtime = EffectRuntime {
1383 progress: transition.progress,
1384 ..effect_runtime(renderer, &id, runtime)
1385 };
1386 compiled[current] = Some(
1387 graph
1388 .add_transition_with_resources(
1389 id,
1390 from,
1391 to,
1392 params,
1393 effect_resources(transition.effect),
1394 runtime,
1395 )
1396 .map_err(GraphError::into_render_error)?,
1397 );
1398 }
1399 }
1400 }
1401 compiled[index].ok_or(RenderError::InvalidGraph)
1402}
1403
1404fn identity_layer(source: RenderNodeId) -> GraphLayer {
1405 GraphLayer {
1406 source,
1407 visual: LayerTransform {
1408 position: [0.5, 0.5],
1409 anchor: [0.5, 0.5],
1410 scale: [1.0, 1.0],
1411 rotation_degrees: 0.0,
1412 opacity: 1.0,
1413 },
1414 blend_mode: BlendMode::Normal,
1415 }
1416}
1417
1418const fn layer_transform(visual: EvaluatedVisual) -> LayerTransform {
1421 LayerTransform {
1422 position: [visual.position.x, visual.position.y],
1423 anchor: [visual.anchor.x, visual.anchor.y],
1424 scale: [visual.scale.x, visual.scale.y],
1425 rotation_degrees: visual.rotation_degrees,
1426 opacity: visual.opacity,
1427 }
1428}
1429
1430fn effect_params<'a>(
1438 renderer: &Renderer,
1439 id: &EffectId,
1440 effect: ActiveEffect<'a>,
1441) -> EffectParams<'a> {
1442 let mut params = EffectParams::with_capacity(effect.authored().params.len());
1443 for (name, sample) in effect.params() {
1444 let mut value = param_value(sample.value);
1445 if sample.interpolated
1446 && let Some(spec) = renderer
1447 .effect_definition(id)
1448 .and_then(|definition| definition.params.iter().find(|spec| spec.name == name))
1449 {
1450 value = clamp_to_range(spec, value);
1451 }
1452 params.insert(name, value);
1453 }
1454 params
1455}
1456
1457fn project_effect_id(renderer: &Renderer, id: &str, version: u32) -> Result<EffectId, RenderError> {
1460 renderer
1461 .project_effect_id(id, version)
1462 .cloned()
1463 .ok_or(RenderError::UnknownEffect)
1464}
1465
1466fn clamp_to_range(spec: &ParamSpec, value: ParamValue) -> ParamValue {
1467 let clamp = |value: f32| {
1468 let value = spec.min.map_or(value, |minimum| value.max(minimum));
1469 spec.max.map_or(value, |maximum| value.min(maximum))
1470 };
1471 match value {
1472 ParamValue::Float(value) => ParamValue::Float(clamp(value)),
1473 ParamValue::Vec2([x, y]) => ParamValue::Vec2([clamp(x), clamp(y)]),
1474 ParamValue::Vec4(values) => ParamValue::Vec4(values.map(clamp)),
1475 other => other,
1476 }
1477}
1478
1479fn effect_resources(effect: ActiveEffect<'_>) -> EffectResources {
1480 effect
1481 .resources()
1482 .fold(EffectResources::new(), |resources, (name, id)| {
1483 resources.with(name, id.as_str())
1484 })
1485}
1486
1487const fn param_value(value: EffectValue) -> ParamValue {
1488 match value {
1489 EffectValue::Float(value) => ParamValue::Float(value),
1490 EffectValue::Vec2(value) => ParamValue::Vec2(value),
1491 EffectValue::Vec4(value) => ParamValue::Vec4(value),
1492 EffectValue::Color(value) => ParamValue::Color(value),
1493 EffectValue::Enum(value) => ParamValue::Enum(value),
1494 EffectValue::Bool(value) => ParamValue::Bool(value),
1495 }
1496}
1497
1498const fn blend_mode(mode: ClipBlendMode) -> BlendMode {
1499 match mode {
1500 ClipBlendMode::Normal => BlendMode::Normal,
1501 ClipBlendMode::Multiply => BlendMode::Multiply,
1502 ClipBlendMode::Screen => BlendMode::Screen,
1503 ClipBlendMode::Overlay => BlendMode::Overlay,
1504 ClipBlendMode::Darken => BlendMode::Darken,
1505 ClipBlendMode::Lighten => BlendMode::Lighten,
1506 ClipBlendMode::ColorDodge => BlendMode::ColorDodge,
1507 ClipBlendMode::ColorBurn => BlendMode::ColorBurn,
1508 ClipBlendMode::HardLight => BlendMode::HardLight,
1509 ClipBlendMode::SoftLight => BlendMode::SoftLight,
1510 ClipBlendMode::Difference => BlendMode::Difference,
1511 ClipBlendMode::Exclusion => BlendMode::Exclusion,
1512 ClipBlendMode::Hue => BlendMode::Hue,
1513 ClipBlendMode::Saturation => BlendMode::Saturation,
1514 ClipBlendMode::Color => BlendMode::Color,
1515 ClipBlendMode::Luminosity => BlendMode::Luminosity,
1516 }
1517}
1518
1519#[cfg(test)]
1520mod tests;