1use alpha2_eval::{
9 ActiveClip, ActiveComposition, ActiveEffect, ActiveTransition, EvaluatedVisual, FrameDesc,
10 GeneratorPlan, ShapesPlan, StepOp, sample_scalar, sample_visual,
11};
12use alpha2_media::{SourceId, YuvEncoding};
13use alpha2_model::{
14 ClipBlendMode, EffectValue, Generator, MaskMode, MatteMode, Rgb, ShapeMask, TextStyle,
15};
16use alpha2_render::{
17 BlendMode, CacheIntent, Color, EffectId, EffectInputs, EffectParams, EffectResources,
18 EffectRuntime, FrameSubmitTimings, GENERATOR_EFFECT_ID, GraphError, GraphLayer, LayerTransform,
19 MASK_COMBINE_EFFECT_IDS, MASK_COMBINE_NAMES, MAX_SHAPE_MASKS, Nv12PlaneTargets, ParamSpec,
20 ParamValue, PendingSourceWork, RenderCacheKey, RenderError, RenderGraph, RenderNodeId,
21 RenderNodeKind, RenderSize, RenderedTexture, Renderer, SourceView, TRACK_MATTE_EFFECT_ID,
22 Yuv420pReadback,
23};
24use alpha2_time::MediaTime;
25
26use crate::TextTexture;
27use crate::layout::draw::LaidOut;
28use std::collections::{HashMap, HashSet};
29use std::time::Duration;
30use web_time::Instant;
31
32pub trait FrameProvider {
40 fn texture_for(&self, source: &SourceId, time: MediaTime) -> Option<SourceView<'_>>;
53
54 fn source_key(&self, _source: &SourceId, _time: MediaTime) -> Option<RenderCacheKey> {
61 None
62 }
63
64 fn pending_source_work(
67 &self,
68 _source: &SourceId,
69 _time: MediaTime,
70 ) -> Option<&PendingSourceWork> {
71 None
72 }
73
74 fn text(&self, _key: u64) -> Option<TextTexture<'_>> {
79 None
80 }
81
82 fn layout(&self, _key: u128) -> Option<std::sync::Arc<crate::PreparedLayout>> {
87 None
88 }
89}
90
91#[derive(Clone, Debug, Default, PartialEq, Eq)]
93pub struct RenderReport {
94 pub missing_sources: Vec<SourceId>,
96}
97
98impl RenderReport {
99 #[must_use]
101 pub fn is_complete(&self) -> bool {
102 self.missing_sources.is_empty()
103 }
104}
105
106#[derive(Debug)]
108pub struct FrameRender {
109 pub report: RenderReport,
111 pub texture: Option<RenderedTexture>,
113}
114
115impl FrameRender {
116 #[must_use]
118 pub fn is_complete(&self) -> bool {
119 self.texture.is_some()
120 }
121}
122
123#[derive(Debug)]
125pub struct FrameYuv420pRender {
126 pub report: RenderReport,
128 pub readback: Option<Yuv420pReadback>,
130}
131
132impl FrameYuv420pRender {
133 #[must_use]
135 pub fn is_complete(&self) -> bool {
136 self.readback.is_some()
137 }
138}
139
140#[derive(Debug)]
142#[cfg_attr(not(feature = "wgpu-interop"), doc(hidden))]
143pub struct FrameNv12Render {
144 pub report: RenderReport,
146 pub submitted: bool,
148}
149
150#[derive(Debug)]
152pub struct FrameGraph<'a> {
153 pub report: RenderReport,
155 pub graph: Option<RenderGraph<'a>>,
157 pub pending: Vec<&'a PendingSourceWork>,
160}
161
162pub trait FrameRenderer {
166 #[must_use]
172 fn inspect_frame(&self, frame: &FrameDesc, frames: &dyn FrameProvider) -> RenderReport;
173
174 fn build_frame_graph<'a>(
185 &self,
186 size: RenderSize,
187 frame: &'a FrameDesc<'a>,
188 frames: &'a dyn FrameProvider,
189 ) -> Result<FrameGraph<'a>, RenderError>;
190
191 fn render_frame(
201 &self,
202 size: RenderSize,
203 frame: &FrameDesc,
204 frames: &dyn FrameProvider,
205 ) -> Result<FrameRender, RenderError>;
206
207 fn render_frame_yuv420p(
216 &self,
217 size: RenderSize,
218 frame: &FrameDesc,
219 frames: &dyn FrameProvider,
220 encoding: YuvEncoding,
221 ) -> Result<FrameYuv420pRender, RenderError>;
222
223 fn render_frame_yuv420p_timed(
233 &self,
234 size: RenderSize,
235 frame: &FrameDesc,
236 frames: &dyn FrameProvider,
237 encoding: YuvEncoding,
238 timings: &mut FrameSubmitTimings,
239 ) -> Result<FrameYuv420pRender, RenderError>;
240
241 #[cfg_attr(not(feature = "wgpu-interop"), doc(hidden))]
252 fn render_frame_nv12_timed(
253 &self,
254 size: RenderSize,
255 frame: &FrameDesc,
256 frames: &dyn FrameProvider,
257 planes: Nv12PlaneTargets<'_>,
258 encoding: YuvEncoding,
259 timings: &mut FrameSubmitTimings,
260 ) -> Result<FrameNv12Render, RenderError>;
261}
262
263impl FrameRenderer for Renderer {
264 fn inspect_frame(&self, frame: &FrameDesc, frames: &dyn FrameProvider) -> RenderReport {
265 available_textures(frame, frames).0
266 }
267
268 fn build_frame_graph<'a>(
269 &self,
270 size: RenderSize,
271 frame: &'a FrameDesc<'a>,
272 frames: &'a dyn FrameProvider,
273 ) -> Result<FrameGraph<'a>, RenderError> {
274 validate_project_effects(self, frame)?;
275 let (report, mut available) = available_textures(frame, frames);
276 if !report.is_complete() {
277 return Ok(FrameGraph {
278 report,
279 graph: None,
280 pending: Vec::new(),
281 });
282 }
283 attach_texts(&mut available, frames, size);
284 let captions = caption_layers(frame, frames, size)?;
285 let graph = frame_graph(self, size, frame, &available, &captions)?;
286 Ok(FrameGraph {
287 report,
288 graph: Some(graph),
289 pending: pending_source_work(&available, frames),
290 })
291 }
292
293 fn render_frame(
294 &self,
295 size: RenderSize,
296 frame: &FrameDesc,
297 frames: &dyn FrameProvider,
298 ) -> Result<FrameRender, RenderError> {
299 let FrameGraph {
300 report,
301 graph,
302 pending,
303 } = self.build_frame_graph(size, frame, frames)?;
304 let texture = match graph {
305 Some(graph) => Some(self.render_graph(&graph, &pending)?),
306 None => None,
307 };
308 Ok(FrameRender { report, texture })
309 }
310
311 fn render_frame_yuv420p(
312 &self,
313 size: RenderSize,
314 frame: &FrameDesc,
315 frames: &dyn FrameProvider,
316 encoding: YuvEncoding,
317 ) -> Result<FrameYuv420pRender, RenderError> {
318 render_frame_yuv420p_measured(self, size, frame, frames, encoding, None)
319 }
320
321 fn render_frame_yuv420p_timed(
322 &self,
323 size: RenderSize,
324 frame: &FrameDesc,
325 frames: &dyn FrameProvider,
326 encoding: YuvEncoding,
327 timings: &mut FrameSubmitTimings,
328 ) -> Result<FrameYuv420pRender, RenderError> {
329 render_frame_yuv420p_measured(self, size, frame, frames, encoding, Some(timings))
330 }
331
332 #[cfg_attr(not(feature = "wgpu-interop"), doc(hidden))]
333 fn render_frame_nv12_timed(
334 &self,
335 size: RenderSize,
336 frame: &FrameDesc,
337 frames: &dyn FrameProvider,
338 planes: Nv12PlaneTargets<'_>,
339 encoding: YuvEncoding,
340 timings: &mut FrameSubmitTimings,
341 ) -> Result<FrameNv12Render, RenderError> {
342 let mut clock = StageClock::start(true);
343 let mut collected = FrameSubmitTimings::default();
344 let FrameGraph {
345 report,
346 graph,
347 pending,
348 } = self.build_frame_graph(size, frame, frames)?;
349 collected.graph_build = clock.lap();
350 let submitted = graph.is_some();
351 if let Some(graph) = graph {
352 self.render_graph_nv12_timed(&graph, &pending, planes, encoding, &mut collected)?;
353 }
354 *timings = collected;
355 Ok(FrameNv12Render { report, submitted })
356 }
357}
358
359fn render_frame_yuv420p_measured(
360 renderer: &Renderer,
361 size: RenderSize,
362 frame: &FrameDesc,
363 frames: &dyn FrameProvider,
364 encoding: YuvEncoding,
365 timings: Option<&mut FrameSubmitTimings>,
366) -> Result<FrameYuv420pRender, RenderError> {
367 let measure = timings.is_some();
368 let mut clock = StageClock::start(measure);
369 let mut collected = FrameSubmitTimings::default();
370 let FrameGraph {
371 report,
372 graph,
373 pending,
374 } = renderer.build_frame_graph(size, frame, frames)?;
375 collected.graph_build = clock.lap();
376 let readback = match graph {
377 Some(graph) if measure => {
378 Some(renderer.render_graph_yuv420p_timed(&graph, &pending, encoding, &mut collected)?)
379 }
380 Some(graph) => Some(renderer.render_graph_yuv420p(&graph, &pending, encoding)?),
381 None => None,
382 };
383 if let Some(timings) = timings {
384 *timings = collected;
385 }
386 Ok(FrameYuv420pRender { report, readback })
387}
388
389#[derive(Copy, Clone)]
391struct StageClock(Option<Instant>);
392
393impl StageClock {
394 fn start(enabled: bool) -> Self {
395 Self(enabled.then(Instant::now))
396 }
397
398 fn lap(&mut self) -> Duration {
400 let Some(mark) = self.0 else {
401 return Duration::ZERO;
402 };
403 let now = Instant::now();
404 self.0 = Some(now);
405 now.duration_since(mark)
406 }
407}
408
409fn validate_project_effects(renderer: &Renderer, frame: &FrameDesc) -> Result<(), RenderError> {
410 require_effects(renderer, frame_effects(frame)).map_err(|_| RenderError::UnknownEffect)
411}
412
413fn frame_effects<'a>(frame: &'a FrameDesc) -> impl Iterator<Item = (&'a str, u32)> {
416 let composition_effects = frame
417 .visual
418 .iter()
419 .flat_map(|clip| clip.composition.into_iter().chain(clip.source_composition))
420 .flat_map(|composition| composition.plan().effects())
421 .map(|effect| (effect.id.as_str(), effect.version));
422 let transition_effects = frame
423 .transitions
424 .iter()
425 .map(|transition| (transition.effect.id(), transition.effect.version()));
426 composition_effects.chain(transition_effects)
427}
428
429pub(crate) fn check_frame_effects(
436 renderer: &Renderer,
437 frame: &FrameDesc,
438) -> Result<(), crate::EngineError> {
439 require_effects(renderer, frame_effects(frame))
440}
441
442pub(crate) fn require_effects<'a>(
444 renderer: &Renderer,
445 effects: impl IntoIterator<Item = (&'a str, u32)>,
446) -> Result<(), crate::EngineError> {
447 for (effect, version) in effects {
448 if renderer.project_effect_id(effect, version).is_none() {
449 return Err(crate::EngineError::UnregisteredEffect {
450 effect: effect.to_owned(),
451 version,
452 });
453 }
454 }
455 Ok(())
456}
457
458fn pending_source_work<'a>(
462 available: &[(&ActiveClip, ClipFrames<'_>)],
463 frames: &'a dyn FrameProvider,
464) -> Vec<&'a PendingSourceWork> {
465 let mut work = Vec::new();
466 for (clip, textures) in available {
467 if textures.source.is_some() {
468 work.extend(frames.pending_source_work(clip.source_id, clip.source_time));
469 }
470 for composition in clip.composition.iter().chain(&clip.source_composition) {
471 for (_, source, time) in composition.media() {
472 work.extend(frames.pending_source_work(source, time));
473 }
474 }
475 }
476 work
477}
478
479#[derive(Copy, Clone, Debug)]
481struct Frame<'t> {
482 view: SourceView<'t>,
483 key: Option<RenderCacheKey>,
484}
485
486impl<'t> Frame<'t> {
487 fn add_to(self, graph: &mut RenderGraph<'t>) -> RenderNodeId {
489 match self.key {
490 Some(key) => graph.add_source_with_key(self.view, key),
491 None => graph.add_source(self.view),
492 }
493 }
494}
495
496#[derive(Copy, Clone, Debug)]
499struct PlacedText<'t> {
500 texture: TextTexture<'t>,
501 visual: LayerTransform,
502}
503
504impl<'t> PlacedText<'t> {
505 fn add_to(self, graph: &mut RenderGraph<'t>) -> Result<RenderNodeId, RenderError> {
508 let source = match self.texture.cache_key {
509 Some(key) => graph.add_source_with_key(self.texture.texture, key),
510 None => graph.add_source(self.texture.texture),
511 };
512 graph
513 .add_composite(vec![GraphLayer {
514 source,
515 visual: self.visual,
516 blend_mode: BlendMode::Normal,
517 }])
518 .map_err(GraphError::into_render_error)
519 }
520}
521
522#[derive(Clone, Debug)]
524enum StepFrame<'t> {
525 Media(Frame<'t>),
527 Text(PlacedText<'t>),
529 Layout(LaidOut<'t>),
531}
532
533#[derive(Clone, Debug, Default)]
535struct ClipFrames<'t> {
536 layer: usize,
541 source: Option<Frame<'t>>,
544 text: Option<PlacedText<'t>>,
546 layout: Option<LaidOut<'t>>,
548 media: Vec<Option<StepFrame<'t>>>,
551 source_media: Vec<Option<StepFrame<'t>>>,
553}
554
555fn available_textures<'frame, 'eval, 'texture>(
558 frame: &'frame FrameDesc<'eval>,
559 frames: &'texture dyn FrameProvider,
560) -> (
561 RenderReport,
562 Vec<(&'frame ActiveClip<'eval>, ClipFrames<'texture>)>,
563) {
564 let mut report = RenderReport::default();
565 let mut available = Vec::with_capacity(frame.visual.len());
566 let mut lookup = |source: &SourceId, time| {
567 let texture = frames.texture_for(source, time);
568 if texture.is_none() && !report.missing_sources.contains(source) {
569 report.missing_sources.push(source.clone());
570 }
571 texture.map(|view| Frame {
572 view,
573 key: frames.source_key(source, time),
574 })
575 };
576 for (layer, clip) in frame.visual.iter().enumerate() {
578 if !frame.draws(clip) {
579 continue;
580 }
581 let mut textures = ClipFrames {
582 layer,
583 ..ClipFrames::default()
584 };
585 let mut complete = true;
586 if clip.needs_source() {
587 textures.source = lookup(clip.source_id, clip.source_time);
588 complete &= textures.source.is_some();
589 }
590 for (composition, media) in [
591 (clip.composition, &mut textures.media),
592 (clip.source_composition, &mut textures.source_media),
593 ] {
594 let Some(composition) = composition else {
595 continue;
596 };
597 *media = vec![None; composition.plan().steps.len()];
598 for (step, source, time) in composition.media() {
599 media[step] = lookup(source, time).map(StepFrame::Media);
600 complete &= media[step].is_some();
601 }
602 }
603 if complete {
604 available.push((clip, textures));
605 }
606 }
607 (report, available)
608}
609
610fn attach_texts<'t>(
614 available: &mut [(&ActiveClip<'_>, ClipFrames<'t>)],
615 frames: &'t dyn FrameProvider,
616 size: RenderSize,
617) {
618 let laid_out = |plan: Option<&GeneratorPlan>, time: MediaTime| {
619 let plan = plan?.layout()?;
620 let layout = frames.layout(crate::layout_key(plan.evaluate(time).key, size))?;
621 LaidOut::attach(layout, frames)
622 };
623 let lookup = |generator: &Generator| match generator {
624 Generator::Text { text, style } => {
625 frames
626 .text(crate::text_key(text, style, size))
627 .map(|texture| PlacedText {
628 texture,
629 visual: text_placement(texture, style, size),
630 })
631 }
632 _ => None,
633 };
634 for (clip, textures) in available {
635 textures.text = clip.generator.and_then(lookup);
636 textures.layout = laid_out(clip.plan, clip.source_time);
637 for (composition, media) in [
638 (clip.composition, &mut textures.media),
639 (clip.source_composition, &mut textures.source_media),
640 ] {
641 let Some(composition) = composition else {
642 continue;
643 };
644 for (step, slot) in composition.plan().steps.iter().zip(media.iter_mut()) {
645 match &step.op {
646 StepOp::Generator {
647 generator: Generator::Layout { .. },
648 start,
649 plan,
650 } => {
651 let time = MediaTime(
652 step.time(composition.time())
653 .ticks()
654 .saturating_add(start.ticks()),
655 );
656 *slot = laid_out(plan.as_deref(), time).map(StepFrame::Layout);
657 }
658 StepOp::Generator { generator, .. } => {
659 *slot = lookup(generator).map(StepFrame::Text);
660 }
661 _ => {}
662 }
663 }
664 }
665 }
666}
667
668fn text_placement(texture: TextTexture<'_>, style: &TextStyle, size: RenderSize) -> LayerTransform {
672 #[allow(clippy::cast_precision_loss)]
673 let scale = [
674 texture.width as f32 / size.width as f32,
675 texture.height as f32 / size.height as f32,
676 ];
677 LayerTransform {
678 position: [style.position.x, style.position.y],
679 anchor: [
680 box_anchor(style.anchor.x, texture.width, texture.margin[0]),
681 box_anchor(style.anchor.y, texture.height, texture.margin[1]),
682 ],
683 scale,
684 rotation_degrees: 0.0,
685 opacity: 1.0,
686 }
687}
688
689struct CaptionLayer<'a> {
691 layers_below: usize,
693 texture: TextTexture<'a>,
694 visual: LayerTransform,
695}
696
697fn caption_graph_layer<'a>(graph: &mut RenderGraph<'a>, caption: &CaptionLayer<'a>) -> GraphLayer {
700 let source = match caption.texture.cache_key {
701 Some(key) => graph.add_source_with_key(caption.texture.texture, key),
702 None => graph.add_source(caption.texture.texture),
703 };
704 GraphLayer {
705 source,
706 visual: caption.visual,
707 blend_mode: BlendMode::Normal,
708 }
709}
710
711fn caption_layers<'a>(
714 frame: &FrameDesc,
715 frames: &'a dyn FrameProvider,
716 size: RenderSize,
717) -> Result<Vec<CaptionLayer<'a>>, RenderError> {
718 frame
719 .captions
720 .iter()
721 .map(|caption| {
722 let texture = frames
723 .text(crate::text_key(caption.text, caption.style, size))
724 .ok_or(RenderError::MissingText)?;
725 Ok(CaptionLayer {
726 layers_below: caption.layers_below,
727 visual: text_placement(texture, caption.style, size),
728 texture,
729 })
730 })
731 .collect()
732}
733
734#[allow(clippy::cast_precision_loss)]
738fn box_anchor(anchor: f32, extent: u32, margin: u32) -> f32 {
739 if margin == 0 {
740 return anchor;
741 }
742 let inner = extent.saturating_sub(margin * 2) as f32;
743 (margin as f32 + anchor * inner) / extent as f32
744}
745
746fn frame_runtime(frame: &FrameDesc, size: RenderSize) -> EffectRuntime {
748 #[allow(clippy::cast_precision_loss)]
749 let ratio = size.width as f32 / size.height as f32;
750 EffectRuntime {
751 time: frame.time,
752 ratio,
753 ..EffectRuntime::default()
754 }
755}
756
757fn frame_graph<'a>(
758 renderer: &Renderer,
759 size: RenderSize,
760 frame: &'a FrameDesc<'a>,
761 available: &[(&'a ActiveClip<'a>, ClipFrames<'a>)],
762 captions: &[CaptionLayer<'a>],
763) -> Result<RenderGraph<'a>, RenderError> {
764 let mut graph = RenderGraph::new(size);
765 graph.set_blend_space(match frame.blend_space {
766 alpha2_model::BlendSpace::LinearLight => alpha2_render::BlendSpace::LinearLight,
767 alpha2_model::BlendSpace::SrgbEncoded => alpha2_render::BlendSpace::SrgbEncoded,
768 });
769 let mut nodes = HashMap::with_capacity(available.len());
770 let mut inputs = Vec::with_capacity(available.len() + frame.transitions.len());
771 let mut coverages = HashMap::new();
773 let runtime = frame_runtime(frame, size);
774 for (clip, textures) in available {
775 if clip.is_matte {
776 continue;
777 }
778 let mut layer = clip_layer(renderer, &mut graph, clip, textures, frame.time, runtime)?;
779 if let Some(matte) = clip.matte {
780 layer = apply_matte(
781 renderer,
782 &mut graph,
783 layer,
784 matte,
785 available,
786 (frame.time, runtime),
787 &mut coverages,
788 )?;
789 }
790 nodes.insert(clip.clip_id, inputs.len());
791 inputs.push(FrameInput {
792 kind: FrameInputKind::Layer(layer),
793 uses_backdrop: clip.blend_mode != ClipBlendMode::Normal,
794 });
795 }
796 let background = background_color(frame.background);
797 let mut suppressed = HashSet::new();
798 for transition in &frame.transitions {
799 let from = *nodes
800 .get(transition.from_clip)
801 .ok_or(RenderError::InvalidGraph)?;
802 let to = *nodes
803 .get(transition.to_clip)
804 .ok_or(RenderError::InvalidGraph)?;
805 let output = inputs.len();
806 inputs.push(FrameInput {
807 kind: FrameInputKind::Transition {
808 from,
809 to,
810 transition,
811 },
812 uses_backdrop: inputs[from].uses_backdrop || inputs[to].uses_backdrop,
813 });
814 nodes.insert(transition.to_clip, output);
815 suppressed.insert(transition.from_clip);
816 }
817 let mut layers = Vec::with_capacity(available.len() + captions.len());
818 let mut captions = captions.iter().peekable();
819 for (clip, textures) in available {
820 while let Some(caption) = captions.next_if(|caption| caption.layers_below <= textures.layer)
823 {
824 layers.push(caption_graph_layer(&mut graph, caption));
825 }
826 if clip.is_matte || suppressed.contains(clip.clip_id) {
827 continue;
828 }
829 let index = *nodes.get(clip.clip_id).ok_or(RenderError::InvalidGraph)?;
830 if let FrameInputKind::Layer(layer) = inputs[index].kind {
831 layers.push(layer);
832 continue;
833 }
834 let uses_backdrop = inputs[index].uses_backdrop;
838 let backdrop = if uses_backdrop {
839 layers.as_slice()
840 } else {
841 &[]
842 };
843 let source = compile_frame_input(
846 &mut graph,
847 &inputs,
848 index,
849 backdrop,
850 uses_backdrop.then_some(background),
851 runtime,
852 renderer,
853 )?;
854 if uses_backdrop {
855 layers.clear();
856 }
857 layers.push(identity_layer(source));
858 }
859 for caption in captions {
860 layers.push(caption_graph_layer(&mut graph, caption));
861 }
862 let output = graph
863 .add_composite_over(layers, background)
864 .map_err(GraphError::into_render_error)?;
865 graph
866 .set_output(output)
867 .map_err(GraphError::into_render_error)?;
868 graph
869 .set_cache_intent(output, CacheIntent::Auto)
870 .map_err(GraphError::into_render_error)?;
871 Ok(graph)
872}
873
874const fn cache_intent(intent: alpha2_model::CacheIntent) -> CacheIntent {
875 match intent {
876 alpha2_model::CacheIntent::Auto => CacheIntent::Auto,
877 alpha2_model::CacheIntent::Pin => CacheIntent::Pin,
878 alpha2_model::CacheIntent::Bypass => CacheIntent::Bypass,
879 }
880}
881
882fn apply_matte<'a, 'c>(
884 renderer: &Renderer,
885 graph: &mut RenderGraph<'a>,
886 mut layer: GraphLayer,
887 matte: &'c alpha2_model::Matte,
888 available: &[(&'a ActiveClip<'a>, ClipFrames<'a>)],
889 (time, runtime): (MediaTime, EffectRuntime),
890 coverages: &mut HashMap<&'c alpha2_model::ClipId, RenderNodeId>,
891) -> Result<GraphLayer, RenderError> {
892 let source = available
893 .iter()
894 .find(|(candidate, _)| candidate.clip_id == &matte.source);
895 match source {
896 Some((source, textures)) => {
897 let coverage = if let Some(coverage) = coverages.get(&matte.source) {
898 *coverage
899 } else {
900 let matte_layer = clip_layer(renderer, graph, source, textures, time, runtime)?;
901 let coverage = alone(graph, matte_layer)?;
902 coverages.insert(&matte.source, coverage);
903 coverage
904 };
905 layer = matted_layer(graph, layer, coverage, matte.mode, runtime)?;
906 }
907 None if is_inverted(matte.mode) => {}
910 None => layer.visual.opacity = 0.0,
911 }
912 Ok(layer)
913}
914
915fn background_color(background: Rgb) -> Color {
917 Color::rgba(
918 f64::from(background.red),
919 f64::from(background.green),
920 f64::from(background.blue),
921 1.0,
922 )
923}
924
925struct FrameInput<'a> {
926 kind: FrameInputKind<'a>,
927 uses_backdrop: bool,
928}
929
930fn clip_layer<'a>(
931 renderer: &Renderer,
932 graph: &mut RenderGraph<'a>,
933 clip: &'a ActiveClip<'a>,
934 textures: &ClipFrames<'a>,
935 time: MediaTime,
936 runtime: EffectRuntime,
937) -> Result<GraphLayer, RenderError> {
938 let mut source = match clip.composition {
939 None => clip_picture(renderer, graph, clip, textures, runtime)?,
940 Some(composition) => plan_node(
941 renderer,
942 graph,
943 composition,
944 &textures.media,
945 Some((clip, textures)),
946 runtime,
947 )?,
948 };
949 if !clip.masks.is_empty() {
950 source = mask_node(graph, source, clip.masks, time, runtime)?;
951 }
952 let intent = cache_intent(clip.cache);
956 if intent == CacheIntent::Bypass || graph.node_kind(source) != Some(RenderNodeKind::Source) {
957 graph
958 .set_cache_intent(source, intent)
959 .map_err(GraphError::into_render_error)?;
960 }
961 Ok(GraphLayer {
962 source,
963 visual: layer_transform(clip.visual.ok_or(RenderError::InvalidVisual)?),
964 blend_mode: blend_mode(clip.blend_mode),
965 })
966}
967
968fn clip_picture<'a>(
971 renderer: &Renderer,
972 graph: &mut RenderGraph<'a>,
973 clip: &'a ActiveClip<'a>,
974 textures: &ClipFrames<'a>,
975 runtime: EffectRuntime,
976) -> Result<RenderNodeId, RenderError> {
977 if let Some(source) = clip.source_composition {
978 return plan_node(
980 renderer,
981 graph,
982 source,
983 &textures.source_media,
984 None,
985 runtime,
986 );
987 }
988 match (clip.generator, textures.source) {
989 (Some(Generator::Text { .. }), _) => {
990 textures.text.ok_or(RenderError::MissingText)?.add_to(graph)
991 }
992 (Some(Generator::Layout { .. }), _) => textures
993 .layout
994 .as_ref()
995 .ok_or(RenderError::MissingText)?
996 .add_to(graph),
997 (Some(Generator::Shapes { .. }), _) => {
998 let plan = clip
999 .plan
1000 .and_then(GeneratorPlan::shapes)
1001 .ok_or(RenderError::InvalidGraph)?;
1002 Ok(shapes_node(graph, plan, clip.source_time))
1003 }
1004 (Some(generator), _) => Ok(generator_node(graph, generator, runtime)),
1005 (None, Some(frame)) => Ok(frame.add_to(graph)),
1006 (None, None) => Err(RenderError::InvalidGraph),
1007 }
1008}
1009
1010fn shapes_node(graph: &mut RenderGraph<'_>, plan: &ShapesPlan, time: MediaTime) -> RenderNodeId {
1014 let size = graph.size();
1015 let canvas = alpha2_geometry::CanvasSize::new(size.width, size.height);
1016 let aspect = f64::from(size.width) / f64::from(size.height);
1017 let items = plan.evaluate(time, aspect);
1018 graph.add_vector(std::sync::Arc::new(alpha2_geometry::build_scene(
1019 &items, canvas,
1020 )))
1021}
1022
1023fn generator_node(
1026 graph: &mut RenderGraph<'_>,
1027 generator: &alpha2_model::Generator,
1028 runtime: EffectRuntime,
1029) -> RenderNodeId {
1030 graph.add_generator(
1031 EffectId::from_static(GENERATOR_EFFECT_ID, 1),
1032 crate::generated::generator_params(generator),
1033 timeless(runtime),
1034 )
1035}
1036
1037fn plan_node<'a>(
1041 renderer: &Renderer,
1042 graph: &mut RenderGraph<'a>,
1043 composition: ActiveComposition<'a>,
1044 media: &[Option<StepFrame<'a>>],
1045 clip: Option<(&'a ActiveClip<'a>, &ClipFrames<'a>)>,
1046 runtime: EffectRuntime,
1047) -> Result<RenderNodeId, RenderError> {
1048 let plan = composition.plan();
1049 let time = composition.time();
1050 let mut built: Vec<RenderNodeId> = Vec::with_capacity(plan.steps.len());
1051 let mut picture = None;
1052 for (index, step) in plan.steps.iter().enumerate() {
1053 let at = step.time(time);
1054 let step_runtime = EffectRuntime {
1055 time: at,
1056 ..runtime
1057 };
1058 let node = match &step.op {
1059 StepOp::Clip => {
1061 if let Some(node) = picture {
1062 node
1063 } else {
1064 let (clip, textures) = clip.ok_or(RenderError::InvalidGraph)?;
1065 *picture.insert(clip_picture(renderer, graph, clip, textures, runtime)?)
1066 }
1067 }
1068 StepOp::Media { .. } => match media.get(index).and_then(Option::as_ref) {
1069 Some(StepFrame::Media(frame)) => frame.add_to(graph),
1070 _ => return Err(RenderError::InvalidGraph),
1071 },
1072 StepOp::Generator {
1073 generator: Generator::Text { .. },
1074 ..
1075 } => match media.get(index).and_then(Option::as_ref) {
1076 Some(StepFrame::Text(text)) => text.add_to(graph)?,
1077 _ => return Err(RenderError::MissingText),
1078 },
1079 StepOp::Generator {
1080 generator: Generator::Layout { .. },
1081 ..
1082 } => match media.get(index).and_then(Option::as_ref) {
1083 Some(StepFrame::Layout(layout)) => layout.add_to(graph)?,
1084 _ => return Err(RenderError::MissingText),
1085 },
1086 op => step_node(renderer, graph, op, &built, at, step_runtime)?,
1087 };
1088 built.push(node);
1089 }
1090 step_input(&built, plan.output)
1091}
1092
1093const _: () = assert!(alpha2_model::MAX_EFFECT_INPUTS == alpha2_render::MAX_EFFECT_INPUTS);
1095
1096fn step_input(built: &[RenderNodeId], step: usize) -> Result<RenderNodeId, RenderError> {
1098 built.get(step).copied().ok_or(RenderError::InvalidGraph)
1099}
1100
1101fn step_node<'a>(
1103 renderer: &Renderer,
1104 graph: &mut RenderGraph<'a>,
1105 op: &'a StepOp,
1106 built: &[RenderNodeId],
1107 time: MediaTime,
1108 runtime: EffectRuntime,
1109) -> Result<RenderNodeId, RenderError> {
1110 let input = |step: usize| step_input(built, step);
1111 let node = match op {
1112 StepOp::Clip | StepOp::Media { .. } => return Err(RenderError::InvalidGraph),
1113 StepOp::Generator {
1114 generator: Generator::Shapes { .. },
1115 start,
1116 plan,
1117 } => {
1118 let plan = plan
1119 .as_deref()
1120 .and_then(GeneratorPlan::shapes)
1121 .ok_or(RenderError::InvalidGraph)?;
1122 let at = MediaTime(time.ticks().saturating_add(start.ticks()));
1123 shapes_node(graph, plan, at)
1124 }
1125 StepOp::Generator { generator, .. } => generator_node(graph, generator, runtime),
1126 StepOp::Effect {
1127 input: from,
1128 inputs,
1129 effect,
1130 } => {
1131 let id = project_effect_id(renderer, &effect.id, effect.version)?;
1132 let active = ActiveEffect::new(effect, time);
1133 let params = effect_params(renderer, &id, active);
1134 let runtime = effect_runtime(renderer, &id, runtime);
1135 let mut named = EffectInputs::new();
1136 for (name, step) in inputs {
1137 named.insert(name.as_str(), input(*step)?);
1138 }
1139 graph.add_effect_with_inputs(
1140 id,
1141 input(*from)?,
1142 named,
1143 params,
1144 effect_resources(active),
1145 runtime,
1146 )
1147 }
1148 .map_err(GraphError::into_render_error)?,
1149 StepOp::Transition {
1150 from,
1151 to,
1152 effect,
1153 progress,
1154 } => {
1155 let id = project_effect_id(renderer, &effect.id, effect.version)?;
1156 let active = ActiveEffect::new(effect, time);
1157 let params = effect_params(renderer, &id, active);
1158 let runtime = EffectRuntime {
1159 progress: sample_scalar(progress, time).clamp(0.0, 1.0),
1160 ..effect_runtime(renderer, &id, runtime)
1161 };
1162 graph.add_transition_with_resources(
1163 id,
1164 input(*from)?,
1165 input(*to)?,
1166 params,
1167 effect_resources(active),
1168 runtime,
1169 )
1170 }
1171 .map_err(GraphError::into_render_error)?,
1172 StepOp::Transform {
1173 input: from,
1174 transform,
1175 } => graph
1176 .add_composite(vec![GraphLayer {
1177 source: input(*from)?,
1178 visual: layer_transform(sample_visual(transform, time, graph_aspect(graph))),
1179 blend_mode: BlendMode::Normal,
1180 }])
1181 .map_err(GraphError::into_render_error)?,
1182 StepOp::Merge { layers } => {
1183 let layers = layers
1184 .iter()
1185 .map(|layer| {
1186 let mut placed = identity_layer(input(layer.input)?);
1187 placed.visual.opacity = sample_scalar(&layer.opacity, time).clamp(0.0, 1.0);
1188 placed.blend_mode = blend_mode(layer.blend_mode);
1189 Ok(placed)
1190 })
1191 .collect::<Result<Vec<_>, RenderError>>()?;
1192 graph
1193 .add_composite(layers)
1194 .map_err(GraphError::into_render_error)?
1195 }
1196 StepOp::Matte {
1197 input: from,
1198 matte,
1199 mode,
1200 } => matte_node(graph, input(*from)?, input(*matte)?, *mode, runtime)?,
1201 StepOp::ShapeMask { input: from, masks } => {
1202 mask_node(graph, input(*from)?, masks, time, runtime)?
1203 }
1204 };
1205 Ok(node)
1206}
1207
1208fn graph_aspect(graph: &RenderGraph<'_>) -> f64 {
1211 let size = graph.size();
1212 f64::from(size.width) / f64::from(size.height)
1213}
1214
1215fn mask_node(
1222 graph: &mut RenderGraph<'_>,
1223 source: RenderNodeId,
1224 masks: &[ShapeMask],
1225 time: MediaTime,
1226 runtime: EffectRuntime,
1227) -> Result<RenderNodeId, RenderError> {
1228 let size = graph.size();
1229 let canvas = alpha2_geometry::CanvasSize::new(size.width, size.height);
1230 let aspect = graph_aspect(graph);
1231 let unit = size.width.min(size.height);
1232 let white = alpha2_model::Paint::Solid {
1233 color: alpha2_model::Rgba::new(1.0, 1.0, 1.0, 1.0),
1234 };
1235 let shown = &masks[..masks.len().min(MAX_SHAPE_MASKS)];
1236 if shown.is_empty() {
1237 return Ok(source);
1238 }
1239 let mut coverages = Vec::with_capacity(shown.len());
1240 for mask in shown {
1241 let outline = alpha2_eval::evaluate_geometry(&mask.shape, time, aspect);
1242 let items = [
1243 alpha2_geometry::EvaluatedItem::Geometry(outline),
1244 alpha2_geometry::EvaluatedItem::Fill {
1245 paint: white.clone(),
1246 opacity: 1.0,
1247 rule: mask.fill_rule,
1248 },
1249 ];
1250 let coverage = graph.add_vector(std::sync::Arc::new(alpha2_geometry::build_scene(
1251 &items, canvas,
1252 )));
1253 #[allow(clippy::cast_precision_loss)]
1254 let sigma = 0.3125 * sample_scalar(&mask.feather, time).max(0.0) * unit as f32;
1255 let coverage = graph
1256 .add_gaussian_pixels(coverage, sigma, timeless(runtime))
1257 .map_err(GraphError::into_render_error)?;
1258 coverages.push((coverage, mask.invert, mask.mode));
1259 }
1260 coverage_mask(graph, source, &coverages)
1261}
1262
1263pub(crate) fn coverage_mask(
1267 graph: &mut RenderGraph<'_>,
1268 source: RenderNodeId,
1269 coverages: &[(RenderNodeId, bool, MaskMode)],
1270) -> Result<RenderNodeId, RenderError> {
1271 if coverages.is_empty() || coverages.len() > MAX_SHAPE_MASKS {
1272 return Err(RenderError::InvalidGraph);
1273 }
1274 let size = graph.size();
1275 #[allow(clippy::cast_precision_loss)]
1276 let runtime = EffectRuntime {
1277 ratio: size.width as f32 / size.height as f32,
1278 ..EffectRuntime::default()
1279 };
1280 let mut inputs = EffectInputs::new();
1281 let mut params = EffectParams::with_capacity(coverages.len() * 2);
1282 for (&(coverage, invert, mode), names) in coverages.iter().zip(MASK_COMBINE_NAMES) {
1283 inputs.insert(names.input, coverage);
1284 params.insert(names.invert, ParamValue::Bool(invert));
1285 params.insert(
1286 names.mode,
1287 ParamValue::Enum(match mode {
1288 MaskMode::Add => 0,
1289 MaskMode::Subtract => 1,
1290 MaskMode::Intersect => 2,
1291 }),
1292 );
1293 }
1294 graph
1295 .add_effect_with_inputs(
1296 EffectId::from_static(MASK_COMBINE_EFFECT_IDS[coverages.len() - 1], 1),
1297 source,
1298 inputs,
1299 params,
1300 EffectResources::new(),
1301 runtime,
1302 )
1303 .map_err(GraphError::into_render_error)
1304}
1305
1306const fn is_inverted(mode: MatteMode) -> bool {
1307 matches!(mode, MatteMode::AlphaInverted | MatteMode::LumaInverted)
1308}
1309
1310fn matted_layer(
1314 graph: &mut RenderGraph<'_>,
1315 layer: GraphLayer,
1316 coverage: RenderNodeId,
1317 mode: MatteMode,
1318 runtime: EffectRuntime,
1319) -> Result<GraphLayer, RenderError> {
1320 let picture = alone(graph, layer)?;
1321 let matted = matte_node(graph, picture, coverage, mode, runtime)?;
1322 Ok(GraphLayer {
1323 blend_mode: layer.blend_mode,
1324 ..identity_layer(matted)
1325 })
1326}
1327
1328fn alone(graph: &mut RenderGraph<'_>, layer: GraphLayer) -> Result<RenderNodeId, RenderError> {
1330 graph
1331 .add_composite(vec![GraphLayer {
1332 blend_mode: BlendMode::Normal,
1333 ..layer
1334 }])
1335 .map_err(GraphError::into_render_error)
1336}
1337
1338fn effect_runtime(renderer: &Renderer, effect: &EffectId, runtime: EffectRuntime) -> EffectRuntime {
1344 if renderer.effect_reads_time(effect) {
1345 runtime
1346 } else {
1347 EffectRuntime {
1348 time: MediaTime::ZERO,
1349 ..runtime
1350 }
1351 }
1352}
1353
1354fn timeless(runtime: EffectRuntime) -> EffectRuntime {
1358 EffectRuntime {
1359 ratio: runtime.ratio,
1360 ..EffectRuntime::default()
1361 }
1362}
1363
1364fn matte_node(
1366 graph: &mut RenderGraph<'_>,
1367 picture: RenderNodeId,
1368 coverage: RenderNodeId,
1369 mode: MatteMode,
1370 runtime: EffectRuntime,
1371) -> Result<RenderNodeId, RenderError> {
1372 graph
1373 .add_transition_with_resources(
1374 EffectId::from_static(TRACK_MATTE_EFFECT_ID, 1),
1375 picture,
1376 coverage,
1377 EffectParams::new()
1378 .with(
1379 "luma",
1380 ParamValue::Bool(matches!(mode, MatteMode::Luma | MatteMode::LumaInverted)),
1381 )
1382 .with("inverted", ParamValue::Bool(is_inverted(mode))),
1383 EffectResources::new(),
1384 EffectRuntime {
1386 progress: 0.5,
1387 ..timeless(runtime)
1388 },
1389 )
1390 .map_err(GraphError::into_render_error)
1391}
1392
1393#[derive(Copy, Clone)]
1394enum FrameInputKind<'a> {
1395 Layer(GraphLayer),
1396 Transition {
1397 from: usize,
1398 to: usize,
1399 transition: &'a ActiveTransition<'a>,
1400 },
1401}
1402
1403fn compile_frame_input<'a>(
1404 graph: &mut RenderGraph<'a>,
1405 inputs: &[FrameInput<'a>],
1406 index: usize,
1407 backdrop: &[GraphLayer],
1408 background: Option<Color>,
1409 runtime: EffectRuntime,
1410 renderer: &Renderer,
1411) -> Result<RenderNodeId, RenderError> {
1412 if index >= inputs.len() {
1413 return Err(RenderError::InvalidGraph);
1414 }
1415 let mut compiled = vec![None; inputs.len()];
1416 let mut pending = vec![(index, false)];
1417 while let Some((current, expanded)) = pending.pop() {
1418 if compiled[current].is_some() {
1419 continue;
1420 }
1421 match inputs[current].kind {
1422 FrameInputKind::Layer(layer) => {
1423 let mut layers = Vec::with_capacity(backdrop.len() + 1);
1424 layers.extend_from_slice(backdrop);
1425 layers.push(layer);
1426 let composite = match background {
1427 Some(background) => graph.add_composite_over(layers, background),
1428 None => graph.add_composite(layers),
1429 };
1430 compiled[current] = Some(composite.map_err(GraphError::into_render_error)?);
1431 }
1432 FrameInputKind::Transition {
1433 from,
1434 to,
1435 transition,
1436 } => {
1437 if from >= inputs.len() || to >= inputs.len() {
1438 return Err(RenderError::InvalidGraph);
1439 }
1440 if !expanded {
1441 pending.push((current, true));
1442 if compiled[to].is_none() {
1443 pending.push((to, false));
1444 }
1445 if compiled[from].is_none() {
1446 pending.push((from, false));
1447 }
1448 continue;
1449 }
1450 let from = compiled[from].ok_or(RenderError::InvalidGraph)?;
1451 let to = compiled[to].ok_or(RenderError::InvalidGraph)?;
1452 let id = project_effect_id(
1453 renderer,
1454 transition.effect.id(),
1455 transition.effect.version(),
1456 )?;
1457 let params = effect_params(renderer, &id, transition.effect);
1458 let runtime = EffectRuntime {
1459 progress: transition.progress,
1460 ..effect_runtime(renderer, &id, runtime)
1461 };
1462 compiled[current] = Some(
1463 graph
1464 .add_transition_with_resources(
1465 id,
1466 from,
1467 to,
1468 params,
1469 effect_resources(transition.effect),
1470 runtime,
1471 )
1472 .map_err(GraphError::into_render_error)?,
1473 );
1474 }
1475 }
1476 }
1477 compiled[index].ok_or(RenderError::InvalidGraph)
1478}
1479
1480fn identity_layer(source: RenderNodeId) -> GraphLayer {
1481 GraphLayer {
1482 source,
1483 visual: LayerTransform {
1484 position: [0.5, 0.5],
1485 anchor: [0.5, 0.5],
1486 scale: [1.0, 1.0],
1487 rotation_degrees: 0.0,
1488 opacity: 1.0,
1489 },
1490 blend_mode: BlendMode::Normal,
1491 }
1492}
1493
1494const fn layer_transform(visual: EvaluatedVisual) -> LayerTransform {
1497 LayerTransform {
1498 position: [visual.position.x, visual.position.y],
1499 anchor: [visual.anchor.x, visual.anchor.y],
1500 scale: [visual.scale.x, visual.scale.y],
1501 rotation_degrees: visual.rotation_degrees,
1502 opacity: visual.opacity,
1503 }
1504}
1505
1506fn effect_params<'a>(
1514 renderer: &Renderer,
1515 id: &EffectId,
1516 effect: ActiveEffect<'a>,
1517) -> EffectParams<'a> {
1518 let mut params = EffectParams::with_capacity(effect.authored().params.len());
1519 for (name, sample) in effect.params() {
1520 let mut value = param_value(sample.value);
1521 if sample.interpolated
1522 && let Some(spec) = renderer
1523 .effect_definition(id)
1524 .and_then(|definition| definition.params.iter().find(|spec| spec.name == name))
1525 {
1526 value = clamp_to_range(spec, value);
1527 }
1528 params.insert(name, value);
1529 }
1530 params
1531}
1532
1533fn project_effect_id(renderer: &Renderer, id: &str, version: u32) -> Result<EffectId, RenderError> {
1536 renderer
1537 .project_effect_id(id, version)
1538 .cloned()
1539 .ok_or(RenderError::UnknownEffect)
1540}
1541
1542fn clamp_to_range(spec: &ParamSpec, value: ParamValue) -> ParamValue {
1543 let clamp = |value: f32| {
1544 let value = spec.min.map_or(value, |minimum| value.max(minimum));
1545 spec.max.map_or(value, |maximum| value.min(maximum))
1546 };
1547 match value {
1548 ParamValue::Float(value) => ParamValue::Float(clamp(value)),
1549 ParamValue::Vec2([x, y]) => ParamValue::Vec2([clamp(x), clamp(y)]),
1550 ParamValue::Vec4(values) => ParamValue::Vec4(values.map(clamp)),
1551 other => other,
1552 }
1553}
1554
1555fn effect_resources(effect: ActiveEffect<'_>) -> EffectResources {
1556 effect
1557 .resources()
1558 .fold(EffectResources::new(), |resources, (name, id)| {
1559 resources.with(name, id.as_str())
1560 })
1561}
1562
1563const fn param_value(value: EffectValue) -> ParamValue {
1564 match value {
1565 EffectValue::Float(value) => ParamValue::Float(value),
1566 EffectValue::Vec2(value) => ParamValue::Vec2(value),
1567 EffectValue::Vec4(value) => ParamValue::Vec4(value),
1568 EffectValue::Color(value) => ParamValue::Color(value),
1569 EffectValue::Enum(value) => ParamValue::Enum(value),
1570 EffectValue::Bool(value) => ParamValue::Bool(value),
1571 }
1572}
1573
1574const fn blend_mode(mode: ClipBlendMode) -> BlendMode {
1575 match mode {
1576 ClipBlendMode::Normal => BlendMode::Normal,
1577 ClipBlendMode::Multiply => BlendMode::Multiply,
1578 ClipBlendMode::Screen => BlendMode::Screen,
1579 ClipBlendMode::Overlay => BlendMode::Overlay,
1580 ClipBlendMode::Darken => BlendMode::Darken,
1581 ClipBlendMode::Lighten => BlendMode::Lighten,
1582 ClipBlendMode::ColorDodge => BlendMode::ColorDodge,
1583 ClipBlendMode::ColorBurn => BlendMode::ColorBurn,
1584 ClipBlendMode::HardLight => BlendMode::HardLight,
1585 ClipBlendMode::SoftLight => BlendMode::SoftLight,
1586 ClipBlendMode::Difference => BlendMode::Difference,
1587 ClipBlendMode::Exclusion => BlendMode::Exclusion,
1588 ClipBlendMode::Hue => BlendMode::Hue,
1589 ClipBlendMode::Saturation => BlendMode::Saturation,
1590 ClipBlendMode::Color => BlendMode::Color,
1591 ClipBlendMode::Luminosity => BlendMode::Luminosity,
1592 }
1593}
1594
1595#[cfg(test)]
1596mod tests;