lumen_engine/node/source/
media_in.rs

1use std::ops::Range;
2
3use crate::error::{MediaError, RenderError};
4use crate::node::{NodeId, NodeParams};
5
6use crate::gpu::{
7    BoundFrame, CompiledOutput, FrameBindContext, GpuCompileNode, GpuCompiledNode, MediaTextureKey,
8    RasterHandle, RasterMetadata,
9};
10
11#[derive(Debug, Clone, Copy, PartialEq, Eq, lumen_macros::NodeEnum)]
12pub enum LoopMode {
13    Clamp,
14    Repeat,
15    PingPong,
16}
17
18impl LoopMode {
19    pub fn from_int(value: i64) -> Self {
20        match value {
21            1 => Self::Repeat,
22            2 => Self::PingPong,
23            _ => Self::Clamp,
24        }
25    }
26}
27
28#[cfg(any(feature = "json", feature = "metadata"))]
29#[derive(lumen_macros::NodeEnum)]
30pub enum MediaInSourceKind {
31    Image = 0,
32    Video = 1,
33}
34
35#[derive(Debug, Clone)]
36pub enum MediaInKind {
37    Image {
38        image_id: String,
39    },
40    Video {
41        stream_id: String,
42        range: Option<Range<u32>>,
43        speed: f32,
44        loop_mode: LoopMode,
45    },
46}
47
48/// Binds an external image or video frame as a GPU texture.
49#[derive(Debug, Clone, lumen_macros::Delegate)]
50pub struct MediaInParams {
51    /// Type of external media source.
52    #[meta(kind = "enum", name = "Media type", enum_type = MediaInSourceKind)]
53    pub kind: i64,
54    /// External media source identifier.
55    #[meta(role = "source_id")]
56    pub source: String,
57    /// First source frame to include.
58    #[meta(min = 0, step = 1)]
59    pub range_start: i64,
60    /// Last source frame to include.
61    #[meta(min = 0, step = 1)]
62    pub range_end: i64,
63    /// Playback speed multiplier.
64    #[meta(step = 0.1)]
65    pub speed: f64,
66    /// Behavior when playback leaves the source range.
67    #[meta(kind = "enum", enum_type = LoopMode)]
68    pub loop_mode: i64,
69}
70
71impl Default for MediaInParams {
72    fn default() -> Self {
73        Self {
74            kind: 0,
75            source: String::new(),
76            range_start: 0,
77            range_end: 0,
78            speed: 1.0,
79            loop_mode: 0,
80        }
81    }
82}
83
84/// Binds an external image or video frame as a GPU texture.
85#[derive(Debug, Clone, lumen_macros::Node)]
86#[node(kind = "media_in", name = "Media In", category = "source")]
87pub struct MediaIn {
88    pub id: NodeId,
89    #[params]
90    pub params: MediaInParamsDelegate,
91}
92
93impl Default for MediaIn {
94    fn default() -> Self {
95        Self {
96            id: NodeId::new(0),
97            params: MediaInParamsDelegate::default(),
98        }
99    }
100}
101
102pub fn resolve_for_context(
103    media_in: &MediaIn,
104    ctx: &crate::expr::ExpressionContext<'_>,
105) -> crate::Result<MediaInKind> {
106    let params = media_in.params.eval(&crate::node::NodeParamEvalContext {
107        node_id: media_in.id,
108        expr: ctx,
109    })?;
110    let kind = params.kind;
111    let source = params.source;
112    let range_start = params.range_start;
113    let range_end = params.range_end;
114    let speed = params.speed as f32;
115    let loop_mode = LoopMode::from_int(params.loop_mode);
116
117    if kind == 1 {
118        Ok(MediaInKind::Video {
119            stream_id: source,
120            range: resolve_range(range_start, range_end),
121            speed,
122            loop_mode,
123        })
124    } else {
125        Ok(MediaInKind::Image { image_id: source })
126    }
127}
128
129pub fn resolve_range(start: i64, end: i64) -> Option<Range<u32>> {
130    if end <= start {
131        return None;
132    }
133    Some(start.max(0) as u32..end.max(0) as u32)
134}
135
136pub fn map_to_source_frame(
137    frame: u32,
138    composition_fps: f32,
139    source_fps: f32,
140    frame_count: u32,
141    range: Option<&Range<u32>>,
142    speed: f32,
143    loop_mode: LoopMode,
144) -> Option<u32> {
145    if frame_count == 0 {
146        return None;
147    }
148    let start = range.map(|range| range.start).unwrap_or(0);
149    let end = range
150        .map(|range| range.end)
151        .unwrap_or(frame_count)
152        .min(frame_count);
153    if end <= start {
154        return None;
155    }
156    let span = end - start;
157    let comp_fps = composition_fps.max(1.0);
158    let media_fps = source_fps.max(comp_fps);
159    let relative =
160        (((frame as f64 / comp_fps as f64) * media_fps as f64) * speed.abs() as f64).floor() as u32;
161    let forward = speed >= 0.0;
162    let mapped = match loop_mode {
163        LoopMode::Clamp => {
164            if forward {
165                start + relative.min(span.saturating_sub(1))
166            } else {
167                end - 1 - relative.min(span.saturating_sub(1))
168            }
169        }
170        LoopMode::Repeat => {
171            let offset = relative % span;
172            start + if forward { offset } else { span - 1 - offset }
173        }
174        LoopMode::PingPong => {
175            let period = span.saturating_mul(2).saturating_sub(2).max(1);
176            let offset = relative % period;
177            let offset = if offset < span {
178                offset
179            } else {
180                period - offset
181            };
182            start + if forward { offset } else { span - 1 - offset }
183        }
184    };
185    (mapped < frame_count).then_some(mapped)
186}
187
188impl GpuCompileNode for MediaIn {
189    fn compile_gpu(
190        &self,
191        ctx: &mut crate::gpu::CompileContext<'_>,
192        port: &crate::node::PortRef,
193    ) -> crate::Result<CompiledOutput> {
194        if port.port != "output" {
195            return Err(ctx.missing_output(self.id, &port.port));
196        }
197
198        let size = self.native_size(ctx).unwrap_or_else(|| {
199            lumen_gpu::Size::new(
200                ctx.composition().render_settings.width.max(1),
201                ctx.composition().render_settings.height.max(1),
202            )
203        });
204        let texture = ctx.builder_mut().texture_for(
205            lumen_gpu::NodeKey(self.id.0),
206            Some(format!("media-in:{}:frame", self.id.0)),
207            lumen_gpu::TextureDesc::sampled(size, lumen_gpu::wgpu::TextureFormat::Rgba8Unorm),
208        );
209        ctx.builder_mut().param(
210            lumen_gpu::ParamKey {
211                owner: lumen_gpu::NodeKey(self.id.0),
212                slot: 0,
213            },
214            lumen_gpu::ParamTarget::Texture(texture),
215        );
216        ctx.register_compiled_node(CompiledMediaInput {
217            node_id: self.id,
218            params: self.params.clone(),
219            texture,
220            size,
221        });
222
223        Ok(CompiledOutput::Raster(RasterHandle {
224            texture,
225            domain: lumen_gpu::TextureDomain::full_frame(size),
226            metadata: RasterMetadata::default(),
227        }))
228    }
229}
230
231impl MediaIn {
232    fn native_size(&self, ctx: &crate::gpu::CompileContext<'_>) -> Option<lumen_gpu::Size> {
233        let media = ctx.media()?;
234        let kind = resolve_for_context(self, &ctx.expr_context(self.id, "source")).ok()?;
235        match kind {
236            MediaInKind::Image { image_id } => {
237                let metadata = media.get_image_resolver(&image_id)?.metadata();
238                Some(lumen_gpu::Size::new(
239                    metadata.width.max(1),
240                    metadata.height.max(1),
241                ))
242            }
243            MediaInKind::Video { stream_id, .. } => {
244                let metadata = media.get_video_resolver(&stream_id)?.metadata();
245                Some(lumen_gpu::Size::new(
246                    metadata.width.max(1),
247                    metadata.height.max(1),
248                ))
249            }
250        }
251    }
252}
253
254#[derive(Debug, Clone)]
255struct CompiledMediaInput {
256    node_id: NodeId,
257    params: MediaInParamsDelegate,
258    texture: lumen_gpu::TextureId,
259    size: lumen_gpu::Size,
260}
261
262impl GpuCompiledNode for CompiledMediaInput {
263    fn node_id(&self) -> NodeId {
264        self.node_id
265    }
266
267    fn bind(&self, ctx: &FrameBindContext<'_>, bound: &mut BoundFrame) -> crate::Result<()> {
268        let media = ctx.media().ok_or_else(|| RenderError::NodeEvaluation {
269            frame: ctx.frame(),
270            node_id: self.node_id,
271            node_kind: "MediaIn",
272            details: "media store is required for media input nodes".to_string(),
273        })?;
274        let node = MediaIn {
275            id: self.node_id,
276            params: self.params.clone(),
277        };
278        let kind = resolve_for_context(&node, &ctx.expr_context(self.node_id, "source"))?;
279        let (frame, key_source, key_frame) = match kind {
280            MediaInKind::Image { image_id } => media
281                .get_image_resolver(&image_id)
282                .ok_or_else(|| MediaError::SourceNotFound {
283                    media_source: image_id.clone(),
284                })?
285                .frame()
286                .map(|frame| (frame, image_id, None))?,
287            MediaInKind::Video {
288                stream_id,
289                range,
290                speed,
291                loop_mode,
292            } => {
293                let resolver = media.get_video_resolver(&stream_id).ok_or_else(|| {
294                    MediaError::SourceNotFound {
295                        media_source: stream_id.clone(),
296                    }
297                })?;
298                let metadata = resolver.metadata();
299                let source_frame = map_to_source_frame(
300                    ctx.frame(),
301                    ctx.expr_context(self.node_id, "source").fps,
302                    metadata.fps,
303                    metadata.frame_count,
304                    range.as_ref(),
305                    speed,
306                    loop_mode,
307                )
308                .ok_or_else(|| MediaError::FrameOutOfRange {
309                    media_source: stream_id.clone(),
310                    frame: ctx.frame(),
311                    frame_count: metadata.frame_count,
312                })?;
313                resolver
314                    .frame(source_frame)
315                    .map(|frame| (frame, stream_id, Some(source_frame)))?
316            }
317        };
318        bound.use_media_texture(
319            self.texture,
320            MediaTextureKey {
321                source: key_source,
322                frame: key_frame,
323                width: self.size.width,
324                height: self.size.height,
325            },
326            frame,
327            self.size,
328        );
329        Ok(())
330    }
331}
332
333#[cfg(test)]
334mod tests {
335    use super::{LoopMode, map_to_source_frame};
336
337    #[test]
338    fn maps_negative_media_speed_in_reverse() {
339        assert_eq!(
340            map_to_source_frame(0, 30.0, 30.0, 10, Some(&(2..7)), -1.0, LoopMode::Clamp),
341            Some(6)
342        );
343        assert_eq!(
344            map_to_source_frame(3, 30.0, 30.0, 10, Some(&(2..7)), -1.0, LoopMode::Clamp),
345            Some(3)
346        );
347        assert_eq!(
348            map_to_source_frame(99, 30.0, 30.0, 10, Some(&(2..7)), -1.0, LoopMode::Clamp),
349            Some(2)
350        );
351    }
352
353    #[test]
354    fn maps_ping_pong_loop_mode() {
355        let frames = (0..8)
356            .map(|frame| map_to_source_frame(frame, 30.0, 30.0, 4, None, 1.0, LoopMode::PingPong))
357            .collect::<Vec<_>>();
358        assert_eq!(
359            frames,
360            vec![
361                Some(0),
362                Some(1),
363                Some(2),
364                Some(3),
365                Some(2),
366                Some(1),
367                Some(0),
368                Some(1)
369            ]
370        );
371    }
372
373    #[test]
374    fn maps_negative_ping_pong_loop_mode() {
375        let frames = (0..8)
376            .map(|frame| map_to_source_frame(frame, 30.0, 30.0, 4, None, -1.0, LoopMode::PingPong))
377            .collect::<Vec<_>>();
378        assert_eq!(
379            frames,
380            vec![
381                Some(3),
382                Some(2),
383                Some(1),
384                Some(0),
385                Some(1),
386                Some(2),
387                Some(3),
388                Some(2)
389            ]
390        );
391    }
392}
lumen_engine/node/source/media_in.rs

lumen_engine/node/source/
media_in.rs
