Skip to main content

framefinery_api/
pipeline.rs

1use std::collections::VecDeque;
2
3use crate::error::Result;
4use crate::{Frame, FrameInfo, MediaError, Packet, RawVideoFrameSource};
5
6/// Counters returned after running a frame-filter pipeline.
7#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
8pub struct FilterPipelineStats {
9    /// Frames pulled from the source.
10    pub input_frames: usize,
11    /// Frames pushed to the sink after every filter stage.
12    pub output_frames: usize,
13}
14
15/// Counters returned after running a frame-to-packet encode pipeline.
16#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
17pub struct EncodePipelineStats {
18    /// Frames pulled from the source.
19    pub input_frames: usize,
20    /// Frames submitted to the encoder after filtering.
21    pub encoded_frames: usize,
22    /// Packets pushed to the sink, including encoder flush packets.
23    pub output_packets: usize,
24}
25
26/// Pull-based source stage for generic media pipelines.
27pub trait Source {
28    /// Item produced by this source.
29    type Output;
30
31    /// Pull the next item, returning `Ok(None)` at clean end of stream.
32    fn pull(&mut self) -> Result<Option<Self::Output>>;
33}
34
35impl<T> Source for Box<T>
36where
37    T: Source + ?Sized,
38{
39    type Output = T::Output;
40
41    fn pull(&mut self) -> Result<Option<Self::Output>> {
42        (**self).pull()
43    }
44}
45
46/// Push-based terminal stage for generic media pipelines.
47pub trait Sink<I> {
48    /// Consume one pipeline item.
49    fn push(&mut self, input: I) -> Result<()>;
50
51    /// Finish the sink after every upstream stage has completed.
52    fn finish(&mut self) -> Result<()> {
53        Ok(())
54    }
55}
56
57/// Frame-to-frame transform stage.
58pub trait Filter {
59    /// Process one frame into zero or more output frames.
60    fn process(&mut self, frame: Frame) -> Result<Vec<Frame>>;
61
62    /// Flush delayed frames at end of stream.
63    fn finish(&mut self) -> Result<Vec<Frame>> {
64        Ok(Vec::new())
65    }
66}
67
68/// Frame-to-packet encoder stage for the generic pipeline helpers.
69pub trait Encoder {
70    /// Encode one frame into zero or more packets.
71    fn encode(&mut self, frame: Frame) -> Result<Vec<Packet>>;
72
73    /// Flush delayed packets at end of stream.
74    fn finish(&mut self) -> Result<Vec<Packet>> {
75        Ok(Vec::new())
76    }
77}
78
79/// Adapter that exposes a [`Source`] of owned frames as a raw-frame callback.
80///
81/// This is useful at API boundaries where frames are naturally produced by a
82/// pipeline stage, but an encoder wants to pull one raw frame into a caller
83/// supplied buffer.
84pub struct FrameSourceRawVideoAdapter<S> {
85    source: S,
86    info: FrameInfo,
87}
88
89impl<S> FrameSourceRawVideoAdapter<S>
90where
91    S: Source<Output = Frame>,
92{
93    /// Create an adapter for a source that must emit frames matching `info`.
94    pub fn new(source: S, info: FrameInfo) -> Self {
95        Self { source, info }
96    }
97
98    /// Return the raw frame metadata expected from the wrapped source.
99    pub const fn info(&self) -> FrameInfo {
100        self.info
101    }
102
103    /// Consume the adapter and return the wrapped source.
104    pub fn into_inner(self) -> S {
105        self.source
106    }
107}
108
109impl<S> RawVideoFrameSource for FrameSourceRawVideoAdapter<S>
110where
111    S: Source<Output = Frame>,
112{
113    fn read_frame(&mut self, output: &mut [u8]) -> Result<bool> {
114        validate_raw_video_output_len(self.info, output)?;
115        let Some(frame) = self.source.pull()? else {
116            return Ok(false);
117        };
118        if frame.info() != self.info {
119            return Err(MediaError::IncompatibleFormat {
120                format: frame.info().format.name(),
121                reason: format!(
122                    "expected {}x{}:{}, got {}x{}:{}",
123                    self.info.width,
124                    self.info.height,
125                    self.info.format,
126                    frame.info().width,
127                    frame.info().height,
128                    frame.info().format
129                ),
130            });
131        }
132        output.copy_from_slice(frame.data());
133        Ok(true)
134    }
135}
136
137/// Raw-frame source wrapper that applies a frame-filter chain on demand.
138///
139/// The wrapper owns only the current input frame and any output frames already
140/// produced by the filters but not yet consumed. It does not materialize the
141/// whole filtered stream, which keeps long native streams and future WASM
142/// capture flows bounded in memory.
143pub struct FilteredRawVideoFrameSource<S> {
144    source: S,
145    input_info: FrameInfo,
146    output_info: FrameInfo,
147    filters: Vec<Box<dyn Filter>>,
148    pending: VecDeque<Frame>,
149    source_finished: bool,
150    flush_index: usize,
151}
152
153impl<S> FilteredRawVideoFrameSource<S>
154where
155    S: RawVideoFrameSource,
156{
157    /// Create a filtered raw-video source from concrete filter objects.
158    pub fn new(
159        source: S,
160        input_info: FrameInfo,
161        output_info: FrameInfo,
162        filters: Vec<Box<dyn Filter>>,
163    ) -> Self {
164        Self {
165            source,
166            input_info,
167            output_info,
168            filters,
169            pending: VecDeque::new(),
170            source_finished: false,
171            flush_index: 0,
172        }
173    }
174
175    /// Frame metadata expected by the wrapped source before filters run.
176    pub const fn input_info(&self) -> FrameInfo {
177        self.input_info
178    }
179
180    /// Frame metadata emitted after filters run.
181    pub const fn output_info(&self) -> FrameInfo {
182        self.output_info
183    }
184
185    /// Consume this adapter and return the wrapped raw-frame source.
186    pub fn into_inner(self) -> S {
187        self.source
188    }
189
190    fn fill_pending(&mut self) -> Result<()> {
191        while self.pending.is_empty() {
192            if !self.source_finished {
193                let mut data = vec![0; self.input_info.expected_len()];
194                if self.source.read_frame(&mut data)? {
195                    let frame = Frame::new(self.input_info, data)?;
196                    let frames = self.process_frames_from(vec![frame], 0)?;
197                    self.queue_frames(frames)?;
198                    continue;
199                }
200                self.source_finished = true;
201            }
202
203            if self.flush_index >= self.filters.len() {
204                break;
205            }
206
207            let index = self.flush_index;
208            let frames = {
209                let filter = &mut self.filters[index];
210                filter.finish()?
211            };
212            let frames = self.process_frames_from(frames, index + 1)?;
213            self.flush_index += 1;
214            self.queue_frames(frames)?;
215        }
216        Ok(())
217    }
218
219    fn process_frames_from(&mut self, mut frames: Vec<Frame>, start: usize) -> Result<Vec<Frame>> {
220        for filter in self.filters.iter_mut().skip(start) {
221            let mut next = Vec::new();
222            for frame in frames {
223                next.extend(filter.process(frame)?);
224            }
225            frames = next;
226            if frames.is_empty() {
227                break;
228            }
229        }
230        Ok(frames)
231    }
232
233    fn queue_frames(&mut self, frames: Vec<Frame>) -> Result<()> {
234        for frame in frames {
235            if frame.info() != self.output_info {
236                return Err(MediaError::IncompatibleFormat {
237                    format: frame.info().format.name(),
238                    reason: format!(
239                        "expected filtered output {}x{}:{}, got {}x{}:{}",
240                        self.output_info.width,
241                        self.output_info.height,
242                        self.output_info.format,
243                        frame.info().width,
244                        frame.info().height,
245                        frame.info().format
246                    ),
247                });
248            }
249            self.pending.push_back(frame);
250        }
251        Ok(())
252    }
253}
254
255impl<S> RawVideoFrameSource for FilteredRawVideoFrameSource<S>
256where
257    S: RawVideoFrameSource,
258{
259    fn read_frame(&mut self, output: &mut [u8]) -> Result<bool> {
260        validate_raw_video_output_len(self.output_info, output)?;
261        self.fill_pending()?;
262        let Some(frame) = self.pending.pop_front() else {
263            return Ok(false);
264        };
265        output.copy_from_slice(frame.data());
266        Ok(true)
267    }
268}
269
270fn validate_raw_video_output_len(info: FrameInfo, output: &[u8]) -> Result<()> {
271    let expected = info.expected_len();
272    let actual = output.len();
273    if actual != expected {
274        return Err(MediaError::BufferLength { expected, actual });
275    }
276    Ok(())
277}
278
279/// Run a source through zero or more frame filters into a frame sink.
280pub fn run_frame_filter_pipeline(
281    source: &mut dyn Source<Output = Frame>,
282    filters: &mut [&mut dyn Filter],
283    sink: &mut dyn Sink<Frame>,
284) -> Result<FilterPipelineStats> {
285    let mut stats = FilterPipelineStats::default();
286    while let Some(frame) = source.pull()? {
287        stats.input_frames += 1;
288        let frames = process_frames_from(vec![frame], 0, filters)?;
289        stats.output_frames += push_frames(frames, sink)?;
290    }
291
292    for filter_index in 0..filters.len() {
293        let frames = filters[filter_index].finish()?;
294        let frames = process_frames_from(frames, filter_index + 1, filters)?;
295        stats.output_frames += push_frames(frames, sink)?;
296    }
297
298    sink.finish()?;
299    Ok(stats)
300}
301
302/// Run a source through zero or more frame filters, an encoder, and a packet sink.
303pub fn run_frame_encode_pipeline(
304    source: &mut dyn Source<Output = Frame>,
305    filters: &mut [&mut dyn Filter],
306    encoder: &mut dyn Encoder,
307    sink: &mut dyn Sink<Packet>,
308) -> Result<EncodePipelineStats> {
309    let mut stats = EncodePipelineStats::default();
310    while let Some(frame) = source.pull()? {
311        stats.input_frames += 1;
312        let frames = process_frames_from(vec![frame], 0, filters)?;
313        stats.encoded_frames += encode_frames(frames, encoder, sink, &mut stats)?;
314    }
315
316    for filter_index in 0..filters.len() {
317        let frames = filters[filter_index].finish()?;
318        let frames = process_frames_from(frames, filter_index + 1, filters)?;
319        stats.encoded_frames += encode_frames(frames, encoder, sink, &mut stats)?;
320    }
321
322    for packet in encoder.finish()? {
323        stats.output_packets += 1;
324        sink.push(packet)?;
325    }
326    sink.finish()?;
327    Ok(stats)
328}
329
330fn process_frames_from(
331    mut frames: Vec<Frame>,
332    start: usize,
333    filters: &mut [&mut dyn Filter],
334) -> Result<Vec<Frame>> {
335    for filter in filters.iter_mut().skip(start) {
336        let mut next = Vec::new();
337        for frame in frames {
338            next.extend(filter.process(frame)?);
339        }
340        frames = next;
341    }
342    Ok(frames)
343}
344
345fn push_frames(frames: Vec<Frame>, sink: &mut dyn Sink<Frame>) -> Result<usize> {
346    let count = frames.len();
347    for frame in frames {
348        sink.push(frame)?;
349    }
350    Ok(count)
351}
352
353fn encode_frames(
354    frames: Vec<Frame>,
355    encoder: &mut dyn Encoder,
356    sink: &mut dyn Sink<Packet>,
357    stats: &mut EncodePipelineStats,
358) -> Result<usize> {
359    let frame_count = frames.len();
360    for frame in frames {
361        for packet in encoder.encode(frame)? {
362            stats.output_packets += 1;
363            sink.push(packet)?;
364        }
365    }
366    Ok(frame_count)
367}
368
369#[cfg(test)]
370mod tests {
371    use super::*;
372    use crate::{FrameInfo, MediaError, PixelFormat, StreamId};
373
374    struct VecFrameSource {
375        frames: std::vec::IntoIter<Frame>,
376    }
377
378    impl VecFrameSource {
379        fn new(frames: Vec<Frame>) -> Self {
380            Self {
381                frames: frames.into_iter(),
382            }
383        }
384    }
385
386    struct VecRawFrameSource {
387        info: FrameInfo,
388        frames: VecDeque<Vec<u8>>,
389        reads: usize,
390    }
391
392    impl VecRawFrameSource {
393        fn new(info: FrameInfo, frames: Vec<Vec<u8>>) -> Self {
394            Self {
395                info,
396                frames: frames.into(),
397                reads: 0,
398            }
399        }
400    }
401
402    impl RawVideoFrameSource for VecRawFrameSource {
403        fn read_frame(&mut self, output: &mut [u8]) -> Result<bool> {
404            assert_eq!(output.len(), self.info.expected_len());
405            let Some(frame) = self.frames.pop_front() else {
406                return Ok(false);
407            };
408            output.copy_from_slice(&frame);
409            self.reads += 1;
410            Ok(true)
411        }
412    }
413
414    impl Source for VecFrameSource {
415        type Output = Frame;
416
417        fn pull(&mut self) -> Result<Option<Self::Output>> {
418            Ok(self.frames.next())
419        }
420    }
421
422    #[derive(Default)]
423    struct VecFrameSink {
424        frames: Vec<Frame>,
425        finished: bool,
426    }
427
428    impl Sink<Frame> for VecFrameSink {
429        fn push(&mut self, input: Frame) -> Result<()> {
430            self.frames.push(input);
431            Ok(())
432        }
433
434        fn finish(&mut self) -> Result<()> {
435            self.finished = true;
436            Ok(())
437        }
438    }
439
440    #[derive(Default)]
441    struct VecPacketSink {
442        packets: Vec<Packet>,
443        finished: bool,
444    }
445
446    impl Sink<Packet> for VecPacketSink {
447        fn push(&mut self, input: Packet) -> Result<()> {
448            self.packets.push(input);
449            Ok(())
450        }
451
452        fn finish(&mut self) -> Result<()> {
453            self.finished = true;
454            Ok(())
455        }
456    }
457
458    struct DuplicateFilter;
459
460    impl Filter for DuplicateFilter {
461        fn process(&mut self, frame: Frame) -> Result<Vec<Frame>> {
462            Ok(vec![frame.clone(), frame])
463        }
464    }
465
466    struct PassthroughFilter;
467
468    impl Filter for PassthroughFilter {
469        fn process(&mut self, frame: Frame) -> Result<Vec<Frame>> {
470            Ok(vec![frame])
471        }
472    }
473
474    struct FlushingFilter {
475        info: FrameInfo,
476    }
477
478    impl Filter for FlushingFilter {
479        fn process(&mut self, frame: Frame) -> Result<Vec<Frame>> {
480            Ok(vec![frame])
481        }
482
483        fn finish(&mut self) -> Result<Vec<Frame>> {
484            Ok(vec![Frame::blank(self.info)])
485        }
486    }
487
488    struct PacketizingEncoder {
489        next_pts: i64,
490    }
491
492    impl Encoder for PacketizingEncoder {
493        fn encode(&mut self, frame: Frame) -> Result<Vec<Packet>> {
494            let packet = Packet::new(
495                StreamId(0),
496                Some(crate::Timestamp(self.next_pts)),
497                frame.into_data(),
498            );
499            self.next_pts += 1;
500            Ok(vec![packet])
501        }
502
503        fn finish(&mut self) -> Result<Vec<Packet>> {
504            Ok(vec![Packet::new(StreamId(0), None, b"eos".to_vec())])
505        }
506    }
507
508    fn test_frame(fill: u8) -> Frame {
509        let info = FrameInfo::new(2, 2, PixelFormat::Rgb24).unwrap();
510        Frame::new(info, vec![fill; info.expected_len()]).unwrap()
511    }
512
513    #[test]
514    fn filter_pipeline_runs_filters_and_flushes_in_order() {
515        let info = FrameInfo::new(2, 2, PixelFormat::Rgb24).unwrap();
516        let mut source = VecFrameSource::new(vec![test_frame(7)]);
517        let mut duplicate = DuplicateFilter;
518        let mut flush = FlushingFilter { info };
519        let mut filters: Vec<&mut dyn Filter> = vec![&mut duplicate, &mut flush];
520        let mut sink = VecFrameSink::default();
521
522        let stats =
523            run_frame_filter_pipeline(&mut source, filters.as_mut_slice(), &mut sink).unwrap();
524
525        assert_eq!(
526            stats,
527            FilterPipelineStats {
528                input_frames: 1,
529                output_frames: 3,
530            }
531        );
532        assert_eq!(sink.frames.len(), 3);
533        assert!(sink.finished);
534        assert_eq!(sink.frames[0].data(), &[7; 12]);
535        assert_eq!(sink.frames[2].data(), &[0; 12]);
536    }
537
538    #[test]
539    fn encode_pipeline_pushes_encoder_packets_and_finish_packet() {
540        let mut source = VecFrameSource::new(vec![test_frame(1), test_frame(2)]);
541        let mut passthrough = PassthroughFilter;
542        let mut filters: Vec<&mut dyn Filter> = vec![&mut passthrough];
543        let mut encoder = PacketizingEncoder { next_pts: 0 };
544        let mut sink = VecPacketSink::default();
545
546        let stats =
547            run_frame_encode_pipeline(&mut source, filters.as_mut_slice(), &mut encoder, &mut sink)
548                .unwrap();
549
550        assert_eq!(
551            stats,
552            EncodePipelineStats {
553                input_frames: 2,
554                encoded_frames: 2,
555                output_packets: 3,
556            }
557        );
558        assert_eq!(sink.packets.len(), 3);
559        assert!(sink.finished);
560        assert_eq!(sink.packets[0].pts, Some(crate::Timestamp(0)));
561        assert_eq!(sink.packets[2].data, b"eos");
562    }
563
564    #[test]
565    fn passthrough_filter_preserves_frame() {
566        let frame = test_frame(9);
567        let mut filter = PassthroughFilter;
568        let out = filter.process(frame.clone()).unwrap();
569        assert_eq!(out, vec![frame]);
570    }
571
572    #[test]
573    fn frame_source_raw_video_adapter_reads_one_frame_at_a_time() {
574        let first = test_frame(3);
575        let second = test_frame(4);
576        let info = first.info();
577        let mut source = FrameSourceRawVideoAdapter::new(
578            VecFrameSource::new(vec![first.clone(), second.clone()]),
579            info,
580        );
581        let mut output = vec![0; info.expected_len()];
582
583        assert!(source.read_frame(&mut output).unwrap());
584        assert_eq!(output, first.data());
585        assert!(source.read_frame(&mut output).unwrap());
586        assert_eq!(output, second.data());
587        assert!(!source.read_frame(&mut output).unwrap());
588    }
589
590    #[test]
591    fn filtered_raw_video_source_pulls_input_on_demand() {
592        let info = FrameInfo::new(2, 2, PixelFormat::Rgb24).unwrap();
593        let first = vec![5; info.expected_len()];
594        let second = vec![6; info.expected_len()];
595        let source = VecRawFrameSource::new(info, vec![first.clone(), second.clone()]);
596        let filters: Vec<Box<dyn Filter>> = vec![Box::new(PassthroughFilter)];
597        let mut filtered = FilteredRawVideoFrameSource::new(source, info, info, filters);
598        let mut output = vec![0; info.expected_len()];
599
600        assert!(filtered.read_frame(&mut output).unwrap());
601        assert_eq!(output, first);
602        let source = filtered.into_inner();
603        assert_eq!(source.reads, 1);
604    }
605
606    #[test]
607    fn source_errors_stop_pipeline_before_sink_finish() {
608        struct FailingSource;
609
610        impl Source for FailingSource {
611            type Output = Frame;
612
613            fn pull(&mut self) -> Result<Option<Self::Output>> {
614                Err(MediaError::Message("source failed".to_string()))
615            }
616        }
617
618        let mut source = FailingSource;
619        let mut sink = VecFrameSink::default();
620        let err = run_frame_filter_pipeline(&mut source, &mut [], &mut sink).unwrap_err();
621
622        assert_eq!(err.to_string(), "source failed");
623        assert!(!sink.finished);
624    }
625}