ez_ffmpeg/core/analysis/crop/
mod.rs

1//! Native luma crop / letterbox detection.
2//!
3//! `VideoDetector::Crop` and [`CropDetectionOptions`] scan decoded **progressive**
4//! Y planes in Rust. Interlaced frames (`AV_FRAME_FLAG_INTERLACED`) fail the
5//! job as [`Error::AnalysisFrame`]; fields are not modeled as a pair of
6//! half-height scans. The scanner
7//! does not require FFmpeg's GPL `cropdetect` filter. Coordinates are not guaranteed to match
8//! FFmpeg bit-for-bit: `round` expands outward so content is never cut,
9//! `limit == 0` treats every sample as active, a finite temporal window is
10//! used instead of a permanent historical maximum, and consensus is an
11//! independent-median of the window (not FFmpeg's historical-max).
12//! [`CropDetectionOptions::skip_initial`] drops real video frames before
13//! format / hardware / interlace validation and before any luma or scene
14//! handling, so a scene cut during skip does not reset crop state and a
15//! known-bad leading frame can be stepped over. Flush / props-only markers
16//! are triaged first and do not consume the skip budget.
17//!
18//! Crop events and observations are published only for frames that carry a
19//! timestamp: a fully timestamp-less stream still updates crop state but
20//! never publishes (mixed-PTS streams publish on the timestamped frames).
21//!
22//! Users who need the historical `lavfi.cropdetect.*` values can still attach
23//! an explicit `cropdetect` filter graph;
24//! [`MetadataEventFilter`](crate::core::analysis::filter::MetadataEventFilter)
25//! keeps parsing those keys when native crop detection is not configured.
26//!
27//! Operators can time this scanner against a GPL `ffmpeg` binary (when
28//! `ffmpeg -filters` lists `cropdetect`) with the ignored integration test:
29//! `cargo test --release --test crop_parity_bench -- --ignored`.
30
31mod luma;
32mod scan;
33mod stability;
34
35use crate::core::analysis::event::Timestamp;
36use crate::core::analysis::report::{AnalysisReport, CropSuggestion};
37use crate::error::{Error, Result};
38use ffmpeg_next::Frame;
39use luma::{LumaAccess, LumaView};
40use scan::{legacy_limit, resolve_threshold, scan_boundary_bands, ScanConfig, ThresholdBand};
41use stability::Stability;
42use std::sync::{Arc, Mutex};
43use std::time::Duration;
44
45/// How a luma sample is classified as near-black.
46#[derive(Debug, Clone, Copy, PartialEq)]
47#[non_exhaustive]
48pub enum CropLumaThreshold {
49    /// Fraction of the full digital code range, in `0.0..=1.0`.
50    ///
51    /// `0.0` is a sentinel: every sample is treated as active (full frame).
52    Normalized(f32),
53    /// Raw luma code after unpacking the stored sample.
54    RawCode(u16),
55    /// Fraction above nominal black in the declared signal range.
56    ///
57    /// `0.0` is nominal black itself (16 for limited 8-bit, 64 for limited
58    /// 10-bit, 0 for full range). Unlike [`Normalized`](Self::Normalized),
59    /// it is **not** the full-frame sentinel.
60    AboveNominalBlack(f32),
61}
62
63/// Runtime handle for changing the luma threshold between frames.
64///
65/// Only `limit` is mutable at runtime, matching the public cropdetect command
66/// surface. Invalid updates return an error and leave the previous value
67/// unchanged.
68#[derive(Clone)]
69pub struct CropDetectionControl {
70    inner: Arc<Mutex<CropLumaThreshold>>,
71}
72
73impl std::fmt::Debug for CropDetectionControl {
74    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
75        f.debug_struct("CropDetectionControl")
76            .field("threshold", &self.threshold())
77            .finish()
78    }
79}
80
81impl CropDetectionControl {
82    /// Creates a control holding `initial` after validating it.
83    pub fn new(initial: CropLumaThreshold) -> Result<Self> {
84        validate_threshold(initial)?;
85        Ok(Self {
86            inner: Arc::new(Mutex::new(initial)),
87        })
88    }
89
90    /// Replaces the threshold. On validation failure the previous value is kept.
91    pub fn set_threshold(&self, value: CropLumaThreshold) -> Result<()> {
92        validate_threshold(value)?;
93        let mut guard = self.inner.lock().unwrap_or_else(|e| e.into_inner());
94        *guard = value;
95        Ok(())
96    }
97
98    /// Current threshold.
99    pub fn threshold(&self) -> CropLumaThreshold {
100        *self.inner.lock().unwrap_or_else(|e| e.into_inner())
101    }
102}
103
104/// Builder for native crop detection. Fields are private so new knobs can be
105/// added without breaking source compatibility.
106#[derive(Debug, Clone)]
107pub struct CropDetectionOptions {
108    threshold: CropLumaThreshold,
109    round: u32,
110    reset_every: u32,
111    skip_initial: u32,
112    active_tolerance: f32,
113    soft_margin: f32,
114    temporal_window: Duration,
115    max_border_fraction: f32,
116    control: Option<CropDetectionControl>,
117}
118
119impl Default for CropDetectionOptions {
120    fn default() -> Self {
121        Self::new()
122    }
123}
124
125impl CropDetectionOptions {
126    /// Defaults matching `limit=24`, `round=16`, `reset=0`, plus `skip=2`.
127    pub fn new() -> Self {
128        Self {
129            threshold: CropLumaThreshold::Normalized(24.0 / 255.0),
130            round: 16,
131            reset_every: 0,
132            skip_initial: 2,
133            active_tolerance: scan::DEFAULT_ACTIVE_TOLERANCE,
134            soft_margin: 4.0 / 255.0,
135            temporal_window: Duration::from_millis(500),
136            max_border_fraction: scan::DEFAULT_MAX_BORDER,
137            control: None,
138        }
139    }
140
141    /// Maps the historical [`crate::analysis::VideoDetector::Crop`] integers.
142    pub fn from_legacy(limit: u32, round: u32, reset: u32) -> Self {
143        let mut opts = Self::new();
144        opts.threshold = legacy_limit(limit);
145        opts.round = round;
146        opts.reset_every = reset;
147        opts.skip_initial = 2;
148        opts
149    }
150
151    /// Hard luma threshold used to classify near-black samples.
152    pub fn threshold(mut self, value: CropLumaThreshold) -> Self {
153        self.threshold = value;
154        self
155    }
156
157    /// Width/height multiple after outward expansion. `0`/`1` skip extra multiples.
158    pub fn round(mut self, multiple: u32) -> Self {
159        self.round = multiple;
160        self
161    }
162
163    /// Clear temporal evidence every `frames` evaluated frames (`0` = never).
164    /// The current stable rectangle is kept.
165    pub fn reset_every(mut self, frames: u32) -> Self {
166        self.reset_every = frames;
167        self
168    }
169
170    /// Skip the first `frames` real video frames. The default is 2.
171    ///
172    /// Skipped frames do not contribute luma reads, crop state, scene-cut
173    /// resets, crop events, or format / hardware / interlace validation.
174    /// Flush markers are not counted. Bootstrap still needs three
175    /// high-confidence candidates after skip, so with the default the first
176    /// crop event is at the earliest the 5th real frame.
177    pub fn skip_initial(mut self, frames: u32) -> Self {
178        self.skip_initial = frames;
179        self
180    }
181
182    /// Fraction of a line that may be active and still count as a black bar.
183    ///
184    /// Values above `0.5` have no further effect: a line whose weighted
185    /// activity exceeds half its samples is always classified as content.
186    pub fn active_tolerance(mut self, fraction: f32) -> Self {
187        self.active_tolerance = fraction;
188        self
189    }
190
191    /// Extra luma codes above the hard threshold treated as a soft band.
192    pub fn soft_margin(mut self, fraction: f32) -> Self {
193        self.soft_margin = fraction;
194        self
195    }
196
197    /// Sliding window of high-confidence candidates used for hysteresis.
198    pub fn temporal_window(mut self, duration: Duration) -> Self {
199        self.temporal_window = duration;
200        self
201    }
202
203    /// Maximum fraction of each dimension searched as a border band.
204    pub fn max_border_fraction(mut self, fraction: f32) -> Self {
205        self.max_border_fraction = fraction;
206        self
207    }
208
209    /// Share a handle that can change the luma threshold between frames.
210    pub fn threshold_control(mut self, control: CropDetectionControl) -> Self {
211        self.control = Some(control);
212        self
213    }
214
215    pub(crate) fn validate(&self) -> Result<()> {
216        validate_threshold(self.threshold)?;
217        for (v, what) in [
218            (self.round, "crop round"),
219            (self.reset_every, "crop reset"),
220            (self.skip_initial, "crop skip_initial"),
221        ] {
222            if v > i32::MAX as u32 {
223                return Err(Error::InvalidRecipeArg(format!(
224                    "{what} must be <= {}, got {v}",
225                    i32::MAX
226                )));
227            }
228        }
229        if !self.active_tolerance.is_finite() || !(0.0..=1.0).contains(&self.active_tolerance) {
230            return Err(Error::InvalidRecipeArg(format!(
231                "crop active_tolerance must be in 0.0..=1.0, got {}",
232                self.active_tolerance
233            )));
234        }
235        if !self.soft_margin.is_finite() || self.soft_margin < 0.0 {
236            return Err(Error::InvalidRecipeArg(format!(
237                "crop soft_margin must be finite and >= 0, got {}",
238                self.soft_margin
239            )));
240        }
241        if !self.max_border_fraction.is_finite()
242            || !(0.05..=0.49).contains(&self.max_border_fraction)
243        {
244            return Err(Error::InvalidRecipeArg(format!(
245                "crop max_border_fraction must be in 0.05..=0.49, got {}",
246                self.max_border_fraction
247            )));
248        }
249        Ok(())
250    }
251}
252
253fn validate_threshold(value: CropLumaThreshold) -> Result<()> {
254    match value {
255        CropLumaThreshold::Normalized(f) | CropLumaThreshold::AboveNominalBlack(f) => {
256            if !f.is_finite() || !(0.0..=1.0).contains(&f) {
257                Err(Error::InvalidRecipeArg(format!(
258                    "crop luma threshold fraction must be finite in 0.0..=1.0, got {f}"
259                )))
260            } else {
261                Ok(())
262            }
263        }
264        CropLumaThreshold::RawCode(_) => Ok(()),
265    }
266}
267
268/// Half-open raw bounds, before `round` / chroma expansion.
269#[derive(Debug, Clone, Copy, PartialEq, Eq)]
270pub struct CropRawBounds {
271    pub left: i32,
272    pub top: i32,
273    pub right_exclusive: i32,
274    pub bottom_exclusive: i32,
275}
276
277/// One published crop observation (raw + aligned).
278#[derive(Debug, Clone, Copy, PartialEq)]
279pub struct CropObservation {
280    pub at: Timestamp,
281    pub raw: CropRawBounds,
282    pub aligned: CropSuggestion,
283}
284
285/// [`AnalysisReport`] plus the last raw/aligned crop observation.
286#[derive(Debug, Clone, PartialEq)]
287pub struct DetailedAnalysisReport {
288    pub report: AnalysisReport,
289    pub last_crop_observation: Option<CropObservation>,
290}
291
292/// Native crop scanner stored on [`crate::analysis::MetadataEventFilter`].
293pub(crate) struct CropScanner {
294    options: CropDetectionOptions,
295    control: CropDetectionControl,
296    last_threshold: CropLumaThreshold,
297    stability: Stability,
298    #[cfg(test)]
299    last_probe_count: u32,
300}
301
302impl CropScanner {
303    pub(crate) fn new(options: CropDetectionOptions) -> Result<Self> {
304        options.validate()?;
305        let control = match options.control.clone() {
306            Some(c) => c,
307            None => CropDetectionControl::new(options.threshold)?,
308        };
309        let initial = control.threshold();
310        let window_us = options.temporal_window.as_micros().min(i64::MAX as u128) as i64;
311        Ok(Self {
312            stability: Stability::new(
313                options.skip_initial,
314                options.reset_every,
315                window_us,
316                options.round,
317            ),
318            options,
319            control,
320            last_threshold: initial,
321            #[cfg(test)]
322            last_probe_count: 0,
323        })
324    }
325
326    pub(crate) fn process_frame(
327        &mut self,
328        frame: &Frame,
329        frame_ts: Option<Timestamp>,
330        scene_changed: bool,
331    ) -> Result<Option<(CropSuggestion, CropObservation)>> {
332        if LumaView::is_passthrough_marker(frame) {
333            return Ok(self.publish(frame_ts));
334        }
335
336        if self.stability.skip_due() {
337            self.stability.consume_skip();
338            return Ok(None);
339        }
340
341        let luma = match LumaView::try_from_frame(frame)
342            .map_err(|e| Error::AnalysisFrame(e.to_string().into_boxed_str()))?
343        {
344            Some(luma) => luma,
345            None => return Ok(self.publish(frame_ts)),
346        };
347
348        self.stability.set_geometry(
349            luma.frame_width() as i32,
350            luma.frame_height() as i32,
351            luma.chroma_grid(),
352        );
353
354        self.stability.on_evaluated_frame();
355
356        let snapshot = self.control.threshold();
357        if snapshot != self.last_threshold {
358            self.stability.clear_evidence();
359            self.last_threshold = snapshot;
360        }
361
362        let full = CropRawBounds {
363            left: 0,
364            top: 0,
365            right_exclusive: luma.frame_width() as i32,
366            bottom_exclusive: luma.frame_height() as i32,
367        };
368        if scene_changed {
369            self.stability.reset_scene(full);
370        }
371
372        let band = resolve_band(snapshot, &luma, self.options.soft_margin)?;
373        let cfg = ScanConfig {
374            threshold: band,
375            active_tolerance: self.options.active_tolerance,
376            max_border_fraction: self.options.max_border_fraction,
377        };
378        if let Some(candidate) = scan_boundary_bands(&luma, &cfg) {
379            if candidate.reliable {
380                self.stability
381                    .observe(candidate.raw, frame_ts.map(|t| t.time_us));
382            }
383        }
384        #[cfg(test)]
385        {
386            self.last_probe_count = luma.probe_count();
387        }
388
389        if !scene_changed {
390            self.stability.maybe_periodic_reset();
391        }
392
393        Ok(self.publish(frame_ts))
394    }
395
396    fn publish(
397        &mut self,
398        frame_ts: Option<Timestamp>,
399    ) -> Option<(CropSuggestion, CropObservation)> {
400        let (_, mut obs) = self.stability.current_aligned()?;
401        let ts = frame_ts?;
402        obs.at = ts;
403        Some((obs.aligned, obs))
404    }
405
406    #[cfg(test)]
407    pub(crate) fn last_probe_count(&self) -> u32 {
408        self.last_probe_count
409    }
410
411    #[cfg(test)]
412    pub(crate) fn process_luma<L: LumaAccess>(
413        &mut self,
414        luma: &L,
415        time_us: Option<i64>,
416        scene_changed: bool,
417    ) -> Option<(CropSuggestion, CropObservation)> {
418        #[cfg(test)]
419        {
420            self.last_probe_count = 0;
421        }
422        if self.stability.skip_due() {
423            self.stability.consume_skip();
424            return None;
425        }
426        self.stability.set_geometry(
427            luma.frame_width() as i32,
428            luma.frame_height() as i32,
429            luma.chroma_grid(),
430        );
431        self.stability.on_evaluated_frame();
432        let snapshot = self.control.threshold();
433        if snapshot != self.last_threshold {
434            self.stability.clear_evidence();
435            self.last_threshold = snapshot;
436        }
437        let full = CropRawBounds {
438            left: 0,
439            top: 0,
440            right_exclusive: luma.frame_width() as i32,
441            bottom_exclusive: luma.frame_height() as i32,
442        };
443        if scene_changed {
444            self.stability.reset_scene(full);
445        }
446        let band = resolve_band(snapshot, luma, self.options.soft_margin).ok()?;
447        let cfg = ScanConfig {
448            threshold: band,
449            active_tolerance: self.options.active_tolerance,
450            max_border_fraction: self.options.max_border_fraction,
451        };
452        if let Some(candidate) = scan_boundary_bands(luma, &cfg) {
453            if candidate.reliable {
454                self.stability.observe(candidate.raw, time_us);
455            }
456        }
457        #[cfg(test)]
458        {
459            self.last_probe_count = luma.probe_count();
460        }
461        if !scene_changed {
462            self.stability.maybe_periodic_reset();
463        }
464        let ts = time_us.map(|us| Timestamp {
465            time_us: us,
466            pts: Some(us),
467            time_base: Some((1, 1_000_000)),
468        });
469        self.publish(ts)
470    }
471}
472
473fn resolve_band<L: LumaAccess>(
474    spec: CropLumaThreshold,
475    luma: &L,
476    soft_margin: f32,
477) -> Result<ThresholdBand> {
478    resolve_threshold(spec, luma.bit_depth(), luma.signal_range(), soft_margin)
479        .map_err(Error::InvalidRecipeArg)
480}
481
482#[cfg(test)]
483mod tests;
ez_ffmpeg/core/analysis/crop/mod.rs

ez_ffmpeg/core/analysis/crop/
mod.rs
