ffcine_stabilize/
lib.rs

1//! Video stabilization: optical-flow motion detect on the GPU, camera path smoothing on the CPU,
2//! and a per-frame warp ([`Mat3`], output pixel -> source pixel) for the grade pass.
3//!
4//! 1. `detect` (feature `nvof`): NVDEC decode, `scale_cuda` to an analysis size, NVIDIA
5//!    optical flow between consecutive frames, a robust similarity per frame
6//!    ([`motion::estimate`]). Cached on disk by source and parameters (`detect_cached`).
7//! 2. [`stabilize`]: the camera path, vidstab's Gaussian smoothing, and the static zoom that
8//!    keeps every frame free of black borders. Cheap, so it is recomputed every time.
9//! 3. [`hook::warp_hook`]: the warps as an mpv `.hook` for libplacebo.
10//!
11//! [`gyro`] is the other way to get the motions of step 1, from what the camera recorded;
12//! `detect_motion` chooses between them.
13
14// Numeric code: pixel counts and grid indices convert to f64 and back all over, always in range.
15#![allow(
16    clippy::cast_precision_loss,
17    clippy::cast_possible_truncation,
18    clippy::cast_sign_loss,
19    clippy::many_single_char_names
20)]
21
22pub mod cache;
23// Public for ffcine-render's NVOF denoise motion; the driver API is raw by nature.
24#[cfg(feature = "nvof")]
25#[allow(clippy::missing_safety_doc, clippy::not_unsafe_ptr_arg_deref)]
26pub mod cuda;
27#[cfg(feature = "nvof")]
28mod detect;
29pub mod fx;
30pub mod gyro;
31pub mod hook;
32pub mod motion;
33#[cfg(feature = "nvof")]
34pub mod nvof;
35#[cfg(feature = "telemetry")]
36mod parser;
37pub mod path;
38#[cfg(feature = "nvof")]
39mod read;
40
41#[cfg(feature = "nvof")]
42pub use cache::detect_cached;
43pub use cache::motions_text;
44#[cfg(feature = "nvof")]
45pub use detect::{analysis_size, detect};
46pub use gyro::GyroParams;
47pub use motion::{EstimateParams, Motion};
48pub use path::Zoom;
49
50use std::fmt;
51use std::str::FromStr;
52
53use ffcine_core::Error;
54use ffcine_core::color::named_enum;
55use ffcine_core::geom::Mat3;
56use ffcine_core::probe::Rational;
57
58/// The frames to analyse, indexed like render.py's shots: `-ss start` on the input, then
59/// `setpts=PTS-STARTPTS,fps=rate`, `frames` frames (the last repeats if the source runs out).
60#[derive(Debug, Clone, Copy, PartialEq)]
61pub struct SourceRange {
62    /// Seconds from the start of the file.
63    pub start: f64,
64    pub frames: usize,
65    pub rate: Rational,
66}
67
68named_enum!(
69    /// Optical flow engine speed/quality trade-off (NVOF perf level).
70    #[derive(Default)]
71    FlowQuality {
72        #[default]
73        Slow = "slow",
74        Medium = "medium",
75        Fast = "fast",
76    }
77);
78
79#[derive(Debug, Clone, Copy, PartialEq)]
80pub struct DetectParams {
81    /// Analysis width in pixels (height keeps the aspect, rounded to even). Flow runs at this
82    /// size; motions are rescaled to source pixels.
83    pub analysis_width: u32,
84    pub quality: FlowQuality,
85    /// Flow vector spacing in analysis pixels: 1, 2 or 4.
86    pub grid: u32,
87    pub estimate: EstimateParams,
88}
89
90impl Default for DetectParams {
91    fn default() -> Self {
92        Self {
93            analysis_width: 1920,
94            quality: FlowQuality::Slow,
95            grid: 4,
96            estimate: EstimateParams::default(),
97        }
98    }
99}
100
101/// What detect found: per output frame, the motion from it to the previous frame in source
102/// pixels (`motions[0]` is the identity).
103#[derive(Debug, Clone, PartialEq)]
104pub struct Detection {
105    pub width: u32,
106    pub height: u32,
107    pub analysis_width: u32,
108    pub analysis_height: u32,
109    pub motions: Vec<Motion>,
110}
111
112named_enum!(
113    /// Where a stabilizer's camera motion comes from.
114    #[derive(Default)]
115    MotionSource {
116        /// Gyro if the file has telemetry `auto` trusts, else optical flow.
117        #[default]
118        Auto = "auto",
119        /// The file's gyro or attitude telemetry; an error if it has none.
120        Gyro = "gyro",
121        /// Optical flow.
122        Flow = "flow",
123    }
124);
125
126/// Which motion source to use, and how.
127#[derive(Debug, Clone, Copy, PartialEq, Default)]
128pub struct MotionOptions {
129    pub source: MotionSource,
130    pub gyro: GyroParams,
131}
132
133/// What `detect_motion` found, and how.
134#[derive(Debug, Clone, PartialEq)]
135pub struct MotionDetection {
136    pub detection: Detection,
137    /// [`MotionSource::Gyro`] or [`MotionSource::Flow`], never `Auto`.
138    pub source: MotionSource,
139    /// Why `auto` fell back to flow.
140    pub fallback: Option<String>,
141    /// Read from the cache.
142    pub cached: bool,
143}
144
145/// The camera motion of `range` from the source `options` chooses, through the cache if `cache`.
146/// With [`MotionSource::Auto`], a file with no telemetry it trusts goes to optical flow and says
147/// why in [`MotionDetection::fallback`].
148#[cfg(feature = "nvof")]
149pub fn detect_motion(
150    path: &std::path::Path,
151    range: &SourceRange,
152    params: &DetectParams,
153    options: &MotionOptions,
154    cache: bool,
155) -> ffcine_core::Result<MotionDetection> {
156    let flow = |fallback| -> ffcine_core::Result<MotionDetection> {
157        let (detection, cached) = if cache {
158            detect_cached(path, range, params)?
159        } else {
160            (detect(path, range, params)?, false)
161        };
162        Ok(MotionDetection {
163            detection,
164            source: MotionSource::Flow,
165            fallback,
166            cached,
167        })
168    };
169    if options.source == MotionSource::Flow {
170        return flow(None);
171    }
172    let strict = options.source == MotionSource::Gyro;
173    let gyro = || -> Result<Detection, String> {
174        let (telemetry, size) = read::read(path, range, strict)?;
175        if let Some(why) = telemetry.auto_refusal.filter(|_| !strict) {
176            return Err(why.into());
177        }
178        gyro::detection(&telemetry, range, &options.gyro, size)
179    };
180    let got = if cache {
181        cache::gyro_cached(path, range, options, gyro)?
182    } else {
183        gyro().map(|d| (d, false))
184    };
185    match got {
186        Ok((detection, cached)) => Ok(MotionDetection {
187            detection,
188            source: MotionSource::Gyro,
189            fallback: None,
190            cached,
191        }),
192        Err(why) if strict => Err(Error::Pipeline(format!("gyro: {why}"))),
193        Err(why) => flow(Some(why)),
194    }
195}
196
197/// Smoothing and zoom; the defaults are render.py's (`smoothing` 20, optzoom 1).
198#[derive(Debug, Clone, Copy, PartialEq)]
199pub struct StabilizeParams {
200    /// Gaussian half-window in frames, as vidstab's `smoothing`.
201    pub smoothing: u32,
202    pub zoom: Zoom,
203}
204
205impl Default for StabilizeParams {
206    fn default() -> Self {
207        Self {
208            smoothing: 20,
209            zoom: Zoom::default(),
210        }
211    }
212}
213
214#[derive(Debug, Clone, PartialEq)]
215pub struct Stabilization {
216    /// Per output frame: output pixel -> source pixel, zoom included.
217    pub transforms: Vec<Mat3>,
218    /// The static zoom applied (1.0 = none).
219    pub zoom: f64,
220    /// The zoom needed for no black borders (`INFINITY` if no zoom suffices).
221    pub min_zoom: f64,
222}
223
224/// Smooths the detected camera path and computes the per-frame warps.
225pub fn stabilize(det: &Detection, params: &StabilizeParams) -> Stabilization {
226    let (w, h) = (f64::from(det.width), f64::from(det.height));
227    let centre = [w / 2.0, h / 2.0];
228    let actual = path::camera_path(&det.motions, centre);
229    let smooth = path::gaussian(&actual, params.smoothing);
230    let mut transforms = path::corrections(&actual, &smooth, centre);
231    let min_zoom = path::min_zoom(&transforms, w, h);
232    let zoom = path::choose_zoom(params.zoom, min_zoom);
233    path::apply_zoom(&mut transforms, centre, zoom);
234    Stabilization {
235        transforms,
236        zoom,
237        min_zoom,
238    }
239}
240
241/// Folds a static source crop `(x, y, width, height)` (source pixels) into `transforms`, for a
242/// pass that renders the full `width x height` frame: output pixel `d` of the full-size
243/// intermediate maps to crop position `(x + d.x * cw / w, y + d.y * ch / h)`, then through the
244/// warp. libplacebo only shows the `MAIN` hook the crop, so the grade pass crops this way instead.
245pub fn fold_crop(transforms: &[Mat3], crop: [f64; 4], width: u32, height: u32) -> Vec<Mat3> {
246    let [x, y, cw, ch] = crop;
247    let a = [
248        [cw / f64::from(width), 0.0, x],
249        [0.0, ch / f64::from(height), y],
250        [0.0, 0.0, 1.0],
251    ];
252    transforms
253        .iter()
254        .map(|m| ffcine_core::geom::mul(m, &a))
255        .collect()
256}
257
258/// Warps as text: per frame, the row-major 3x3 matrix (output pixel -> source pixel).
259pub fn transforms_text(transforms: &[Mat3]) -> String {
260    use std::fmt::Write as _;
261    let mut s =
262        String::from("# frame m00 m01 m02 m10 m11 m12 m20 m21 m22; output px -> source px\n");
263    for (k, m) in transforms.iter().enumerate() {
264        let _ = write!(s, "{k}");
265        for v in m.iter().flatten() {
266            let _ = write!(s, " {v}");
267        }
268        s.push('\n');
269    }
270    s
271}
272
273/// High-frequency residual motion of a clip, the stabilization quality metric: RMS of the
274/// frame-to-frame motion minus its Gaussian-smoothed (`sigma` frames) version.
275#[derive(Debug, Clone, Copy, PartialEq)]
276pub struct Jitter {
277    /// Pixels.
278    pub tx: f64,
279    pub ty: f64,
280    /// Degrees.
281    pub angle: f64,
282    /// Frame pairs that counted.
283    pub pairs: usize,
284    /// Their mean motion-fit confidence.
285    pub confidence: f64,
286}
287
288/// [`Jitter`] of `motions` (as from `detect`), skipping the frame pairs ending at the indices
289/// in `exclude` (cuts, punch-ins) in both the smoothing and the RMS.
290pub fn jitter(motions: &[Motion], sigma: f64, exclude: &[usize]) -> Jitter {
291    let keep: Vec<usize> = (1..motions.len())
292        .filter(|i| !exclude.contains(i))
293        .collect();
294    let series = |f: fn(&Motion) -> f64| -> f64 {
295        let reach = (3.0 * sigma).ceil() as usize;
296        let mut sum2 = 0.0;
297        for &i in &keep {
298            let (mut s, mut ws) = (0.0, 0.0);
299            for &j in &keep {
300                if j + reach >= i && j <= i + reach {
301                    let d = j as f64 - i as f64;
302                    let w = (-d * d / (2.0 * sigma * sigma)).exp();
303                    s += w * f(&motions[j]);
304                    ws += w;
305                }
306            }
307            sum2 += (f(&motions[i]) - s / ws).powi(2);
308        }
309        (sum2 / keep.len().max(1) as f64).sqrt()
310    };
311    Jitter {
312        tx: series(|m| m.tx),
313        ty: series(|m| m.ty),
314        angle: series(|m| m.angle.to_degrees()),
315        pairs: keep.len(),
316        confidence: keep.iter().map(|&i| motions[i].confidence).sum::<f64>()
317            / keep.len().max(1) as f64,
318    }
319}
320
321#[cfg(test)]
322mod tests {
323    use super::*;
324    use ffcine_core::geom;
325
326    fn shaky(n: usize) -> Vec<Motion> {
327        (0..n)
328            .map(|i| {
329                if i == 0 {
330                    return Motion::IDENTITY;
331                }
332                let t = i as f64;
333                Motion {
334                    // A slow pan plus shake.
335                    tx: -3.0 + 2.5 * (t * 1.7).sin(),
336                    ty: 1.5 * (t * 2.3).cos(),
337                    angle: 0.002 * (t * 1.1).sin(),
338                    scale: 1.0,
339                    confidence: 1.0,
340                }
341            })
342            .collect()
343    }
344
345    #[test]
346    fn stabilizing_removes_jitter() {
347        let det = Detection {
348            width: 3840,
349            height: 2160,
350            analysis_width: 1920,
351            analysis_height: 1080,
352            motions: shaky(120),
353        };
354        let before = jitter(&det.motions, 5.0, &[]);
355        let st = stabilize(&det, &StabilizeParams::default());
356        assert_eq!(st.transforms.len(), 120);
357        assert!(st.zoom > 1.0 && st.zoom < 1.05, "{st:?}");
358        // The stabilized clip's own motion: what output frame k shows, mapped to frame k-1.
359        let c = [1920.0, 1080.0];
360        let after: Vec<Motion> = (0..120)
361            .map(|k| {
362                if k == 0 {
363                    return Motion::IDENTITY;
364                }
365                let m = det.motions[k].to_mat3(c);
366                let prev = geom::inverse(&st.transforms[k - 1]).unwrap();
367                let rel = geom::mul(&prev, &geom::mul(&m, &st.transforms[k]));
368                let (tx, ty, angle, scale) = geom::decompose_similarity(&rel, c);
369                Motion {
370                    tx,
371                    ty,
372                    angle,
373                    scale,
374                    confidence: 1.0,
375                }
376            })
377            .collect();
378        let after = jitter(&after, 5.0, &[]);
379        assert!(after.tx < before.tx * 0.1, "{before:?} -> {after:?}");
380        assert!(after.ty < before.ty * 0.1, "{before:?} -> {after:?}");
381        assert!(after.angle < before.angle * 0.1, "{before:?} -> {after:?}");
382        assert_eq!(jitter(&det.motions, 5.0, &[50]).pairs, 118);
383    }
384
385    #[cfg(feature = "nvof")]
386    #[test]
387    fn no_gyro_is_an_error_only_when_asked_for() {
388        let dir = std::env::temp_dir().join(format!("ffcine-motion-{}", std::process::id()));
389        std::fs::create_dir_all(&dir).unwrap();
390        let f = dir.join("not-a-video.mp4");
391        std::fs::write(&f, b"not a video").unwrap();
392        let range = SourceRange {
393            start: 0.0,
394            frames: 2,
395            rate: Rational { num: 25, den: 1 },
396        };
397        let run = |source| {
398            let options = MotionOptions {
399                source,
400                ..MotionOptions::default()
401            };
402            detect_motion(&f, &range, &DetectParams::default(), &options, false)
403                .unwrap_err()
404                .to_string()
405        };
406        // Asked for gyro: the gyro path's error. Auto: on to flow, which fails on this file itself.
407        assert!(run(MotionSource::Gyro).contains("gyro: "));
408        assert!(!run(MotionSource::Auto).contains("gyro: "));
409        assert!(!run(MotionSource::Flow).contains("gyro: "));
410        std::fs::remove_dir_all(&dir).unwrap();
411    }
412
413    #[test]
414    fn crop_folds_before_the_warp() {
415        let t = [geom::translation(10.0, -4.0)];
416        let f = fold_crop(&t, [100.0, 50.0, 1920.0, 1080.0], 3840, 2160);
417        // Full-frame output corner (3840, 2160) is crop corner (2020, 1130), then the warp.
418        let p = geom::apply(&f[0], [3840.0, 2160.0]);
419        assert!(
420            (p[0] - 2030.0).abs() < 1e-9 && (p[1] - 1126.0).abs() < 1e-9,
421            "{p:?}"
422        );
423    }
424}
ffcine_stabilize/lib.rs

ffcine_stabilize/
lib.rs
