ffcine_breakdown/
lib.rs

1//! `ffcine breakdown`: the shot list of a reference video (a reel, a trailer), for recreating it.
2//!
3//! One decode at analysis size (NVDEC with its hardware resize when it takes the clip, else the
4//! CPU) feeds the scene detector of `ffcine triage` frame by frame, the camera motion estimate
5//! ([`motion`]), per-frame sharpness and colour ([`colour`]). From the cuts come the shots; a
6//! second decode at keyframe size writes each shot's first, sharpest-middle and last frame. The
7//! soundtrack's beats ([`ffcine_audio::beats`]) say whether the cuts land on them ([`rhythm`]),
8//! batched faster-whisper (as triage) transcribes the speech, and `tesseract`, when on `PATH`,
9//! reads on-screen text off the keyframes. Out come `breakdown.json`, `contact-sheet.jpg`
10//! ([`sheet`]) and `shotlist.md` ([`shotlist`]); see `docs/breakdown.md`.
11
12// Frame maths on pixel counts and seconds.
13#![allow(
14    clippy::many_single_char_names,
15    clippy::cast_precision_loss,
16    clippy::cast_possible_truncation,
17    clippy::cast_sign_loss
18)]
19
20pub mod colour;
21#[cfg(feature = "ffmpeg")]
22mod decode;
23pub mod motion;
24pub mod ocr;
25pub mod rhythm;
26#[cfg(feature = "ffmpeg")]
27mod run;
28pub mod sheet;
29pub mod shotlist;
30
31use std::path::{Path, PathBuf};
32
33use serde::{Deserialize, Serialize};
34
35pub use colour::Colour;
36pub use ffcine_triage::Transcript;
37pub use motion::{Motion, MotionKind, MotionParams};
38pub use rhythm::Music;
39#[cfg(feature = "ffmpeg")]
40pub use run::{Options, run};
41
42/// `breakdown.json` format version.
43pub const VERSION: u32 = 1;
44
45#[derive(Debug, thiserror::Error)]
46pub enum Error {
47    #[error(transparent)]
48    Io(#[from] std::io::Error),
49    #[error(transparent)]
50    Json(#[from] serde_json::Error),
51    #[error("decode: {0}")]
52    Decode(String),
53    #[error("{0}")]
54    Invalid(String),
55}
56
57impl From<ffcine_triage::Error> for Error {
58    fn from(e: ffcine_triage::Error) -> Self {
59        match e {
60            ffcine_triage::Error::Io(e) => Self::Io(e),
61            ffcine_triage::Error::Json(e) => Self::Json(e),
62            ffcine_triage::Error::Decode(m) => Self::Decode(m),
63            e => Self::Invalid(e.to_string()),
64        }
65    }
66}
67
68pub type Result<T, E = Error> = std::result::Result<T, E>;
69
70/// A shot's stills, relative to the output directory.
71#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
72pub struct Keyframes {
73    /// The representative frame: the sharpest of the middle of the shot.
74    pub key: String,
75    /// Its time, seconds.
76    pub key_time: f64,
77    pub first: String,
78    pub last: String,
79}
80
81/// One shot, between two cuts.
82#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
83pub struct Shot {
84    /// 1-based.
85    pub n: usize,
86    /// In and out, seconds (out is the next shot's in).
87    pub start: f64,
88    pub end: f64,
89    pub duration: f64,
90    /// First frame and one past the last, counted from 0.
91    pub frames: [usize; 2],
92    pub keyframes: Keyframes,
93    pub motion: Motion,
94    pub colour: Colour,
95    /// Whether the cut into this shot is on a beat (none for the first shot or without music).
96    #[serde(skip_serializing_if = "Option::is_none")]
97    pub cut_on_beat: Option<bool>,
98    /// Nearest beat minus the cut, seconds.
99    #[serde(skip_serializing_if = "Option::is_none")]
100    pub beat_offset: Option<f64>,
101    /// Length in beats.
102    #[serde(skip_serializing_if = "Option::is_none")]
103    pub beats: Option<f64>,
104    /// Words spoken in the shot.
105    #[serde(skip_serializing_if = "String::is_empty", default)]
106    pub speech: String,
107    /// On-screen text read off the keyframe.
108    #[serde(skip_serializing_if = "String::is_empty", default)]
109    pub text: String,
110}
111
112/// Shot lengths over the whole video.
113#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
114pub struct Pacing {
115    pub shots: usize,
116    /// Seconds.
117    pub mean: f64,
118    pub median: f64,
119    pub shortest: f64,
120    pub longest: f64,
121    pub cuts_per_minute: f64,
122}
123
124/// `breakdown.json`.
125#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
126pub struct Breakdown {
127    pub version: u32,
128    pub source: PathBuf,
129    /// Seconds.
130    pub duration: f64,
131    pub fps: f64,
132    pub width: u32,
133    pub height: u32,
134    pub frames: usize,
135    /// `nvdec` or `cpu`.
136    pub decoder: String,
137    pub pacing: Pacing,
138    /// Cut times, seconds.
139    pub cuts: Vec<f64>,
140    pub shots: Vec<Shot>,
141    /// None without audio or with `--no-beats`.
142    pub music: Option<Music>,
143    /// None without speech or with `--no-transcribe`.
144    pub transcript: Option<Transcript>,
145    /// Contact sheet pages, relative to the output directory (empty if none was written).
146    pub contact_sheets: Vec<String>,
147    pub shotlist: String,
148}
149
150impl Breakdown {
151    pub fn load(path: &Path) -> Result<Self> {
152        Ok(serde_json::from_str(&std::fs::read_to_string(path)?)?)
153    }
154}
155
156/// A shot's frame range and its chosen stills, before the analysis is attached.
157#[derive(Debug, Clone, Copy, PartialEq, Eq)]
158pub struct Span {
159    /// First frame and one past the last.
160    pub a: usize,
161    pub b: usize,
162    pub first: usize,
163    pub key: usize,
164    pub last: usize,
165}
166
167/// Splits `n` frames at `cuts` (frame indices where a shot starts, ascending) and picks each
168/// shot's stills: one frame in from each end (so a soft cut doesn't leak in), and the sharpest
169/// frame of the middle 60 %, favouring the centre.
170pub fn spans(n: usize, cuts: &[usize], sharpness: &[f32]) -> Vec<Span> {
171    let mut starts = vec![0];
172    starts.extend(cuts.iter().copied().filter(|&c| c > 0 && c < n));
173    starts.dedup();
174    let mut out = Vec::with_capacity(starts.len());
175    for (i, &a) in starts.iter().enumerate() {
176        let b = starts.get(i + 1).copied().unwrap_or(n);
177        if b <= a {
178            continue;
179        }
180        let len = b - a;
181        let inset = usize::from(len > 4);
182        let (lo, hi) = (a + len / 5, (a + len * 4 / 5).max(a + len / 5 + 1).min(b));
183        let key = (lo..hi)
184            .max_by(|&x, &y| {
185                let score = |f: usize| {
186                    let rel = (f - a) as f32 / len as f32;
187                    sharpness.get(f).copied().unwrap_or(0.0) * (1.0 - (rel - 0.5).abs())
188                };
189                score(x).total_cmp(&score(y))
190            })
191            .unwrap_or(a + len / 2);
192        out.push(Span {
193            a,
194            b,
195            first: a + inset,
196            key,
197            last: b - 1 - inset,
198        });
199    }
200    out
201}
202
203/// Median of sorted values: the mean of the middle two for an even count.
204fn median(sorted: &[f64]) -> f64 {
205    let n = sorted.len();
206    match n {
207        0 => 0.0,
208        _ if n.is_multiple_of(2) => f64::midpoint(sorted[n / 2 - 1], sorted[n / 2]),
209        _ => sorted[n / 2],
210    }
211}
212
213/// Shot-length statistics.
214pub fn pacing(shots: &[Shot], duration: f64) -> Pacing {
215    let mut d: Vec<f64> = shots.iter().map(|s| s.duration).collect();
216    d.sort_by(f64::total_cmp);
217    let r2 = |v: f64| (v * 100.0).round() / 100.0;
218    let n = d.len();
219    Pacing {
220        shots: n,
221        mean: r2(if n == 0 {
222            0.0
223        } else {
224            d.iter().sum::<f64>() / n as f64
225        }),
226        median: r2(median(&d)),
227        shortest: r2(d.first().copied().unwrap_or(0.0)),
228        longest: r2(d.last().copied().unwrap_or(0.0)),
229        cuts_per_minute: r2(if duration > 0.0 {
230            n.saturating_sub(1) as f64 * 60.0 / duration
231        } else {
232            0.0
233        }),
234    }
235}
236
237/// The words of `t` whose middle falls in `[start, end)`, with the text of segments that have
238/// no word times, in time order.
239pub fn words_in(t: &Transcript, start: f64, end: f64) -> String {
240    let inside = |a: f64, b: f64| (start..end).contains(&f64::midpoint(a, b));
241    let mut parts: Vec<(f64, &str)> = Vec::new();
242    for seg in &t.segments {
243        if seg.words.is_empty() {
244            if inside(seg.start, seg.end) {
245                parts.push((seg.start, seg.text.trim()));
246            }
247        } else {
248            parts.extend(
249                seg.words
250                    .iter()
251                    .filter(|w| inside(w.start, w.end))
252                    .map(|w| (w.start, w.text.trim())),
253            );
254        }
255    }
256    parts.sort_by(|a, b| a.0.total_cmp(&b.0));
257    parts
258        .iter()
259        .map(|p| p.1)
260        .filter(|s| !s.is_empty())
261        .collect::<Vec<_>>()
262        .join(" ")
263}
264
265/// `m:ss.cc`.
266pub fn clock(t: f64) -> String {
267    let cs = (t.max(0.0) * 100.0).round() as u64;
268    format!("{}:{:02}.{:02}", cs / 6000, cs / 100 % 60, cs % 100)
269}
270
271#[cfg(test)]
272mod tests {
273    use super::*;
274
275    #[test]
276    fn splits_at_cuts() {
277        let mut sharp = vec![1.0; 30];
278        sharp[14] = 9.0;
279        let s = spans(30, &[0, 10, 20, 20], &sharp);
280        assert_eq!(s.len(), 3);
281        assert_eq!((s[0].a, s[0].b, s[0].first, s[0].last), (0, 10, 1, 8));
282        assert_eq!(s[1].key, 14);
283        assert_eq!(s[2].b, 30);
284        let one = spans(3, &[], &[]);
285        assert_eq!((one[0].first, one[0].key, one[0].last), (0, 1, 2));
286        let tiny = spans(1, &[], &[]);
287        assert_eq!((tiny[0].first, tiny[0].key, tiny[0].last), (0, 0, 0));
288    }
289
290    #[test]
291    fn clocks() {
292        assert_eq!(clock(0.0), "0:00.00");
293        assert_eq!(clock(83.456), "1:23.46");
294    }
295
296    #[test]
297    fn even_median() {
298        assert!((median(&[1.0, 3.0]) - 2.0).abs() < 1e-9);
299        assert!((median(&[1.0, 2.0, 9.0]) - 2.0).abs() < 1e-9);
300        assert!(median(&[]).abs() < 1e-9);
301    }
302
303    #[test]
304    fn speech_keeps_untimed_segments() {
305        use ffcine_triage::index::{Segment, Word};
306        let w = |s: f64, t: &str| Word {
307            start: s,
308            end: s + 0.2,
309            text: format!(" {t}"),
310            prob: 1.0,
311        };
312        let t = Transcript {
313            language: "en".into(),
314            model: "m".into(),
315            segments: vec![
316                Segment {
317                    start: 0.0,
318                    end: 0.5,
319                    text: " a b".into(),
320                    words: vec![w(0.0, "a"), w(0.3, "b")],
321                },
322                Segment {
323                    start: 0.6,
324                    end: 0.9,
325                    text: " untimed".into(),
326                    words: vec![],
327                },
328            ],
329        };
330        assert_eq!(words_in(&t, 0.0, 1.0), "a b untimed");
331        assert_eq!(words_in(&t, 0.25, 1.0), "b untimed");
332    }
333}
ffcine_breakdown/lib.rs

ffcine_breakdown/
lib.rs
