ffcine_triage/
lib.rs

1//! `ffcine triage`: a searchable clip index from proxies, before anything is rendered.
2//!
3//! Per clip, from the DJI `.LRF` proxy when there is one: a contact strip and scene cuts
4//! ([`scene`]), a speech map ([`vad`]), runaway/dead/dark flags ([`flags`]), and a word-timed
5//! transcript from one batched GPU whisper run ([`transcribe`]). The result is `index.json`
6//! ([`index`]), searchable with [`search`](mod@search), and a static clip page ([`page`]) for picking clips.
7//! [`snap`](mod@snap) moves proposed in/out points onto word boundaries and silence.
8//!
9//! Clips come from a directory scan or a list ([`clips`]) until the `ffcine-ingest` catalog lands;
10//! see `docs/triage.md`.
11
12// Signal and frame maths on sample counts, pixel sizes and seconds.
13#![allow(
14    clippy::many_single_char_names,
15    clippy::cast_precision_loss,
16    clippy::cast_possible_truncation,
17    clippy::cast_sign_loss
18)]
19
20pub mod clips;
21#[cfg(feature = "ffmpeg")]
22pub mod decode;
23pub mod flags;
24pub mod index;
25pub mod location;
26pub mod page;
27pub mod scene;
28pub mod search;
29pub mod snap;
30pub mod strip;
31pub mod transcribe;
32pub mod vad;
33
34pub use index::{Clip, Flag, FlagKind, Index, Span, Transcript, Word, timecode};
35pub use search::{Hit, search};
36pub use snap::{Prefer, SnapParams, snap};
37
38#[derive(Debug, thiserror::Error)]
39pub enum Error {
40    #[error(transparent)]
41    Io(#[from] std::io::Error),
42    #[error(transparent)]
43    Json(#[from] serde_json::Error),
44    #[error("decode: {0}")]
45    Decode(String),
46    #[error("whisper: {0}")]
47    Whisper(String),
48    #[error("{0}")]
49    Invalid(String),
50}
51
52pub type Result<T, E = Error> = std::result::Result<T, E>;
53
54#[cfg(feature = "ffmpeg")]
55pub use run::{Options, Summary, run};
56
57#[cfg(feature = "ffmpeg")]
58mod run {
59    use std::path::{Path, PathBuf};
60    use std::sync::Mutex;
61    use std::sync::atomic::{AtomicUsize, Ordering};
62
63    use serde::{Deserialize, Serialize};
64
65    use crate::clips::{self, ClipFile};
66    use crate::decode::{self, ScanParams};
67    use crate::flags::FlagParams;
68    use crate::index::{self, Clip, Index};
69    use crate::transcribe::{Whisper, WhisperParams};
70    use crate::vad::{self, VadParams};
71    use crate::{Result, page, strip};
72
73    #[derive(Debug, Clone)]
74    pub struct Options {
75        /// Footage directory.
76        pub dir: PathBuf,
77        /// Clip list instead of the catalog or a scan of `dir` (one path per line).
78        pub list: Option<PathBuf>,
79        /// Scan `dir` even when it has an ingest `catalog.json`.
80        pub no_catalog: bool,
81        /// Output directory (index.json, index.html, strips/, clips/).
82        pub out: PathBuf,
83        /// Clips analysed at once.
84        pub jobs: usize,
85        pub scan: ScanParams,
86        pub vad: VadParams,
87        pub flags: FlagParams,
88        /// Look up clip locations in Immich.
89        pub immich: Option<crate::location::ImmichParams>,
90        /// `None` skips transcription.
91        pub whisper: Option<WhisperParams>,
92        /// Clips with less VAD speech than this (seconds) aren't transcribed.
93        pub min_speech: f64,
94        /// Re-analyse clips even when the cache matches.
95        pub force: bool,
96        /// Page title.
97        pub title: String,
98    }
99
100    #[derive(Debug, Default)]
101    pub struct Summary {
102        pub clips: usize,
103        pub analysed: usize,
104        pub transcribed: usize,
105        pub failed: Vec<(String, String)>,
106        pub index: PathBuf,
107        pub page: PathBuf,
108    }
109
110    #[derive(Serialize, Deserialize)]
111    struct Cached {
112        version: u32,
113        clip: Clip,
114    }
115
116    fn mtime(p: &Path) -> Result<(u64, i64)> {
117        let m = std::fs::metadata(p)?;
118        let t = m
119            .modified()?
120            .duration_since(std::time::UNIX_EPOCH)
121            .map_or(0, |d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX));
122        Ok((m.len(), t))
123    }
124
125    fn load_cached(path: &Path, f: &ClipFile, size: u64, mt: i64) -> Option<Clip> {
126        let c: Cached = serde_json::from_str(&std::fs::read_to_string(path).ok()?).ok()?;
127        (c.version == index::VERSION
128            && c.clip.size == size
129            && c.clip.mtime == mt
130            && c.clip.analysed == f.analysed())
131        .then_some(c.clip)
132    }
133
134    fn save_cached(path: &Path, clip: &Clip) -> Result<()> {
135        index::write_json(
136            path,
137            &Cached {
138                version: index::VERSION,
139                clip: clip.clone(),
140            },
141        )
142    }
143
144    fn analyse(f: &ClipFile, o: &Options, pcm: Option<&Path>, size: u64, mt: i64) -> Result<Clip> {
145        let analysed = f.analysed();
146        let use_gpu = o.scan.gpu && f.proxy.is_none();
147        let v = decode::scan_video(analysed, use_gpu, &o.scan)?;
148        let (width, height, fps) = if f.proxy.is_some() {
149            decode::probe_video(&f.source).unwrap_or((v.width, v.height, v.fps))
150        } else {
151            (v.width, v.height, v.fps)
152        };
153        let strip_name = format!("strips/{}.jpg", f.id);
154        let frames: Vec<strip::Frame<'_>> = v
155            .thumbs
156            .iter()
157            .map(|t| strip::Frame {
158                w: t.w,
159                h: t.h,
160                rgb: &t.rgb,
161            })
162            .collect();
163        let strip = if frames.is_empty() {
164            None
165        } else {
166            strip::write_strip(&o.out.join(&strip_name), &frames)?;
167            Some(strip_name)
168        };
169        let audio = decode::scan_audio(analysed, pcm)?;
170        let recorded = f
171            .local
172            .clone()
173            .or_else(|| clips::dji_name(&f.id).map(|(t, _)| t));
174        Ok(Clip {
175            id: f.id.clone(),
176            source: f.source.clone(),
177            analysed: analysed.to_owned(),
178            size,
179            mtime: mt,
180            duration: (v.duration * 1000.0).round() / 1000.0,
181            fps: (fps * 1000.0).round() / 1000.0,
182            width: Some(width),
183            height: Some(height),
184            recorded,
185            utc: f.utc.clone(),
186            take: None,
187            strip,
188            strip_times: v
189                .thumbs
190                .iter()
191                .map(|t| (t.t * 100.0).round() / 100.0)
192                .collect(),
193            cuts: v.cuts,
194            activity: v.activity,
195            luma: v.luma,
196            has_audio: audio.is_some(),
197            speech: audio
198                .map(|a| vad::speech(&a.db, &o.vad))
199                .unwrap_or_default(),
200            transcript: None,
201            flags: Vec::new(),
202            location: None,
203        })
204    }
205
206    #[allow(clippy::too_many_lines)] // the run, top to bottom
207    pub fn run(o: &Options) -> Result<Summary> {
208        decode::init()?;
209        let catalog = if o.list.is_none() && !o.no_catalog {
210            clips::from_catalog(&o.dir)?
211        } else {
212            None
213        };
214        let files = match (&o.list, catalog) {
215            (Some(l), _) => clips::from_list(l)?,
216            (None, Some(c)) => {
217                eprintln!("{} clips from the ingest catalog", c.len());
218                c
219            }
220            (None, None) => clips::discover(&o.dir, &[o.out.as_path()])?,
221        };
222        let cache_dir = o.out.join("clips");
223        let scratch = o.out.join(".scratch");
224        for d in [&o.out, &o.out.join("strips"), &cache_dir, &scratch] {
225            std::fs::create_dir_all(d)?;
226        }
227        let n = files.len();
228        let slots: Vec<Mutex<Option<Clip>>> = (0..n).map(|_| Mutex::new(None)).collect();
229        let failed = Mutex::new(Vec::new());
230        let next = AtomicUsize::new(0);
231        let done = AtomicUsize::new(0);
232        let analysed = AtomicUsize::new(0);
233        let pcm_path = |id: &str| scratch.join(format!("{id}.f32"));
234        let want_pcm = o.whisper.is_some();
235
236        std::thread::scope(|s| {
237            for _ in 0..o.jobs.max(1) {
238                s.spawn(|| {
239                    loop {
240                        let i = next.fetch_add(1, Ordering::Relaxed);
241                        let Some(f) = files.get(i) else { break };
242                        let cache = cache_dir.join(format!("{}.json", f.id));
243                        let r = mtime(&f.source).and_then(|(size, mt)| {
244                            if !o.force
245                                && let Some(c) = load_cached(&cache, f, size, mt)
246                            {
247                                return Ok(c);
248                            }
249                            let pcm = want_pcm.then(|| pcm_path(&f.id));
250                            let c = analyse(f, o, pcm.as_deref(), size, mt)?;
251                            save_cached(&cache, &c)?;
252                            analysed.fetch_add(1, Ordering::Relaxed);
253                            Ok(c)
254                        });
255                        let k = done.fetch_add(1, Ordering::Relaxed) + 1;
256                        match r {
257                            Ok(c) => {
258                                eprintln!(
259                                    "[{k}/{n}] {} {} cuts {} speech {:.1} s",
260                                    c.id,
261                                    index::timecode(c.duration),
262                                    c.cuts.len(),
263                                    c.speech_seconds()
264                                );
265                                if let Ok(mut slot) = slots[i].lock() {
266                                    *slot = Some(c);
267                                }
268                            }
269                            Err(e) => {
270                                eprintln!("[{k}/{n}] {}: {e}", f.id);
271                                if let Ok(mut fl) = failed.lock() {
272                                    fl.push((f.id.clone(), e.to_string()));
273                                }
274                            }
275                        }
276                    }
277                });
278            }
279        });
280
281        let mut clips: Vec<Clip> = slots
282            .into_iter()
283            .filter_map(|m| m.into_inner().ok().flatten())
284            .collect();
285        let mut failed = failed.into_inner().unwrap_or_default();
286
287        // Takes and flags are cheap and depend on the whole set: always recomputed.
288        let keys: Vec<_> = clips
289            .iter()
290            .map(|c| (c.id.clone(), c.recorded.clone(), c.duration))
291            .collect();
292        let takes = clips::takes(&keys, 3.0);
293        for (c, t) in clips.iter_mut().zip(&takes) {
294            c.take =
295                (*t != c.id || takes.iter().filter(|x| *x == t).count() > 1).then(|| t.clone());
296        }
297        let take_len = |t: &Option<String>, own: f64| {
298            t.as_ref().map_or(own, |t| {
299                clips
300                    .iter()
301                    .filter(|c| c.take.as_ref() == Some(t))
302                    .map(|c| c.duration)
303                    .sum()
304            })
305        };
306        let lens: Vec<f64> = clips
307            .iter()
308            .map(|c| take_len(&c.take, c.duration))
309            .collect();
310        for (c, len) in clips.iter_mut().zip(lens) {
311            c.flags = crate::flags::flags(c, len, &o.flags);
312        }
313
314        if let Some(ip) = &o.immich {
315            let before: Vec<bool> = clips.iter().map(|c| c.location.is_some()).collect();
316            match crate::location::locate(&mut clips, ip) {
317                Ok(n) => eprintln!("located {n} clips"),
318                Err(e) => eprintln!("no locations: {e}"),
319            }
320            for (c, had) in clips.iter().zip(before) {
321                if !had && c.location.is_some() {
322                    save_cached(&cache_dir.join(format!("{}.json", c.id)), c)?;
323                }
324            }
325        }
326
327        let mut transcribed = 0;
328        if let Some(wp) = &o.whisper {
329            let todo: Vec<usize> = (0..clips.len())
330                .filter(|&i| {
331                    clips[i].transcript.is_none() && clips[i].speech_seconds() >= o.min_speech
332                })
333                .collect();
334            if !todo.is_empty() {
335                eprintln!(
336                    "transcribing {} clips ({}, {})",
337                    todo.len(),
338                    wp.model,
339                    wp.device
340                );
341                match Whisper::start(wp, &scratch) {
342                    Ok(mut w) => {
343                        for (k, &i) in todo.iter().enumerate() {
344                            let c = &mut clips[i];
345                            let pcm = pcm_path(&c.id);
346                            let r = (if pcm.is_file() {
347                                Ok(())
348                            } else {
349                                decode::scan_audio(&c.analysed, Some(&pcm)).map(drop)
350                            })
351                            .and_then(|()| w.transcribe(&c.id, &pcm));
352                            match r {
353                                Ok(t) => {
354                                    eprintln!(
355                                        "[{}/{}] {} {}: {} words",
356                                        k + 1,
357                                        todo.len(),
358                                        c.id,
359                                        t.language,
360                                        t.words().count()
361                                    );
362                                    c.transcript = Some(t);
363                                    save_cached(&cache_dir.join(format!("{}.json", c.id)), c)?;
364                                    transcribed += 1;
365                                    let _ = std::fs::remove_file(&pcm);
366                                }
367                                Err(e) => {
368                                    eprintln!("{}: {e}", c.id);
369                                    failed.push((c.id.clone(), e.to_string()));
370                                    if matches!(e, crate::Error::Whisper(ref m) if m.starts_with("worker exited"))
371                                    {
372                                        break;
373                                    }
374                                }
375                            }
376                        }
377                    }
378                    Err(e) => {
379                        eprintln!("no transcripts: {e}");
380                        failed.push(("whisper".into(), e.to_string()));
381                    }
382                }
383            }
384        }
385        let _ = std::fs::remove_dir_all(&scratch);
386
387        let index = Index {
388            version: index::VERSION,
389            root: o.dir.clone(),
390            clips,
391        };
392        let index_path = o.out.join("index.json");
393        index.save(&index_path)?;
394        let page_path = o.out.join("index.html");
395        std::fs::write(&page_path, page::render(&index, &o.title)?)?;
396        Ok(Summary {
397            clips: index.clips.len(),
398            analysed: analysed.into_inner(),
399            transcribed,
400            failed,
401            index: index_path,
402            page: page_path,
403        })
404    }
405}
ffcine_triage/lib.rs

ffcine_triage/
lib.rs
