Skip to main content

video_analysis_test_support/
lib.rs

1//! Public API for the video-analysis-test-support crate.
2
3pub mod surface;
4use std::fs;
5use std::path::{Path, PathBuf};
6use std::process::{Command, Stdio};
7
8use num_rational::Rational64;
9use video_analysis_core::{
10    AnalysisEvent, FramePosition, Observation, ObservationKind, OwnedTextSegment, OwnedVideoFrame,
11    PixelFormat, Scene, SceneDetector, ScenePipeline, TextSegment, Timebase, Timestamp,
12};
13use video_analysis_dataset::{
14    AnalysisDataset, DatasetRecord, FeatureRecord, FeatureValue, SceneRecord, TextSegmentRecord,
15};
16use video_analysis_radiance_fields::{CameraModel, ColorRgb, Vec2, Vec3};
17use video_analysis_radiance_io::{
18    ColmapCamera, ColmapDataset, ColmapImage, ColmapPoint2d, ColmapPoint3d, ColmapTrackElement,
19    NerfstudioFrame, NerfstudioTransforms,
20};
21
22/// Constant for me at the zoo wikimedia URL.
23pub const ME_AT_THE_ZOO_WIKIMEDIA_URL: &str =
24    "https://upload.wikimedia.org/wikipedia/commons/e/e0/Me_at_the_zoo.webm";
25
26/// Returns timestamp.
27pub fn timestamp(seconds: i64) -> Timestamp {
28    Timestamp::new(seconds, Timebase::new(1, 1))
29}
30
31/// Returns timestamp at.
32pub fn timestamp_at(pts: i64, num: i32, den: i32) -> Timestamp {
33    Timestamp::new(pts, Timebase::new(num, den))
34}
35
36/// Returns frame position.
37pub fn frame_position(frame_index: u64) -> FramePosition {
38    FramePosition {
39        frame_index,
40        timestamp: timestamp(frame_index as i64),
41    }
42}
43
44/// Returns frame position at rate.
45pub fn frame_position_at_rate(frame_index: u64, frame_rate: i64) -> FramePosition {
46    FramePosition::from_frame_index(frame_index, Rational64::new(frame_rate, 1))
47}
48
49/// Returns scene.
50pub fn scene(start: u64, end: u64) -> Scene {
51    Scene {
52        start: frame_position(start),
53        end: frame_position(end),
54    }
55}
56
57/// Returns RGB frame.
58pub fn rgb_frame(width: u32, height: u32, frame_index: u64, rgb: [u8; 3]) -> OwnedVideoFrame {
59    let mut data = Vec::with_capacity(width as usize * height as usize * 3);
60    for _ in 0..(width as usize * height as usize) {
61        data.extend_from_slice(&rgb);
62    }
63    OwnedVideoFrame {
64        position: frame_position(frame_index),
65        width,
66        height,
67        pixel_format: PixelFormat::Rgb24,
68        stride: width as usize * 3,
69        data,
70    }
71}
72
73/// Returns checkerboard frame.
74pub fn checkerboard_frame(width: u32, height: u32, frame_index: u64) -> OwnedVideoFrame {
75    let mut data = Vec::with_capacity(width as usize * height as usize * 3);
76    for y in 0..height {
77        for x in 0..width {
78            let value = if (x + y) % 2 == 0 { 255 } else { 0 };
79            data.extend_from_slice(&[value, value, value]);
80        }
81    }
82    OwnedVideoFrame {
83        position: frame_position(frame_index),
84        width,
85        height,
86        pixel_format: PixelFormat::Rgb24,
87        stride: width as usize * 3,
88        data,
89    }
90}
91
92/// Specification for deterministic synthetic video fixtures.
93#[derive(Debug, Clone)]
94pub struct SyntheticVideoSpec {
95    /// Frames per second.
96    pub fps: Rational64,
97    /// Frame width in pixels.
98    pub width: u32,
99    /// Frame height in pixels.
100    pub height: u32,
101    /// Number of frames to generate.
102    pub frame_count: u64,
103    /// Ordered scene patterns. Later patterns override earlier pixels.
104    pub patterns: Vec<ScenePattern>,
105}
106
107impl SyntheticVideoSpec {
108    /// Creates a new synthetic video spec with a dark base frame.
109    pub fn new(width: u32, height: u32, frame_count: u64, fps: Rational64) -> Self {
110        Self {
111            fps,
112            width,
113            height,
114            frame_count,
115            patterns: vec![ScenePattern::SolidColor {
116                start_frame: 0,
117                end_frame: frame_count,
118                rgb: [32, 32, 32],
119            }],
120        }
121    }
122
123    /// Adds a pattern to this spec.
124    pub fn pattern(mut self, pattern: ScenePattern) -> Self {
125        self.patterns.push(pattern);
126        self
127    }
128}
129
130/// Deterministic frame patterns for scene-detection tests and benchmarks.
131#[derive(Debug, Clone)]
132pub enum ScenePattern {
133    /// Fills a frame range with one color.
134    SolidColor {
135        /// Inclusive start frame.
136        start_frame: u64,
137        /// Exclusive end frame.
138        end_frame: u64,
139        /// RGB color.
140        rgb: [u8; 3],
141    },
142    /// Applies a two-color hard cut at `at_frame`.
143    HardCut {
144        /// Cut frame.
145        at_frame: u64,
146        /// Color before the cut.
147        before: [u8; 3],
148        /// Color from the cut onward.
149        after: [u8; 3],
150    },
151    /// Applies a temporary flash.
152    Flash {
153        /// Inclusive start frame.
154        start_frame: u64,
155        /// Flash length in frames.
156        length: u64,
157        /// Flash color.
158        rgb: [u8; 3],
159    },
160    /// Creates a fade out/in around the darkest midpoint.
161    FadeOutIn {
162        /// Inclusive start frame.
163        start_frame: u64,
164        /// Exclusive end frame.
165        end_frame: u64,
166        /// Bright scene color.
167        high: u8,
168        /// Dark fade color.
169        low: u8,
170    },
171    /// Applies a luminance ramp.
172    LuminanceRamp {
173        /// Inclusive start frame.
174        start_frame: u64,
175        /// Exclusive end frame.
176        end_frame: u64,
177        /// Starting luma.
178        from: u8,
179        /// Ending luma.
180        to: u8,
181    },
182    /// Applies a deterministic stripe pattern with shifted luma distribution.
183    HistogramShift {
184        /// Inclusive start frame.
185        start_frame: u64,
186        /// Exclusive end frame.
187        end_frame: u64,
188    },
189    /// Applies a deterministic checker texture offset that changes perceptual hashes.
190    TextureShift {
191        /// Inclusive start frame.
192        start_frame: u64,
193        /// Exclusive end frame.
194        end_frame: u64,
195        /// Checker offset.
196        offset: u32,
197    },
198}
199
200/// Generates deterministic synthetic video frames.
201pub fn synthetic_frames(spec: &SyntheticVideoSpec) -> Vec<OwnedVideoFrame> {
202    (0..spec.frame_count)
203        .map(|frame_index| {
204            let mut frame =
205                rgb_frame_at_rate(spec.width, spec.height, frame_index, spec.fps, [32, 32, 32]);
206            for pattern in &spec.patterns {
207                apply_pattern(&mut frame, spec.frame_count, pattern);
208            }
209            frame
210        })
211        .collect()
212}
213
214/// Runs a scene detector on pre-generated frames using the public scene pipeline.
215pub fn run_detector_on_frames<D: SceneDetector + 'static>(
216    detector: D,
217    frames: &[OwnedVideoFrame],
218) -> video_analysis_core::Result<video_analysis_core::DetectionResult> {
219    let mut pipeline = ScenePipeline::builder()
220        .detector(detector)
221        .start_in_scene(true)
222        .build()?;
223    for frame in frames {
224        pipeline.process_frame(frame.clone())?;
225    }
226    pipeline.finish_detection()
227}
228
229/// Asserts scene start boundaries by frame index.
230pub fn assert_scene_boundaries(
231    result: &video_analysis_core::DetectionResult,
232    expected_start_frames: &[u64],
233) {
234    let actual = result
235        .scenes
236        .iter()
237        .map(|scene| scene.start.frame_index)
238        .collect::<Vec<_>>();
239    assert_eq!(actual, expected_start_frames);
240}
241
242/// Asserts a metric row contains a key.
243pub fn assert_metric_present(metrics: &video_analysis_core::MetricsStore, frame: u64, key: &str) {
244    let row = metrics
245        .rows()
246        .get(&frame)
247        .unwrap_or_else(|| panic!("missing metric row for frame {frame}"));
248    assert!(
249        row.contains_key(key),
250        "missing metric key `{key}` for frame {frame}; row keys: {:?}",
251        row.keys().collect::<Vec<_>>()
252    );
253}
254
255fn rgb_frame_at_rate(
256    width: u32,
257    height: u32,
258    frame_index: u64,
259    fps: Rational64,
260    rgb: [u8; 3],
261) -> OwnedVideoFrame {
262    let mut frame = rgb_frame(width, height, frame_index, rgb);
263    frame.position = FramePosition::from_frame_index(frame_index, fps);
264    frame
265}
266
267fn apply_pattern(frame: &mut OwnedVideoFrame, frame_count: u64, pattern: &ScenePattern) {
268    let frame_index = frame.position.frame_index;
269    match *pattern {
270        ScenePattern::SolidColor {
271            start_frame,
272            end_frame,
273            rgb,
274        } if in_range(frame_index, start_frame, end_frame) => fill_frame(frame, rgb),
275        ScenePattern::HardCut {
276            at_frame,
277            before,
278            after,
279        } => {
280            if frame_index < at_frame {
281                fill_frame(frame, before);
282            } else if frame_index < frame_count {
283                fill_frame(frame, after);
284            }
285        }
286        ScenePattern::Flash {
287            start_frame,
288            length,
289            rgb,
290        } if in_range(frame_index, start_frame, start_frame.saturating_add(length)) => {
291            fill_frame(frame, rgb);
292        }
293        ScenePattern::FadeOutIn {
294            start_frame,
295            end_frame,
296            high,
297            low,
298        } if in_range(frame_index, start_frame, end_frame) => {
299            let len = end_frame.saturating_sub(start_frame).max(1);
300            let local = frame_index.saturating_sub(start_frame);
301            let midpoint = len / 2;
302            let edge_weight = if local <= midpoint {
303                1.0 - (local as f32 / midpoint.max(1) as f32)
304            } else {
305                (local.saturating_sub(midpoint)) as f32 / (len - midpoint).max(1) as f32
306            };
307            let value = low as f32 + (high as f32 - low as f32) * edge_weight.clamp(0.0, 1.0);
308            fill_frame(frame, [value as u8, value as u8, value as u8]);
309        }
310        ScenePattern::LuminanceRamp {
311            start_frame,
312            end_frame,
313            from,
314            to,
315        } if in_range(frame_index, start_frame, end_frame) => {
316            let len = end_frame.saturating_sub(start_frame).max(1);
317            let local = frame_index.saturating_sub(start_frame);
318            let t = local as f32 / len as f32;
319            let value = from as f32 + (to as f32 - from as f32) * t;
320            fill_frame(frame, [value as u8, value as u8, value as u8]);
321        }
322        ScenePattern::HistogramShift {
323            start_frame,
324            end_frame,
325        } if in_range(frame_index, start_frame, end_frame) => {
326            for y in 0..frame.height {
327                for x in 0..frame.width {
328                    let value = if (x / 4 + y / 4) % 3 == 0 { 16 } else { 236 };
329                    set_pixel(frame, x, y, [value, value, value]);
330                }
331            }
332        }
333        ScenePattern::TextureShift {
334            start_frame,
335            end_frame,
336            offset,
337        } if in_range(frame_index, start_frame, end_frame) => {
338            for y in 0..frame.height {
339                for x in 0..frame.width {
340                    let value = ((x * 5 + y * 3 + offset * 17) % 256) as u8;
341                    set_pixel(frame, x, y, [value, value, value]);
342                }
343            }
344        }
345        _ => {}
346    }
347}
348
349fn in_range(frame_index: u64, start_frame: u64, end_frame: u64) -> bool {
350    frame_index >= start_frame && frame_index < end_frame
351}
352
353fn fill_frame(frame: &mut OwnedVideoFrame, rgb: [u8; 3]) {
354    for pixel in frame.data.chunks_exact_mut(3) {
355        pixel.copy_from_slice(&rgb);
356    }
357}
358
359fn set_pixel(frame: &mut OwnedVideoFrame, x: u32, y: u32, rgb: [u8; 3]) {
360    let offset = y as usize * frame.stride + x as usize * 3;
361    frame.data[offset..offset + 3].copy_from_slice(&rgb);
362}
363
364/// Returns owned text segment.
365pub fn owned_text_segment(index: u64, text: impl Into<String>) -> OwnedTextSegment {
366    OwnedTextSegment::new(index, text).timestamp(timestamp(index as i64))
367}
368
369/// Returns text segment.
370pub fn text_segment(index: u64, text: &str) -> TextSegment<'_> {
371    TextSegment {
372        segment_index: index,
373        timestamp: Some(timestamp(index as i64)),
374        text,
375        language: Some("en"),
376        is_final: true,
377    }
378}
379
380/// Returns dataset with scene text and feature.
381pub fn dataset_with_scene_text_and_feature() -> AnalysisDataset {
382    let mut dataset = AnalysisDataset::empty();
383    let scene = scene(0, 2);
384    let text = text_segment(0, "Rust video analysis test fixture.");
385    dataset.push(DatasetRecord::Scene(SceneRecord::from_scene(0, &scene)));
386    dataset.push(DatasetRecord::TextSegment(TextSegmentRecord::from_segment(
387        "transcript",
388        &text,
389    )));
390    dataset.extend_observations([Observation::new("fixture", ObservationKind::Text)
391        .at_timestamp(timestamp(0))
392        .in_scene(0)
393        .label("fixture")
394        .text(text.text)]);
395    dataset
396        .extend_events([AnalysisEvent::new("fixture", "text:fixture").at_timestamp(timestamp(0))]);
397    dataset.push(DatasetRecord::Feature(
398        FeatureRecord::new("fixture.vector", FeatureValue::Vector(vec![1.0, 2.0, 3.0]))
399            .scope("global")
400            .timestamp(timestamp(0)),
401    ));
402    dataset
403}
404
405/// Returns comfy workflow JSON.
406pub fn comfy_workflow_json() -> &'static str {
407    r#"{
408  "version": 0.4,
409  "nodes": [
410    {"id": 1, "type": "LoadImage", "outputs": [{"name": "IMAGE", "type": "IMAGE", "links": [1]}]},
411    {"id": 2, "type": "SaveImage", "inputs": [{"name": "images", "type": "IMAGE", "link": 1}]}
412  ],
413  "links": [[1, 1, 0, 2, 0, "IMAGE"]],
414  "last_node_id": 2,
415  "last_link_id": 1
416}"#
417}
418
419/// Returns comfy prompt JSON.
420pub fn comfy_prompt_json() -> &'static str {
421    r#"{
422  "1": {"class_type": "LoadImage", "inputs": {"image": "input.png"}},
423  "2": {"class_type": "SaveImage", "inputs": {"images": ["1", 0]}}
424}"#
425}
426
427/// Returns minimal COLMAP dataset.
428pub fn minimal_colmap_dataset() -> ColmapDataset {
429    ColmapDataset {
430        cameras: vec![ColmapCamera {
431            id: 1,
432            model: CameraModel::Pinhole,
433            raw_model: "PINHOLE".to_string(),
434            width: 64,
435            height: 48,
436            params: vec![50.0, 50.0, 32.0, 24.0],
437        }],
438        images: vec![ColmapImage {
439            id: 1,
440            qw: 1.0,
441            qx: 0.0,
442            qy: 0.0,
443            qz: 0.0,
444            tx: 0.0,
445            ty: 0.0,
446            tz: 0.0,
447            camera_id: 1,
448            name: "frame_0001.png".to_string(),
449            points2d: vec![ColmapPoint2d {
450                xy: Vec2::new(32.0, 24.0),
451                point3d_id: Some(1),
452            }],
453        }],
454        points: vec![ColmapPoint3d {
455            id: 1,
456            xyz: Vec3::new(0.0, 0.0, 1.0),
457            color: ColorRgb::new(1.0, 0.0, 0.0),
458            error: 0.1,
459            track: vec![ColmapTrackElement {
460                image_id: 1,
461                point2d_index: 0,
462            }],
463        }],
464    }
465}
466
467/// Returns minimal Nerfstudio transforms.
468pub fn minimal_nerfstudio_transforms() -> NerfstudioTransforms {
469    NerfstudioTransforms {
470        camera_model: Some("PINHOLE".to_string()),
471        fl_x: Some(50.0),
472        fl_y: Some(50.0),
473        cx: Some(32.0),
474        cy: Some(24.0),
475        w: Some(64),
476        h: Some(48),
477        frames: vec![NerfstudioFrame {
478            file_path: "images/frame_0001.png".to_string(),
479            transform_matrix: [
480                [1.0, 0.0, 0.0, 0.0],
481                [0.0, 1.0, 0.0, 0.0],
482                [0.0, 0.0, 1.0, 0.0],
483                [0.0, 0.0, 0.0, 1.0],
484            ],
485            fl_x: None,
486            fl_y: None,
487            cx: None,
488            cy: None,
489            w: None,
490            h: None,
491        }],
492    }
493}
494
495/// Returns require command.
496pub fn require_command(command: &str) -> PathBuf {
497    find_command(command).unwrap_or_else(|| panic!("required command `{command}` is unavailable"))
498}
499
500/// Returns find command.
501pub fn find_command(command: &str) -> Option<PathBuf> {
502    let path = Path::new(command);
503    if path.components().count() > 1 && is_executable_command(path) {
504        return Some(path.to_path_buf());
505    }
506    std::env::var_os("PATH").and_then(|paths| {
507        std::env::split_paths(&paths)
508            .map(|dir| dir.join(command))
509            .find(|candidate| is_executable_command(candidate))
510    })
511}
512
513/// Returns command succeeds.
514pub fn command_succeeds(command: &str, args: &[&str]) -> bool {
515    Command::new(command)
516        .args(args)
517        .stdin(Stdio::null())
518        .stdout(Stdio::null())
519        .stderr(Stdio::null())
520        .status()
521        .map(|status| status.success())
522        .unwrap_or(false)
523}
524
525/// Returns find python with modules.
526pub fn find_python_with_modules(modules: &[&str]) -> Option<PathBuf> {
527    let import = format!("import {}", modules.join(", "));
528    [
529        PathBuf::from("python3"),
530        PathBuf::from("python"),
531        PathBuf::from(".external-test-tools/model-python-venv/bin/python"),
532    ]
533    .into_iter()
534    .find(|candidate| {
535        Command::new(candidate)
536            .args(["-c", &import])
537            .stdin(Stdio::null())
538            .stdout(Stdio::null())
539            .stderr(Stdio::null())
540            .status()
541            .map(|status| status.success())
542            .unwrap_or(false)
543    })
544}
545
546/// Returns ensure me at the zoo fixture.
547pub fn ensure_me_at_the_zoo_fixture() -> PathBuf {
548    if let Some(path) = std::env::var_os("ME_AT_THE_ZOO_VIDEO_PATH") {
549        let path = PathBuf::from(path);
550        assert_nonempty_file(&path);
551        return path;
552    }
553
554    let url = std::env::var("ME_AT_THE_ZOO_VIDEO_URL")
555        .unwrap_or_else(|_| ME_AT_THE_ZOO_WIKIMEDIA_URL.to_string());
556    let cache_dir = std::env::temp_dir().join("video-analysis-fixtures");
557    fs::create_dir_all(&cache_dir).expect("expected fixture cache directory to be writable");
558    let path = cache_dir.join("me-at-the-zoo.webm");
559    if path.exists() {
560        assert_nonempty_file(&path);
561        return path;
562    }
563
564    download_to_path(&url, &path);
565    assert_nonempty_file(&path);
566    path
567}
568
569/// Asserts that a file exists and is nonempty.
570pub fn assert_nonempty_file(path: impl AsRef<Path>) {
571    let path = path.as_ref();
572    let metadata = fs::metadata(path)
573        .unwrap_or_else(|err| panic!("expected `{}` metadata: {err}", path.display()));
574    assert!(
575        metadata.is_file() && metadata.len() > 0,
576        "expected `{}` to be a non-empty file",
577        path.display()
578    );
579}
580
581fn is_executable_command(path: &Path) -> bool {
582    path.is_file()
583}
584
585fn download_to_path(url: &str, path: &Path) {
586    let unique = std::time::SystemTime::now()
587        .duration_since(std::time::UNIX_EPOCH)
588        .expect("expected system time after unix epoch")
589        .as_nanos();
590    let tmp = path.with_extension(format!("part-{unique}-{}", std::process::id()));
591    if let Some(parent) = path.parent() {
592        fs::create_dir_all(parent)
593            .unwrap_or_else(|err| panic!("expected `{}` directory: {err}", parent.display()));
594    }
595
596    if path.exists() {
597        assert_nonempty_file(path);
598        return;
599    }
600
601    let curl = find_command("curl");
602    let wget = find_command("wget");
603    let status = if let Some(curl) = curl {
604        Command::new(curl)
605            .args(["-L", "--fail", "--silent", "--show-error", "-o"])
606            .arg(&tmp)
607            .arg(url)
608            .status()
609    } else if let Some(wget) = wget {
610        Command::new(wget).args(["-O"]).arg(&tmp).arg(url).status()
611    } else {
612        panic!("required command `curl` or `wget` is unavailable");
613    }
614    .unwrap_or_else(|err| panic!("expected fixture download command to start: {err}"));
615
616    assert!(
617        status.success(),
618        "expected fixture download from `{url}` to succeed"
619    );
620    if path.exists() {
621        let _ = fs::remove_file(&tmp);
622        assert_nonempty_file(path);
623        return;
624    }
625    fs::rename(&tmp, path).unwrap_or_else(|err| {
626        panic!(
627            "expected downloaded fixture `{}` to move into place: {err}",
628            path.display()
629        )
630    });
631}
632
633#[cfg(test)]
634mod tests {
635    use super::*;
636
637    #[test]
638    fn builders_create_valid_core_fixtures() {
639        let frame = checkerboard_frame(4, 2, 3);
640        assert_eq!(frame.data.len(), 4 * 2 * 3);
641        assert_eq!(frame.as_frame().width, 4);
642        let dataset = dataset_with_scene_text_and_feature();
643        assert_eq!(dataset.scenes().count(), 1);
644        assert_eq!(dataset.features().count(), 1);
645    }
646}