1pub mod surface;
4use std::fs;
5use std::path::{Path, PathBuf};
6use std::process::{Command, Stdio};
7
8use num_rational::Rational64;
9use video_analysis_core::{
10 AnalysisEvent, FramePosition, Observation, ObservationKind, OwnedTextSegment, OwnedVideoFrame,
11 PixelFormat, Scene, SceneDetector, ScenePipeline, TextSegment, Timebase, Timestamp,
12};
13use video_analysis_dataset::{
14 AnalysisDataset, DatasetRecord, FeatureRecord, FeatureValue, SceneRecord, TextSegmentRecord,
15};
16use video_analysis_radiance_fields::{CameraModel, ColorRgb, Vec2, Vec3};
17use video_analysis_radiance_io::{
18 ColmapCamera, ColmapDataset, ColmapImage, ColmapPoint2d, ColmapPoint3d, ColmapTrackElement,
19 NerfstudioFrame, NerfstudioTransforms,
20};
21
22pub const ME_AT_THE_ZOO_WIKIMEDIA_URL: &str =
24 "https://upload.wikimedia.org/wikipedia/commons/e/e0/Me_at_the_zoo.webm";
25
26pub fn timestamp(seconds: i64) -> Timestamp {
28 Timestamp::new(seconds, Timebase::new(1, 1))
29}
30
31pub fn timestamp_at(pts: i64, num: i32, den: i32) -> Timestamp {
33 Timestamp::new(pts, Timebase::new(num, den))
34}
35
36pub fn frame_position(frame_index: u64) -> FramePosition {
38 FramePosition {
39 frame_index,
40 timestamp: timestamp(frame_index as i64),
41 }
42}
43
44pub fn frame_position_at_rate(frame_index: u64, frame_rate: i64) -> FramePosition {
46 FramePosition::from_frame_index(frame_index, Rational64::new(frame_rate, 1))
47}
48
49pub fn scene(start: u64, end: u64) -> Scene {
51 Scene {
52 start: frame_position(start),
53 end: frame_position(end),
54 }
55}
56
57pub fn rgb_frame(width: u32, height: u32, frame_index: u64, rgb: [u8; 3]) -> OwnedVideoFrame {
59 let mut data = Vec::with_capacity(width as usize * height as usize * 3);
60 for _ in 0..(width as usize * height as usize) {
61 data.extend_from_slice(&rgb);
62 }
63 OwnedVideoFrame {
64 position: frame_position(frame_index),
65 width,
66 height,
67 pixel_format: PixelFormat::Rgb24,
68 stride: width as usize * 3,
69 data,
70 }
71}
72
73pub fn checkerboard_frame(width: u32, height: u32, frame_index: u64) -> OwnedVideoFrame {
75 let mut data = Vec::with_capacity(width as usize * height as usize * 3);
76 for y in 0..height {
77 for x in 0..width {
78 let value = if (x + y) % 2 == 0 { 255 } else { 0 };
79 data.extend_from_slice(&[value, value, value]);
80 }
81 }
82 OwnedVideoFrame {
83 position: frame_position(frame_index),
84 width,
85 height,
86 pixel_format: PixelFormat::Rgb24,
87 stride: width as usize * 3,
88 data,
89 }
90}
91
92#[derive(Debug, Clone)]
94pub struct SyntheticVideoSpec {
95 pub fps: Rational64,
97 pub width: u32,
99 pub height: u32,
101 pub frame_count: u64,
103 pub patterns: Vec<ScenePattern>,
105}
106
107impl SyntheticVideoSpec {
108 pub fn new(width: u32, height: u32, frame_count: u64, fps: Rational64) -> Self {
110 Self {
111 fps,
112 width,
113 height,
114 frame_count,
115 patterns: vec![ScenePattern::SolidColor {
116 start_frame: 0,
117 end_frame: frame_count,
118 rgb: [32, 32, 32],
119 }],
120 }
121 }
122
123 pub fn pattern(mut self, pattern: ScenePattern) -> Self {
125 self.patterns.push(pattern);
126 self
127 }
128}
129
130#[derive(Debug, Clone)]
132pub enum ScenePattern {
133 SolidColor {
135 start_frame: u64,
137 end_frame: u64,
139 rgb: [u8; 3],
141 },
142 HardCut {
144 at_frame: u64,
146 before: [u8; 3],
148 after: [u8; 3],
150 },
151 Flash {
153 start_frame: u64,
155 length: u64,
157 rgb: [u8; 3],
159 },
160 FadeOutIn {
162 start_frame: u64,
164 end_frame: u64,
166 high: u8,
168 low: u8,
170 },
171 LuminanceRamp {
173 start_frame: u64,
175 end_frame: u64,
177 from: u8,
179 to: u8,
181 },
182 HistogramShift {
184 start_frame: u64,
186 end_frame: u64,
188 },
189 TextureShift {
191 start_frame: u64,
193 end_frame: u64,
195 offset: u32,
197 },
198}
199
200pub fn synthetic_frames(spec: &SyntheticVideoSpec) -> Vec<OwnedVideoFrame> {
202 (0..spec.frame_count)
203 .map(|frame_index| {
204 let mut frame =
205 rgb_frame_at_rate(spec.width, spec.height, frame_index, spec.fps, [32, 32, 32]);
206 for pattern in &spec.patterns {
207 apply_pattern(&mut frame, spec.frame_count, pattern);
208 }
209 frame
210 })
211 .collect()
212}
213
214pub fn run_detector_on_frames<D: SceneDetector + 'static>(
216 detector: D,
217 frames: &[OwnedVideoFrame],
218) -> video_analysis_core::Result<video_analysis_core::DetectionResult> {
219 let mut pipeline = ScenePipeline::builder()
220 .detector(detector)
221 .start_in_scene(true)
222 .build()?;
223 for frame in frames {
224 pipeline.process_frame(frame.clone())?;
225 }
226 pipeline.finish_detection()
227}
228
229pub fn assert_scene_boundaries(
231 result: &video_analysis_core::DetectionResult,
232 expected_start_frames: &[u64],
233) {
234 let actual = result
235 .scenes
236 .iter()
237 .map(|scene| scene.start.frame_index)
238 .collect::<Vec<_>>();
239 assert_eq!(actual, expected_start_frames);
240}
241
242pub fn assert_metric_present(metrics: &video_analysis_core::MetricsStore, frame: u64, key: &str) {
244 let row = metrics
245 .rows()
246 .get(&frame)
247 .unwrap_or_else(|| panic!("missing metric row for frame {frame}"));
248 assert!(
249 row.contains_key(key),
250 "missing metric key `{key}` for frame {frame}; row keys: {:?}",
251 row.keys().collect::<Vec<_>>()
252 );
253}
254
255fn rgb_frame_at_rate(
256 width: u32,
257 height: u32,
258 frame_index: u64,
259 fps: Rational64,
260 rgb: [u8; 3],
261) -> OwnedVideoFrame {
262 let mut frame = rgb_frame(width, height, frame_index, rgb);
263 frame.position = FramePosition::from_frame_index(frame_index, fps);
264 frame
265}
266
267fn apply_pattern(frame: &mut OwnedVideoFrame, frame_count: u64, pattern: &ScenePattern) {
268 let frame_index = frame.position.frame_index;
269 match *pattern {
270 ScenePattern::SolidColor {
271 start_frame,
272 end_frame,
273 rgb,
274 } if in_range(frame_index, start_frame, end_frame) => fill_frame(frame, rgb),
275 ScenePattern::HardCut {
276 at_frame,
277 before,
278 after,
279 } => {
280 if frame_index < at_frame {
281 fill_frame(frame, before);
282 } else if frame_index < frame_count {
283 fill_frame(frame, after);
284 }
285 }
286 ScenePattern::Flash {
287 start_frame,
288 length,
289 rgb,
290 } if in_range(frame_index, start_frame, start_frame.saturating_add(length)) => {
291 fill_frame(frame, rgb);
292 }
293 ScenePattern::FadeOutIn {
294 start_frame,
295 end_frame,
296 high,
297 low,
298 } if in_range(frame_index, start_frame, end_frame) => {
299 let len = end_frame.saturating_sub(start_frame).max(1);
300 let local = frame_index.saturating_sub(start_frame);
301 let midpoint = len / 2;
302 let edge_weight = if local <= midpoint {
303 1.0 - (local as f32 / midpoint.max(1) as f32)
304 } else {
305 (local.saturating_sub(midpoint)) as f32 / (len - midpoint).max(1) as f32
306 };
307 let value = low as f32 + (high as f32 - low as f32) * edge_weight.clamp(0.0, 1.0);
308 fill_frame(frame, [value as u8, value as u8, value as u8]);
309 }
310 ScenePattern::LuminanceRamp {
311 start_frame,
312 end_frame,
313 from,
314 to,
315 } if in_range(frame_index, start_frame, end_frame) => {
316 let len = end_frame.saturating_sub(start_frame).max(1);
317 let local = frame_index.saturating_sub(start_frame);
318 let t = local as f32 / len as f32;
319 let value = from as f32 + (to as f32 - from as f32) * t;
320 fill_frame(frame, [value as u8, value as u8, value as u8]);
321 }
322 ScenePattern::HistogramShift {
323 start_frame,
324 end_frame,
325 } if in_range(frame_index, start_frame, end_frame) => {
326 for y in 0..frame.height {
327 for x in 0..frame.width {
328 let value = if (x / 4 + y / 4) % 3 == 0 { 16 } else { 236 };
329 set_pixel(frame, x, y, [value, value, value]);
330 }
331 }
332 }
333 ScenePattern::TextureShift {
334 start_frame,
335 end_frame,
336 offset,
337 } if in_range(frame_index, start_frame, end_frame) => {
338 for y in 0..frame.height {
339 for x in 0..frame.width {
340 let value = ((x * 5 + y * 3 + offset * 17) % 256) as u8;
341 set_pixel(frame, x, y, [value, value, value]);
342 }
343 }
344 }
345 _ => {}
346 }
347}
348
349fn in_range(frame_index: u64, start_frame: u64, end_frame: u64) -> bool {
350 frame_index >= start_frame && frame_index < end_frame
351}
352
353fn fill_frame(frame: &mut OwnedVideoFrame, rgb: [u8; 3]) {
354 for pixel in frame.data.chunks_exact_mut(3) {
355 pixel.copy_from_slice(&rgb);
356 }
357}
358
359fn set_pixel(frame: &mut OwnedVideoFrame, x: u32, y: u32, rgb: [u8; 3]) {
360 let offset = y as usize * frame.stride + x as usize * 3;
361 frame.data[offset..offset + 3].copy_from_slice(&rgb);
362}
363
364pub fn owned_text_segment(index: u64, text: impl Into<String>) -> OwnedTextSegment {
366 OwnedTextSegment::new(index, text).timestamp(timestamp(index as i64))
367}
368
369pub fn text_segment(index: u64, text: &str) -> TextSegment<'_> {
371 TextSegment {
372 segment_index: index,
373 timestamp: Some(timestamp(index as i64)),
374 text,
375 language: Some("en"),
376 is_final: true,
377 }
378}
379
380pub fn dataset_with_scene_text_and_feature() -> AnalysisDataset {
382 let mut dataset = AnalysisDataset::empty();
383 let scene = scene(0, 2);
384 let text = text_segment(0, "Rust video analysis test fixture.");
385 dataset.push(DatasetRecord::Scene(SceneRecord::from_scene(0, &scene)));
386 dataset.push(DatasetRecord::TextSegment(TextSegmentRecord::from_segment(
387 "transcript",
388 &text,
389 )));
390 dataset.extend_observations([Observation::new("fixture", ObservationKind::Text)
391 .at_timestamp(timestamp(0))
392 .in_scene(0)
393 .label("fixture")
394 .text(text.text)]);
395 dataset
396 .extend_events([AnalysisEvent::new("fixture", "text:fixture").at_timestamp(timestamp(0))]);
397 dataset.push(DatasetRecord::Feature(
398 FeatureRecord::new("fixture.vector", FeatureValue::Vector(vec![1.0, 2.0, 3.0]))
399 .scope("global")
400 .timestamp(timestamp(0)),
401 ));
402 dataset
403}
404
405pub fn comfy_workflow_json() -> &'static str {
407 r#"{
408 "version": 0.4,
409 "nodes": [
410 {"id": 1, "type": "LoadImage", "outputs": [{"name": "IMAGE", "type": "IMAGE", "links": [1]}]},
411 {"id": 2, "type": "SaveImage", "inputs": [{"name": "images", "type": "IMAGE", "link": 1}]}
412 ],
413 "links": [[1, 1, 0, 2, 0, "IMAGE"]],
414 "last_node_id": 2,
415 "last_link_id": 1
416}"#
417}
418
419pub fn comfy_prompt_json() -> &'static str {
421 r#"{
422 "1": {"class_type": "LoadImage", "inputs": {"image": "input.png"}},
423 "2": {"class_type": "SaveImage", "inputs": {"images": ["1", 0]}}
424}"#
425}
426
427pub fn minimal_colmap_dataset() -> ColmapDataset {
429 ColmapDataset {
430 cameras: vec![ColmapCamera {
431 id: 1,
432 model: CameraModel::Pinhole,
433 raw_model: "PINHOLE".to_string(),
434 width: 64,
435 height: 48,
436 params: vec![50.0, 50.0, 32.0, 24.0],
437 }],
438 images: vec![ColmapImage {
439 id: 1,
440 qw: 1.0,
441 qx: 0.0,
442 qy: 0.0,
443 qz: 0.0,
444 tx: 0.0,
445 ty: 0.0,
446 tz: 0.0,
447 camera_id: 1,
448 name: "frame_0001.png".to_string(),
449 points2d: vec![ColmapPoint2d {
450 xy: Vec2::new(32.0, 24.0),
451 point3d_id: Some(1),
452 }],
453 }],
454 points: vec![ColmapPoint3d {
455 id: 1,
456 xyz: Vec3::new(0.0, 0.0, 1.0),
457 color: ColorRgb::new(1.0, 0.0, 0.0),
458 error: 0.1,
459 track: vec![ColmapTrackElement {
460 image_id: 1,
461 point2d_index: 0,
462 }],
463 }],
464 }
465}
466
467pub fn minimal_nerfstudio_transforms() -> NerfstudioTransforms {
469 NerfstudioTransforms {
470 camera_model: Some("PINHOLE".to_string()),
471 fl_x: Some(50.0),
472 fl_y: Some(50.0),
473 cx: Some(32.0),
474 cy: Some(24.0),
475 w: Some(64),
476 h: Some(48),
477 frames: vec![NerfstudioFrame {
478 file_path: "images/frame_0001.png".to_string(),
479 transform_matrix: [
480 [1.0, 0.0, 0.0, 0.0],
481 [0.0, 1.0, 0.0, 0.0],
482 [0.0, 0.0, 1.0, 0.0],
483 [0.0, 0.0, 0.0, 1.0],
484 ],
485 fl_x: None,
486 fl_y: None,
487 cx: None,
488 cy: None,
489 w: None,
490 h: None,
491 }],
492 }
493}
494
495pub fn require_command(command: &str) -> PathBuf {
497 find_command(command).unwrap_or_else(|| panic!("required command `{command}` is unavailable"))
498}
499
500pub fn find_command(command: &str) -> Option<PathBuf> {
502 let path = Path::new(command);
503 if path.components().count() > 1 && is_executable_command(path) {
504 return Some(path.to_path_buf());
505 }
506 std::env::var_os("PATH").and_then(|paths| {
507 std::env::split_paths(&paths)
508 .map(|dir| dir.join(command))
509 .find(|candidate| is_executable_command(candidate))
510 })
511}
512
513pub fn command_succeeds(command: &str, args: &[&str]) -> bool {
515 Command::new(command)
516 .args(args)
517 .stdin(Stdio::null())
518 .stdout(Stdio::null())
519 .stderr(Stdio::null())
520 .status()
521 .map(|status| status.success())
522 .unwrap_or(false)
523}
524
525pub fn find_python_with_modules(modules: &[&str]) -> Option<PathBuf> {
527 let import = format!("import {}", modules.join(", "));
528 [
529 PathBuf::from("python3"),
530 PathBuf::from("python"),
531 PathBuf::from(".external-test-tools/model-python-venv/bin/python"),
532 ]
533 .into_iter()
534 .find(|candidate| {
535 Command::new(candidate)
536 .args(["-c", &import])
537 .stdin(Stdio::null())
538 .stdout(Stdio::null())
539 .stderr(Stdio::null())
540 .status()
541 .map(|status| status.success())
542 .unwrap_or(false)
543 })
544}
545
546pub fn ensure_me_at_the_zoo_fixture() -> PathBuf {
548 if let Some(path) = std::env::var_os("ME_AT_THE_ZOO_VIDEO_PATH") {
549 let path = PathBuf::from(path);
550 assert_nonempty_file(&path);
551 return path;
552 }
553
554 let url = std::env::var("ME_AT_THE_ZOO_VIDEO_URL")
555 .unwrap_or_else(|_| ME_AT_THE_ZOO_WIKIMEDIA_URL.to_string());
556 let cache_dir = std::env::temp_dir().join("video-analysis-fixtures");
557 fs::create_dir_all(&cache_dir).expect("expected fixture cache directory to be writable");
558 let path = cache_dir.join("me-at-the-zoo.webm");
559 if path.exists() {
560 assert_nonempty_file(&path);
561 return path;
562 }
563
564 download_to_path(&url, &path);
565 assert_nonempty_file(&path);
566 path
567}
568
569pub fn assert_nonempty_file(path: impl AsRef<Path>) {
571 let path = path.as_ref();
572 let metadata = fs::metadata(path)
573 .unwrap_or_else(|err| panic!("expected `{}` metadata: {err}", path.display()));
574 assert!(
575 metadata.is_file() && metadata.len() > 0,
576 "expected `{}` to be a non-empty file",
577 path.display()
578 );
579}
580
581fn is_executable_command(path: &Path) -> bool {
582 path.is_file()
583}
584
585fn download_to_path(url: &str, path: &Path) {
586 let unique = std::time::SystemTime::now()
587 .duration_since(std::time::UNIX_EPOCH)
588 .expect("expected system time after unix epoch")
589 .as_nanos();
590 let tmp = path.with_extension(format!("part-{unique}-{}", std::process::id()));
591 if let Some(parent) = path.parent() {
592 fs::create_dir_all(parent)
593 .unwrap_or_else(|err| panic!("expected `{}` directory: {err}", parent.display()));
594 }
595
596 if path.exists() {
597 assert_nonempty_file(path);
598 return;
599 }
600
601 let curl = find_command("curl");
602 let wget = find_command("wget");
603 let status = if let Some(curl) = curl {
604 Command::new(curl)
605 .args(["-L", "--fail", "--silent", "--show-error", "-o"])
606 .arg(&tmp)
607 .arg(url)
608 .status()
609 } else if let Some(wget) = wget {
610 Command::new(wget).args(["-O"]).arg(&tmp).arg(url).status()
611 } else {
612 panic!("required command `curl` or `wget` is unavailable");
613 }
614 .unwrap_or_else(|err| panic!("expected fixture download command to start: {err}"));
615
616 assert!(
617 status.success(),
618 "expected fixture download from `{url}` to succeed"
619 );
620 if path.exists() {
621 let _ = fs::remove_file(&tmp);
622 assert_nonempty_file(path);
623 return;
624 }
625 fs::rename(&tmp, path).unwrap_or_else(|err| {
626 panic!(
627 "expected downloaded fixture `{}` to move into place: {err}",
628 path.display()
629 )
630 });
631}
632
633#[cfg(test)]
634mod tests {
635 use super::*;
636
637 #[test]
638 fn builders_create_valid_core_fixtures() {
639 let frame = checkerboard_frame(4, 2, 3);
640 assert_eq!(frame.data.len(), 4 * 2 * 3);
641 assert_eq!(frame.as_frame().width, 4);
642 let dataset = dataset_with_scene_text_and_feature();
643 assert_eq!(dataset.scenes().count(), 1);
644 assert_eq!(dataset.features().count(), 1);
645 }
646}