liboxen 0.65.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112

//! Helper functions to get metadata from audio files.
//!

use crate::{error::OxenError, model::metadata::MetadataAudio};

use lofty::file::{AudioFile, FileType};
use lofty::probe::Probe;
use std::io::{BufReader, Read, Seek};

/// Detects the audio metadata of the file `data` reads, in the format `extension` indicates.
pub fn get_metadata(data: impl Read + Seek, extension: &str) -> Result<MetadataAudio, OxenError> {
    let data = BufReader::new(data);
    let probe = match FileType::from_ext(extension) {
        Some(file_type) => Probe::with_file_type(data, file_type),
        None => Probe::new(data),
    };
    match probe.read() {
        Ok(tagged_file) => {
            let properties = tagged_file.properties();
            let duration = properties.duration();
            let seconds = duration.as_secs_f64();
            let rate = properties.sample_rate().unwrap_or(0);
            let channels = properties.channels().unwrap_or(0);

            Ok(MetadataAudio::new(
                seconds,
                channels as usize,
                rate as usize,
            ))
        }
        Err(err) => Err(OxenError::internal_error(format!(
            "Could not read audio stream as {extension}: {err}"
        ))),
    }
}

#[cfg(test)]
mod tests {
    use crate::model::EntryDataType;
    use crate::model::metadata::MetadataAudio;
    use crate::model::metadata::generic_metadata::GenericMetadata;
    use crate::repositories;
    use crate::test;
    use crate::util;

    use approx::assert_relative_eq;

    #[test]
    fn test_get_metadata_audio_flac() {
        let file = test::test_audio_file_with_name("121-121726-0005.flac");
        let metadata = repositories::metadata::get(file).unwrap();

        println!("metadata: {metadata:?}");

        assert_eq!(metadata.size, 37096);
        assert_eq!(metadata.data_type, EntryDataType::Audio);
        assert_eq!(metadata.mime_type, "audio/x-flac");

        assert!(metadata.metadata.is_some());
        let metadata: MetadataAudio = match metadata.metadata.unwrap() {
            GenericMetadata::MetadataAudio(metadata) => metadata,
            _ => panic!("Wrong metadata type"),
        };

        assert_eq!(metadata.audio.num_channels, 1);
        assert_eq!(metadata.audio.sample_rate, 16000);
        assert_relative_eq!(metadata.audio.num_seconds, 3.1);
    }

    #[test]
    fn test_get_metadata_audio_wav() {
        let file = test::test_audio_file_with_name("121-121726-0005.wav");
        let metadata = repositories::metadata::get(&file).unwrap();

        println!("metadata: {metadata:?}");

        assert_eq!(metadata.size, 99278);
        assert_eq!(metadata.data_type, EntryDataType::Audio);
        assert_eq!(metadata.mime_type, "audio/x-wav");

        assert!(metadata.metadata.is_some());
        let metadata: MetadataAudio = match metadata.metadata.unwrap() {
            GenericMetadata::MetadataAudio(metadata) => metadata,
            _ => panic!("Wrong metadata type"),
        };

        assert_eq!(metadata.audio.num_channels, 1);
        assert_eq!(metadata.audio.sample_rate, 16000);
        assert_relative_eq!(metadata.audio.num_seconds, 3.1);

        test::run_empty_dir_test(|dir| {
            let version_file = dir.join("data");
            util::fs::copy(&file, &version_file)?;
            let (_, data_type, staged) =
                repositories::metadata::inspect_file(&version_file, &file)?;
            assert_eq!(data_type, EntryDataType::Audio);
            let Some(GenericMetadata::MetadataAudio(staged)) = staged else {
                panic!("a copy with no extension takes the format its name gives, got {staged:?}");
            };
            assert_eq!(staged.audio.sample_rate, metadata.audio.sample_rate);
            assert_eq!(staged.audio.num_channels, metadata.audio.num_channels);

            let missing = dir.join("missing.wav");
            assert!(
                repositories::metadata::inspect_file(&missing, &missing).is_err(),
                "a file that cannot be opened fails its inspection"
            );
            Ok(())
        })
        .expect("the extensionless copy's metadata matches the named file's");
    }
}