dspk-io 0.0.10

Recording file formats for dspk: raw binary and Zarr traces, format detection, chunk caching
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150

//! Format detection: [`open`] picks the reader from the path.

use std::path::Path;

use dspk_core::{DspError, DspResult, RecordingSource};

use super::format::Format;
use super::sources::SourceEntry;
use crate::registry::formats;

/// The format of `path`: the first compiled-in [`Format`] that detects it.
pub fn detect(path: &Path) -> DspResult<&'static dyn Format> {
    detect_in(formats(), path)
}

/// The format of `path`: the first of `formats` that detects it. For crates that add formats to
/// the generic ones (e.g. `dspk_synapse::io`).
pub fn detect_in(formats: &[&'static dyn Format], path: &Path) -> DspResult<&'static dyn Format> {
    if !path.exists() {
        return Err(DspError::Io(format!("{} does not exist", path.display())));
    }
    formats.iter().copied().find(|f| f.detect(path)).ok_or_else(|| {
        let names: Vec<&str> = formats.iter().map(|f| f.name()).collect();
        DspError::UnsupportedFormat(format!(
            "cannot tell the format of {} (compiled-in formats: {}; a raw binary file needs a JSON sidecar)",
            path.display(),
            names.join(", ")
        ))
    })
}

/// Opens the default source of `path` (see [`Format::open_default`]).
pub fn open(path: &Path) -> DspResult<Box<dyn RecordingSource>> {
    detect(path)?.open_default(path)
}

/// Every source of `path`, from metadata only.
pub fn sources(path: &Path) -> DspResult<Vec<SourceEntry>> {
    detect(path)?.sources(path)
}

/// Opens source `id` of `path` (an id from [`sources`]).
pub fn open_source(path: &Path, id: &str) -> DspResult<Box<dyn RecordingSource>> {
    detect(path)?.open(path, id)
}

#[cfg(test)]
mod tests {
    use super::*;
    use dspk_core::{MemoryOrder, SampleFormat};
    #[cfg(feature = "zarr")]
    use crate::core::CachedRecording;
    use crate::generic::raw::write_raw;
    #[cfg(feature = "zarr")]
    use crate::generic::zarr_traces::write_zarr;
    use crate::test_support::{assert_native_matches, assert_stored_matches};
    use dspk_core::MemoryRecording;

    /// `channels` × 3 000 samples at 30 kHz: a different sine per channel, in µV.
    fn source(channels: usize) -> MemoryRecording {
        const SAMPLES: usize = 3_000;
        let data = (0..channels)
            .flat_map(|c| (0..SAMPLES).map(move |t| 100.0 * ((t as f32) * 0.01 * (c + 1) as f32).sin()))
            .collect();
        MemoryRecording::new("source", data, channels, 30_000.0).unwrap()
    }

    #[test]
    fn test_open_detects_formats() {
        let dir = std::env::temp_dir().join(format!("dspk_io_open_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        let src = source(4);

        let bin = dir.join("rec.bin");
        write_raw(&src, &bin, SampleFormat::I16, MemoryOrder::TimeMajor, 0.25, 1000, |_, _| {}).unwrap();
        let rec = open(&bin).unwrap();
        assert_eq!(rec.info().format, SampleFormat::I16);
        assert_eq!(rec.info().samples, src.info().samples);

        #[cfg(feature = "zarr")]
        {
            let z = dir.join("rec.zarr");
            write_zarr(&src, &z, &crate::ZarrOptions { chunk_samples: 1000, ..Default::default() }, |_, _| {}).unwrap();
            assert_eq!(open(&z).unwrap().info().channel_count(), 4);
        }

        std::fs::write(dir.join("other.dat"), [0u8; 8]).unwrap();
        assert!(matches!(open(&dir.join("other.dat")), Err(DspError::UnsupportedFormat(_))));
        assert!(open(&dir.join("missing.bin")).is_err());
        std::fs::remove_dir_all(&dir).unwrap();
    }

    #[test]
    fn test_single_recording_is_one_source() {
        let dir = std::env::temp_dir().join(format!("dspk_io_src_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        let src = source(3);
        let bin = dir.join("rec.bin");
        write_raw(&src, &bin, SampleFormat::F32, MemoryOrder::ChannelMajor, 1.0, 1000, |_, _| {}).unwrap();

        let list = sources(&bin).unwrap();
        assert_eq!(list.len(), 1);
        assert_eq!((list[0].id.as_str(), list[0].channels), (crate::core::sources::MAIN, 3));
        // The sidecar keeps the source's unit
        assert_eq!(list[0].unit, src.info().channels[0].unit);
        assert_eq!(open_source(&bin, crate::core::sources::MAIN).unwrap().info().channel_count(), 3);
        assert!(open_source(&bin, "nope").is_err());
        std::fs::remove_dir_all(&dir).unwrap();
    }

    #[test]
    fn stored_reads_match_scaled_reads() {
        let dir = std::env::temp_dir().join(format!("dspk_io_stored_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        let src = source(5);
        let channels = [4usize, 0, 2];

        for (name, format, order, gain) in [
            ("tm_i16.bin", SampleFormat::I16, MemoryOrder::TimeMajor, 0.25),
            ("cm_i16.bin", SampleFormat::I16, MemoryOrder::ChannelMajor, 0.5),
            ("cm_f32.bin", SampleFormat::F32, MemoryOrder::ChannelMajor, 1.0),
            ("tm_i32.bin", SampleFormat::I32, MemoryOrder::TimeMajor, 0.01),
        ] {
            let bin = dir.join(name);
            write_raw(&src, &bin, format, order, gain, 700, |_, _| {}).unwrap();
            let rec = open(&bin).unwrap();
            assert_stored_matches(rec.as_ref(), &channels, 100..1_900);
            assert_native_matches(rec.as_ref(), 100..1_900);
            // A sliced view maps channels and samples onto its parent
            let parent: std::sync::Arc<dyn RecordingSource> = std::sync::Arc::from(rec);
            let sliced = dspk_core::SlicedRecording::new(parent, 50..2_000, Some(vec![3, 1, 4])).unwrap();
            assert_stored_matches(&sliced, &[2, 0], 10..900);
        }

        // In-memory f32 µV with unit gain: the default implementation
        assert_stored_matches(&src, &channels, 0..500);

        #[cfg(feature = "zarr")]
        {
            let z = dir.join("rec.zarr");
            write_zarr(&src, &z, &crate::ZarrOptions { chunk_samples: 700, ..Default::default() }, |_, _| {}).unwrap();
            assert_stored_matches(open(&z).unwrap().as_ref(), &channels, 100..1_900);
            assert_native_matches(open(&z).unwrap().as_ref(), 100..1_900);
            let cached = CachedRecording::wrap(open(&z).unwrap(), 1 << 20);
            assert_native_matches(cached.as_ref(), 100..1_900);
            assert_stored_matches(cached.as_ref(), &channels, 100..1_900);
        }
        std::fs::remove_dir_all(&dir).unwrap();
    }
}