Skip to main content

openreadout_spikeglx/
lib.rs

1//! Reader for SpikeGLX recordings: a `.meta` text file beside a headerless interleaved int16
2//! `.bin` (Neuropixels imec AP/LF streams, NI-DAQ and OneBox streams).
3//!
4//! Layout and vocabulary: `docs/formats/spikeglx.md`. Provenance: `docs/provenance/spikeglx.md`
5//! (the SpikeGLX authors' public metadata documentation). One stream (one `.bin`) is one trace
6//! with a single sweep; channels are scaled to µV (imec) or V (NI, OneBox) as the documentation
7//! specifies; status (SY) and digital (XD) words are returned raw.
8//!
9//! # Example
10//!
11//! Every reader implements [`openreadout_core::FormatReader`]: `descriptor`
12//! says what it reads, `sniff` recognises a file from its first bytes, and `open` returns a
13//! [`openreadout_core::Dataset`] with a header-only `info()` summary and lazy data
14//! access.
15//!
16//! ```
17//! use openreadout_core::FormatReader;
18//! use openreadout_spikeglx::SpikeGlxReader;
19//!
20//! let format = SpikeGlxReader.descriptor();
21//! assert_eq!(format.id, openreadout_spikeglx::FORMAT_ID);
22//! println!("{} ({}): .{}", format.name, format.family, format.extensions.join(", ."));
23//! ```
24//!
25//! Reading a file:
26//!
27//! ```no_run
28//! use std::path::Path;
29//!
30//! use openreadout_core::FormatReader;
31//! use openreadout_spikeglx::SpikeGlxReader;
32//!
33//! let mut dataset = SpikeGlxReader.open(Path::new("run_g0_t0.imec0.ap.bin"))?;
34//! let info = dataset.info()?; // headers only
35//! let trace = &info.traces[0];
36//! println!("{} sweep(s) at {} Hz, {} channel(s)", trace.sweep_count, trace.sample_rate_hz, trace.channels.len());
37//! // The first 1000 samples of sweep 0, every channel, scaled to physical units.
38//! let samples = dataset.read_trace(0, 0, 0, 1000)?;
39//! for (channel, values) in trace.channels.iter().zip(&samples.channels) {
40//!     println!("{} ({}): {:?}", channel.name, channel.unit.as_deref().unwrap_or("?"), values.first());
41//! }
42//! # Ok::<(), openreadout_core::Error>(())
43//! ```
44//!
45//! Applications that accept any instrument file usually register every reader in an
46//! [`openreadout_core::Registry`] and let it detect the format; that is what the
47//! `openreadout` command-line tool does.
48//!
49//! # API stability
50//!
51//! The supported API is what this page documents. The parser modules are public only so that
52//! tests and fuzz targets can reach them: they are hidden from this documentation and may change
53//! in any release.
54#![forbid(unsafe_code)]
55#![warn(missing_docs)]
56
57mod assurance;
58#[doc(hidden)]
59pub mod dataset;
60#[doc(hidden)]
61pub mod meta;
62
63use std::path::Path;
64
65use openreadout_core::Result;
66use openreadout_core::model::{DetectConfidence, FormatDescriptor};
67use openreadout_core::reader::{Dataset, Detection, FormatReader, has_extension};
68use openreadout_core::source::Input;
69
70#[doc(hidden)]
71pub use dataset::{MAX_META_LEN, SpikeGlxDataset, stream_paths};
72#[doc(hidden)]
73pub use meta::{
74    ChannelKind, Meta, SavedChannel, StreamKind, default_max_int, fixed_gain, has_selectable_gain,
75    imec_scale, list_elements, ni_scale, parse_meta, parse_subset, saved_channels,
76};
77
78/// Format id used on the command line and in JSON.
79pub const FORMAT_ID: &str = "spikeglx";
80
81/// True when `.meta` text looks like SpikeGLX metadata.
82pub fn looks_like_meta(text: &[u8]) -> bool {
83    let t = String::from_utf8_lossy(&text[..text.len().min(1 << 16)]);
84    t.contains("nSavedChans=") && (t.contains("typeThis=") || t.contains("SampRate="))
85}
86
87/// Files a SpikeGLX run directory may hold besides the stream pairs: `.meta` files (read with
88/// their `.bin`), logs, notes and text outputs.
89pub const SESSION_COMPANIONS: [&str; 6] = ["meta", "txt", "log", "json", "md", "csv"];
90
91/// A probe subdirectory of a run: its name ends in `_imec` and a probe number.
92fn is_probe_dir_name(name: &str) -> bool {
93    name.rsplit_once("_imec")
94        .is_some_and(|(_, n)| !n.is_empty() && n.bytes().all(|b| b.is_ascii_digit()))
95}
96
97/// The stream `.bin` files (each with its `.meta`) of a SpikeGLX run directory, or `None` when
98/// `dir` is not one (see [`openreadout_core::session::session_files`]). The run directory
99/// `<run>_g0/` holds the NI-DAQ and OneBox streams; each probe's streams are in a subdirectory
100/// named `…_imec<N>` (normally `<run>_g0_imec<N>/`), which is read too. Other subdirectories
101/// (post-processing outputs) are ignored, and a directory that only wraps a run directory is not
102/// a run itself.
103pub fn session_files(dir: &Path) -> Option<Vec<std::path::PathBuf>> {
104    session_files_in(&Input::local(dir))
105}
106
107fn session_files_in(input: &Input) -> Option<Vec<std::path::PathBuf>> {
108    let dir = input.path();
109    let fs = input.fs();
110    let data = |p: &Path| has_extension(p, &["bin"]) && fs.is_file(&p.with_extension("meta"));
111    let own = |d: &Path| {
112        openreadout_core::session::session_files_in(fs, d, &data, &SESSION_COMPANIONS, 0)
113    };
114    let mut files = Vec::new();
115    // the run directory's own files; a directory with only probe subdirectories is fine
116    if let Some(v) = own(dir) {
117        files.extend(v);
118    } else {
119        let foreign = fs
120            .read_dir(dir)
121            .ok()?
122            .filter_map(std::result::Result::ok)
123            .any(|e| {
124                let n = e.file_name().to_string_lossy().into_owned();
125                e.file_type().is_ok_and(|t| t.is_file())
126                    && !n.starts_with('.')
127                    && !has_extension(&e.path(), &["ok"])
128                    && !has_extension(&e.path(), &SESSION_COMPANIONS)
129            });
130        if foreign {
131            return None;
132        }
133    }
134    let mut probes: Vec<std::path::PathBuf> = fs
135        .read_dir(dir)
136        .ok()?
137        .filter_map(std::result::Result::ok)
138        .filter(|e| {
139            e.file_type().is_ok_and(|t| t.is_dir())
140                && is_probe_dir_name(&e.file_name().to_string_lossy())
141        })
142        .map(|e| e.path())
143        .collect();
144    probes.sort();
145    for p in probes {
146        files.extend(own(&p)?);
147    }
148    files.sort();
149    (!files.is_empty()).then_some(files)
150}
151
152/// Open a SpikeGLX run directory: every stream (imec AP and LF bands, NI-DAQ, OneBox) is a
153/// trace of its own, with its own rate and sample clock; nothing is resampled or realigned.
154pub fn open_session(dir: &Path) -> Result<openreadout_core::session::SessionDataset> {
155    open_session_input(&Input::local(dir))
156}
157
158fn open_session_input(input: &Input) -> Result<openreadout_core::session::SessionDataset> {
159    let dir = input.path();
160    let files = session_files_in(input).ok_or_else(|| {
161        openreadout_core::Error::unsupported(
162            FORMAT_ID,
163            "a directory that is not a SpikeGLX run",
164            "A SpikeGLX run directory holds .bin/.meta stream pairs (probe streams may sit one level down); open a single .bin or .meta otherwise.",
165        )
166    })?;
167    let mut members: Vec<Box<dyn Dataset>> = Vec::new();
168    for f in &files {
169        members.push(Box::new(SpikeGlxDataset::open_input(&input.with_path(f))?));
170    }
171    openreadout_core::session::SessionDataset::new(
172        FORMAT_ID,
173        dir,
174        members,
175        openreadout_core::session::Combine::Separate,
176    )
177}
178
179/// The SpikeGLX reader.
180#[derive(Debug, Default, Clone, Copy)]
181pub struct SpikeGlxReader;
182
183impl FormatReader for SpikeGlxReader {
184    fn assurance(&self) -> Option<&'static openreadout_core::assurance::AssuranceProfile> {
185        Some(&assurance::SPIKEGLX)
186    }
187
188    fn descriptor(&self) -> FormatDescriptor {
189        FormatDescriptor {
190            id: FORMAT_ID.into(),
191            name: "SpikeGLX (.bin + .meta)".into(),
192            vendor: "SpikeGLX (HHMI Janelia), Neuropixels / NI-DAQ / OneBox".into(),
193            extensions: vec!["bin".into(), "meta".into()],
194            family: "electrophysiology".into(),
195            can_read: true,
196            can_write: false,
197            confidence: assurance::SPIKEGLX.confidence,
198            known_gaps: vec![
199                "A run directory is one session with one trace per stream (AP, LF, NI, OneBox), each on its own clock; streams are not aligned on their sync pulses".into(),
200                "Probe sites come from ~snsGeomMap (µm) or ~snsShankMap (grid column/row, not converted to µm); imro bank/reference settings are reported in `info --view full`, not normalized".into(),
201                "imMaxInt defaults for probe types other than NP 1.0 and 2.0 (types 21/24) are inferred from the ProbeTable when the metadata omits it".into(),
202            ],
203        }
204    }
205
206    fn sniff(&self, head: &[u8], path: &Path) -> Option<Detection> {
207        self.sniff_input(head, &Input::local(path))
208    }
209
210    /// A `.bin` is recognised by the `.meta` next to it, looked up in the input's namespace.
211    fn sniff_input(&self, head: &[u8], input: &Input) -> Option<Detection> {
212        let path = input.path();
213        if input.is_dir() {
214            // Definite once a stream's .meta reads as SpikeGLX metadata, so that a run directory
215            // holding its probe subdirectory stays one data set (not walked into by `index`).
216            let files = session_files_in(input)?;
217            let signed = files.iter().take(8).any(|f| {
218                dataset::read_meta(input.fs(), &f.with_extension("meta"))
219                    .is_ok_and(|t| t.len() as u64 <= MAX_META_LEN && looks_like_meta(&t))
220            });
221            return Some(Detection {
222                format_id: FORMAT_ID,
223                confidence: if signed {
224                    DetectConfidence::Definite
225                } else {
226                    DetectConfidence::Likely
227                },
228                note: Some("SpikeGLX run directory (every stream read as one session)".into()),
229            });
230        }
231        if has_extension(path, &["meta"]) && looks_like_meta(head) {
232            return Some(Detection {
233                format_id: FORMAT_ID,
234                confidence: DetectConfidence::Definite,
235                note: None,
236            });
237        }
238        if has_extension(path, &["bin"]) {
239            let (_, meta) = stream_paths(path);
240            if let Ok(text) = dataset::read_meta(input.fs(), &meta)
241                && text.len() as u64 <= MAX_META_LEN
242                && looks_like_meta(&text)
243            {
244                return Some(Detection {
245                    format_id: FORMAT_ID,
246                    confidence: DetectConfidence::Definite,
247                    note: None,
248                });
249            }
250        }
251        None
252    }
253
254    fn open(&self, path: &Path) -> Result<Box<dyn Dataset>> {
255        self.open_input(&Input::local(path))
256    }
257
258    fn open_input(&self, input: &Input) -> Result<Box<dyn Dataset>> {
259        if input.is_dir() {
260            return Ok(Box::new(open_session_input(input)?));
261        }
262        Ok(Box::new(SpikeGlxDataset::open_input(input)?))
263    }
264
265    fn reads_any_source(&self) -> bool {
266        true
267    }
268}