use openreadout_core::source::SourceFile as File;
use std::collections::BTreeMap;
use std::path::Path;
use openreadout_core::bytes::{Block, read_block};
use openreadout_core::model::Finding;
use openreadout_core::{Error, Result};
use crate::FORMAT_ID;
pub const PLX_MAGIC: [u8; 4] = *b"PLEX";
pub const FILE_HEADER_LEN: u64 = 7504;
pub const SPIKE_HEADER_LEN: u64 = 1020;
pub const EVENT_HEADER_LEN: u64 = 296;
pub const CONTINUOUS_HEADER_LEN: u64 = 296;
pub const BLOCK_HEADER_LEN: u64 = 16;
pub const BLOCK_SPIKE: u16 = 1;
pub const BLOCK_EVENT: u16 = 4;
pub const BLOCK_CONTINUOUS: u16 = 5;
pub const MAX_CHANNEL_HEADERS: i32 = 1 << 16;
#[derive(Debug, Clone, Default)]
pub struct PlxHeader {
pub version: i32,
pub comment: String,
pub clock_hz: i32,
pub spike_channel_count: i32,
pub event_channel_count: i32,
pub continuous_channel_count: i32,
pub waveform_points: i32,
pub pre_threshold_points: i32,
pub recorded_at: Option<String>,
pub waveform_rate_hz: i32,
pub last_timestamp: f64,
pub electrodes_per_channel: u8,
pub data_electrodes_per_channel: u8,
pub spike_bits: u8,
pub continuous_bits: u8,
pub spike_max_mv: u16,
pub continuous_max_mv: u16,
pub spike_preamp_gain: u16,
pub acquired_with: String,
pub processed_with: String,
}
#[derive(Debug, Clone, Default)]
pub struct SpikeChannel {
pub name: String,
pub signal_name: String,
pub channel: i32,
pub gain: i32,
pub filter: i32,
pub threshold: i32,
pub unit_count: i32,
pub comment: String,
}
#[derive(Debug, Clone, Default)]
pub struct EventChannel {
pub name: String,
pub channel: i32,
pub comment: String,
}
#[derive(Debug, Clone, Default)]
pub struct ContinuousChannel {
pub name: String,
pub channel: i32,
pub rate_hz: i32,
pub gain: i32,
pub enabled: bool,
pub preamp_gain: i32,
pub comment: String,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct SampleBlock {
pub offset: u64,
pub timestamp: u64,
pub samples: u32,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct BlockList {
chunks: Vec<Vec<u8>>,
groups: Vec<(SampleBlock, u32, u32)>,
len: usize,
last: Steps,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
struct Steps {
block: SampleBlock,
offset_step: u64,
timestamp_step: u64,
}
impl Steps {
fn start(block: SampleBlock) -> Self {
Steps {
block,
offset_step: 0,
timestamp_step: 0,
}
}
}
impl BlockList {
const GROUP: usize = 64;
const CHUNK: usize = 64 << 10;
const MAX_BLOCK_BYTES: usize = 30;
pub fn len(&self) -> usize {
self.len
}
pub fn is_empty(&self) -> bool {
self.len == 0
}
pub fn push(&mut self, b: SampleBlock) {
let prev = self.last;
match self.chunks.last_mut() {
Some(out) if !self.len.is_multiple_of(Self::GROUP) => {
let offset_step = b.offset.wrapping_sub(prev.block.offset);
let timestamp_step = b.timestamp.wrapping_sub(prev.block.timestamp);
put_varint(out, zigzag(offset_step.wrapping_sub(prev.offset_step)));
put_varint(
out,
zigzag(timestamp_step.wrapping_sub(prev.timestamp_step)),
);
put_varint(
out,
zigzag(u64::from(b.samples).wrapping_sub(u64::from(prev.block.samples))),
);
self.last = Steps {
block: b,
offset_step,
timestamp_step,
};
}
_ => {
let room = self
.chunks
.last()
.map_or(0, |c| Self::CHUNK.saturating_sub(c.len()));
if room < Self::GROUP * Self::MAX_BLOCK_BYTES {
self.chunks.push(Vec::with_capacity(Self::CHUNK));
}
let chunk = self.chunks.len() - 1;
let pos = self.chunks[chunk].len();
self.groups.push((
b,
u32::try_from(chunk).unwrap_or(u32::MAX),
u32::try_from(pos).unwrap_or(u32::MAX),
));
self.last = Steps::start(b);
}
}
self.len += 1;
}
pub fn range(&self, first: usize, count: usize) -> impl Iterator<Item = SampleBlock> + '_ {
let end = first.saturating_add(count).min(self.len);
let group = first / Self::GROUP;
BlockIter {
list: self,
next: group * Self::GROUP,
end,
last: Steps::default(),
bytes: &[],
pos: 0,
}
.skip(first - group * Self::GROUP)
}
pub fn iter(&self) -> impl Iterator<Item = SampleBlock> + '_ {
self.range(0, self.len)
}
}
struct BlockIter<'a> {
list: &'a BlockList,
next: usize,
end: usize,
last: Steps,
bytes: &'a [u8],
pos: usize,
}
impl Iterator for BlockIter<'_> {
type Item = SampleBlock;
fn next(&mut self) -> Option<SampleBlock> {
if self.next >= self.end {
return None;
}
let i = self.next;
self.next += 1;
if i.is_multiple_of(BlockList::GROUP) {
let (b, chunk, pos) = *self.list.groups.get(i / BlockList::GROUP)?;
self.bytes = self.list.chunks.get(chunk as usize)?;
self.pos = pos as usize;
self.last = Steps::start(b);
return Some(b);
}
let prev = self.last;
let offset_step = prev
.offset_step
.wrapping_add(unzigzag(get_varint(self.bytes, &mut self.pos)?));
let timestamp_step = prev
.timestamp_step
.wrapping_add(unzigzag(get_varint(self.bytes, &mut self.pos)?));
let samples = u64::from(prev.block.samples)
.wrapping_add(unzigzag(get_varint(self.bytes, &mut self.pos)?));
let b = SampleBlock {
offset: prev.block.offset.wrapping_add(offset_step),
timestamp: prev.block.timestamp.wrapping_add(timestamp_step),
samples: u32::try_from(samples).ok()?,
};
self.last = Steps {
block: b,
offset_step,
timestamp_step,
};
Some(b)
}
}
fn zigzag(v: u64) -> u64 {
(v << 1) ^ (v >> 63).wrapping_neg()
}
fn unzigzag(v: u64) -> u64 {
(v >> 1) ^ (v & 1).wrapping_neg()
}
fn put_varint(out: &mut Vec<u8>, mut v: u64) {
while v >= 0x80 {
out.push((v as u8) | 0x80);
v >>= 7;
}
out.push(v as u8);
}
fn get_varint(b: &[u8], pos: &mut usize) -> Option<u64> {
let mut v = 0u64;
for shift in (0..64).step_by(7) {
let byte = *b.get(*pos)?;
*pos += 1;
v |= u64::from(byte & 0x7F) << shift;
if byte < 0x80 {
return Some(v);
}
}
None
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct EventRecord {
pub timestamp: u64,
pub channel: u16,
pub value: u16,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Run {
pub first_block: usize,
pub block_count: usize,
pub samples: u64,
pub timestamp: u64,
}
#[derive(Debug, Clone, Default)]
pub struct PlxIndex {
pub continuous: BTreeMap<u16, BlockList>,
pub spikes: Vec<u64>,
pub max_waveform: u32,
pub events: Vec<EventRecord>,
pub block_counts: BTreeMap<u16, u64>,
pub data_end: u64,
pub findings: Vec<Finding>,
}
#[derive(Debug, Clone, Default)]
pub struct PlxFile {
pub header: PlxHeader,
pub spike_channels: Vec<SpikeChannel>,
pub event_channels: Vec<EventChannel>,
pub continuous_channels: Vec<ContinuousChannel>,
pub data_start: u64,
pub file_len: u64,
pub index: PlxIndex,
}
fn need<T>(b: &Block, at: u64, read: fn(&Block, u64) -> Option<T>) -> Result<T> {
read(b, at).ok_or_else(|| Error::corrupt_at(FORMAT_ID, at, "header cut off"))
}
pub fn parse_file_header(b: &Block) -> Result<PlxHeader> {
if b.slice(0, 4) != Some(&PLX_MAGIC[..]) {
return Err(Error::corrupt_at(FORMAT_ID, 0, "no PLEX signature"));
}
if (b.len() as u64) < FILE_HEADER_LEN {
return Err(Error::corrupt_at(
FORMAT_ID,
b.len() as u64,
format!(
"the file is {} bytes, shorter than the {FILE_HEADER_LEN}-byte file header",
b.len()
),
));
}
let version = need(b, 4, Block::i32_at)?;
let date: Vec<i32> = (0..6)
.map(|k| need(b, 160 + 4 * k, Block::i32_at))
.collect::<Result<_>>()?;
let recorded_at = ((1900..=2200).contains(&date[0])
&& (1..=12).contains(&date[1])
&& (1..=31).contains(&date[2])
&& (0..24).contains(&date[3])
&& (0..60).contains(&date[4])
&& (0..62).contains(&date[5]))
.then(|| {
format!(
"{:04}-{:02}-{:02}T{:02}:{:02}:{:02}",
date[0], date[1], date[2], date[3], date[4], date[5]
)
});
let v103 = version >= 103;
let u8f = |at| if v103 { b.u8_at(at).unwrap_or(0) } else { 0 };
let u16f = |at, min| {
if version >= min {
b.u16_at(at).unwrap_or(0)
} else {
0
}
};
Ok(PlxHeader {
version,
comment: b.text_at(8, 128).unwrap_or_default(),
clock_hz: need(b, 136, Block::i32_at)?,
spike_channel_count: need(b, 140, Block::i32_at)?,
event_channel_count: need(b, 144, Block::i32_at)?,
continuous_channel_count: need(b, 148, Block::i32_at)?,
waveform_points: need(b, 152, Block::i32_at)?,
pre_threshold_points: need(b, 156, Block::i32_at)?,
recorded_at,
waveform_rate_hz: need(b, 188, Block::i32_at)?,
last_timestamp: b.f64_at(192).unwrap_or(0.0),
electrodes_per_channel: u8f(200),
data_electrodes_per_channel: u8f(201),
spike_bits: u8f(202),
continuous_bits: u8f(203),
spike_max_mv: u16f(204, 103),
continuous_max_mv: u16f(206, 103),
spike_preamp_gain: u16f(208, 105),
acquired_with: if version >= 106 {
b.text_at(210, 18).unwrap_or_default()
} else {
String::new()
},
processed_with: if version >= 106 {
b.text_at(228, 18).unwrap_or_default()
} else {
String::new()
},
})
}
impl PlxHeader {
pub fn continuous_scale(&self, c: &ContinuousChannel) -> f64 {
let gain = f64::from(c.gain);
match self.version {
..=101 => 5000.0 / (2048.0 * gain * 1000.0),
102 => 5000.0 / (2048.0 * gain * f64::from(c.preamp_gain)),
_ => {
f64::from(self.continuous_max_mv)
/ (0.5
* f64::from(2u32.pow(u32::from(self.spike_bits.min(31))))
* gain
* f64::from(c.preamp_gain))
}
}
}
pub fn spike_scale(&self, c: &SpikeChannel) -> f64 {
let gain = f64::from(c.gain);
let full = f64::from(2u32.pow(u32::from(self.spike_bits.min(31))));
match self.version {
..=102 => 3000.0 / (2048.0 * gain * 1000.0),
103 | 104 => f64::from(self.spike_max_mv) / (0.5 * full * gain * 1000.0),
_ => {
f64::from(self.spike_max_mv)
/ (0.5 * full * gain * f64::from(self.spike_preamp_gain))
}
}
}
}
pub fn parse_plx(f: &mut File, path: &Path, file_len: u64) -> Result<PlxFile> {
let hb = read_block(f, path, 0, FILE_HEADER_LEN, file_len)?;
let header = parse_file_header(&hb)?;
for (n, what) in [
(header.spike_channel_count, "spike"),
(header.event_channel_count, "event"),
(header.continuous_channel_count, "continuous"),
] {
if !(0..=MAX_CHANNEL_HEADERS).contains(&n) {
return Err(Error::corrupt_at(
FORMAT_ID,
140,
format!("{n} {what} channel headers"),
));
}
}
let ns = u64::try_from(header.spike_channel_count).unwrap_or(0);
let ne = u64::try_from(header.event_channel_count).unwrap_or(0);
let nc = u64::try_from(header.continuous_channel_count).unwrap_or(0);
let data_start = FILE_HEADER_LEN
+ ns * SPIKE_HEADER_LEN
+ ne * EVENT_HEADER_LEN
+ nc * CONTINUOUS_HEADER_LEN;
if data_start > file_len {
return Err(Error::corrupt_at(
FORMAT_ID,
file_len,
format!(
"the channel headers end at byte {data_start}, past the end of the file ({file_len} bytes)"
),
));
}
let b = read_block(
f,
path,
FILE_HEADER_LEN,
data_start - FILE_HEADER_LEN,
file_len,
)?;
let v105 = header.version >= 105;
let mut at = FILE_HEADER_LEN;
let mut spike_channels = Vec::new();
for _ in 0..ns {
spike_channels.push(SpikeChannel {
name: b.text_at(at, 32).unwrap_or_default(),
signal_name: b.text_at(at + 32, 32).unwrap_or_default(),
channel: need(&b, at + 64, Block::i32_at)?,
gain: need(&b, at + 80, Block::i32_at)?,
filter: need(&b, at + 84, Block::i32_at)?,
threshold: need(&b, at + 88, Block::i32_at)?,
unit_count: need(&b, at + 96, Block::i32_at)?,
comment: if v105 {
b.text_at(at + 848, 128).unwrap_or_default()
} else {
String::new()
},
});
at += SPIKE_HEADER_LEN;
}
let mut event_channels = Vec::new();
for _ in 0..ne {
event_channels.push(EventChannel {
name: b.text_at(at, 32).unwrap_or_default(),
channel: need(&b, at + 32, Block::i32_at)?,
comment: if v105 {
b.text_at(at + 36, 128).unwrap_or_default()
} else {
String::new()
},
});
at += EVENT_HEADER_LEN;
}
let mut continuous_channels = Vec::new();
for _ in 0..nc {
continuous_channels.push(ContinuousChannel {
name: b.text_at(at, 32).unwrap_or_default(),
channel: need(&b, at + 32, Block::i32_at)?,
rate_hz: need(&b, at + 36, Block::i32_at)?,
gain: need(&b, at + 40, Block::i32_at)?,
enabled: need(&b, at + 44, Block::i32_at)? != 0,
preamp_gain: need(&b, at + 48, Block::i32_at)?,
comment: if v105 {
b.text_at(at + 56, 128).unwrap_or_default()
} else {
String::new()
},
});
at += CONTINUOUS_HEADER_LEN;
}
let index = walk_blocks(f, path, data_start, file_len)?;
Ok(PlxFile {
header,
spike_channels,
event_channels,
continuous_channels,
data_start,
file_len,
index,
})
}
pub fn block_header(b: &Block, at: u64) -> Option<(u16, u64, u16, u16, u16, u16)> {
let kind = b.u16_at(at)?;
let high = u64::from(b.u16_at(at + 2)?);
let low = u64::from(b.u32_at(at + 4)?);
Some((
kind,
(high << 32) | low,
b.u16_at(at + 8)?,
b.u16_at(at + 10)?,
b.u16_at(at + 12)?,
b.u16_at(at + 14)?,
))
}
pub fn walk_blocks(f: &mut File, path: &Path, start: u64, file_len: u64) -> Result<PlxIndex> {
const CHUNK: u64 = 8 << 20;
let mut ix = PlxIndex::default();
let mut pos = start;
let mut buf = Block::default();
while pos < file_len {
if pos + BLOCK_HEADER_LEN > file_len {
ix.findings.push(
Finding::error(
"truncated",
format!(
"{} bytes after the last whole data block (a block header is cut off)",
file_len - pos
),
)
.at(pos),
);
break;
}
if pos < buf.origin || pos + BLOCK_HEADER_LEN > buf.end() {
buf = read_block(f, path, pos, CHUNK, file_len)?;
}
let Some((kind, ts, ch, unit, waves, words)) = block_header(&buf, pos) else {
break;
};
let samples = u64::from(waves) * u64::from(words);
let len = BLOCK_HEADER_LEN + 2 * samples;
if pos + len > file_len {
ix.findings.push(
Finding::error(
"truncated",
format!(
"the data block at byte {pos} declares {samples} samples but the file ends {} bytes later (cut off)",
file_len - pos
),
)
.at(pos),
);
break;
}
*ix.block_counts.entry(kind).or_default() += 1;
match kind {
BLOCK_SPIKE => {
ix.spikes.push(pos);
ix.max_waveform = ix
.max_waveform
.max(u32::try_from(samples).unwrap_or(u32::MAX));
}
BLOCK_EVENT => ix.events.push(EventRecord {
timestamp: ts,
channel: ch,
value: unit,
}),
BLOCK_CONTINUOUS => ix.continuous.entry(ch).or_default().push(SampleBlock {
offset: pos + BLOCK_HEADER_LEN,
timestamp: ts,
samples: u32::try_from(samples).unwrap_or(u32::MAX),
}),
other => {
ix.findings.push(
Finding::error(
"bad_block",
format!("data block of unknown type {other} at byte {pos}; the rest of the file is not read"),
)
.at(pos),
);
break;
}
}
pos += len;
ix.data_end = pos;
}
if ix.data_end == 0 {
ix.data_end = start;
}
Ok(ix)
}
pub fn runs(
blocks: impl IntoIterator<Item = SampleBlock>,
clock_hz: f64,
rate_hz: f64,
) -> Vec<Run> {
let mut out: Vec<Run> = Vec::new();
let ticks = if rate_hz > 0.0 {
clock_hz / rate_hz
} else {
0.0
};
let mut expect: Option<f64> = None;
for (i, b) in blocks.into_iter().enumerate() {
let start = b.timestamp as f64;
let joins = expect.is_some_and(|e| (start - e).abs() <= ticks / 2.0);
match out.last_mut() {
Some(r) if joins => {
r.block_count += 1;
r.samples += u64::from(b.samples);
}
_ => out.push(Run {
first_block: i,
block_count: 1,
samples: u64::from(b.samples),
timestamp: b.timestamp,
}),
}
expect = Some(start + f64::from(b.samples) * ticks);
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn block_list_round_trips() {
let mut want = Vec::new();
let mut list = BlockList::default();
let (mut offset, mut ts) = (1000u64, 5u64 << 33);
for i in 0..50_000u64 {
offset += 36 + (i * 7919) % 5000;
ts = match i {
500 => ts + (1 << 40),
_ if i % 97 == 0 => ts - 3,
_ => ts + 400,
};
let b = SampleBlock {
offset,
timestamp: ts,
samples: (10 + i % 3) as u32,
};
want.push(b);
list.push(b);
}
assert_eq!(list.len(), want.len());
assert_eq!(list.iter().collect::<Vec<_>>(), want);
for (first, count) in [
(0, 1),
(63, 2),
(64, 64),
(130, 500),
(49_990, 50),
(50_000, 5),
] {
let got: Vec<_> = list.range(first, count).collect();
let end = (first + count).min(want.len());
assert_eq!(got, want[first.min(end)..end], "range {first}+{count}");
}
assert!(BlockList::default().iter().next().is_none());
}
#[test]
fn block_list_stays_small() {
let mut list = BlockList::default();
let (mut offset, mut ts) = (0u64, 0u64);
let n = 1_000_000u64;
for i in 0..n {
offset += 36 + 104 * ((i * 2_654_435_761) % 3);
ts += 400;
list.push(SampleBlock {
offset,
timestamp: ts,
samples: 10,
});
}
let bytes = list.chunks.iter().map(Vec::capacity).sum::<usize>()
+ list.groups.capacity() * std::mem::size_of::<(SampleBlock, u32, u32)>();
assert!(
bytes as u64 <= 6 * n,
"{bytes} bytes for {n} blocks ({} per block)",
bytes as f64 / n as f64
);
assert_eq!(
list.range(999_999, 1).next().map(|b| b.offset),
Some(offset)
);
}
}