use std::collections::BTreeMap;
use std::path::Path;
use openreadout_core::bytes::{Block, read_block};
use openreadout_core::model::Finding;
use openreadout_core::source::SourceFile;
use openreadout_core::{Error, Result};
use super::TDT_FORMAT_ID;
pub const TSQ_RECORD_LEN: u64 = 40;
pub const MAX_TSQ_RECORDS: u64 = 50_000_000;
pub const EVENT_STROBE_ON: i32 = 0x101;
pub const EVENT_STROBE_OFF: i32 = 0x102;
pub const EVENT_SCALAR: i32 = 0x201;
pub const EVENT_STREAM: i32 = 0x8101;
pub const EVENT_SNIPPET: i32 = 0x8201;
pub const EVENT_MARK: i32 = 0x8801;
pub const EVENT_STREAM_MASK: i32 = 0xFF0F;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SampleFormat {
Float32,
Int32,
Int16,
Int8,
Float64,
}
impl SampleFormat {
pub fn from_code(code: i32) -> Option<Self> {
Some(match code {
0 => Self::Float32,
1 => Self::Int32,
2 => Self::Int16,
3 => Self::Int8,
4 => Self::Float64,
_ => return None,
})
}
pub fn width(self) -> u64 {
match self {
Self::Int8 => 1,
Self::Int16 => 2,
Self::Float32 | Self::Int32 => 4,
Self::Float64 => 8,
}
}
pub fn name(self) -> &'static str {
match self {
Self::Float32 => "float32",
Self::Int32 => "int32",
Self::Int16 => "int16",
Self::Int8 => "int8",
Self::Float64 => "float64",
}
}
pub fn decode(self, b: &[u8], out: &mut Vec<f64>) {
match self {
Self::Float32 => out.extend(
b.as_chunks::<4>()
.0
.iter()
.map(|x| f64::from(f32::from_le_bytes(*x))),
),
Self::Int32 => out.extend(
b.as_chunks::<4>()
.0
.iter()
.map(|x| f64::from(i32::from_le_bytes(*x))),
),
Self::Int16 => out.extend(
b.as_chunks::<2>()
.0
.iter()
.map(|x| f64::from(i16::from_le_bytes(*x))),
),
Self::Int8 => out.extend(b.iter().map(|x| f64::from(*x as i8))),
Self::Float64 => {
out.extend(b.as_chunks::<8>().0.iter().map(|x| f64::from_le_bytes(*x)));
}
}
}
}
pub fn store_text(b: [u8; 4]) -> String {
b.iter()
.filter(|c| **c != 0)
.map(|c| char::from(*c))
.collect()
}
#[derive(Debug, Clone, Default)]
pub struct StreamChannel {
pub offsets: Vec<u64>,
pub first_time: f64,
pub last_time: f64,
pub gaps: u64,
}
#[derive(Debug, Clone)]
pub struct StreamStore {
pub name: String,
pub format_code: i32,
pub size_words: i32,
pub sample_rate_hz: f32,
pub channels: BTreeMap<u16, StreamChannel>,
pub unreadable: Option<String>,
}
impl StreamStore {
pub fn format(&self) -> Option<SampleFormat> {
SampleFormat::from_code(self.format_code)
}
pub fn chunk_bytes(&self) -> u64 {
(i64::from(self.size_words) * 4 - TSQ_RECORD_LEN as i64).max(0) as u64
}
pub fn chunk_samples(&self) -> u64 {
self.format().map_or(0, |f| self.chunk_bytes() / f.width())
}
}
#[derive(Debug, Clone, Copy)]
pub struct Snippet {
pub time: f64,
pub store: u16,
pub channel: u16,
pub sort_code: u16,
pub offset: u64,
pub bytes: u32,
pub format_code: i32,
}
#[derive(Debug, Clone, Copy)]
pub struct Strobe {
pub time: f64,
pub store: [u8; 4],
pub onset_store: [u8; 4],
pub value: f64,
pub off: bool,
}
#[derive(Debug, Clone, Default)]
pub struct TsqIndex {
pub file_len: u64,
pub record_count: u64,
pub start_time: Option<f64>,
pub stop_time: Option<f64>,
pub streams: BTreeMap<String, StreamStore>,
pub snippet_stores: Vec<String>,
pub snippets: Vec<Snippet>,
pub strobes: Vec<Strobe>,
pub type_counts: BTreeMap<i32, u64>,
pub findings: Vec<Finding>,
}
impl TsqIndex {
pub fn undecoded_records(&self) -> u64 {
self.type_counts
.iter()
.filter(|(t, _)| !is_known_type(**t))
.map(|(_, n)| *n)
.sum::<u64>()
+ self.type_counts.get(&EVENT_SCALAR).copied().unwrap_or(0)
}
}
fn is_known_type(t: i32) -> bool {
t == 0
|| t & EVENT_STREAM_MASK == EVENT_STREAM
|| matches!(
t,
EVENT_SNIPPET | EVENT_STROBE_ON | EVENT_STROBE_OFF | EVENT_MARK | EVENT_SCALAR
)
}
pub fn looks_like_tsq(head: &[u8]) -> bool {
let at = |o: usize| -> Option<[u8; 4]> { head.get(o..o + 4).and_then(|b| b.try_into().ok()) };
matches!(
(at(4), at(44), at(48)),
(Some([0, 0, 0, 0]), Some(t), Some([1, 0, 0, 0])) if i32::from_le_bytes(t) == EVENT_MARK
)
}
#[allow(clippy::many_single_char_names)] pub fn parse_tsq(f: &mut SourceFile, path: &Path, file_len: u64) -> Result<TsqIndex> {
let n = file_len / TSQ_RECORD_LEN;
if n > MAX_TSQ_RECORDS {
return Err(Error::unsupported(
TDT_FORMAT_ID,
format!("a .tsq index of {n} records"),
format!(
"{} holds more than {MAX_TSQ_RECORDS} records; split the block or check that it is a .tsq file",
path.display()
),
));
}
if n < 2 {
return Err(Error::corrupt(
TDT_FORMAT_ID,
format!(
"{} is too short for a .tsq index ({file_len} bytes)",
path.display()
),
));
}
let mut out = TsqIndex {
file_len,
record_count: n,
..TsqIndex::default()
};
if !file_len.is_multiple_of(TSQ_RECORD_LEN) {
out.findings.push(
Finding::warning(
"partial_record",
format!(
"{} bytes after the last whole record",
file_len % TSQ_RECORD_LEN
),
)
.at(n * TSQ_RECORD_LEN),
);
}
let window = 1u64 << 22;
let mut buf = Block::default();
let mut snippet_index: BTreeMap<[u8; 4], u16> = BTreeMap::new();
for k in 0..n {
let at = k * TSQ_RECORD_LEN;
if at + TSQ_RECORD_LEN > buf.end() || at < buf.origin {
buf = read_block(f, path, at, window, file_len)?;
}
let Some(r) = buf.slice(at, TSQ_RECORD_LEN as usize) else {
return Err(Error::corrupt_at(TDT_FORMAT_ID, at, "index record cut off"));
};
let i32_at = |o: usize| i32::from_le_bytes([r[o], r[o + 1], r[o + 2], r[o + 3]]);
let u16_at = |o: usize| u16::from_le_bytes([r[o], r[o + 1]]);
let mut eight = [0u8; 8];
eight.copy_from_slice(&r[16..24]);
let time = f64::from_le_bytes(eight);
eight.copy_from_slice(&r[24..32]);
let offset_bytes = eight;
let size = i32_at(0);
let kind = i32_at(4);
let name = [r[8], r[9], r[10], r[11]];
let channel = u16_at(12);
let sort_code = u16_at(14);
let format_code = i32_at(32);
let rate = f32::from_le_bytes([r[36], r[37], r[38], r[39]]);
*out.type_counts.entry(kind).or_insert(0) += 1;
if k == 0 {
if kind != 0 || u64::try_from(size).ok() != Some(file_len) {
out.findings.push(Finding::warning(
"index_header",
format!("record 0 has type {kind} and size {size}, not type 0 and the file length {file_len}"),
));
}
continue;
}
if kind == EVENT_MARK {
match name {
[1, 0, 0, 0] => out.start_time = Some(time),
[2, 0, 0, 0] => out.stop_time = Some(time),
_ => {}
}
} else if kind & EVENT_STREAM_MASK == EVENT_STREAM {
let store = store_text(name);
let s = out
.streams
.entry(store.clone())
.or_insert_with(|| StreamStore {
name: store.clone(),
format_code,
size_words: size,
sample_rate_hz: rate,
channels: BTreeMap::new(),
unreadable: None,
});
if s.unreadable.is_none() {
if s.format_code != format_code {
s.unreadable = Some(format!(
"its data format changes from {} to {format_code}",
s.format_code
));
} else if s.size_words != size {
s.unreadable = Some(format!(
"its chunk size changes from {} to {size} words",
s.size_words
));
} else if s.format().is_none() {
s.unreadable = Some(format!("data format code {format_code} is not known"));
} else if s.chunk_samples() == 0 {
s.unreadable = Some(format!("chunks of {size} words hold no samples"));
}
}
let per_chunk = s.chunk_samples() as f64 / f64::from(s.sample_rate_hz);
let c = s.channels.entry(channel).or_insert_with(|| StreamChannel {
first_time: time,
last_time: time,
..StreamChannel::default()
});
if !c.offsets.is_empty() && time - c.last_time > 1.5 * per_chunk {
c.gaps += 1;
}
c.last_time = time;
c.offsets.push(u64::from_le_bytes(offset_bytes));
} else if kind == EVENT_SNIPPET {
let next = u16::try_from(snippet_index.len()).unwrap_or(u16::MAX);
let store = *snippet_index.entry(name).or_insert_with(|| {
out.snippet_stores.push(store_text(name));
next
});
out.snippets.push(Snippet {
time,
store,
channel,
sort_code,
offset: u64::from_le_bytes(offset_bytes),
bytes: u32::try_from((i64::from(size) * 4 - TSQ_RECORD_LEN as i64).max(0))
.unwrap_or(0),
format_code,
});
} else if kind == EVENT_STROBE_ON || kind == EVENT_STROBE_OFF {
out.strobes.push(Strobe {
time,
store: name,
onset_store: [r[12], r[13], r[14], r[15]],
value: f64::from_le_bytes(offset_bytes),
off: kind == EVENT_STROBE_OFF,
});
}
}
if out.start_time.is_none() {
out.findings.push(Finding::warning(
"no_start_mark",
"the index has no block start mark; times count from the first record",
));
}
if out.stop_time.is_none() {
out.findings.push(Finding::warning(
"no_stop_mark",
"the index has no block stop mark (the recording may not have been closed)",
));
}
Ok(out)
}