use std::collections::{BTreeMap, HashMap};
use std::path::{Path, PathBuf};
use crate::analysis::burst::{
compute_asdr, detect_single_channel_bursts, multi_channel_burst_profile, AsdrTrace,
BurstProfileOptions, MultiChannelBurst, MultiChannelBurstOptions, SingleChannelBurst,
SingleChannelBurstOptions,
};
use crate::analysis::filter::{FilterAlgorithm, FilterChain, LowPassKind};
use crate::analysis::spike::{
attach_waveforms, calc_thresholds, detect_spikes, filter_spikes, parse_channel_values,
SpikeDetectOptions, SpikeFilterOptions, SpikeTrain, ThresholdMethod, ThresholdOptions,
};
use crate::analysis::streaming::{
apply_max_spikes_per_channel, per_channel_values, ChannelStreamState, TraceSegment,
};
use crate::error::Result;
use crate::header::Med64Header;
use crate::recording::Recording;
#[cfg(feature = "layouts")]
use crate::layout::{analysis_channels, detect_layout, RecordingLayout};
#[cfg(feature = "histograms")]
use crate::analysis::burst::{amplitude_histogram, isi_histogram, Histogram, HistogramOptions};
#[cfg(feature = "correlation")]
use crate::analysis::burst::{
burst_onset_correlogram, synchrony_index, BurstOnsetCorrelogram, ChannelSynchrony,
CorrelationOptions,
};
#[cfg(feature = "parallel")]
use rayon::prelude::*;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AnalysisStage {
ReadingBlocks,
Detecting,
Bursts,
Done,
}
#[derive(Debug, Clone)]
pub struct AnalysisProgress {
pub path: PathBuf,
pub blocks_done: usize,
pub blocks_total: Option<usize>,
pub stage: AnalysisStage,
}
#[derive(Debug, Clone)]
pub struct AnalysisOptions {
pub highpass_hz: Option<f64>,
pub lowpass_hz: Option<f64>,
pub lowpass_kind: LowPassKind,
pub threshold_method: ThresholdMethod,
pub threshold_percent: f64,
pub threshold_window_sec: f64,
pub plus_thresholds_mv: Option<Vec<f64>>,
pub minus_thresholds_mv: Option<Vec<f64>>,
pub channel_spike_limit_hz: f64,
pub pre_ms: f64,
pub post_ms: f64,
pub pre_ms_per_channel: Option<Vec<f64>>,
pub post_ms_per_channel: Option<Vec<f64>>,
pub spike_filter: SpikeFilterOptions,
pub enabled_channels: Option<Vec<usize>>,
pub max_blocks: Option<usize>,
pub max_spikes_per_channel: Option<usize>,
pub time_from_ms: f64,
pub time_to_ms: Option<f64>,
pub freq_window_ms: f64,
pub keep_filtered_rows: usize,
pub retain_filtered: bool,
pub oversample: usize,
pub reduce: usize,
pub min_channel_rate_hz: f64,
pub max_channel_rate_hz: f64,
pub burst_min_duration_ms: Option<f64>,
pub burst_max_duration_ms: Option<f64>,
pub reset_filter_each_block: bool,
pub filter_algorithm: FilterAlgorithm,
pub single_burst: SingleChannelBurstOptions,
pub multi_burst: MultiChannelBurstOptions,
pub per_trace: bool,
pub detect_plus: bool,
pub detect_minus: bool,
#[cfg(feature = "layouts")]
pub layout: Option<RecordingLayout>,
#[cfg(feature = "correlation")]
pub correlation: CorrelationOptions,
#[cfg(feature = "histograms")]
pub histograms: HistogramOptions,
}
impl Default for AnalysisOptions {
fn default() -> Self {
Self {
highpass_hz: Some(300.0),
lowpass_hz: Some(5000.0),
lowpass_kind: LowPassKind::Butterworth,
threshold_method: ThresholdMethod::Median,
threshold_percent: 500.0,
threshold_window_sec: 1.0,
plus_thresholds_mv: None,
minus_thresholds_mv: None,
channel_spike_limit_hz: 400.0,
pre_ms: 1.0,
post_ms: 2.5,
pre_ms_per_channel: None,
post_ms_per_channel: None,
spike_filter: SpikeFilterOptions::default(),
enabled_channels: None,
max_blocks: None,
max_spikes_per_channel: None,
time_from_ms: 0.0,
time_to_ms: None,
freq_window_ms: 1000.0,
keep_filtered_rows: 0,
retain_filtered: false,
oversample: 1,
reduce: 1,
min_channel_rate_hz: 0.0,
max_channel_rate_hz: f64::MAX,
burst_min_duration_ms: None,
burst_max_duration_ms: None,
reset_filter_each_block: false,
filter_algorithm: FilterAlgorithm::Biquad,
single_burst: SingleChannelBurstOptions::default(),
multi_burst: MultiChannelBurstOptions::default(),
per_trace: true,
detect_plus: true,
detect_minus: true,
#[cfg(feature = "layouts")]
layout: None,
#[cfg(feature = "correlation")]
correlation: CorrelationOptions::default(),
#[cfg(feature = "histograms")]
histograms: HistogramOptions::default(),
}
}
}
impl AnalysisOptions {
pub fn reference() -> Self {
Self {
highpass_hz: Some(100.0),
lowpass_hz: Some(2000.0),
threshold_method: ThresholdMethod::Median,
threshold_percent: 500.0,
threshold_window_sec: 0.0,
channel_spike_limit_hz: 400.0,
per_trace: false,
retain_filtered: false,
reset_filter_each_block: true,
filter_algorithm: FilterAlgorithm::SecondPole,
..Default::default()
}
}
}
#[derive(Debug, Clone)]
pub struct SpikeFrequencyBin {
pub channel: usize,
pub start_ms: f64,
pub end_ms: f64,
pub spike_count: usize,
pub rate_hz: f64,
}
#[derive(Debug, Clone)]
pub struct FilteredSample {
pub time_ms: f64,
pub channel: usize,
pub millivolts: f64,
}
#[derive(Debug, Clone)]
pub struct AnalysisReport {
pub path: std::path::PathBuf,
pub header: Med64Header,
pub spikes: SpikeTrain,
pub single_bursts: Vec<SingleChannelBurst>,
pub multi_bursts: Vec<MultiChannelBurst>,
pub asdr: AsdrTrace,
pub frequency_bins: Vec<SpikeFrequencyBin>,
pub filtered_preview: Vec<FilteredSample>,
pub filtered_by_channel: std::collections::BTreeMap<usize, Vec<f64>>,
pub raster: crate::analysis::SpikeRaster,
pub burst_profile: crate::analysis::BurstProfile,
#[cfg(feature = "correlation")]
pub correlogram: BurstOnsetCorrelogram,
#[cfg(feature = "correlation")]
pub synchrony: Vec<ChannelSynchrony>,
#[cfg(feature = "histograms")]
pub isi_histogram: Histogram,
#[cfg(feature = "histograms")]
pub amplitude_histogram: Histogram,
}
pub fn run_analysis(path: impl AsRef<Path>, options: &AnalysisOptions) -> Result<AnalysisReport> {
run_analysis_with_progress(path, options, |_| {})
}
pub fn run_analysis_with_progress<F>(
path: impl AsRef<Path>,
options: &AnalysisOptions,
on_progress: F,
) -> Result<AnalysisReport>
where
F: FnMut(AnalysisProgress),
{
let path = path.as_ref();
let recording = crate::formats::open_recording_any(path)?;
run_analysis_on_recording(recording, path, options, on_progress)
}
pub fn run_analysis_on_recording<F>(
mut recording: Recording,
path: &Path,
options: &AnalysisOptions,
mut on_progress: F,
) -> Result<AnalysisReport>
where
F: FnMut(AnalysisProgress),
{
let header = recording.header().clone();
let over = options.oversample.max(1);
let red = options.reduce.max(1);
let sample_rate = header.sampling_frequency_hz * over as f64 / red as f64;
let interval_ms = 1000.0 / sample_rate;
let num_ch = header.num_channel_slots();
let channels: Vec<usize> = resolve_channels(path, &header, options, num_ch);
recording.set_channel_filter(channels.clone());
let hp = options.highpass_hz.unwrap_or(100.0);
let lp = options.lowpass_hz.unwrap_or(5000.0);
let pre_ms = options
.pre_ms_per_channel
.clone()
.unwrap_or_else(|| per_channel_values(options.pre_ms, &channels, num_ch));
let post_ms = options
.post_ms_per_channel
.clone()
.unwrap_or_else(|| per_channel_values(options.post_ms, &channels, num_ch));
let threshold_opts = ThresholdOptions {
method: options.threshold_method,
magnification_percent: options.threshold_percent,
..Default::default()
};
let refractory = (sample_rate / options.channel_spike_limit_hz.max(1.0)).round() as usize;
let window_samples = (options.threshold_window_sec * sample_rate).round() as usize;
let mut segment = TraceSegment::default();
let mut train = SpikeTrain::default();
let mut filtered_preview = Vec::new();
let mut filtered_by_channel: std::collections::BTreeMap<usize, Vec<f64>> =
std::collections::BTreeMap::new();
let mut session_end = 0.0;
let blocks_total = options.max_blocks.or(Some(header.number_of_real_data_blocks as usize));
on_progress(AnalysisProgress {
path: path.to_path_buf(),
blocks_done: 0,
blocks_total,
stage: AnalysisStage::ReadingBlocks,
});
for (block_index, block_result) in recording.blocks().enumerate() {
if options.max_blocks.is_some_and(|max| block_index >= max) {
break;
}
let block = block_result?;
let block_start = block.block_offset_ms + block.trace_offset_ms;
let block_end = block_start + block.samples.len() as f64 * interval_ms;
session_end = block_end;
if block_end < options.time_from_ms {
continue;
}
if options.time_to_ms.is_some_and(|to| block_start > to) {
break;
}
if options.per_trace && segment.trace != block.trace_number && !segment.channels.is_empty()
{
if options.retain_filtered {
for (ch, state) in &segment.channels {
filtered_by_channel
.entry(*ch)
.or_default()
.extend(state.samples.iter().copied());
}
}
let mut part = segment.flush(
&header,
interval_ms,
&threshold_opts,
options.plus_thresholds_mv.as_deref(),
options.minus_thresholds_mv.as_deref(),
&pre_ms,
&post_ms,
window_samples,
refractory,
options.filter_algorithm == FilterAlgorithm::SecondPole,
options.detect_plus,
options.detect_minus,
);
train.spikes.append(&mut part.spikes);
segment.channels.clear();
}
segment.trace = block.trace_number;
segment.trace_offset_ms = block.trace_offset_ms;
for (col, &channel) in block.channel_indices.iter().enumerate() {
if !channels.contains(&channel) {
continue;
}
let mv: Vec<f64> =
block.samples.iter().map(|row| header.counts_to_millivolts(row[col])).collect();
let mv = resample_mv(&mv, over, red);
let state = segment.channels.entry(channel).or_insert_with(|| {
ChannelStreamState::new(
channel,
sample_rate,
hp,
lp,
options.lowpass_kind,
options.filter_algorithm,
)
});
if options.reset_filter_each_block {
state.filter_reset();
}
state.push_block(&mv);
if options.keep_filtered_rows > 0 && filtered_preview.len() < options.keep_filtered_rows
{
if let Some(state) = segment.channels.get(&channel) {
let last = state.samples.last().copied().unwrap_or(0.0);
filtered_preview.push(FilteredSample {
time_ms: block_end - interval_ms,
channel,
millivolts: last,
});
}
}
}
on_progress(AnalysisProgress {
path: path.to_path_buf(),
blocks_done: block_index + 1,
blocks_total,
stage: AnalysisStage::ReadingBlocks,
});
}
on_progress(AnalysisProgress {
path: path.to_path_buf(),
blocks_done: blocks_total.unwrap_or(0),
blocks_total,
stage: AnalysisStage::Detecting,
});
if !segment.channels.is_empty() {
if options.retain_filtered {
for (ch, state) in &segment.channels {
filtered_by_channel.entry(*ch).or_default().extend(state.samples.iter().copied());
}
}
let mut part = segment.flush(
&header,
interval_ms,
&threshold_opts,
options.plus_thresholds_mv.as_deref(),
options.minus_thresholds_mv.as_deref(),
&pre_ms,
&post_ms,
window_samples,
refractory,
options.filter_algorithm == FilterAlgorithm::SecondPole,
options.detect_plus,
options.detect_minus,
);
train.spikes.append(&mut part.spikes);
}
train.spikes.sort_by(|a, b| {
a.within_trace_time_ms
.partial_cmp(&b.within_trace_time_ms)
.unwrap()
.then(a.trace.cmp(&b.trace))
.then(a.channel.cmp(&b.channel))
});
train = filter_spikes(train, &options.spike_filter);
train = apply_max_spikes_per_channel(train, options.max_spikes_per_channel);
let trace_duration = options.time_to_ms.unwrap_or_else(|| {
if options.max_blocks.is_some() {
session_end
} else {
session_end.max(header.real_trace_duration_ms)
}
}) - options.time_from_ms;
train = apply_channel_rate_gate(
train,
trace_duration.max(1e-9),
options.min_channel_rate_hz,
options.max_channel_rate_hz,
);
let frequency_bins = compute_spike_frequencies(&train, options.freq_window_ms);
let mut single_opts = options.single_burst.clone();
if single_opts.trace_end_ms.is_none() {
single_opts.trace_end_ms = Some(trace_duration.max(0.0));
}
on_progress(AnalysisProgress {
path: path.to_path_buf(),
blocks_done: blocks_total.unwrap_or(0),
blocks_total,
stage: AnalysisStage::Bursts,
});
let single_bursts = filter_burst_duration(
detect_single_channel_bursts(&train, &single_opts),
options.burst_min_duration_ms,
options.burst_max_duration_ms,
);
let asdr = compute_asdr(&train, trace_duration.max(0.0), &options.multi_burst);
let multi_bursts = asdr.bursts.clone();
let raster = crate::analysis::spike_raster(&train);
let burst_profile =
multi_channel_burst_profile(&asdr, &multi_bursts, &BurstProfileOptions::default());
on_progress(AnalysisProgress {
path: path.to_path_buf(),
blocks_done: blocks_total.unwrap_or(0),
blocks_total,
stage: AnalysisStage::Done,
});
Ok(AnalysisReport {
path: path.to_path_buf(),
header,
spikes: train.clone(),
single_bursts: single_bursts.clone(),
multi_bursts,
asdr,
frequency_bins,
filtered_preview,
filtered_by_channel,
raster,
burst_profile,
#[cfg(feature = "correlation")]
correlogram: burst_onset_correlogram(&single_bursts, &options.correlation),
#[cfg(feature = "correlation")]
synchrony: synchrony_index(&single_bursts, &options.correlation),
#[cfg(feature = "histograms")]
isi_histogram: isi_histogram(&train, &options.histograms),
#[cfg(feature = "histograms")]
amplitude_histogram: amplitude_histogram(&train, &options.histograms),
})
}
pub fn run_analysis_from_spikes(
path: impl AsRef<Path>,
header: Med64Header,
train: SpikeTrain,
options: &AnalysisOptions,
) -> Result<AnalysisReport> {
let path = path.as_ref();
let mut train = filter_spikes(train, &options.spike_filter);
train = crate::analysis::streaming::apply_max_spikes_per_channel(
train,
options.max_spikes_per_channel,
);
let trace_duration = options.time_to_ms.unwrap_or(
header
.real_trace_duration_ms
.max(train.spikes.iter().map(|s| s.within_trace_time_ms).fold(0.0, f64::max)),
) - options.time_from_ms;
let frequency_bins = compute_spike_frequencies(&train, options.freq_window_ms);
let mut single_opts = options.single_burst.clone();
if single_opts.trace_end_ms.is_none() {
single_opts.trace_end_ms = Some(trace_duration.max(0.0));
}
let single_bursts = filter_burst_duration(
detect_single_channel_bursts(&train, &single_opts),
options.burst_min_duration_ms,
options.burst_max_duration_ms,
);
let asdr = compute_asdr(&train, trace_duration.max(0.0), &options.multi_burst);
let multi_bursts = asdr.bursts.clone();
let raster = crate::analysis::spike_raster(&train);
let burst_profile =
multi_channel_burst_profile(&asdr, &multi_bursts, &BurstProfileOptions::default());
Ok(AnalysisReport {
path: path.to_path_buf(),
header,
spikes: train.clone(),
single_bursts: single_bursts.clone(),
multi_bursts,
asdr,
frequency_bins,
filtered_preview: Vec::new(),
filtered_by_channel: std::collections::BTreeMap::new(),
raster,
burst_profile,
#[cfg(feature = "correlation")]
correlogram: burst_onset_correlogram(&single_bursts, &options.correlation),
#[cfg(feature = "correlation")]
synchrony: synchrony_index(&single_bursts, &options.correlation),
#[cfg(feature = "histograms")]
isi_histogram: isi_histogram(&train, &options.histograms),
#[cfg(feature = "histograms")]
amplitude_histogram: amplitude_histogram(&train, &options.histograms),
})
}
fn resolve_channels(
path: &Path,
header: &Med64Header,
options: &AnalysisOptions,
num_ch: usize,
) -> Vec<usize> {
if let Some(ref ch) = options.enabled_channels {
return ch.clone();
}
#[cfg(feature = "layouts")]
if let Some(layout) = options.layout {
return analysis_channels(layout, header);
}
#[cfg(feature = "layouts")]
{
let layout = detect_layout(header, path);
if layout != RecordingLayout::Med64Standard {
return analysis_channels(layout, header);
}
}
header.active_channel_indices_for(num_ch)
}
pub fn compute_spike_frequencies(train: &SpikeTrain, window_ms: f64) -> Vec<SpikeFrequencyBin> {
if train.is_empty() || window_ms <= 0.0 {
return Vec::new();
}
let mut counts: BTreeMap<(usize, usize), usize> = BTreeMap::new();
for spike in &train.spikes {
let idx = (spike.within_trace_time_ms / window_ms) as usize;
*counts.entry((spike.channel, idx)).or_default() += 1;
}
let mut out = Vec::new();
for ((channel, idx), count) in counts {
let start = idx as f64 * window_ms;
let end = start + window_ms;
out.push(SpikeFrequencyBin {
channel,
start_ms: start,
end_ms: end,
spike_count: count,
rate_hz: count as f64 / (window_ms / 1000.0),
});
}
out.sort_by(|a, b| {
a.channel.cmp(&b.channel).then(a.start_ms.partial_cmp(&b.start_ms).unwrap())
});
out
}
fn resample_mv(mv: &[f64], oversample: usize, reduce: usize) -> Vec<f64> {
let over = oversample.max(1);
let red = reduce.max(1);
let mut up = Vec::new();
if over == 1 {
up.extend_from_slice(mv);
} else if mv.len() < 2 {
up.extend_from_slice(mv);
} else {
for w in mv.windows(2) {
up.push(w[0]);
for k in 1..over {
let t = k as f64 / over as f64;
up.push(w[0] * (1.0 - t) + w[1] * t);
}
}
if let Some(&last) = mv.last() {
up.push(last);
}
}
if red == 1 {
up
} else {
up.into_iter().step_by(red).collect()
}
}
fn apply_channel_rate_gate(
train: SpikeTrain,
duration_sec: f64,
min_hz: f64,
max_hz: f64,
) -> SpikeTrain {
if min_hz <= 0.0 && max_hz >= f64::MAX / 2.0 {
return train;
}
let mut counts = std::collections::BTreeMap::<usize, usize>::new();
for s in &train.spikes {
*counts.entry(s.channel).or_default() += 1;
}
let keep: std::collections::BTreeSet<usize> = counts
.into_iter()
.filter(|(_, n)| {
let rate = *n as f64 / duration_sec.max(1e-9);
rate >= min_hz && rate <= max_hz
})
.map(|(ch, _)| ch)
.collect();
SpikeTrain { spikes: train.spikes.into_iter().filter(|s| keep.contains(&s.channel)).collect() }
}
fn filter_burst_duration(
bursts: Vec<SingleChannelBurst>,
min_ms: Option<f64>,
max_ms: Option<f64>,
) -> Vec<SingleChannelBurst> {
bursts
.into_iter()
.filter(|b| {
min_ms.map(|m| b.duration_ms >= m).unwrap_or(true)
&& max_ms.map(|m| b.duration_ms <= m).unwrap_or(true)
})
.collect()
}
pub fn spike_timestamp_table_header() -> &'static str {
"channel,time_of_day,within_session_time,within_session_time_ms,within_trace_time_ms,cluster_id,trace_num,pre_ms,post_ms,y0_mV,y1_mV"
}
pub fn spike_timestamp_csv_header() -> &'static str {
spike_timestamp_table_header()
}
pub fn format_spike_csv_row(header: &Med64Header, spike: &crate::analysis::spike::Spike) -> String {
let time_of_day = format!(
"{:02}:{:02}:{:02}.{:03}",
header.start_hour(),
header.start_minute(),
header.start_second(),
header.start_millisecond()
);
let within_session = format!(
"{:02}:{:02}:{:02}.{:03}",
((spike.within_session_time_ms / 3_600_000.0) as u64) % 24,
((spike.within_session_time_ms / 60_000.0) as u64) % 60,
((spike.within_session_time_ms / 1000.0) as u64) % 60,
(spike.within_session_time_ms % 1000.0) as u64
);
let pre = spike.waveform_times_ms.first().copied().unwrap_or(0.0).abs();
let post = spike.waveform_times_ms.last().copied().unwrap_or(0.0).abs();
let y0 = spike.waveform_mv.first().copied().unwrap_or(0.0);
let y1 = spike.waveform_mv.last().copied().unwrap_or(spike.peak_mv);
format!(
"{},{},{},{:.3},{:.3},{},{},{:.3},{:.3},{:.6},{:.6}",
spike.channel + 1,
time_of_day,
within_session,
spike.within_session_time_ms,
spike.within_trace_time_ms,
0, spike.trace + 1,
pre,
post,
y0,
y1
)
}
pub fn spike_timestamp_preamble(header: &Med64Header) -> String {
let tz = format_timezone_offset(header.start_timezone_minutes);
let session_start = format!(
"{:04}/{}/{:02} {:02}:{:02}:{:02} {tz}",
header.start_year,
header.start_month,
header.start_day,
header.start_hour(),
header.start_minute(),
header.start_second(),
);
format!(
"File Format Version,20071201\nSession Start Time,{session_start}\nTrace Duration (ms),{:.3}\n",
header.real_trace_duration_ms
)
}
fn format_timezone_offset(minutes: i16) -> String {
let sign = if minutes >= 0 { '+' } else { '-' };
let abs = minutes.unsigned_abs();
format!("{sign}{:02}:{:02}", abs / 60, abs % 60)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::analysis::spike::ThresholdMethod;
#[test]
fn analyzes_sample_data1_subset() {
let report = run_analysis(
crate::fixture_paths::sample_data1(),
&AnalysisOptions {
max_blocks: Some(2),
highpass_hz: Some(100.0),
lowpass_hz: Some(2000.0),
plus_thresholds_mv: Some(vec![0.5; 64]),
minus_thresholds_mv: Some(vec![-0.5; 64]),
threshold_method: ThresholdMethod::Fixed,
per_trace: true,
..Default::default()
},
)
.unwrap();
assert!(report.header.sampling_frequency_hz > 0.0);
}
#[test]
fn per_trace_finds_spikes_sample_data6() {
let report = run_analysis(
crate::fixture_paths::sample_data6_cap(),
&AnalysisOptions {
max_blocks: Some(5),
highpass_hz: Some(100.0),
lowpass_hz: Some(2000.0),
threshold_method: ThresholdMethod::Median,
..Default::default()
},
)
.unwrap();
assert!(!report.spikes.is_empty());
}
}