use std::ops::Deref;
use ndarray::{Array2, Array3, s};
use crate::powerset::PowersetMapping;
use super::ChunkLayout;
pub(in crate::pipeline) struct PendingEmbedding<'a> {
pub chunk_idx: usize,
pub speaker_idx: usize,
pub audio: &'a [f32],
pub mask: Vec<f32>,
pub clean_mask: Vec<f32>,
}
pub(in crate::pipeline) struct PendingSplitEmbedding {
pub chunk_idx: usize,
pub speaker_idx: usize,
pub fbank_idx: usize,
pub weights: Vec<f32>,
}
#[derive(Debug, Clone)]
pub struct DecodedSegmentations(pub Array3<f32>);
impl Deref for DecodedSegmentations {
type Target = Array3<f32>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
#[derive(Debug, Clone)]
pub struct ChunkEmbeddings(pub Array3<f32>);
impl Deref for ChunkEmbeddings {
type Target = Array3<f32>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
#[derive(Debug, Clone)]
pub struct SpeakerCountTrack(pub Vec<usize>);
impl Deref for SpeakerCountTrack {
type Target = Vec<usize>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
#[derive(Debug, Clone)]
pub struct ChunkSpeakerClusters(pub Array2<i32>);
impl Deref for ChunkSpeakerClusters {
type Target = Array2<i32>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
#[derive(Debug, Clone)]
pub struct DiscreteDiarization(pub Array2<f32>);
impl Deref for DiscreteDiarization {
type Target = Array2<f32>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl DiscreteDiarization {
pub fn to_segments(&self) -> Vec<crate::segment::Segment> {
self.to_segments_with(
crate::pipeline::FRAME_STEP_SECONDS,
crate::pipeline::FRAME_DURATION_SECONDS,
)
}
pub fn to_segments_with(
&self,
frame_step_seconds: f64,
frame_duration_seconds: f64,
) -> Vec<crate::segment::Segment> {
crate::segment::to_segments(&self.0, frame_step_seconds, frame_duration_seconds)
}
}
#[derive(Debug, Clone)]
pub(crate) struct FrameActivations(pub(crate) Array2<f32>);
impl Deref for FrameActivations {
type Target = Array2<f32>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
pub(in crate::pipeline) struct RawSegmentationWindows(pub Vec<Array2<f32>>);
impl RawSegmentationWindows {
pub(in crate::pipeline) fn decode(
self,
powerset: &PowersetMapping,
) -> Result<DecodedSegmentations, crate::powerset::PowersetDecodeError> {
let mut windows = self.0.into_iter();
let Some(first_window) = windows.next() else {
return Ok(DecodedSegmentations(Array3::zeros((0, 0, 0))));
};
let num_windows = windows.len() + 1;
let first = powerset.hard_decode(&first_window)?;
let mut stacked = Array3::<f32>::zeros((num_windows, first.nrows(), first.ncols()));
stacked.slice_mut(s![0, .., ..]).assign(&first);
for (window_idx, window) in windows.enumerate() {
let decoded = powerset.hard_decode(&window)?;
stacked
.slice_mut(s![window_idx + 1, .., ..])
.assign(&decoded);
}
Ok(DecodedSegmentations(stacked))
}
}
pub struct BatchInput<'a> {
pub audio: &'a [f32],
pub file_id: &'a str,
}
#[derive(Clone)]
pub struct InferenceArtifacts {
pub(in crate::pipeline) layout: ChunkLayout,
pub(in crate::pipeline) segmentations: DecodedSegmentations,
pub(in crate::pipeline) embeddings: ChunkEmbeddings,
#[cfg(feature = "_metrics")]
pub(in crate::pipeline) stage_timings: Option<InferenceStageTimings>,
}
impl InferenceArtifacts {
pub(in crate::pipeline) fn try_new(
layout: ChunkLayout,
segmentations: DecodedSegmentations,
embeddings: ChunkEmbeddings,
#[cfg(feature = "_metrics")] stage_timings: Option<InferenceStageTimings>,
) -> Result<Self, super::PipelineError> {
let chunks = segmentations.0.shape()[0];
let speakers = segmentations.0.shape()[2];
if embeddings.0.shape()[0] != chunks {
return Err(super::PipelineError::Invariant(format!(
"embedding chunks {} do not match segmentation chunks {chunks}",
embeddings.0.shape()[0]
)));
}
if layout.start_frames.len() != chunks {
return Err(super::PipelineError::Invariant(format!(
"layout chunks {} do not match segmentation chunks {chunks}",
layout.start_frames.len()
)));
}
if embeddings.0.shape()[1] != speakers {
return Err(super::PipelineError::Invariant(format!(
"embedding speakers {} do not match segmentation speakers {speakers}",
embeddings.0.shape()[1]
)));
}
if embeddings.0.shape()[2] != crate::inference::embedding::EMBEDDING_WIDTH {
return Err(super::PipelineError::Invariant(format!(
"embedding width {} is invalid",
embeddings.0.shape()[2]
)));
}
Ok(Self {
layout,
segmentations,
embeddings,
#[cfg(feature = "_metrics")]
stage_timings,
})
}
pub(in crate::pipeline) fn empty_with_layout(layout: ChunkLayout) -> Self {
Self {
layout: layout.with_num_chunks(0),
segmentations: DecodedSegmentations(ndarray::Array3::zeros((0, 0, 0))),
embeddings: ChunkEmbeddings(ndarray::Array3::zeros((0, 0, 0))),
#[cfg(feature = "_metrics")]
stage_timings: None,
}
}
#[cfg(feature = "_metrics")]
#[cfg_attr(docsrs, doc(cfg(feature = "_metrics")))]
pub const fn stage_timings(&self) -> Option<InferenceStageTimings> {
self.stage_timings
}
#[cfg(feature = "_metrics")]
#[cfg_attr(docsrs, doc(cfg(feature = "_metrics")))]
pub const fn segmentations(&self) -> &DecodedSegmentations {
&self.segmentations
}
#[cfg(feature = "_metrics")]
#[cfg_attr(docsrs, doc(cfg(feature = "_metrics")))]
pub const fn embeddings(&self) -> &ChunkEmbeddings {
&self.embeddings
}
#[cfg(feature = "_metrics")]
#[cfg_attr(docsrs, doc(cfg(feature = "_metrics")))]
pub fn usable_training_embedding_count(
&self,
clean_frame_duration: super::super::CleanFrameDuration,
) -> usize {
self.embeddings
.training_set(&self.segmentations, clean_frame_duration)
.0
.nrows()
}
}
#[cfg(feature = "_metrics")]
#[cfg_attr(docsrs, doc(cfg(feature = "_metrics")))]
#[derive(Clone, Copy, Debug)]
pub struct InferenceStageTimings {
pub segmentation_seconds: f64,
pub embedding_seconds: f64,
pub prediction_seconds: f64,
pub filterbank_preparation_seconds: f64,
pub mask_preparation_seconds: f64,
pub total_seconds: f64,
pub chunk_count: usize,
pub pipelined: bool,
}
pub struct DiarizationResult {
pub segmentations: DecodedSegmentations,
pub embeddings: ChunkEmbeddings,
pub speaker_count: SpeakerCountTrack,
pub hard_clusters: ChunkSpeakerClusters,
pub discrete_diarization: DiscreteDiarization,
pub segments: Vec<crate::segment::Segment>,
pub exclusive_segments: Vec<crate::segment::Segment>,
}
impl DiarizationResult {
pub fn rttm(&self, file_id: &str) -> String {
crate::segment::to_rttm(&self.segments, file_id)
}
}
pub(in crate::pipeline) enum InferencePath {
Sequential,
Concurrent,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub(in crate::pipeline) enum EmbeddingPath {
Masked,
Split,
MultiMask,
}
#[cfg(test)]
mod tests {
use super::super::layout::ChunkLayout;
use super::*;
use crate::inference::embedding::EMBEDDING_WIDTH;
use ndarray::Array3;
#[test]
fn try_new_rejects_mismatched_chunk_counts() {
let layout = ChunkLayout::new(1.0, 16_000, 160_000, 2);
let segmentations = DecodedSegmentations(Array3::zeros((2, 4, 3)));
let embeddings = ChunkEmbeddings(Array3::zeros((1, 3, EMBEDDING_WIDTH)));
assert!(
InferenceArtifacts::try_new(
layout,
segmentations,
embeddings,
#[cfg(feature = "_metrics")]
None,
)
.is_err()
);
}
#[test]
fn try_new_rejects_invalid_embedding_width() {
let layout = ChunkLayout::new(1.0, 16_000, 160_000, 1);
let segmentations = DecodedSegmentations(Array3::zeros((1, 4, 3)));
let embeddings = ChunkEmbeddings(Array3::zeros((1, 3, 8)));
assert!(
InferenceArtifacts::try_new(
layout,
segmentations,
embeddings,
#[cfg(feature = "_metrics")]
None,
)
.is_err()
);
}
#[test]
fn try_new_accepts_matching_extents() {
let layout = ChunkLayout::new(1.0, 16_000, 160_000, 1);
let segmentations = DecodedSegmentations(Array3::zeros((1, 4, 3)));
let embeddings = ChunkEmbeddings(Array3::zeros((1, 3, EMBEDDING_WIDTH)));
assert!(
InferenceArtifacts::try_new(
layout,
segmentations,
embeddings,
#[cfg(feature = "_metrics")]
None,
)
.is_ok()
);
}
#[test]
fn empty_artifacts_have_zero_chunks() {
let artifacts =
InferenceArtifacts::empty_with_layout(ChunkLayout::new(1.0, 16_000, 160_000, 4));
assert_eq!(artifacts.segmentations.0.shape()[0], 0);
assert_eq!(artifacts.embeddings.0.shape()[0], 0);
assert!(artifacts.layout.start_frames.is_empty());
}
}