use crate::cognitive::CognitiveOutputSpeech;
use crate::cognitive_output_audio::CognitiveOutputAudio;
use crate::peer_input_audio::{PEER_INPUT_AUDIO_CHUNK_DURATION, PeerInputAudio};
use crate::plugin::Plugin;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use std::sync::Arc;
pub fn calculate_chunk_indices(base_index: u64, start_secs: f64, end_secs: f64) -> (u64, u64) {
let chunk_duration_secs = PEER_INPUT_AUDIO_CHUNK_DURATION.as_secs_f64();
let start_index = base_index + (start_secs / chunk_duration_secs).floor() as u64;
let end_index = base_index + (end_secs / chunk_duration_secs).ceil() as u64;
(start_index, end_index)
}
#[derive(Clone)]
pub struct SpeechDetected(Arc<tokio::sync::Notify>);
impl SpeechDetected {
pub fn new(notify: Arc<tokio::sync::Notify>) -> Self {
Self(notify)
}
pub fn notify(&self) {
self.0.notify_waiters();
}
}
#[derive(
Serialize,
Deserialize,
PartialEq,
Eq,
Hash,
Debug,
Clone,
JsonSchema,
derive_more::Display,
derive_more::From,
derive_more::Deref,
)]
pub struct SpeakerId(pub String);
impl SpeakerId {
pub fn new(speaker_id: String) -> Self {
Self(speaker_id)
}
}
#[derive(Deserialize, Serialize, Default, Clone, Debug, JsonSchema)]
pub struct Word {
pub start_index: Option<u64>,
pub end_index: Option<u64>,
pub word: String,
pub speaker_hint: Option<String>,
}
#[derive(Deserialize, Serialize, Default, Clone, Debug, JsonSchema)]
pub struct SpeechTranscript {
pub start_index: u64,
pub end_index: u64,
pub transcript: String,
pub words: Vec<Word>,
}
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
pub enum InputVoiceAudio {
Voice(PeerInputAudioIndexed),
NoVoice(PeerInputAudioIndexed),
}
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
pub struct PeerInputAudioIndexed {
pub audio: PeerInputAudio,
pub index: u64,
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
pub struct CosineSimilarity(pub f32);
#[derive(Clone, Debug)]
pub struct WordOverlap {
pub start_index: u64,
pub end_index: u64,
pub overlaps: std::collections::HashMap<InternalSpeaker, u64>,
pub word: String,
}
pub type SpeakerHeuristicFn =
dyn Fn(&[WordOverlap], &[SpeakerSegment]) -> Vec<Option<SpeakerId>> + Send + Sync;
#[derive(Clone)]
pub struct SpeakerHeuristicCallback(std::sync::Arc<SpeakerHeuristicFn>);
impl std::fmt::Debug for SpeakerHeuristicCallback {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SpeakerHeuristicCallback")
.finish_non_exhaustive()
}
}
impl SpeakerHeuristicCallback {
pub fn new<F>(callback: F) -> Self
where
F: Fn(&[WordOverlap], &[SpeakerSegment]) -> Vec<Option<SpeakerId>> + Send + Sync + 'static,
{
Self(std::sync::Arc::new(callback))
}
pub fn evaluate(
&self,
words: &[WordOverlap],
segments: &[SpeakerSegment],
) -> Vec<Option<SpeakerId>> {
(self.0)(words, segments)
}
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub enum InternalSpeaker {
Unknown(Option<(SpeakerId, CosineSimilarity)>),
Recognized(SpeakerId),
}
impl PartialEq for InternalSpeaker {
fn eq(&self, other: &Self) -> bool {
match (self, other) {
(InternalSpeaker::Unknown(a), InternalSpeaker::Unknown(b)) => match (a, b) {
(Some((id_a, score_a)), Some((id_b, score_b))) => {
id_a == id_b && score_a.0 == score_b.0
}
(None, None) => true,
_ => false,
},
(InternalSpeaker::Recognized(a), InternalSpeaker::Recognized(b)) => a == b,
_ => false,
}
}
}
impl Eq for InternalSpeaker {}
impl std::hash::Hash for InternalSpeaker {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
match self {
InternalSpeaker::Unknown(opt) => {
0_u8.hash(state);
if let Some((id, score)) = opt {
1_u8.hash(state);
id.hash(state);
score.0.to_bits().hash(state);
} else {
0_u8.hash(state);
}
}
InternalSpeaker::Recognized(id) => {
1_u8.hash(state);
id.hash(state);
}
}
}
}
impl From<InternalSpeaker> for crate::peer_input::Speaker {
fn from(val: InternalSpeaker) -> Self {
match val {
InternalSpeaker::Unknown(None) => crate::peer_input::Speaker::Unknown(None),
InternalSpeaker::Unknown(Some((id, _))) => crate::peer_input::Speaker::Unknown(Some(
SpeakerId(format!("Maybe {}", id).to_string()),
)),
InternalSpeaker::Recognized(id) => {
crate::peer_input::Speaker::Recognized(SpeakerId(id.to_string()))
}
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct SpeakerSegment {
pub speaker: InternalSpeaker,
pub start_index: u64,
pub end_index: u64,
}
impl From<InputVoiceAudio> for PeerInputAudio {
fn from(input_voice_audio: InputVoiceAudio) -> Self {
match input_voice_audio {
InputVoiceAudio::Voice(peer_input_audio) => peer_input_audio.audio,
InputVoiceAudio::NoVoice(peer_input_audio) => peer_input_audio.audio,
}
}
}
impl From<InputVoiceAudio> for [i32; crate::peer_input_audio::PEER_INPUT_AUDIO_CHUNK_SIZE] {
fn from(input_voice_audio: InputVoiceAudio) -> Self {
let peer_input_audio: PeerInputAudio = input_voice_audio.into();
peer_input_audio.into()
}
}
use crate::sync::{broadcast, mpsc};
use async_trait::async_trait;
#[async_trait]
pub trait STTPlugin: Plugin + Send + Sync {
async fn start(
&self,
audio_rx: mpsc::Receiver<InputVoiceAudio>,
transcript_tx: mpsc::Sender<SpeechTranscript>,
speech_detected: SpeechDetected,
) -> Result<(), String>;
}
#[async_trait]
pub trait TTSPlugin: Plugin + Send + Sync {
async fn start(
&self,
speech_rx: broadcast::Receiver<CognitiveOutputSpeech>,
audio_tx: mpsc::Sender<CognitiveOutputAudio>,
) -> Result<(), String>;
}
#[async_trait]
pub trait DiarizationPlugin: Plugin + Send + Sync {
async fn start(
&self,
audio_rx: broadcast::Receiver<InputVoiceAudio>,
segment_tx: mpsc::Sender<SpeakerSegment>,
) -> Result<(), String>;
fn heuristic(&self) -> Option<crate::speech_to_text::SpeakerHeuristicCallback> {
None
}
}