kcode_speaker_v3_analysis/
lib.rs

1use serde::{Deserialize, Deserializer, Serialize, Serializer};
2use std::{collections::BTreeSet, error::Error, fmt, str::FromStr};
3
4pub const MAX_AUDIO_DURATION_MS: u64 = 150_000;
5pub const OGG_MEDIA_TYPE: &str = "audio/ogg";
6pub const FEATURE_SCHEMA_REVISION: &str = "speaker-v3-features-24-r1";
7pub const GEMINI_TRANSCRIPT_PROMPT_REVISION: &str = "speaker-v3-gemini-transcript-r1";
8pub const GEMINI_FEATURE_PROMPT_ONE_REVISION: &str = "speaker-v3-gemini-feature-1-r2";
9pub const GEMINI_FEATURE_PROMPT_TWO_REVISION: &str = "speaker-v3-gemini-feature-2-r2";
10pub const GEMINI_FEATURE_PROMPT_THREE_REVISION: &str = "speaker-v3-gemini-feature-3-r2";
11pub const GPT_STRUCTURING_PROMPT_REVISION: &str = "speaker-v3-gpt-structure-r2";
12
13pub const GEMINI_FEATURE_PROMPT_REVISIONS: [&str; 3] = [
14    GEMINI_FEATURE_PROMPT_ONE_REVISION,
15    GEMINI_FEATURE_PROMPT_TWO_REVISION,
16    GEMINI_FEATURE_PROMPT_THREE_REVISION,
17];
18
19pub const FEATURE_NAMES: [&str; 24] = [
20    "median_f0_hz",
21    "high_front_vowel_f1_hz",
22    "high_back_vowel_f2_hz",
23    "spectral_tilt_db_per_octave",
24    "cepstral_peak_prominence_db",
25    "foreign_accentedness_1_to_9",
26    "dominant_rhotic_realization",
27    "unstressed_vowel_reduction_percent",
28    "high_front_vowel_f2_hz",
29    "low_vowel_f1_hz",
30    "h1_minus_h2_db",
31    "rhotic_f3_minus_f2_hz",
32    "word_initial_t_vot_ms",
33    "dominant_lateral_realization",
34    "monophthongization_percent",
35    "vocal_gender_presentation",
36    "low_vowel_f2_hz",
37    "high_back_vowel_f1_hz",
38    "mean_formant_dispersion_hz",
39    "creaky_phonation_percent",
40    "hypernasality_0_to_4",
41    "sibilant_center_of_gravity_hz",
42    "consonant_cluster_reduction_percent",
43    "perceived_vocal_age_years",
44];
45
46const NUMERIC_FEATURE_NAMES: [&str; 22] = [
47    "median_f0_hz",
48    "high_front_vowel_f1_hz",
49    "high_back_vowel_f2_hz",
50    "spectral_tilt_db_per_octave",
51    "cepstral_peak_prominence_db",
52    "foreign_accentedness_1_to_9",
53    "unstressed_vowel_reduction_percent",
54    "high_front_vowel_f2_hz",
55    "low_vowel_f1_hz",
56    "h1_minus_h2_db",
57    "rhotic_f3_minus_f2_hz",
58    "word_initial_t_vot_ms",
59    "monophthongization_percent",
60    "vocal_gender_presentation",
61    "low_vowel_f2_hz",
62    "high_back_vowel_f1_hz",
63    "mean_formant_dispersion_hz",
64    "creaky_phonation_percent",
65    "hypernasality_0_to_4",
66    "sibilant_center_of_gravity_hz",
67    "consonant_cluster_reduction_percent",
68    "perceived_vocal_age_years",
69];
70
71pub const GEMINI_TRANSCRIPT_PROMPT: &str = include_str!("gemini-transcript-prompt.txt");
72pub const GEMINI_FEATURE_PROMPT_ONE: &str = include_str!("gemini-feature-1-prompt.txt");
73pub const GEMINI_FEATURE_PROMPT_TWO: &str = include_str!("gemini-feature-2-prompt.txt");
74pub const GEMINI_FEATURE_PROMPT_THREE: &str = include_str!("gemini-feature-3-prompt.txt");
75pub const GPT_STRUCTURING_PROMPT: &str = include_str!("gpt-structuring-prompt.txt");
76
77#[derive(Debug, Clone, PartialEq, Eq)]
78pub enum ValidationError {
79    InvalidOgg,
80    InvalidDuration(u64),
81    ByteLengthOverflow,
82    Blank(&'static str),
83    InvalidSpeakerLabel(String),
84    DuplicateSpeakerLabel(LocalSpeakerLabel),
85    NonFiniteFeature(&'static str),
86}
87
88impl fmt::Display for ValidationError {
89    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
90        match self {
91            Self::InvalidOgg => formatter.write_str("audio is not a complete Ogg first page"),
92            Self::InvalidDuration(value) => write!(formatter, "invalid audio duration: {value} ms"),
93            Self::ByteLengthOverflow => formatter.write_str("audio byte length exceeds u64"),
94            Self::Blank(field) => write!(formatter, "{field} is blank"),
95            Self::InvalidSpeakerLabel(value) => write!(formatter, "invalid speaker label: {value}"),
96            Self::DuplicateSpeakerLabel(value) => {
97                write!(formatter, "duplicate speaker label: {value}")
98            }
99            Self::NonFiniteFeature(field) => write!(formatter, "{field} is not finite"),
100        }
101    }
102}
103
104impl Error for ValidationError {}
105
106#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
107pub struct OggAudioMetadata {
108    duration_ms: u64,
109    byte_length: u64,
110    filename: Option<String>,
111}
112
113impl OggAudioMetadata {
114    pub fn from_bytes(
115        bytes: &[u8],
116        duration_ms: u64,
117        filename: Option<String>,
118    ) -> Result<Self, ValidationError> {
119        validate_duration(duration_ms)?;
120        validate_optional_text(filename.as_deref(), "filename")?;
121        if bytes.len() < 27 || &bytes[..4] != b"OggS" || bytes[4] != 0 {
122            return Err(ValidationError::InvalidOgg);
123        }
124        let body_start = 27 + bytes[26] as usize;
125        if bytes.len() < body_start {
126            return Err(ValidationError::InvalidOgg);
127        }
128        let body_length: usize = bytes[27..body_start]
129            .iter()
130            .map(|value| *value as usize)
131            .sum();
132        if bytes.len() < body_start + body_length {
133            return Err(ValidationError::InvalidOgg);
134        }
135        Ok(Self {
136            duration_ms,
137            byte_length: u64::try_from(bytes.len())
138                .map_err(|_| ValidationError::ByteLengthOverflow)?,
139            filename,
140        })
141    }
142
143    pub fn validate(&self) -> Result<(), ValidationError> {
144        validate_duration(self.duration_ms)?;
145        validate_optional_text(self.filename.as_deref(), "filename")?;
146        (self.byte_length >= 27)
147            .then_some(())
148            .ok_or(ValidationError::InvalidOgg)
149    }
150
151    pub fn duration_ms(&self) -> u64 {
152        self.duration_ms
153    }
154
155    pub fn byte_length(&self) -> u64 {
156        self.byte_length
157    }
158
159    pub fn filename(&self) -> Option<&str> {
160        self.filename.as_deref()
161    }
162
163    pub fn media_type(&self) -> &'static str {
164        OGG_MEDIA_TYPE
165    }
166}
167
168fn validate_duration(duration_ms: u64) -> Result<(), ValidationError> {
169    (1..=MAX_AUDIO_DURATION_MS)
170        .contains(&duration_ms)
171        .then_some(())
172        .ok_or(ValidationError::InvalidDuration(duration_ms))
173}
174
175fn validate_optional_text(value: Option<&str>, field: &'static str) -> Result<(), ValidationError> {
176    if value.is_some_and(|text| text.trim().is_empty()) {
177        return Err(ValidationError::Blank(field));
178    }
179    Ok(())
180}
181
182fn validate_text(value: &str, field: &'static str) -> Result<(), ValidationError> {
183    (!value.trim().is_empty())
184        .then_some(())
185        .ok_or(ValidationError::Blank(field))
186}
187
188#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
189pub struct LocalSpeakerLabel(u32);
190
191impl LocalSpeakerLabel {
192    pub fn new(number: u32) -> Result<Self, ValidationError> {
193        (number > 0)
194            .then_some(Self(number))
195            .ok_or_else(|| ValidationError::InvalidSpeakerLabel("Speaker 0".into()))
196    }
197
198    pub fn number(self) -> u32 {
199        self.0
200    }
201}
202
203impl fmt::Display for LocalSpeakerLabel {
204    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
205        write!(formatter, "Speaker {}", self.0)
206    }
207}
208
209impl FromStr for LocalSpeakerLabel {
210    type Err = ValidationError;
211
212    fn from_str(value: &str) -> Result<Self, Self::Err> {
213        let number = value
214            .strip_prefix("Speaker ")
215            .and_then(|value| value.parse::<u32>().ok())
216            .filter(|value| *value > 0)
217            .ok_or_else(|| ValidationError::InvalidSpeakerLabel(value.into()))?;
218        Ok(Self(number))
219    }
220}
221
222impl Serialize for LocalSpeakerLabel {
223    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
224        serializer.serialize_str(&self.to_string())
225    }
226}
227
228impl<'de> Deserialize<'de> for LocalSpeakerLabel {
229    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
230        String::deserialize(deserializer)?
231            .parse()
232            .map_err(serde::de::Error::custom)
233    }
234}
235
236#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
237#[serde(rename_all = "snake_case")]
238pub enum VocalGenderPresentation {
239    StronglyFeminine,
240    Feminine,
241    Androgynous,
242    Masculine,
243    StronglyMasculine,
244}
245
246impl VocalGenderPresentation {
247    pub fn numeric_value(self) -> f64 {
248        match self {
249            Self::StronglyFeminine => -2.0,
250            Self::Feminine => -1.0,
251            Self::Androgynous => 0.0,
252            Self::Masculine => 1.0,
253            Self::StronglyMasculine => 2.0,
254        }
255    }
256}
257
258#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
259pub struct FeatureVector24 {
260    pub median_f0_hz: Option<f64>,
261    pub high_front_vowel_f1_hz: Option<f64>,
262    pub high_back_vowel_f2_hz: Option<f64>,
263    pub spectral_tilt_db_per_octave: Option<f64>,
264    pub cepstral_peak_prominence_db: Option<f64>,
265    pub foreign_accentedness_1_to_9: Option<f64>,
266    pub dominant_rhotic_realization: Option<String>,
267    pub unstressed_vowel_reduction_percent: Option<f64>,
268    pub high_front_vowel_f2_hz: Option<f64>,
269    pub low_vowel_f1_hz: Option<f64>,
270    pub h1_minus_h2_db: Option<f64>,
271    pub rhotic_f3_minus_f2_hz: Option<f64>,
272    pub word_initial_t_vot_ms: Option<f64>,
273    pub dominant_lateral_realization: Option<String>,
274    pub monophthongization_percent: Option<f64>,
275    pub vocal_gender_presentation: Option<VocalGenderPresentation>,
276    pub low_vowel_f2_hz: Option<f64>,
277    pub high_back_vowel_f1_hz: Option<f64>,
278    pub mean_formant_dispersion_hz: Option<f64>,
279    pub creaky_phonation_percent: Option<f64>,
280    pub hypernasality_0_to_4: Option<f64>,
281    pub sibilant_center_of_gravity_hz: Option<f64>,
282    pub consonant_cluster_reduction_percent: Option<f64>,
283    pub perceived_vocal_age_years: Option<f64>,
284}
285
286impl FeatureVector24 {
287    pub fn validate(&self) -> Result<(), ValidationError> {
288        for (name, value) in NUMERIC_FEATURE_NAMES.into_iter().zip(self.numeric_values()) {
289            if value.is_some_and(|number| !number.is_finite()) {
290                return Err(ValidationError::NonFiniteFeature(name));
291            }
292        }
293        validate_optional_text(
294            self.dominant_rhotic_realization.as_deref(),
295            "dominant_rhotic_realization",
296        )?;
297        validate_optional_text(
298            self.dominant_lateral_realization.as_deref(),
299            "dominant_lateral_realization",
300        )
301    }
302
303    pub fn numeric_values(&self) -> [Option<f64>; 22] {
304        [
305            self.median_f0_hz,
306            self.high_front_vowel_f1_hz,
307            self.high_back_vowel_f2_hz,
308            self.spectral_tilt_db_per_octave,
309            self.cepstral_peak_prominence_db,
310            self.foreign_accentedness_1_to_9,
311            self.unstressed_vowel_reduction_percent,
312            self.high_front_vowel_f2_hz,
313            self.low_vowel_f1_hz,
314            self.h1_minus_h2_db,
315            self.rhotic_f3_minus_f2_hz,
316            self.word_initial_t_vot_ms,
317            self.monophthongization_percent,
318            self.vocal_gender_presentation
319                .map(VocalGenderPresentation::numeric_value),
320            self.low_vowel_f2_hz,
321            self.high_back_vowel_f1_hz,
322            self.mean_formant_dispersion_hz,
323            self.creaky_phonation_percent,
324            self.hypernasality_0_to_4,
325            self.sibilant_center_of_gravity_hz,
326            self.consonant_cluster_reduction_percent,
327            self.perceived_vocal_age_years,
328        ]
329    }
330
331    pub fn nominal_values(&self) -> [Option<&str>; 2] {
332        [
333            self.dominant_rhotic_realization.as_deref(),
334            self.dominant_lateral_realization.as_deref(),
335        ]
336    }
337
338    pub fn present_feature_count(&self) -> u8 {
339        let numeric = self
340            .numeric_values()
341            .into_iter()
342            .filter(Option::is_some)
343            .count();
344        let nominal = self
345            .nominal_values()
346            .into_iter()
347            .filter(Option::is_some)
348            .count();
349        (numeric + nominal) as u8
350    }
351}
352
353#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
354pub struct StructuredSpeaker {
355    pub speaker: LocalSpeakerLabel,
356    pub language: String,
357    pub features: FeatureVector24,
358    pub features_usable_for_training: bool,
359}
360
361impl StructuredSpeaker {
362    pub fn validate(&self) -> Result<(), ValidationError> {
363        validate_text(&self.language, "language")?;
364        self.features.validate()
365    }
366}
367
368#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
369pub struct StructuredAnalysis {
370    pub transcript: String,
371    pub speakers: Vec<StructuredSpeaker>,
372}
373
374impl StructuredAnalysis {
375    pub fn validate(&self) -> Result<(), ValidationError> {
376        validate_text(&self.transcript, "transcript")?;
377        let mut labels = BTreeSet::new();
378        for speaker in &self.speakers {
379            speaker.validate()?;
380            if !labels.insert(speaker.speaker) {
381                return Err(ValidationError::DuplicateSpeakerLabel(speaker.speaker));
382            }
383        }
384        Ok(())
385    }
386}
387
388#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
389pub struct GeminiCohort {
390    pub model_id: String,
391    pub transcript_prompt_revision: String,
392    pub feature_prompt_revisions: [String; 3],
393    pub feature_schema_revision: String,
394}
395
396impl GeminiCohort {
397    pub fn new(model_id: impl Into<String>) -> Self {
398        Self {
399            model_id: model_id.into(),
400            transcript_prompt_revision: GEMINI_TRANSCRIPT_PROMPT_REVISION.into(),
401            feature_prompt_revisions: GEMINI_FEATURE_PROMPT_REVISIONS.map(str::to_owned),
402            feature_schema_revision: FEATURE_SCHEMA_REVISION.into(),
403        }
404    }
405
406    pub fn validate(&self) -> Result<(), ValidationError> {
407        validate_text(&self.model_id, "gemini_model_id")?;
408        validate_text(
409            &self.transcript_prompt_revision,
410            "transcript_prompt_revision",
411        )?;
412        for revision in &self.feature_prompt_revisions {
413            validate_text(revision, "feature_prompt_revision")?;
414        }
415        validate_text(&self.feature_schema_revision, "feature_schema_revision")
416    }
417}
418
419#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
420pub struct StructurerProvenance {
421    pub model_id: String,
422    pub prompt_revision: String,
423}
424
425impl StructurerProvenance {
426    pub fn new(model_id: impl Into<String>) -> Self {
427        Self {
428            model_id: model_id.into(),
429            prompt_revision: GPT_STRUCTURING_PROMPT_REVISION.into(),
430        }
431    }
432
433    pub fn validate(&self) -> Result<(), ValidationError> {
434        validate_text(&self.model_id, "structurer_model_id")?;
435        validate_text(&self.prompt_revision, "structurer_prompt_revision")
436    }
437}
438
439#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
440pub struct AnalysisEnvelope {
441    pub audio: OggAudioMetadata,
442    pub analysis: StructuredAnalysis,
443    pub gemini: GeminiCohort,
444    pub structurer: StructurerProvenance,
445}
446
447impl AnalysisEnvelope {
448    pub fn validate(&self) -> Result<(), ValidationError> {
449        self.audio.validate()?;
450        self.analysis.validate()?;
451        self.gemini.validate()?;
452        self.structurer.validate()
453    }
454}
455
456#[cfg(test)]
457mod tests {
458    use super::*;
459
460    fn ogg(body_length: u8) -> Vec<u8> {
461        let mut bytes = vec![0; 28 + body_length as usize];
462        bytes[..4].copy_from_slice(b"OggS");
463        bytes[4] = 0;
464        bytes[26] = 1;
465        bytes[27] = body_length;
466        bytes
467    }
468
469    fn speaker(number: u32) -> StructuredSpeaker {
470        StructuredSpeaker {
471            speaker: LocalSpeakerLabel::new(number).unwrap(),
472            language: "English".into(),
473            features: FeatureVector24::default(),
474            features_usable_for_training: true,
475        }
476    }
477
478    #[test]
479    fn ogg_metadata_checks_page_and_duration() {
480        let bytes = ogg(3);
481        let metadata =
482            OggAudioMetadata::from_bytes(&bytes, MAX_AUDIO_DURATION_MS, Some("voice.ogg".into()))
483                .unwrap();
484        assert_eq!(metadata.media_type(), OGG_MEDIA_TYPE);
485        assert_eq!(metadata.byte_length(), bytes.len() as u64);
486        assert_eq!(metadata.filename(), Some("voice.ogg"));
487        assert_eq!(
488            OggAudioMetadata::from_bytes(&bytes, 0, None),
489            Err(ValidationError::InvalidDuration(0))
490        );
491        assert_eq!(
492            OggAudioMetadata::from_bytes(&bytes[..bytes.len() - 1], 1, None),
493            Err(ValidationError::InvalidOgg)
494        );
495        let mut wrong = bytes;
496        wrong[4] = 1;
497        assert_eq!(
498            OggAudioMetadata::from_bytes(&wrong, 1, None),
499            Err(ValidationError::InvalidOgg)
500        );
501    }
502
503    #[test]
504    fn speaker_labels_have_one_exact_form() {
505        let label = LocalSpeakerLabel::new(12).unwrap();
506        assert_eq!(label.to_string(), "Speaker 12");
507        assert_eq!("Speaker 12".parse(), Ok(label));
508        assert!("speaker 12".parse::<LocalSpeakerLabel>().is_err());
509        assert_eq!(serde_json::to_string(&label).unwrap(), "\"Speaker 12\"");
510        assert_eq!(
511            serde_json::from_str::<LocalSpeakerLabel>("\"Speaker 12\"").unwrap(),
512            label
513        );
514    }
515
516    #[test]
517    fn features_validate_and_project_in_frozen_order() {
518        let features = FeatureVector24 {
519            median_f0_hz: Some(100.0),
520            dominant_rhotic_realization: Some("tap".into()),
521            vocal_gender_presentation: Some(VocalGenderPresentation::Masculine),
522            perceived_vocal_age_years: Some(30.0),
523            ..FeatureVector24::default()
524        };
525        assert_eq!(features.numeric_values()[0], Some(100.0));
526        assert_eq!(features.numeric_values()[13], Some(1.0));
527        assert_eq!(features.numeric_values()[21], Some(30.0));
528        assert_eq!(features.nominal_values(), [Some("tap"), None]);
529        assert_eq!(features.present_feature_count(), 4);
530        assert!(features.validate().is_ok());
531        let invalid = FeatureVector24 {
532            hypernasality_0_to_4: Some(f64::NAN),
533            ..FeatureVector24::default()
534        };
535        assert_eq!(
536            invalid.validate(),
537            Err(ValidationError::NonFiniteFeature("hypernasality_0_to_4"))
538        );
539        let blank = FeatureVector24 {
540            dominant_lateral_realization: Some(" ".into()),
541            ..FeatureVector24::default()
542        };
543        assert_eq!(
544            blank.validate(),
545            Err(ValidationError::Blank("dominant_lateral_realization"))
546        );
547    }
548
549    #[test]
550    fn structured_analysis_validates_and_round_trips() {
551        let analysis = StructuredAnalysis {
552            transcript: "[high] Speaker 1: hello world".into(),
553            speakers: vec![speaker(1)],
554        };
555        analysis.validate().unwrap();
556        let encoded = serde_json::to_vec(&analysis).unwrap();
557        let decoded: StructuredAnalysis = serde_json::from_slice(&encoded).unwrap();
558        assert_eq!(decoded, analysis);
559        assert!(decoded.speakers[0].features_usable_for_training);
560        let duplicate = StructuredAnalysis {
561            transcript: "speech".into(),
562            speakers: vec![speaker(1), speaker(1)],
563        };
564        assert!(matches!(
565            duplicate.validate(),
566            Err(ValidationError::DuplicateSpeakerLabel(_))
567        ));
568    }
569
570    #[test]
571    fn provenance_uses_current_revisions_and_validates() {
572        let cohort = GeminiCohort::new("gemini-model");
573        assert_eq!(
574            cohort.feature_prompt_revisions,
575            GEMINI_FEATURE_PROMPT_REVISIONS.map(str::to_owned)
576        );
577        cohort.validate().unwrap();
578        StructurerProvenance::new("gpt-5.6").validate().unwrap();
579        assert_eq!(
580            GeminiCohort::new(" ").validate(),
581            Err(ValidationError::Blank("gemini_model_id"))
582        );
583    }
584
585    #[test]
586    fn prompts_cover_the_frozen_workflow() {
587        let packets = [
588            GEMINI_FEATURE_PROMPT_ONE,
589            GEMINI_FEATURE_PROMPT_TWO,
590            GEMINI_FEATURE_PROMPT_THREE,
591        ];
592        let groups = [
593            &FEATURE_NAMES[..8],
594            &FEATURE_NAMES[8..16],
595            &FEATURE_NAMES[16..],
596        ];
597        let mut shared_prefix = None;
598        for (prompt, names) in packets.into_iter().zip(groups) {
599            let (prefix, suffix) = prompt.split_once("Requested features:\n\n").unwrap();
600            assert!(prefix.contains("{{TRANSCRIPT}}"));
601            assert_eq!(shared_prefix.get_or_insert(prefix), &prefix);
602            for name in names {
603                assert!(suffix.contains(name));
604            }
605            assert!(suffix.contains("Target speaker:\n{{TARGET_SPEAKER}}"));
606        }
607        assert!(GEMINI_TRANSCRIPT_PROMPT.contains("[high] Speaker N:"));
608        assert!(GEMINI_TRANSCRIPT_PROMPT.contains("feature estimation"));
609        assert!(GPT_STRUCTURING_PROMPT.contains("record_speaker_analysis"));
610        assert!(GPT_STRUCTURING_PROMPT.contains("15 consecutive spoken words"));
611    }
612}
kcode_speaker_v3_analysis/lib.rs

kcode_speaker_v3_analysis/
lib.rs
