native_whisperx/config/
parity.rs

1//! Parity harness configuration and reports for WhisperX comparison fixtures.
2
3use std::path::PathBuf;
4
5use serde::{Deserialize, Serialize};
6use text_transcripts::TranscriptionContract;
7
8use super::defaults::{default_gating, default_true};
9use super::{
10    AlignmentConfig, AsrConfig, DiarizationConfig, ExternalWhisperxConfig, NativeWhisperxReport,
11    OutputConfig, OutputFormat, TranslationConfig, VadConfig,
12};
13
14#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
15#[serde(rename_all = "camelCase")]
16pub struct ExpectedOutputFile {
17    pub format: OutputFormat,
18    pub path: PathBuf,
19    #[serde(default)]
20    pub comparison: OutputComparisonMode,
21    #[serde(default = "default_true")]
22    pub gating: bool,
23}
24
25#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
26#[serde(rename_all = "camelCase")]
27pub enum OutputComparisonMode {
28    #[default]
29    Exact,
30    JsonSemantic,
31    SubtitleSemantic,
32}
33
34#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
35#[serde(rename_all = "camelCase")]
36pub struct ExpectedOutputComparison {
37    pub format: OutputFormat,
38    #[serde(default)]
39    pub comparison: OutputComparisonMode,
40    #[serde(default = "default_true")]
41    pub gating: bool,
42    pub expected_path: PathBuf,
43    pub actual_path: Option<PathBuf>,
44    pub passed: bool,
45    pub difference: Option<String>,
46}
47
48#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
49#[serde(rename_all = "camelCase")]
50pub enum ExpectedTranscriptTarget {
51    #[default]
52    Native,
53    Whisperx,
54}
55
56#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
57#[serde(rename_all = "camelCase")]
58pub struct ParityConfig {
59    pub input: PathBuf,
60    #[serde(default)]
61    pub expected_json: Option<PathBuf>,
62    #[serde(default)]
63    pub expected_target: ExpectedTranscriptTarget,
64    #[serde(default)]
65    pub comparison: ParityComparisonConfig,
66    #[serde(default)]
67    pub native_asr: AsrConfig,
68    #[serde(default)]
69    pub translation: TranslationConfig,
70    #[serde(default)]
71    pub vad: VadConfig,
72    #[serde(default)]
73    pub alignment: AlignmentConfig,
74    #[serde(default)]
75    pub diarization: DiarizationConfig,
76    #[serde(default)]
77    pub whisperx_diarization: Option<DiarizationConfig>,
78    #[serde(default)]
79    pub whisperx: ExternalWhisperxConfig,
80    #[serde(default)]
81    pub language: Option<String>,
82    #[serde(default)]
83    pub output: OutputConfig,
84}
85
86#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
87#[serde(rename_all = "camelCase")]
88pub struct ParityFixtureSuite {
89    #[serde(default)]
90    pub fixtures: Vec<ParityFixtureCase>,
91    #[serde(default)]
92    pub multi_input_fixtures: Vec<ParityMultiInputFixtureCase>,
93}
94
95#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
96#[serde(rename_all = "camelCase")]
97pub struct ParityFixtureCase {
98    pub name: String,
99    #[serde(default = "default_gating")]
100    pub gating: bool,
101    pub input: PathBuf,
102    #[serde(default)]
103    pub clip_seconds: Option<f64>,
104    #[serde(default)]
105    pub timeout_seconds: Option<u64>,
106    #[serde(default)]
107    pub expected_json: Option<PathBuf>,
108    #[serde(default)]
109    pub expected_target: ExpectedTranscriptTarget,
110    #[serde(default)]
111    pub comparison: ParityComparisonConfig,
112    #[serde(default)]
113    pub expected_outputs: Vec<ExpectedOutputFile>,
114    #[serde(default)]
115    pub native_asr: AsrConfig,
116    #[serde(default)]
117    pub translation: TranslationConfig,
118    #[serde(default)]
119    pub vad: VadConfig,
120    #[serde(default)]
121    pub alignment: AlignmentConfig,
122    #[serde(default)]
123    pub diarization: DiarizationConfig,
124    #[serde(default)]
125    pub whisperx_diarization: Option<DiarizationConfig>,
126    #[serde(default)]
127    pub whisperx: ExternalWhisperxConfig,
128    #[serde(default)]
129    pub language: Option<String>,
130    #[serde(default)]
131    pub output: OutputConfig,
132    #[serde(default)]
133    pub required_diagnostics: Vec<String>,
134}
135
136#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
137#[serde(rename_all = "camelCase")]
138pub struct ParityMultiInputFixtureCase {
139    pub name: String,
140    #[serde(default = "default_gating")]
141    pub gating: bool,
142    pub inputs: Vec<PathBuf>,
143    #[serde(default)]
144    pub clip_seconds_per_input: Option<f64>,
145    #[serde(default)]
146    pub timeout_seconds: Option<u64>,
147    #[serde(default)]
148    pub native_asr: AsrConfig,
149    #[serde(default)]
150    pub translation: TranslationConfig,
151    #[serde(default)]
152    pub vad: VadConfig,
153    #[serde(default)]
154    pub alignment: AlignmentConfig,
155    #[serde(default)]
156    pub diarization: DiarizationConfig,
157    #[serde(default)]
158    pub whisperx_diarization: Option<DiarizationConfig>,
159    #[serde(default)]
160    pub whisperx: ExternalWhisperxConfig,
161    #[serde(default)]
162    pub language: Option<String>,
163    #[serde(default)]
164    pub output: OutputConfig,
165    #[serde(default)]
166    pub required_diagnostics: Vec<String>,
167}
168
169#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
170#[serde(rename_all = "camelCase")]
171pub struct ParityComparisonConfig {
172    #[serde(default = "default_true")]
173    pub text: bool,
174    #[serde(default = "default_true")]
175    pub language: bool,
176    #[serde(default = "default_true")]
177    pub segment_text: bool,
178    #[serde(default = "default_true")]
179    pub word_text: bool,
180    #[serde(default = "default_true")]
181    pub char_count: bool,
182    #[serde(default = "default_true")]
183    pub char_content: bool,
184    #[serde(default = "default_true")]
185    pub segment_count: bool,
186    #[serde(default = "default_true")]
187    pub word_count: bool,
188    #[serde(default = "default_true")]
189    pub segment_timing: bool,
190    #[serde(default = "default_true")]
191    pub word_timing: bool,
192    #[serde(default = "default_true")]
193    pub speaker_turns: bool,
194    #[serde(default = "default_true")]
195    pub vad_segments: bool,
196    #[serde(default = "default_true")]
197    pub vad_segment_timing: bool,
198    #[serde(default = "default_true")]
199    pub vad_segment_count: bool,
200}
201
202impl Default for ParityComparisonConfig {
203    fn default() -> Self {
204        Self {
205            text: true,
206            language: true,
207            segment_text: true,
208            word_text: true,
209            char_count: true,
210            char_content: true,
211            segment_count: true,
212            word_count: true,
213            segment_timing: true,
214            word_timing: true,
215            speaker_turns: true,
216            vad_segments: true,
217            vad_segment_timing: true,
218            vad_segment_count: true,
219        }
220    }
221}
222
223#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
224#[serde(rename_all = "camelCase")]
225pub struct ParityFixtureSuiteReport {
226    pub passed: bool,
227    pub cases: Vec<ParityFixtureCaseReport>,
228}
229
230#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
231#[serde(rename_all = "camelCase")]
232pub struct ParityFixtureCaseReport {
233    pub name: String,
234    #[serde(default)]
235    pub gating: bool,
236    pub passed: bool,
237    #[serde(default, skip_serializing_if = "Option::is_none")]
238    pub started_at: Option<String>,
239    #[serde(default, skip_serializing_if = "Option::is_none")]
240    pub elapsed_seconds: Option<f64>,
241    #[serde(default)]
242    pub timed_out: bool,
243    #[serde(default, skip_serializing_if = "Option::is_none")]
244    pub report: Option<ParityReport>,
245    #[serde(default)]
246    pub missing_required_diagnostics: Vec<String>,
247    #[serde(default)]
248    pub expected_output_matches: Vec<ExpectedOutputComparison>,
249    #[serde(default, skip_serializing_if = "Option::is_none")]
250    pub error: Option<String>,
251    #[serde(default)]
252    pub failure_summary: Vec<String>,
253}
254
255#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
256#[serde(rename_all = "camelCase")]
257pub struct ParityPreflightReport {
258    pub passed: bool,
259    pub manifest: PathBuf,
260    pub root: PathBuf,
261    pub whisperx_command: PathBuf,
262    pub model_dir: PathBuf,
263    pub source_checkout_tag: Option<String>,
264    pub cases: Vec<ParityPreflightCaseReport>,
265}
266
267#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
268#[serde(rename_all = "camelCase")]
269pub struct ParityPreflightCaseReport {
270    pub name: String,
271    pub gating: bool,
272    pub passed: bool,
273    pub missing: Vec<String>,
274    pub warnings: Vec<String>,
275}
276
277#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
278#[serde(rename_all = "camelCase")]
279pub struct ParityReport {
280    pub native_report: NativeWhisperxReport,
281    pub whisperx_report: NativeWhisperxReport,
282    #[serde(default)]
283    pub expected: Option<TranscriptionContract>,
284    #[serde(default)]
285    pub expected_target: ExpectedTranscriptTarget,
286    pub comparison: ParityComparison,
287    pub expected_segment_count_matches: Option<bool>,
288    pub expected_text_matches: Option<bool>,
289}
290
291#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
292#[serde(rename_all = "camelCase")]
293pub struct ParityComparison {
294    pub text_matches: bool,
295    #[serde(default)]
296    pub language_matches: Option<bool>,
297    #[serde(default)]
298    pub segment_text_matches: Option<bool>,
299    #[serde(default)]
300    pub word_text_matches: Option<bool>,
301    #[serde(default)]
302    pub char_count_matches: Option<bool>,
303    #[serde(default)]
304    pub char_content_matches: Option<bool>,
305    pub segment_count_matches: bool,
306    pub word_count_matches: bool,
307    pub segment_timing_matches: bool,
308    pub word_timing_matches: bool,
309    pub speaker_turns_match: bool,
310    #[serde(default)]
311    pub vad_segment_count_matches: Option<bool>,
312    #[serde(default)]
313    pub vad_segment_timing_matches: Option<bool>,
314    pub confidence_compared: bool,
315    pub passed: bool,
316    pub tolerance: ParityTolerance,
317    #[serde(default)]
318    pub differences: Vec<String>,
319    #[serde(default)]
320    pub diagnostic_differences: Vec<String>,
321}
322
323#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
324#[serde(rename_all = "camelCase")]
325pub struct ParityTolerance {
326    pub segment_seconds: f64,
327    pub word_seconds: f64,
328}
329
330impl Default for ParityTolerance {
331    fn default() -> Self {
332        Self {
333            segment_seconds: 0.100,
334            word_seconds: 0.050,
335        }
336    }
337}
native_whisperx/config/parity.rs

native_whisperx/config/
parity.rs
