espeak-ng 0.2.0

Pure Rust port of eSpeak NG text-to-speech
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
//! Parser for spectral frame sequences in the espeak-ng `phondata` binary.

// src/synthesize/phondata.rs
//
// Parse spectral frame sequences from the espeak-ng `phondata` binary file.
//
// The phondata file contains precomputed acoustic data for each phoneme.
// Frame sequences are stored as either:
//   - SPECT_SEQ  (non-Klatt):  4-byte header + N × 44-byte frame_t2 structs
//   - SPECT_SEQK (Klatt):      4-byte header + N × 64-byte frame_t  structs
//
// Which variant is used is determined by `frame[0].frflags & FRFLAG_KLATT`.
//
// References: synthesize.h (frame_t, SPECT_SEQ, SPECT_SEQK)

/// Maximum frames in a spectral sequence (N_SEQ_FRAMES in synthesize.h)
pub const N_SEQ_FRAMES: usize = 25;

/// FRFLAG_KLATT: frame uses extra Klatt parameters (64-byte layout)
pub const FRFLAG_KLATT: u16 = 0x01;
/// FRFLAG_VOWEL_CENTRE: marks the centre point of a vowel
pub const FRFLAG_VOWEL_CENTRE: u16 = 0x02;
/// FRFLAG_BREAK_LF: break, but keep F3 and above continuous
pub const FRFLAG_BREAK_LF: u16 = 0x08;
/// FRFLAG_BREAK: don't merge with next frame
pub const FRFLAG_BREAK: u16 = 0x10;
/// FRFLAG_FORMANT_RATE: allow an increased rate of formant-frequency change
pub const FRFLAG_FORMANT_RATE: u16 = 0x20;

/// Size of the non-Klatt frame_t2 struct in C (bytes)
pub const FRAME_T2_SIZE: usize = 44;
/// Size of the full Klatt frame_t struct in C (bytes)
pub const FRAME_T_SIZE: usize = 64;

// ---------------------------------------------------------------------------
// SpectFrame — a single formant synthesis frame
// ---------------------------------------------------------------------------

/// Spectral synthesis frame — mirrors `frame_t` from synthesize.h.
///
/// For non-Klatt sequences only the first set of fields (matching `frame_t2`)
/// are meaningful.
#[derive(Debug, Clone, Default)]
pub struct SpectFrame {
    /// Frame flags (FRFLAG_* bitmask)
    pub frflags: u16,
    /// Formant frequencies F0–F6 in Hz
    pub ffreq: [i16; 7],
    /// Frame duration in STEPSIZE units (64 samples ≈ 2.9 ms at 22050 Hz)
    pub length: u8,
    /// Relative amplitude (root-mean-square)
    pub rms: u8,
    /// Formant heights (amplitude of each formant), F0–F7
    pub fheight: [u8; 8],
    /// Formant widths / 4, F0–F5
    pub fwidth: [u8; 6],
    /// Right-side formant widths / 4, F0–F2
    pub fright: [u8; 3],
    /// Klatt bandwidths / 2: BNZ, F1, F2, F3
    pub bw: [u8; 4],
    /// Klatt parameters: AV, FNZ, Tilt, Aspr, Skew
    pub klattp: [u8; 5],
    /// Extended Klatt parameters, continuing `klattp` from index 5:
    /// Kopen, AVp, Fric, FricBP, Turb
    pub klattp2: [u8; 5],
    /// Klatt parallel amplitudes F0–F6
    pub klatt_ap: [u8; 7],
    /// Klatt parallel bandwidths / 2, F0–F6
    pub klatt_bp: [u8; 7],
}

impl SpectFrame {
    /// Parse a non-Klatt frame_t2 from 44 bytes.
    pub fn from_bytes_t2(data: &[u8]) -> Option<Self> {
        if data.len() < FRAME_T2_SIZE { return None; }
        let mut f = SpectFrame::default();
        f.frflags = u16::from_le_bytes([data[0], data[1]]);
        for i in 0..7 {
            f.ffreq[i] = i16::from_le_bytes([data[2 + i*2], data[3 + i*2]]);
        }
        // offset 16
        f.length = data[16];
        f.rms    = data[17];
        // offset 18
        f.fheight.copy_from_slice(&data[18..26]);
        // offset 26
        f.fwidth.copy_from_slice(&data[26..32]);
        // offset 32
        f.fright.copy_from_slice(&data[32..35]);
        // offset 35
        f.bw.copy_from_slice(&data[35..39]);
        // offset 39
        f.klattp.copy_from_slice(&data[39..44]);
        Some(f)
    }

    /// Serialize a non-Klatt `frame_t2` to 44 bytes — the inverse of
    /// [`from_bytes_t2`](Self::from_bytes_t2).  Part of the `phondata`
    /// acoustic-data compiler (§1.6).
    pub fn to_bytes_t2(&self) -> [u8; FRAME_T2_SIZE] {
        let mut d = [0u8; FRAME_T2_SIZE];
        d[0..2].copy_from_slice(&self.frflags.to_le_bytes());
        for i in 0..7 {
            d[2 + i * 2..4 + i * 2].copy_from_slice(&self.ffreq[i].to_le_bytes());
        }
        d[16] = self.length;
        d[17] = self.rms;
        d[18..26].copy_from_slice(&self.fheight);
        d[26..32].copy_from_slice(&self.fwidth);
        d[32..35].copy_from_slice(&self.fright);
        d[35..39].copy_from_slice(&self.bw);
        d[39..44].copy_from_slice(&self.klattp);
        d
    }

    /// Serialize a Klatt `frame_t` to 64 bytes — the inverse of
    /// [`from_bytes_t`](Self::from_bytes_t).  The first 44 bytes match
    /// [`to_bytes_t2`](Self::to_bytes_t2); the tail adds the Klatt parameters.
    pub fn to_bytes_t(&self) -> [u8; FRAME_T_SIZE] {
        let mut d = [0u8; FRAME_T_SIZE];
        d[..FRAME_T2_SIZE].copy_from_slice(&self.to_bytes_t2());
        d[44..49].copy_from_slice(&self.klattp2);
        d[49..56].copy_from_slice(&self.klatt_ap);
        d[56..63].copy_from_slice(&self.klatt_bp);
        // d[63] = spare (0)
        d
    }

    /// Parse a Klatt frame_t from 64 bytes.
    pub fn from_bytes_t(data: &[u8]) -> Option<Self> {
        if data.len() < FRAME_T_SIZE { return None; }
        let mut f = SpectFrame::default();
        f.frflags = u16::from_le_bytes([data[0], data[1]]);
        for i in 0..7 {
            f.ffreq[i] = i16::from_le_bytes([data[2 + i*2], data[3 + i*2]]);
        }
        f.length = data[16];
        f.rms    = data[17];
        f.fheight.copy_from_slice(&data[18..26]);
        f.fwidth.copy_from_slice(&data[26..32]);
        f.fright.copy_from_slice(&data[32..35]);
        f.bw.copy_from_slice(&data[35..39]);
        f.klattp.copy_from_slice(&data[39..44]);
        f.klattp2.copy_from_slice(&data[44..49]);
        f.klatt_ap.copy_from_slice(&data[49..56]);
        f.klatt_bp.copy_from_slice(&data[56..63]);
        // data[63] = spare
        Some(f)
    }

    /// Return the F1 frequency in Hz (for synthesis).
    #[inline]
    pub fn f1_hz(&self) -> f64 { self.ffreq[1] as f64 }

    /// Return the F2 frequency in Hz.
    #[inline]
    pub fn f2_hz(&self) -> f64 { self.ffreq[2] as f64 }

    /// Return the F3 frequency in Hz.
    #[inline]
    pub fn f3_hz(&self) -> f64 { self.ffreq[3] as f64 }

    /// Return duration in samples at 22050 Hz (STEPSIZE = 64 samples).
    #[inline]
    pub fn dur_samples(&self, speed_factor: f64) -> usize {
        let base = (self.length as usize) * 64;
        ((base as f64 * speed_factor) as usize).max(1)
    }
}

/// Compile a non-Klatt `SPECT_SEQ` to `phondata` bytes — the inverse of
/// [`SpectSeq::parse`], and the acoustic-data half of the phoneme compiler
/// (`compiledata.c`, §1.6).  Header = `length_total(i16)` (sum of frame lengths)
/// + `n_frames(u8)` + `sqflags(u8)`, followed by the 44-byte `frame_t2` entries.
pub fn compile_spect_seq(frames: &[SpectFrame]) -> Vec<u8> {
    // Klatt vs non-Klatt is chosen (and detected on read) from the first frame's
    // FRFLAG_KLATT bit, matching `SpectSeq::parse`.
    let is_klatt = frames.first().is_some_and(|f| f.frflags & FRFLAG_KLATT != 0);
    let frame_size = if is_klatt { FRAME_T_SIZE } else { FRAME_T2_SIZE };
    let length_total: i16 = frames.iter().map(|f| f.length as i16).sum();
    let mut out = Vec::with_capacity(4 + frames.len() * frame_size);
    out.extend_from_slice(&length_total.to_le_bytes());
    out.push(frames.len() as u8);
    out.push(0); // sqflags
    for f in frames {
        if is_klatt {
            out.extend_from_slice(&f.to_bytes_t());
        } else {
            out.extend_from_slice(&f.to_bytes_t2());
        }
    }
    out
}

// ---------------------------------------------------------------------------
// SpectSeq — sequence of frames for one phoneme
// ---------------------------------------------------------------------------

/// A spectral frame sequence as stored in phondata.
///
/// Mirrors `SPECT_SEQ` / `SPECT_SEQK` from synthesize.h.
#[derive(Debug, Clone)]
pub struct SpectSeq {
    /// The individual frames (in display order).
    pub frames: Vec<SpectFrame>,
    /// True if this is a Klatt sequence (frame_t, 64 bytes each).
    pub is_klatt: bool,
}

impl SpectSeq {
    /// Parse a `SPECT_SEQ` or `SPECT_SEQK` from `phondata` at a given offset.
    ///
    /// Returns `None` if the data is truncated or the frame count is 0.
    pub fn parse(phondata: &[u8], offset: usize) -> Option<Self> {
        // Header: length_total(i16) + n_frames(u8) + sqflags(u8) = 4 bytes
        if offset + 4 > phondata.len() { return None; }
        let data = &phondata[offset..];
        let n_frames = data[2] as usize;
        if n_frames == 0 || n_frames > N_SEQ_FRAMES { return None; }

        // Determine Klatt vs non-Klatt from first frame's frflags
        if data.len() < 4 + 2 { return None; } // need at least frflags of frame 0
        let first_frflags = u16::from_le_bytes([data[4], data[5]]);
        let is_klatt = (first_frflags & FRFLAG_KLATT) != 0;
        let frame_size = if is_klatt { FRAME_T_SIZE } else { FRAME_T2_SIZE };

        if data.len() < 4 + n_frames * frame_size { return None; }

        let mut frames = Vec::with_capacity(n_frames);
        for i in 0..n_frames {
            let base = 4 + i * frame_size;
            let frame_data = &data[base..base + frame_size];
            let frame = if is_klatt {
                SpectFrame::from_bytes_t(frame_data)?
            } else {
                SpectFrame::from_bytes_t2(frame_data)?
            };
            frames.push(frame);
        }

        Some(SpectSeq { frames, is_klatt })
    }

    /// True if the sequence is intended for a voiced phoneme.
    ///
    /// A sequence is considered voiced if any frame has non-zero klattp[0] (AV).
    /// For non-Klatt frames, we check whether F1 is non-zero.
    pub fn is_voiced(&self) -> bool {
        self.frames.iter().any(|f| {
            f.klattp[0] > 0 || f.ffreq[1] > 0
        })
    }
}

// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn spect_seq_compile_round_trip() {
        let mut f1 = SpectFrame::default();
        f1.frflags = 0x0002; // not FRFLAG_KLATT → frame_t2
        f1.ffreq = [120, 730, 1090, 2440, 3400, 4000, 4500];
        f1.length = 20;
        f1.rms = 40;
        f1.fheight = [1, 2, 3, 4, 5, 6, 7, 8];
        f1.fwidth = [9, 10, 11, 12, 13, 14];
        f1.fright = [1, 2, 3];
        f1.bw = [4, 5, 6, 7];
        f1.klattp = [50, 1, 2, 3, 4];
        let mut f2 = f1.clone();
        f2.ffreq[1] = 700;
        f2.length = 25;

        let frames = vec![f1.clone(), f2.clone()];
        let bytes = compile_spect_seq(&frames);
        let seq = SpectSeq::parse(&bytes, 0).expect("parse compiled SPECT_SEQ");

        assert!(!seq.is_klatt);
        assert_eq!(seq.frames.len(), 2);
        // Full byte-level round-trip of each frame.
        assert_eq!(seq.frames[0].to_bytes_t2(), f1.to_bytes_t2());
        assert_eq!(seq.frames[1].to_bytes_t2(), f2.to_bytes_t2());
        assert_eq!(seq.frames[0].length, 20);
        assert_eq!(seq.frames[1].ffreq[1], 700);
    }

    #[test]
    fn spect_seqk_klatt_compile_round_trip() {
        // A Klatt sequence (first frame carries FRFLAG_KLATT → 64-byte frame_t).
        let mut f = SpectFrame::default();
        f.frflags = FRFLAG_KLATT;
        f.ffreq = [130, 700, 1200, 2500, 3300, 3900, 4400];
        f.length = 18;
        f.rms = 45;
        f.klattp = [50, 1, 2, 3, 4];
        f.klattp2 = [5, 6, 7, 8, 9];
        f.klatt_ap = [1, 2, 3, 4, 5, 6, 7];
        f.klatt_bp = [8, 9, 10, 11, 12, 13, 14];

        let frames = vec![f.clone(), f.clone()];
        let bytes = compile_spect_seq(&frames);
        let seq = SpectSeq::parse(&bytes, 0).expect("parse compiled SPECT_SEQK");
        assert!(seq.is_klatt, "should be detected as a Klatt sequence");
        assert_eq!(seq.frames.len(), 2);
        assert_eq!(seq.frames[0].to_bytes_t(), f.to_bytes_t());
        assert_eq!(seq.frames[0].klatt_bp, [8, 9, 10, 11, 12, 13, 14]);
    }

    // Minimal valid SPECT_SEQ (non-Klatt, 1 frame):
    // Header: length_total=0 (i16 LE), n_frames=1, sqflags=0
    // Frame (44 bytes): frflags=0, ffreq=[500,1500,2500,...], length=4, rms=100, ...
    fn make_seq_1frame() -> Vec<u8> {
        let mut data = vec![0u8; 4 + FRAME_T2_SIZE];
        data[2] = 1; // n_frames
        // Frame starts at offset 4
        // frflags = 0 (non-Klatt)
        // ffreq[1] = 500 Hz (F1)
        let f1 = 500i16.to_le_bytes();
        let f2 = 1500i16.to_le_bytes();
        data[4+2..4+4].copy_from_slice(&[0,0]); // ffreq[0] = 0
        data[4+4..4+6].copy_from_slice(&f1);    // ffreq[1] = 500
        data[4+6..4+8].copy_from_slice(&f2);    // ffreq[2] = 1500
        data[4+16] = 4;  // length = 4 steps
        data[4+17] = 80; // rms = 80
        data
    }

    #[test]
    fn parse_basic_seq() {
        let raw = make_seq_1frame();
        let seq = SpectSeq::parse(&raw, 0).expect("should parse");
        assert_eq!(seq.frames.len(), 1);
        assert!(!seq.is_klatt);
        assert_eq!(seq.frames[0].f1_hz(), 500.0);
        assert_eq!(seq.frames[0].f2_hz(), 1500.0);
        assert_eq!(seq.frames[0].length, 4);
        assert_eq!(seq.frames[0].rms, 80);
    }

    #[test]
    fn parse_too_short_returns_none() {
        let raw = vec![0u8; 3];
        assert!(SpectSeq::parse(&raw, 0).is_none());
    }

    #[test]
    fn parse_zero_frames_returns_none() {
        let raw = vec![0u8; 8]; // header with n_frames=0
        assert!(SpectSeq::parse(&raw, 0).is_none());
    }

    #[test]
    fn parse_at_offset() {
        let mut raw = vec![0xff_u8; 8]; // garbage prefix
        let seq_data = make_seq_1frame();
        raw.extend_from_slice(&seq_data);
        let seq = SpectSeq::parse(&raw, 8).expect("should parse at offset 8");
        assert_eq!(seq.frames.len(), 1);
        assert_eq!(seq.frames[0].f1_hz(), 500.0);
    }

    #[test]
    fn frame_t2_round_trip() {
        let mut raw = [0u8; FRAME_T2_SIZE];
        // frflags = 2
        raw[0] = 2; raw[1] = 0;
        // ffreq[0]=100, ffreq[1]=500, ffreq[2]=1500
        raw[2] = 100; raw[3] = 0;
        raw[4] = 244; raw[5] = 1;  // 500 = 0x01F4
        raw[6] = 220; raw[7] = 5;  // 1500 = 0x05DC
        raw[16] = 8;  // length
        raw[17] = 60; // rms
        let frame = SpectFrame::from_bytes_t2(&raw).unwrap();
        assert_eq!(frame.frflags, 2);
        assert_eq!(frame.ffreq[1], 500);
        assert_eq!(frame.ffreq[2], 1500);
        assert_eq!(frame.length, 8);
        assert_eq!(frame.rms, 60);
    }

    #[test]
    fn frame_t_round_trip() {
        let mut raw = [0u8; FRAME_T_SIZE];
        raw[0] = 1; // frflags = FRFLAG_KLATT
        let frame = SpectFrame::from_bytes_t(&raw).unwrap();
        assert_eq!(frame.frflags, 1);
    }

    #[test]
    fn klatt_seq_detected() {
        let mut raw = vec![0u8; 4 + FRAME_T_SIZE];
        raw[2] = 1; // n_frames=1
        raw[4] = 1; // first frame frflags bit 0 = FRFLAG_KLATT
        let seq = SpectSeq::parse(&raw, 0).expect("should parse");
        assert!(seq.is_klatt);
        assert_eq!(seq.frames.len(), 1);
    }

    #[test]
    fn voiced_detection() {
        let mut raw = make_seq_1frame();
        // set klattp[0] (AV) = 0 initially → not voiced
        // klattp starts at offset 4+39 = 43
        raw[4+39] = 0;
        let seq_unvoiced = SpectSeq::parse(&raw, 0).unwrap();
        // F1 is 500, so is_voiced via ffreq check
        assert!(seq_unvoiced.is_voiced(), "non-zero F1 → voiced");

        // Now set F1 = 0 and AV = 0 → unvoiced
        raw[4+4] = 0; raw[4+5] = 0; // ffreq[1] = 0
        let seq_v2 = SpectSeq::parse(&raw, 0).unwrap();
        assert!(!seq_v2.is_voiced());
    }

    #[test]
    fn dur_samples_speed_factor() {
        let mut f = SpectFrame::default();
        f.length = 4;
        // 4 * 64 = 256 samples at speed 1.0
        assert_eq!(f.dur_samples(1.0), 256);
        // double speed → 128
        assert_eq!(f.dur_samples(0.5), 128);
    }
}