1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
//! A loaded Qwen3-ASR model, with what it needs to transcribe.
use burn::{
Tensor,
tensor::Device,
};
use super::{
Qwen3AsrFrontEnd,
Qwen3AsrFrontEndConfig,
Qwen3AsrFrontEndMeta,
Qwen3AsrGreedyDecoderConfig,
Qwen3AsrPromptConfig,
Qwen3AsrTranscript,
blocks::{
Qwen3Asr,
Qwen3AsrAudioEncoderMeta,
Qwen3AsrMeta,
},
};
use crate::{
burner::module::ModuleInit,
errors::BunsenResult,
kits::tokens::ByteLevelVocabulary,
};
/// A Qwen3-ASR model, its front end, its vocabulary, its prompt and its
/// decoder: what a pretrained loads to, and everything a transcription
/// needs.
#[derive(Debug)]
pub struct Qwen3AsrBundle {
/// The model.
pub model: Qwen3Asr,
/// The feature extractor the model was trained behind.
pub front_end: Qwen3AsrFrontEnd,
/// The bytes the model's token ids spell.
pub vocab: ByteLevelVocabulary,
/// The chat template around the audio.
pub prompt: Qwen3AsrPromptConfig,
/// The decode's budget and end tokens.
pub decoder: Qwen3AsrGreedyDecoderConfig,
}
impl Qwen3AsrBundle {
/// The longest window [`transcribe_long`](Self::transcribe_long) is
/// meant to take, in seconds: Qwen3-ASR was trained on utterances of at
/// most 30 s.
pub const MAX_WINDOW_SECONDS: usize = 30;
/// A bundle of `model` and `vocab`, behind the default front end, prompt
/// and decoder.
///
/// # Errors
/// As [`Qwen3AsrFrontEndConfig`]'s `try_init`.
pub fn new(
model: Qwen3Asr,
vocab: ByteLevelVocabulary,
device: &Device,
) -> BunsenResult<Self> {
Ok(Self {
model,
front_end: Qwen3AsrFrontEndConfig::new().try_init(device)?,
vocab,
prompt: Qwen3AsrPromptConfig::new(),
decoder: Qwen3AsrGreedyDecoderConfig::new(),
})
}
/// Transcribes one utterance, greedily.
///
/// # Arguments
/// * `samples` - mono audio at the front end's
/// [`sample_rate`](Qwen3AsrFrontEndMeta::sample_rate); at most
/// [`MAX_WINDOW_SECONDS`](Self::MAX_WINDOW_SECONDS) is the model's range.
///
/// # Errors
/// As [`Qwen3AsrFrontEnd::forward`]; [`InvalidResource`] if a decoded
/// token is outside the vocabulary.
///
/// [`InvalidResource`]: crate::errors::BunsenErrorKind::InvalidResource
pub fn transcribe(
&self,
samples: &[f32],
device: &Device,
) -> BunsenResult<Qwen3AsrTranscript> {
let (features, valid) = self.front_end.forward(samples, device)?;
self.transcribe_features(features, valid)
}
/// Transcribes one utterance's features, greedily.
///
/// # Arguments
/// * `features` - `[n_mels, frames]`, as the front end packages them.
/// * `valid` - the frames the encoder reads; past them, to the end of the
/// last chunk, is padding the encoder's stem sees.
///
/// # Errors
/// As [`transcribe`](Self::transcribe), less the front end's.
pub fn transcribe_features(
&self,
features: Tensor<2>,
valid: usize,
) -> BunsenResult<Qwen3AsrTranscript> {
let audio = self.model.encode_audio(features, valid);
let n_audio = self.model.audio_tower.output_len(valid);
let ids = self.prompt.prompt(n_audio, self.model.audio_token());
let embeds = self.model.prompt_embeds(&ids, audio)?;
let tokens = self.decoder.init().decode(&self.model, embeds);
self.prompt.parse(tokens, &self.decoder.eos, &self.vocab)
}
/// How each token of a reply attended to the utterance's audio: the
/// decoder's attention weights from the position that predicts the token
/// onto the audio placeholders, one per encoder frame.
///
/// One uncached pass over the prompt and the reply, the reply's tokens
/// fed as the greedy decode fed them; the decode itself is not rerun.
///
/// # Arguments
/// * `features`, `valid` - as
/// [`transcribe_features`](Self::transcribe_features) took them.
/// * `reply` - the tokens the decode generated for them.
/// * `layers` - the decoder layers, by index.
///
/// # Returns
/// For each of `layers`, `[n_heads, reply.len(), n_audio]`; row `k` is
/// the distribution token `k` was predicted under, restricted to the
/// audio (its sum is the attention the audio received).
///
/// # Errors
/// As [`Qwen3Asr::prompt_embeds`].
///
/// # Panics
/// If `reply` is empty, or a layer index is past the last layer.
pub fn audio_attention(
&self,
features: Tensor<2>,
valid: usize,
reply: &[usize],
layers: &[usize],
) -> BunsenResult<Vec<Tensor<3>>> {
assert!(!reply.is_empty(), "an empty reply attends to nothing");
let audio = self.model.encode_audio(features, valid);
let n_audio = self.model.audio_tower.output_len(valid);
let mut ids = self.prompt.prompt(n_audio, self.model.audio_token());
let prompt_len = ids.len();
ids.extend_from_slice(&reply[..reply.len() - 1]);
let embeds = self.model.prompt_embeds(&ids, audio)?;
let lm = &self.model.language_model;
let r_emb = lm.rotary(ids.len(), &embeds.device());
let audio_at = self.prompt.prefix.len();
Ok(lm
.attention_weights(
embeds,
&r_emb,
layers,
prompt_len - 1..prompt_len - 1 + reply.len(),
audio_at..audio_at + n_audio,
)
.into_iter()
.map(|w| w.squeeze_dim(0))
.collect())
}
/// Transcribes one waveform of any length, in windows of at most
/// `max_window` samples.
///
/// A recording longer than the model's range is cut into windows, each
/// cut at the quietest mel frame of the window's last sixth
/// ([`quiet_cuts`](crate::ops::signal::quiet_cuts)), and each window is
/// transcribed alone. The texts are joined with spaces; the language is
/// the first window's that names one.
///
/// # Errors
/// As [`transcribe`](Self::transcribe).
///
/// # Panics
/// If `max_window` is shorter than six mel frames.
pub fn transcribe_long(
&self,
samples: &[f32],
max_window: usize,
device: &Device,
) -> BunsenResult<Qwen3AsrTranscript> {
let windows = crate::ops::signal::quiet_cuts(samples, max_window, self.front_end.hop());
let mut joined = Qwen3AsrTranscript {
tokens: Vec::new(),
language: None,
text: String::new(),
};
let mut texts = Vec::new();
for (a, b) in windows {
let t = self.transcribe(&samples[a..b], device)?;
joined.tokens.extend(t.tokens);
joined.language = joined.language.or(t.language);
if !t.text.is_empty() {
texts.push(t.text);
}
}
joined.text = texts.join(" ");
Ok(joined)
}
/// [`MAX_WINDOW_SECONDS`](Self::MAX_WINDOW_SECONDS), in samples.
pub fn max_window(&self) -> usize {
Self::MAX_WINDOW_SECONDS * self.front_end.sample_rate()
}
}