1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
//! Explicit synchronous extension for the existing prepared PQ2 activation path.
use bonsai_ternary::Q8KBlock;
use super::{PQ2, PTQ1, PreparedMatvec, Q8Block, dimensions, finite};
use crate::{Result, invalid, model::Tensor};
/// Scoped executor for original PQ2 rows and already prepared `Q8_K` activations.
///
/// The graph retains transforms, activation quantization, adapter application and
/// all mutable model state. An executor must preserve original per-128-block
/// scale/reduction order, including `fused` multiply-add selection. It must not
/// retain borrowed slices after return; uncertain native consumers require owned
/// staging retained through verified teardown by an external process supervisor.
/// No implementation or successful invocation establishes device qualification.
pub trait Pq2ProjectionExecutor {
/// Stable arithmetic/placement identity, excluding unverified device claims.
fn identity(&self) -> &str;
/// Remaining submissions admitted by this caller-bound executor session.
///
/// `None` denotes an opaque capacity. A bounded executor must report counts
/// for every selected projection, allowing complete-work admission before
/// the first submission. The caller must bind the native session's actual
/// remaining capacity, including work performed before borrowing it.
fn operation_capacity(&self) -> Option<u64> {
None
}
/// Stable submission count for one selected projection, or unknown.
///
/// When capacity is bounded, an unknown count must reject admission. Counts
/// include every row tile, original block and arithmetic lane composition.
fn operation_count(&self, _tensor: &Tensor) -> Option<u64> {
None
}
/// Decide eligibility from validated geometry before any device submission.
///
/// `false` keeps this whole projection on its configured CPU backend. A later
/// execution error is propagated without CPU replay or partial continuation.
/// Eligibility and [`Self::operation_count`] must remain stable for the
/// complete borrowed graph scope, whose admission may cache these decisions.
fn supports(&self, tensor: &Tensor) -> bool;
/// Replace the complete projection using the existing exact quantized codes.
///
/// # Errors
/// Rejects unavailable resources, cancellation, native failures and malformed
/// output. The caller clears output and closes its scoped decoder on failure.
fn project(
&mut self,
tensor: &Tensor,
activation: &[Q8KBlock],
fused: bool,
output: &mut Vec<f32>,
) -> Result<()>;
/// Opt in to PTQ1 or odd-block PQ2 with original prepared `Q8_0` codes.
///
/// Existing executors leave these paths on CPU. Eligibility must precede
/// submission and must not depend on a failed device attempt.
/// Eligibility and operation counts must remain stable for the complete
/// borrowed graph scope.
fn supports_q8(&self, _tensor: &Tensor) -> bool {
false
}
/// Execute a complete `Q8_0` projection without changing its float order.
///
/// When `pinned` and PTQ1, retain four independent DOTPROD lanes across the
/// complete row: each per-32 group's lane sums positions `4*i..4*i+4` and
/// `4*i+16..4*i+20`, FMA into that lane, then reduce adjacent pairs once.
/// Otherwise retain per-32 scale multiplication, per-128 group summation and
/// outer weight-scale multiplication in their original nonfused order.
///
/// # Errors
/// Rejects unsupported arithmetic, submission failures and invalid output.
fn project_q8(
&mut self,
_tensor: &Tensor,
_activation: &[Q8Block],
_pinned: bool,
_output: &mut Vec<f32>,
) -> Result<()> {
Err(invalid("projection executor does not support Q8_0"))
}
/// Opt in to direct full-`K` `F32` activation submission for `PQ2_0` tensors.
///
/// Existing bitplane and integer-guarded executors leave this `false`.
fn supports_f32(&self, _tensor: &Tensor) -> bool {
false
}
/// Execute a complete `PQ2_0` projection directly from the `F32` activation vector.
///
/// # Errors
/// Rejects unsupported arithmetic, submission failures and invalid output.
fn project_f32(
&mut self,
_tensor: &Tensor,
_activation: &[f32],
_pinned: bool,
_output: &mut Vec<f32>,
) -> Result<()> {
Err(invalid("projection executor does not support direct F32"))
}
/// Opt in to batched direct `F32` activation submission for a group of
/// compatible sibling `PQ2_0` projections sharing one input vector.
fn supports_f32_group(&self, _tensors: &[&Tensor]) -> bool {
false
}
/// Execute a group of compatible `PQ2_0` projections sharing one `F32`
/// activation vector in a single batched executor submission.
///
/// # Errors
/// Rejects unsupported groups, submission failures and invalid output.
fn project_f32_group(
&mut self,
_tensors: &[&Tensor],
_activation: &[f32],
_pinned: bool,
_outputs: &mut [&mut Vec<f32>],
) -> Result<()> {
Err(invalid(
"projection executor does not support batched F32 groups",
))
}
}
impl PreparedMatvec<'_> {
/// Execute a group of compatible `PQ2_0` sibling projections in a single
/// batched executor call when supported by `executor`.
///
/// # Errors
/// Rejects incompatible preparation, executor failures or incomplete output.
pub fn matvec_group_with_executor(
&mut self,
tensors: &[&Tensor],
outputs: &mut [&mut Vec<f32>],
executor: &mut dyn Pq2ProjectionExecutor,
) -> Result<()> {
if tensors.is_empty() || tensors.len() != outputs.len() {
return Err(invalid("mismatched projection group outputs"));
}
for output in outputs.iter_mut() {
output.clear();
}
let result = (|| {
let mut expected_rows = Vec::new();
expected_rows.try_reserve_exact(tensors.len())?;
for &tensor in tensors {
let (columns, rows, _) = dimensions(tensor)?;
if !self.is_compatible(tensor)?
|| tensor.kind != PQ2
|| !columns.is_multiple_of(128)
{
return Err(invalid(
"incompatible prepared projection executor activation",
));
}
expected_rows.push(rows);
}
executor.project_f32_group(
tensors,
self.input,
self.scratch.pinned_accumulation,
outputs,
)?;
for (output, rows) in outputs.iter().zip(expected_rows) {
if output.len() != rows {
return Err(invalid("projection executor returned incomplete output"));
}
for &value in output.iter() {
finite(value)?;
}
}
Ok(())
})();
if result.is_err() {
for output in outputs.iter_mut() {
output.clear();
}
}
result
}
/// Execute an eligible original-PQ2 projection through a scoped executor.
///
/// PTQ1 and odd-block PQ2 use `Q8_0` only with an explicit executor opt-in.
/// Other types retain configured CPU execution.
/// Eligibility is decided before submission. Executor errors never trigger
/// replay. Complete finite output is required before publication; CPU backend
/// statistics do not count work performed by an external executor.
///
/// # Errors
/// Rejects incompatible preparation, executor failures or incomplete output.
pub fn matvec_with_executor(
&mut self,
tensor: &Tensor,
output: &mut Vec<f32>,
executor: &mut dyn Pq2ProjectionExecutor,
) -> Result<()> {
output.clear();
let result = (|| {
let (columns, rows, _) = dimensions(tensor)?;
if !self.is_compatible(tensor)? {
return Err(invalid(
"incompatible prepared projection executor activation",
));
}
if tensor.kind == PQ2 && columns.is_multiple_of(128) && executor.supports_f32(tensor) {
executor.project_f32(
tensor,
self.input,
self.scratch.pinned_accumulation,
output,
)?;
} else if tensor.kind == PQ2 && columns.is_multiple_of(256) && executor.supports(tensor)
{
if !self.quantized {
return Err(invalid(
"prepared projection executor activation requires quantization",
));
}
executor.project(
tensor,
&self.scratch.q8k,
self.scratch.pinned_accumulation,
output,
)?;
} else if (tensor.kind == PTQ1 || tensor.kind == PQ2 && !columns.is_multiple_of(256))
&& executor.supports_q8(tensor)
{
if !self.quantized {
return Err(invalid(
"prepared projection executor activation requires quantization",
));
}
executor.project_q8(
tensor,
&self.scratch.q8,
self.scratch.pinned_accumulation,
output,
)?;
} else {
return self.matvec_into(tensor, output);
}
if output.len() != rows {
return Err(invalid("projection executor returned incomplete output"));
}
for &value in output.iter() {
finite(value)?;
}
Ok(())
})();
if result.is_err() {
output.clear();
}
result
}
}