1use std::collections::{BTreeMap, BTreeSet};
6use std::sync::{Arc, Mutex, MutexGuard};
7
8use llama_cpp_4::context::tensor_transaction::MAX_TENSOR_ROWS;
9use llama_cpp_4::context::{
10 CapturedTensorData, TensorAccess, TensorDataMut, TensorElementType, TensorRowMapping,
11 TensorSelector, TensorTransaction, TensorTransactionError, TensorTransactionHandler,
12 TensorTransactions, TensorWriteback, TransactionalTensorCapture,
13};
14use logit_loom::{
15 ActivationCapturePlanV1, ActivationCapturePositionsV1, ActivationCaptureReceiptV1,
16 ActivationCaptureRecordV1, ActivationCapturedDataV1, ActivationInvocationReceiptV1,
17 ActivationInvocationRowV1, ActivationOperatorV1, ActivationPhaseV1, ActivationProgramReceiptV1,
18 ActivationProgramV1, ActivationTelemetryDispositionV1, ActivationVectorBankV1,
19 ActivationVectorNormalizationV1, Digest, MAX_ACTIVATION_INVOCATION_RECEIPTS,
20 SpeculationTelemetryResolutionV1, TextTensorElementTypeV1, TextTensorSiteV1,
21 activation_f32_row_identity,
22};
23
24use crate::{Error, Model};
25
26#[derive(Clone, Debug, PartialEq, Eq)]
33pub struct LlamaCppTensorProfile {
34 selector_implementation: Digest,
35 moe_layers: Vec<u32>,
36 identity: Digest,
37}
38
39impl LlamaCppTensorProfile {
40 pub fn layer_outputs(model: &Model) -> Result<Self, Error> {
48 Self::new(model, Vec::new())
49 }
50
51 pub fn new(model: &Model, moe_layers: Vec<u32>) -> Result<Self, Error> {
59 if moe_layers.windows(2).any(|window| window[0] >= window[1])
60 || moe_layers
61 .iter()
62 .any(|layer| *layer >= model.topology().layers)
63 || (!moe_layers.is_empty() && model.topology().experts.is_none())
64 {
65 return Err(Error::Invalid(
66 "MoE profile layers must be unique, ordered, in range, and expert-backed"
67 .to_owned(),
68 ));
69 }
70 let selector_implementation = model.tensor_selector_implementation().clone();
71 let identity = Digest::of_serializable(
72 "llamacpp-tensor-profile-v1",
73 &(
74 model.topology().digest()?,
75 &selector_implementation,
76 &moe_layers,
77 ),
78 )?;
79 Ok(Self {
80 selector_implementation,
81 moe_layers,
82 identity,
83 })
84 }
85
86 pub const fn identity(&self) -> &Digest {
88 &self.identity
89 }
90
91 fn supports(&self, site: &TextTensorSiteV1) -> bool {
92 match site {
93 TextTensorSiteV1::LayerOutput { .. } => true,
94 TextTensorSiteV1::RouterLogits { layer, selector }
95 | TextTensorSiteV1::RouterProbabilities { layer, selector }
96 | TextTensorSiteV1::SelectedExperts { layer, selector } => {
97 selector == &self.selector_implementation
98 && self.moe_layers.binary_search(layer).is_ok()
99 }
100 TextTensorSiteV1::Named { selector, .. } => selector == &self.selector_implementation,
101 }
102 }
103}
104
105#[derive(Clone, Debug, PartialEq)]
107pub struct ActivationConfiguration {
108 topology: logit_loom::TextModelTopologyV1,
109 program: ActivationProgramV1,
110 program_identity: Digest,
111 captures: Vec<(Digest, ActivationCapturePlanV1)>,
112 banks: BTreeMap<Digest, ActivationVectorBankV1>,
113 profile: LlamaCppTensorProfile,
114}
115
116impl ActivationConfiguration {
117 pub fn new(
128 model: &Model,
129 profile: LlamaCppTensorProfile,
130 program: ActivationProgramV1,
131 capture_plans: Vec<ActivationCapturePlanV1>,
132 vector_banks: Vec<ActivationVectorBankV1>,
133 ) -> Result<Self, Error> {
134 if profile.selector_implementation != *model.tensor_selector_implementation() {
135 return Err(Error::Incompatible(
136 "tensor profile belongs to another selector implementation".to_owned(),
137 ));
138 }
139 let topology = model.topology().clone();
140 let program_identity = program.digest_for(&topology)?;
141
142 let mut captures = Vec::with_capacity(capture_plans.len());
143 for plan in capture_plans {
144 let identity = plan.digest_for(&topology)?;
145 if let Some((prior, _)) = captures.last()
146 && prior >= &identity
147 {
148 return Err(Error::Invalid(
149 "capture plans must have unique canonical identity order".to_owned(),
150 ));
151 }
152 captures.push((identity, plan));
153 }
154 let capture_identities = captures
155 .iter()
156 .map(|(identity, _)| identity.clone())
157 .collect::<Vec<_>>();
158 if capture_identities != program.observations {
159 return Err(Error::Incompatible(
160 "activation capture plans differ from program observations".to_owned(),
161 ));
162 }
163
164 let mut banks = BTreeMap::new();
165 for bank in vector_banks {
166 let identity = bank.digest_for(&topology)?;
167 if banks.insert(identity, bank).is_some() {
168 return Err(Error::Invalid(
169 "activation vector-bank identities must be unique".to_owned(),
170 ));
171 }
172 }
173 let referenced = program
174 .operations
175 .iter()
176 .map(|operation| operation.vector_bank.clone())
177 .collect::<BTreeSet<_>>();
178 if referenced.len() != banks.len()
179 || referenced
180 .iter()
181 .any(|identity| !banks.contains_key(identity))
182 {
183 return Err(Error::Incompatible(
184 "activation vector banks must exactly resolve program references".to_owned(),
185 ));
186 }
187 for operation in &program.operations {
188 let bank = banks.get(&operation.vector_bank).ok_or_else(|| {
189 Error::Incompatible("activation vector bank is unavailable".to_owned())
190 })?;
191 if operation.operator == ActivationOperatorV1::ScaledProjectionRemoval
192 && bank.normalization != ActivationVectorNormalizationV1::UnitL2
193 {
194 return Err(Error::Invalid(
195 "projection removal requires a unit-L2 vector bank".to_owned(),
196 ));
197 }
198 for site in &operation.sites {
199 validate_profile_site(&profile, site)?;
200 if bank.site_family != site.family()
201 || bank.row_elements != site.row_elements(&topology)?
202 || site.layer().and_then(|layer| bank.row(layer)).is_none()
203 {
204 return Err(Error::Incompatible(
205 "activation vector bank does not cover an operation site".to_owned(),
206 ));
207 }
208 }
209 }
210 for (_, plan) in &captures {
211 for site in &plan.sites {
212 validate_profile_site(&profile, site)?;
213 if plan.retention == logit_loom::ActivationCaptureRetentionV1::Statistics
214 && site.element_type() != TextTensorElementTypeV1::F32
215 {
216 return Err(Error::Invalid(
217 "statistics capture requires f32 tensor sites".to_owned(),
218 ));
219 }
220 }
221 }
222 Ok(Self {
223 topology,
224 program,
225 program_identity,
226 captures,
227 banks,
228 profile,
229 })
230 }
231
232 pub const fn program_identity(&self) -> &Digest {
234 &self.program_identity
235 }
236
237 pub const fn profile_identity(&self) -> &Digest {
239 self.profile.identity()
240 }
241
242 pub(crate) fn lower(
243 self,
244 maximum_rows: u32,
245 ) -> Result<(ActivationController, TensorTransactions), Error> {
246 let maximum_rows = usize::try_from(maximum_rows)
247 .map_err(|_| Error::Invalid("activation batch rows exceed usize".to_owned()))?;
248 if maximum_rows == 0 || maximum_rows > MAX_TENSOR_ROWS {
249 return Err(Error::Invalid(format!(
250 "activation batch rows must be between 1 and {MAX_TENSOR_ROWS}"
251 )));
252 }
253
254 let program = Arc::new(self.program);
255 let banks = Arc::new(self.banks);
256 let captures = Arc::new(self.captures);
257 let mut selected_sites = BTreeSet::new();
258 for operation in &program.operations {
259 selected_sites.extend(operation.sites.iter().cloned());
260 }
261 for (_, plan) in captures.iter() {
262 selected_sites.extend(plan.sites.iter().cloned());
263 }
264
265 let mutable_sites = program
266 .operations
267 .iter()
268 .flat_map(|operation| operation.sites.iter().cloned())
269 .collect::<BTreeSet<_>>();
270 let retained_sites = captures
271 .iter()
272 .flat_map(|(_, plan)| plan.sites.iter().cloned())
273 .collect::<BTreeSet<_>>();
274
275 let mut site_bindings = BTreeMap::new();
276 for site in selected_sites {
277 validate_profile_site(&self.profile, &site)?;
278 let name = native_tensor_name(&site);
279 let binding = SiteBinding {
280 site: site.clone(),
281 name: name.clone(),
282 };
283 if let Some(prior) = site_bindings.insert(name.clone(), binding)
284 && prior.site != site
285 {
286 return Err(Error::Incompatible(format!(
287 "multiple tensor sites resolve to native graph node {name}"
288 )));
289 }
290 }
291 let site_bindings = Arc::new(site_bindings);
292 let mut selectors = Vec::with_capacity(site_bindings.len());
293 for binding in site_bindings.values() {
294 let site = &binding.site;
295 let element_type = match site.element_type() {
296 TextTensorElementTypeV1::F32 => TensorElementType::F32,
297 TextTensorElementTypeV1::I32 => TensorElementType::I32,
298 };
299 let access = if mutable_sites.contains(site) {
300 TensorAccess::ReadWriteF32
301 } else {
302 TensorAccess::ReadOnly
303 };
304 selectors.push(
305 TensorSelector::new(
306 binding.name.clone(),
307 element_type,
308 usize::try_from(site.row_elements(&self.topology)?)
309 .map_err(|_| Error::Invalid("tensor row width exceeds usize".to_owned()))?,
310 maximum_rows,
311 access,
312 TensorRowMapping::BatchTokens,
313 retained_sites.contains(site),
314 )
315 .map_err(|error| Error::Invalid(error.to_string()))?,
316 );
317 }
318
319 let state = Arc::new(Mutex::new(ActivationState::default()));
320 let controller = ActivationController {
321 topology: self.topology,
322 program: Arc::clone(&program),
323 program_identity: self.program_identity,
324 captures: Arc::clone(&captures),
325 site_bindings: Arc::clone(&site_bindings),
326 state: Arc::clone(&state),
327 captured_records: Vec::new(),
328 capture_aggregates: BTreeMap::new(),
329 provisional_captures: Vec::new(),
330 provisional_capture_aggregates: BTreeMap::new(),
331 };
332 let transactions = if mutable_sites.is_empty() {
333 TensorTransactions::capture(selectors)
334 } else {
335 TensorTransactions::new(
336 selectors,
337 ActivationHandler {
338 program,
339 banks,
340 sites: site_bindings,
341 state,
342 },
343 )
344 }
345 .map_err(|error| Error::Invalid(error.to_string()))?;
346 Ok((controller, transactions))
347 }
348}
349
350#[derive(Clone, Debug, PartialEq)]
352pub struct ActivationCaptureOutput {
353 pub records: Vec<ActivationCaptureRecordV1>,
355 pub receipts: Vec<ActivationCaptureReceiptV1>,
357}
358
359#[derive(Clone, Debug, PartialEq, Eq)]
361pub struct ActivationProgramOutput {
362 pub invocations: Vec<ActivationInvocationReceiptV1>,
364 pub receipt: ActivationProgramReceiptV1,
366}
367
368#[derive(Clone, Debug)]
369struct SiteBinding {
370 site: TextTensorSiteV1,
371 name: String,
372}
373
374#[derive(Clone, Copy, Debug)]
375pub(crate) struct ActiveDecode {
376 phase: ActivationPhaseV1,
377 last_prefill: bool,
378 disposition: ActivationTelemetryDispositionV1,
379}
380
381#[derive(Debug, Default)]
382struct ActivationState {
383 active: Option<ActiveDecode>,
384 invocations: Vec<ActivationInvocationReceiptV1>,
385 invocation_identities: Vec<Digest>,
386 provisional_invocations: Vec<ProvisionalInvocation>,
387 elements_copied: u64,
388 write_backs: u64,
389}
390
391#[derive(Debug, Default)]
392struct CaptureAggregate {
393 records: Vec<Digest>,
394 elements: u64,
395 retained_bytes: u64,
396}
397
398#[derive(Clone, Debug)]
399struct ProvisionalCapture {
400 record: ActivationCaptureRecordV1,
401 identity: Digest,
402}
403
404#[derive(Clone, Debug)]
405struct ProvisionalInvocation {
406 invocation: ActivationInvocationReceiptV1,
407}
408
409#[derive(Debug)]
410pub(crate) struct ActivationController {
411 topology: logit_loom::TextModelTopologyV1,
412 program: Arc<ActivationProgramV1>,
413 program_identity: Digest,
414 captures: Arc<Vec<(Digest, ActivationCapturePlanV1)>>,
415 site_bindings: Arc<BTreeMap<String, SiteBinding>>,
416 state: Arc<Mutex<ActivationState>>,
417 captured_records: Vec<ActivationCaptureRecordV1>,
418 capture_aggregates: BTreeMap<Digest, CaptureAggregate>,
419 provisional_captures: Vec<ProvisionalCapture>,
420 provisional_capture_aggregates: BTreeMap<Digest, CaptureAggregate>,
421}
422
423impl ActivationController {
424 pub(crate) fn has_last_prefill_capture(&self) -> bool {
425 self.captures
426 .iter()
427 .any(|(_, plan)| plan.positions == ActivationCapturePositionsV1::LastPrefillToken)
428 }
429
430 pub(crate) fn begin_decode(
431 &self,
432 phase: ActivationPhaseV1,
433 last_prefill: bool,
434 disposition: ActivationTelemetryDispositionV1,
435 ) -> Result<(), Error> {
436 if !self.provisional_captures.is_empty() {
437 return Err(Error::Poisoned(
438 "provisional activation captures must be resolved before another decode".to_owned(),
439 ));
440 }
441 let mut state = lock_state(&self.state)?;
442 if state.active.is_some() {
443 return Err(Error::Poisoned(
444 "activation decode phase was already active".to_owned(),
445 ));
446 }
447 if !state.provisional_invocations.is_empty() {
448 return Err(Error::Poisoned(
449 "provisional activation invocations must be resolved before another decode"
450 .to_owned(),
451 ));
452 }
453 state.active = Some(ActiveDecode {
454 phase,
455 last_prefill,
456 disposition,
457 });
458 Ok(())
459 }
460
461 pub(crate) fn end_decode(&self) -> Result<ActiveDecode, Error> {
462 lock_state(&self.state)?
463 .active
464 .take()
465 .ok_or_else(|| Error::Poisoned("activation decode phase was not active".to_owned()))
466 }
467
468 pub(crate) fn consume_captures(
469 &mut self,
470 native: Vec<TransactionalTensorCapture>,
471 decode: ActiveDecode,
472 ) -> Result<(), Error> {
473 let capture_plans = Arc::clone(&self.captures);
474 for capture in native {
475 let site = self
476 .site_bindings
477 .get(&capture.name)
478 .ok_or_else(|| {
479 Error::Incompatible(format!(
480 "native capture {} has no site binding",
481 capture.name
482 ))
483 })?
484 .site
485 .clone();
486 if capture.rows.len() != capture.shape.rows
487 || capture.shape.elements
488 != capture
489 .shape
490 .row_elements
491 .saturating_mul(capture.shape.rows)
492 {
493 return Err(Error::Incompatible(
494 "native capture row accounting is inconsistent".to_owned(),
495 ));
496 }
497 for (row_index, row) in capture.rows.iter().enumerate() {
498 let position = u64::try_from(row.position).map_err(|_| {
499 Error::Incompatible("native capture position is negative".to_owned())
500 })?;
501 let start = row_index
502 .checked_mul(capture.shape.row_elements)
503 .ok_or_else(|| Error::Invalid("capture row offset overflowed".to_owned()))?;
504 let end = start
505 .checked_add(capture.shape.row_elements)
506 .ok_or_else(|| Error::Invalid("capture row end overflowed".to_owned()))?;
507 for (plan_identity, plan) in capture_plans.iter() {
508 if !plan.sites.contains(&site)
509 || !capture_position_selected(plan, position, decode, capture.shape.rows)
510 {
511 continue;
512 }
513 let retained = match &capture.data {
514 CapturedTensorData::F32(values) => {
515 ActivationCapturedDataV1::from_f32(&values[start..end], plan.retention)?
516 }
517 CapturedTensorData::I32(values) => {
518 ActivationCapturedDataV1::from_i32(&values[start..end], plan.retention)?
519 }
520 };
521 let record = ActivationCaptureRecordV1 {
522 plan: plan_identity.clone(),
523 site: site.clone(),
524 position,
525 disposition: decode.disposition,
526 retained,
527 };
528 let record_identity = record.digest_for(plan, &self.topology)?;
529 if decode.disposition == ActivationTelemetryDispositionV1::Provisional {
530 self.push_provisional_capture(
531 plan_identity,
532 plan,
533 record,
534 record_identity,
535 )?;
536 } else {
537 self.commit_capture(plan_identity, plan, record, record_identity)?;
538 }
539 }
540 }
541 }
542 Ok(())
543 }
544
545 fn push_provisional_capture(
546 &mut self,
547 plan_identity: &Digest,
548 plan: &ActivationCapturePlanV1,
549 record: ActivationCaptureRecordV1,
550 identity: Digest,
551 ) -> Result<(), Error> {
552 let elements = u64::from(record.retained.elements());
553 let bytes = retained_bytes(&record.retained);
554 let committed = self.capture_aggregates.get(plan_identity);
555 let pending = self.provisional_capture_aggregates.get(plan_identity);
556 let pending_elements = pending
557 .map_or(0, |aggregate| aggregate.elements)
558 .checked_add(elements)
559 .ok_or_else(|| Error::Invalid("capture element accounting overflowed".to_owned()))?;
560 let pending_bytes = pending
561 .map_or(0, |aggregate| aggregate.retained_bytes)
562 .checked_add(bytes)
563 .ok_or_else(|| {
564 Error::Invalid("capture retained-byte accounting overflowed".to_owned())
565 })?;
566 let total_elements = committed
567 .map_or(0, |aggregate| aggregate.elements)
568 .checked_add(pending_elements)
569 .ok_or_else(|| Error::Invalid("capture element accounting overflowed".to_owned()))?;
570 let total_bytes = committed
571 .map_or(0, |aggregate| aggregate.retained_bytes)
572 .checked_add(pending_bytes)
573 .ok_or_else(|| {
574 Error::Invalid("capture retained-byte accounting overflowed".to_owned())
575 })?;
576 if total_elements > plan.maximum_elements || total_bytes > plan.maximum_retained_bytes {
577 return Err(Error::Invalid(
578 "activation capture exceeded its declared aggregate bound".to_owned(),
579 ));
580 }
581 let aggregate = self
582 .provisional_capture_aggregates
583 .entry(plan_identity.clone())
584 .or_default();
585 aggregate.elements = pending_elements;
586 aggregate.retained_bytes = pending_bytes;
587 aggregate.records.push(identity.clone());
588 self.provisional_captures
589 .push(ProvisionalCapture { record, identity });
590 Ok(())
591 }
592
593 fn commit_capture(
594 &mut self,
595 plan_identity: &Digest,
596 plan: &ActivationCapturePlanV1,
597 record: ActivationCaptureRecordV1,
598 identity: Digest,
599 ) -> Result<(), Error> {
600 let aggregate = self
601 .capture_aggregates
602 .entry(plan_identity.clone())
603 .or_default();
604 let elements = u64::from(record.retained.elements());
605 let bytes = retained_bytes(&record.retained);
606 let next_elements = aggregate
607 .elements
608 .checked_add(elements)
609 .ok_or_else(|| Error::Invalid("capture element accounting overflowed".to_owned()))?;
610 let next_bytes = aggregate.retained_bytes.checked_add(bytes).ok_or_else(|| {
611 Error::Invalid("capture retained-byte accounting overflowed".to_owned())
612 })?;
613 if next_elements > plan.maximum_elements || next_bytes > plan.maximum_retained_bytes {
614 return Err(Error::Invalid(
615 "activation capture exceeded its declared aggregate bound".to_owned(),
616 ));
617 }
618 aggregate.elements = next_elements;
619 aggregate.retained_bytes = next_bytes;
620 aggregate.records.push(identity);
621 self.captured_records.push(record);
622 Ok(())
623 }
624
625 pub(crate) fn resolve_provisional(
626 &mut self,
627 admitted_position_exclusive: u64,
628 ) -> Result<Vec<SpeculationTelemetryResolutionV1>, Error> {
629 let capture_plans = Arc::clone(&self.captures);
630 let mut resolved_captures = Vec::with_capacity(self.provisional_captures.len());
631 for pending in &self.provisional_captures {
632 let (_, plan) = capture_plans
633 .iter()
634 .find(|(identity, _)| identity == &pending.record.plan)
635 .ok_or_else(|| Error::Incompatible("provisional capture has no plan".to_owned()))?;
636 let mut record = pending.record.clone();
637 record.disposition = if record.position < admitted_position_exclusive {
638 ActivationTelemetryDispositionV1::Admitted
639 } else {
640 ActivationTelemetryDispositionV1::Rejected
641 };
642 let identity = record.digest_for(plan, &self.topology)?;
643 let disposition = record.disposition;
644 resolved_captures.push((
645 record,
646 identity.clone(),
647 SpeculationTelemetryResolutionV1 {
648 provisional: pending.identity.clone(),
649 disposition,
650 resolved: identity,
651 },
652 ));
653 }
654
655 let mut state = lock_state(&self.state)?;
656 if state.active.is_some() {
657 return Err(Error::Poisoned(
658 "cannot resolve activation telemetry during a decode".to_owned(),
659 ));
660 }
661 let mut resolved_invocations = Vec::with_capacity(state.provisional_invocations.len());
662 for pending in &state.provisional_invocations {
663 let mut invocation = pending.invocation.clone();
664 for row in &mut invocation.rows {
665 row.disposition = if row.position < admitted_position_exclusive {
666 ActivationTelemetryDispositionV1::Admitted
667 } else {
668 ActivationTelemetryDispositionV1::Rejected
669 };
670 }
671 let identity = invocation.digest()?;
672 resolved_invocations.push((invocation, identity));
673 }
674 let final_invocation_count = state
675 .invocation_identities
676 .len()
677 .checked_add(resolved_invocations.len())
678 .ok_or_else(|| Error::Invalid("activation invocation count overflowed".to_owned()))?;
679 if final_invocation_count > MAX_ACTIVATION_INVOCATION_RECEIPTS {
680 return Err(Error::Invalid(
681 "activation invocation count exceeded its supported bound".to_owned(),
682 ));
683 }
684 let added_elements =
685 resolved_invocations
686 .iter()
687 .try_fold(0_u64, |total, (invocation, _)| {
688 total
689 .checked_add(u64::from(invocation.elements))
690 .ok_or_else(|| {
691 Error::Invalid("activation element accounting overflowed".to_owned())
692 })
693 })?;
694 let final_elements = state
695 .elements_copied
696 .checked_add(added_elements)
697 .ok_or_else(|| Error::Invalid("activation element accounting overflowed".to_owned()))?;
698 let added_write_backs = u64::try_from(resolved_invocations.len())
699 .map_err(|_| Error::Invalid("activation write-back count exceeds u64".to_owned()))?;
700 let final_write_backs = state
701 .write_backs
702 .checked_add(added_write_backs)
703 .ok_or_else(|| {
704 Error::Invalid("activation write-back accounting overflowed".to_owned())
705 })?;
706
707 let mut resolutions = Vec::with_capacity(resolved_captures.len());
708 for (invocation, identity) in resolved_invocations {
709 state.invocations.push(invocation);
710 state.invocation_identities.push(identity);
711 }
712 state.elements_copied = final_elements;
713 state.write_backs = final_write_backs;
714 state.provisional_invocations.clear();
715 drop(state);
716
717 for (record, identity, resolution) in resolved_captures {
718 let (_, plan) = capture_plans
719 .iter()
720 .find(|(plan_identity, _)| plan_identity == &record.plan)
721 .ok_or_else(|| Error::Incompatible("resolved capture has no plan".to_owned()))?;
722 self.commit_capture(&record.plan.clone(), plan, record, identity)?;
723 resolutions.push(resolution);
724 }
725 self.provisional_captures.clear();
726 self.provisional_capture_aggregates.clear();
727 Ok(resolutions)
728 }
729
730 pub(crate) fn take_capture_output(&mut self) -> Result<ActivationCaptureOutput, Error> {
731 if !self.provisional_captures.is_empty() {
732 return Err(Error::Invalid(
733 "cannot drain activation captures before provisional resolution".to_owned(),
734 ));
735 }
736 let mut receipts = Vec::with_capacity(self.capture_aggregates.len());
737 for (plan_identity, aggregate) in std::mem::take(&mut self.capture_aggregates) {
738 let (_, plan) = self
739 .captures
740 .iter()
741 .find(|(identity, _)| identity == &plan_identity)
742 .ok_or_else(|| Error::Incompatible("capture aggregate has no plan".to_owned()))?;
743 let receipt = ActivationCaptureReceiptV1 {
744 plan: plan_identity,
745 records: aggregate.records,
746 elements: aggregate.elements,
747 retained_bytes: aggregate.retained_bytes,
748 };
749 receipt.digest_for(plan)?;
750 receipts.push(receipt);
751 }
752 Ok(ActivationCaptureOutput {
753 records: std::mem::take(&mut self.captured_records),
754 receipts,
755 })
756 }
757
758 pub(crate) fn take_program_output(&self) -> Result<ActivationProgramOutput, Error> {
759 let mut state = lock_state(&self.state)?;
760 if state.active.is_some() {
761 return Err(Error::Invalid(
762 "cannot drain activation receipts during a decode".to_owned(),
763 ));
764 }
765 if !state.provisional_invocations.is_empty() {
766 return Err(Error::Invalid(
767 "cannot drain activation receipts before provisional resolution".to_owned(),
768 ));
769 }
770 let receipt = ActivationProgramReceiptV1 {
771 program: self.program_identity.clone(),
772 topology: self.topology.digest()?,
773 invocations: std::mem::take(&mut state.invocation_identities),
774 elements_copied: std::mem::take(&mut state.elements_copied),
775 write_backs: std::mem::take(&mut state.write_backs),
776 };
777 receipt.digest_for(&self.program)?;
778 Ok(ActivationProgramOutput {
779 invocations: std::mem::take(&mut state.invocations),
780 receipt,
781 })
782 }
783
784 pub(crate) fn reset(&mut self) -> Result<(), Error> {
785 if !self.provisional_captures.is_empty() {
786 return Err(Error::Poisoned(
787 "cannot reset unresolved provisional activation captures".to_owned(),
788 ));
789 }
790 self.captured_records.clear();
791 self.capture_aggregates.clear();
792 self.provisional_capture_aggregates.clear();
793 let mut state = lock_state(&self.state)?;
794 if state.active.is_some() {
795 return Err(Error::Poisoned(
796 "cannot reset activation evidence during a decode".to_owned(),
797 ));
798 }
799 if !state.provisional_invocations.is_empty() {
800 return Err(Error::Poisoned(
801 "cannot reset unresolved provisional activation invocations".to_owned(),
802 ));
803 }
804 state.invocations.clear();
805 state.invocation_identities.clear();
806 state.elements_copied = 0;
807 state.write_backs = 0;
808 Ok(())
809 }
810}
811
812struct ActivationHandler {
813 program: Arc<ActivationProgramV1>,
814 banks: Arc<BTreeMap<Digest, ActivationVectorBankV1>>,
815 sites: Arc<BTreeMap<String, SiteBinding>>,
816 state: Arc<Mutex<ActivationState>>,
817}
818
819impl TensorTransactionHandler for ActivationHandler {
820 #[allow(
821 clippy::too_many_lines,
822 reason = "the callback keeps validation, ordered row transforms, and receipt commit in one transactional scope"
823 )]
824 fn apply(
825 &mut self,
826 mut transaction: TensorTransaction<'_>,
827 ) -> Result<TensorWriteback, TensorTransactionError> {
828 let binding = self
829 .sites
830 .get(transaction.name)
831 .ok_or_else(|| TensorTransactionError::new("selected tensor has no site binding"))?;
832 let TensorDataMut::F32(values) = &mut transaction.data else {
833 return Err(TensorTransactionError::new(
834 "mutable activation transaction is not f32",
835 ));
836 };
837 let mut state = self
838 .state
839 .lock()
840 .map_err(|_| TensorTransactionError::new("activation state mutex was poisoned"))?;
841 let active = state.active.ok_or_else(|| {
842 TensorTransactionError::new("activation callback ran outside an active decode phase")
843 })?;
844 let input = activation_f32_row_identity(values)
845 .map_err(|error| TensorTransactionError::new(error.to_string()))?;
846 let mut changed_rows = Vec::new();
847 for (row_index, row) in transaction.rows.iter().enumerate() {
848 let position = u64::try_from(row.position)
849 .map_err(|_| TensorTransactionError::new("native causal position is negative"))?;
850 let start = row_index
851 .checked_mul(transaction.shape.row_elements)
852 .ok_or_else(|| TensorTransactionError::new("activation row offset overflowed"))?;
853 let end = start
854 .checked_add(transaction.shape.row_elements)
855 .ok_or_else(|| TensorTransactionError::new("activation row end overflowed"))?;
856 let row_values = &mut values[start..end];
857 let mut operations = Vec::new();
858 for (operation_index, operation) in self.program.operations.iter().enumerate() {
859 if operation.phase != active.phase
860 || !operation.positions.contains(position)
861 || !operation.sites.contains(&binding.site)
862 {
863 continue;
864 }
865 let bank = self.banks.get(&operation.vector_bank).ok_or_else(|| {
866 TensorTransactionError::new("activation vector bank is unavailable")
867 })?;
868 let layer = binding.site.layer().ok_or_else(|| {
869 TensorTransactionError::new("mutable activation site has no layer")
870 })?;
871 let vector = bank.row(layer).ok_or_else(|| {
872 TensorTransactionError::new("activation vector row is unavailable")
873 })?;
874 apply_operator(
875 row_values,
876 vector,
877 operation.operator,
878 operation
879 .scale()
880 .map_err(|error| TensorTransactionError::new(error.to_string()))?,
881 )?;
882 operations.push(u32::try_from(operation_index).map_err(|_| {
883 TensorTransactionError::new("activation operation index exceeds u32")
884 })?);
885 }
886 if !operations.is_empty() {
887 let sequence_ids = row
888 .sequence_ids
889 .iter()
890 .map(|sequence| {
891 u32::try_from(*sequence).map_err(|_| {
892 TensorTransactionError::new("native activation sequence id is negative")
893 })
894 })
895 .collect::<Result<Vec<_>, _>>()?;
896 if sequence_ids.windows(2).any(|window| window[0] >= window[1]) {
897 return Err(TensorTransactionError::new(
898 "native activation sequence ids are not canonical",
899 ));
900 }
901 changed_rows.push(ActivationInvocationRowV1 {
902 batch_index: row.batch_index,
903 operations,
904 sequence_ids,
905 position,
906 disposition: active.disposition,
907 });
908 }
909 }
910 let output = activation_f32_row_identity(values)
911 .map_err(|error| TensorTransactionError::new(error.to_string()))?;
912 if changed_rows.is_empty() || input == output {
913 return Ok(TensorWriteback::Unchanged);
914 }
915 let elements = u32::try_from(transaction.shape.elements)
916 .map_err(|_| TensorTransactionError::new("tensor element count exceeds u32"))?;
917 let invocation = ActivationInvocationReceiptV1 {
918 site: binding.site.clone(),
919 phase: active.phase,
920 rows: changed_rows,
921 elements,
922 input,
923 output,
924 };
925 let identity = invocation
926 .digest()
927 .map_err(|error| TensorTransactionError::new(error.to_string()))?;
928 let invocation_count = state
929 .invocation_identities
930 .len()
931 .checked_add(state.provisional_invocations.len())
932 .ok_or_else(|| TensorTransactionError::new("activation invocation count overflowed"))?;
933 if invocation_count >= MAX_ACTIVATION_INVOCATION_RECEIPTS {
934 return Err(TensorTransactionError::new(
935 "activation invocation count exceeded its supported bound",
936 ));
937 }
938 if active.disposition == ActivationTelemetryDispositionV1::Provisional {
939 state
940 .provisional_invocations
941 .push(ProvisionalInvocation { invocation });
942 return Ok(TensorWriteback::Commit);
943 }
944 state.elements_copied = state
945 .elements_copied
946 .checked_add(u64::from(elements))
947 .ok_or_else(|| {
948 TensorTransactionError::new("activation element accounting overflowed")
949 })?;
950 state.write_backs = state.write_backs.checked_add(1).ok_or_else(|| {
951 TensorTransactionError::new("activation write-back accounting overflowed")
952 })?;
953 state.invocation_identities.push(identity);
954 state.invocations.push(invocation);
955 Ok(TensorWriteback::Commit)
956 }
957}
958
959fn validate_profile_site(
960 profile: &LlamaCppTensorProfile,
961 site: &TextTensorSiteV1,
962) -> Result<(), Error> {
963 if profile.supports(site) {
964 Ok(())
965 } else {
966 Err(Error::Incompatible(
967 "tensor site is absent from the exact llama.cpp compatibility profile".to_owned(),
968 ))
969 }
970}
971
972fn native_tensor_name(site: &TextTensorSiteV1) -> String {
973 match site {
974 TextTensorSiteV1::LayerOutput { layer } => format!("l_out-{layer}"),
975 TextTensorSiteV1::RouterLogits { layer, .. } => {
976 format!("ffn_moe_logits-{layer}")
977 }
978 TextTensorSiteV1::RouterProbabilities { layer, .. } => {
979 format!("ffn_moe_probs-{layer}")
980 }
981 TextTensorSiteV1::SelectedExperts { layer, .. } => {
982 format!("ffn_moe_topk-{layer}")
983 }
984 TextTensorSiteV1::Named { name, .. } => name.clone(),
985 }
986}
987
988fn capture_position_selected(
989 plan: &ActivationCapturePlanV1,
990 position: u64,
991 decode: ActiveDecode,
992 native_rows: usize,
993) -> bool {
994 match &plan.positions {
995 ActivationCapturePositionsV1::LastPrefillToken => {
996 decode.phase == ActivationPhaseV1::Prefill && decode.last_prefill && native_rows == 1
997 }
998 ActivationCapturePositionsV1::InclusiveRanges(_) => {
999 plan.positions.contains_explicit(position)
1000 }
1001 }
1002}
1003
1004fn retained_bytes(data: &ActivationCapturedDataV1) -> u64 {
1005 match data {
1006 ActivationCapturedDataV1::Digest { .. } => 0,
1007 ActivationCapturedDataV1::Statistics { .. } => 4 * 4,
1008 ActivationCapturedDataV1::F32Snapshot { values, .. } => u64::try_from(values.len())
1009 .unwrap_or(u64::MAX)
1010 .saturating_mul(4),
1011 ActivationCapturedDataV1::I32Snapshot { values, .. } => u64::try_from(values.len())
1012 .unwrap_or(u64::MAX)
1013 .saturating_mul(4),
1014 }
1015}
1016
1017fn lock_state(
1018 state: &Arc<Mutex<ActivationState>>,
1019) -> Result<MutexGuard<'_, ActivationState>, Error> {
1020 state
1021 .lock()
1022 .map_err(|_| Error::Poisoned("activation callback state was poisoned".to_owned()))
1023}
1024
1025fn apply_operator(
1026 values: &mut [f32],
1027 vector: &[f32],
1028 operator: ActivationOperatorV1,
1029 scale: f32,
1030) -> Result<(), TensorTransactionError> {
1031 if values.len() != vector.len() {
1032 return Err(TensorTransactionError::new(
1033 "activation vector width differs from tensor row",
1034 ));
1035 }
1036 match operator {
1037 ActivationOperatorV1::ScaledAdd => {
1038 for (value, direction) in values.iter_mut().zip(vector) {
1039 *value = finite_f32(f64::from(*value) + f64::from(scale) * f64::from(*direction))?;
1040 }
1041 }
1042 ActivationOperatorV1::ScaledProjectionRemoval => {
1043 let dot =
1044 values
1045 .iter()
1046 .zip(vector)
1047 .try_fold(0.0_f64, |total, (value, direction)| {
1048 let next = total + f64::from(*value) * f64::from(*direction);
1049 next.is_finite().then_some(next).ok_or_else(|| {
1050 TensorTransactionError::new(
1051 "activation projection dot product overflowed",
1052 )
1053 })
1054 })?;
1055 let factor = f64::from(scale) * dot;
1056 if !factor.is_finite() {
1057 return Err(TensorTransactionError::new(
1058 "activation projection factor overflowed",
1059 ));
1060 }
1061 for (value, direction) in values.iter_mut().zip(vector) {
1062 *value = finite_f32(f64::from(*value) - factor * f64::from(*direction))?;
1063 }
1064 }
1065 }
1066 Ok(())
1067}
1068
1069#[allow(
1070 clippy::cast_possible_truncation,
1071 reason = "transaction results intentionally commit to the native f32 tensor contract"
1072)]
1073fn finite_f32(value: f64) -> Result<f32, TensorTransactionError> {
1074 let value = value as f32;
1075 value
1076 .is_finite()
1077 .then_some(value)
1078 .ok_or_else(|| TensorTransactionError::new("activation operator produced non-finite f32"))
1079}
1080
1081#[cfg(test)]
1082mod tests {
1083 use super::*;
1084 use logit_loom::{
1085 ActivationCaptureRetentionV1, TextModelTopologyV1, TextSpeculativeMechanismV1,
1086 };
1087
1088 #[test]
1089 fn scaled_add_is_ordered_and_finite() {
1090 let mut values = [1.0_f32, 2.0];
1091 apply_operator(
1092 &mut values,
1093 &[2.0, -1.0],
1094 ActivationOperatorV1::ScaledAdd,
1095 0.5,
1096 )
1097 .unwrap();
1098 assert_eq!(
1099 values.map(f32::to_bits),
1100 [2.0_f32.to_bits(), 1.5_f32.to_bits()]
1101 );
1102 }
1103
1104 #[test]
1105 fn projection_removal_uses_the_declared_scale() {
1106 let mut values = [2.0_f32, 3.0];
1107 apply_operator(
1108 &mut values,
1109 &[1.0, 0.0],
1110 ActivationOperatorV1::ScaledProjectionRemoval,
1111 0.5,
1112 )
1113 .unwrap();
1114 assert_eq!(
1115 values.map(f32::to_bits),
1116 [1.0_f32.to_bits(), 3.0_f32.to_bits()]
1117 );
1118 }
1119
1120 #[test]
1121 fn operator_failure_does_not_accept_nonfinite_output() {
1122 let mut values = [f32::MAX];
1123 assert!(
1124 apply_operator(
1125 &mut values,
1126 &[f32::MAX],
1127 ActivationOperatorV1::ScaledAdd,
1128 f32::MAX,
1129 )
1130 .is_err()
1131 );
1132 }
1133
1134 #[test]
1135 fn provisional_capture_rows_resolve_against_the_final_causal_prefix() {
1136 let topology = TextModelTopologyV1 {
1137 model: Digest::of_bytes("test-model", b"one"),
1138 backend: Digest::of_bytes("test-backend", b"one"),
1139 architecture_implementation: Digest::of_bytes("test-architecture", b"one"),
1140 layers: 2,
1141 embedding_width: 2,
1142 experts: None,
1143 experts_used: None,
1144 nextn_layers: 1,
1145 supported_speculation: vec![
1146 TextSpeculativeMechanismV1::Mtp,
1147 TextSpeculativeMechanismV1::Eagle3,
1148 ],
1149 };
1150 let site = TextTensorSiteV1::LayerOutput { layer: 0 };
1151 let plan = ActivationCapturePlanV1 {
1152 topology: topology.digest().unwrap(),
1153 sites: vec![site.clone()],
1154 positions: ActivationCapturePositionsV1::InclusiveRanges(vec![
1155 logit_loom::CausalPositionRangeV1 { start: 4, end: 5 },
1156 ]),
1157 retention: ActivationCaptureRetentionV1::Digest,
1158 maximum_elements: 4,
1159 maximum_retained_bytes: 0,
1160 };
1161 let plan_identity = plan.digest_for(&topology).unwrap();
1162 let program = ActivationProgramV1 {
1163 topology: topology.digest().unwrap(),
1164 operations: Vec::new(),
1165 observations: vec![plan_identity.clone()],
1166 };
1167 let program_identity = program.digest_for(&topology).unwrap();
1168 let captures = Arc::new(vec![(plan_identity.clone(), plan.clone())]);
1169 let pending = [4_u64, 5]
1170 .into_iter()
1171 .map(|position| {
1172 let record = ActivationCaptureRecordV1 {
1173 plan: plan_identity.clone(),
1174 site: site.clone(),
1175 position,
1176 disposition: ActivationTelemetryDispositionV1::Provisional,
1177 retained: ActivationCapturedDataV1::from_f32(
1178 &[1.0, 2.0],
1179 ActivationCaptureRetentionV1::Digest,
1180 )
1181 .unwrap(),
1182 };
1183 let identity = record.digest_for(&plan, &topology).unwrap();
1184 ProvisionalCapture { record, identity }
1185 })
1186 .collect::<Vec<_>>();
1187 let mut provisional_aggregates = BTreeMap::new();
1188 provisional_aggregates.insert(
1189 plan_identity,
1190 CaptureAggregate {
1191 records: pending
1192 .iter()
1193 .map(|capture| capture.identity.clone())
1194 .collect(),
1195 elements: 4,
1196 retained_bytes: 0,
1197 },
1198 );
1199 let mut controller = ActivationController {
1200 topology,
1201 program: Arc::new(program),
1202 program_identity,
1203 captures,
1204 site_bindings: Arc::new(BTreeMap::new()),
1205 state: Arc::new(Mutex::new(ActivationState::default())),
1206 captured_records: Vec::new(),
1207 capture_aggregates: BTreeMap::new(),
1208 provisional_captures: pending,
1209 provisional_capture_aggregates: provisional_aggregates,
1210 };
1211
1212 let resolutions = controller.resolve_provisional(5).unwrap();
1213 assert_eq!(resolutions.len(), 2);
1214 let output = controller.take_capture_output().unwrap();
1215 assert_eq!(
1216 output
1217 .records
1218 .iter()
1219 .map(|record| record.disposition)
1220 .collect::<Vec<_>>(),
1221 vec![
1222 ActivationTelemetryDispositionV1::Admitted,
1223 ActivationTelemetryDispositionV1::Rejected,
1224 ]
1225 );
1226 assert_eq!(output.receipts[0].records.len(), 2);
1227 }
1228}