Skip to main content

logit_loom_llamacpp/
activation.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! Topology-bound activation capture and transactional tensor programs.
4
5use std::collections::{BTreeMap, BTreeSet};
6use std::sync::{Arc, Mutex, MutexGuard};
7
8use llama_cpp_4::context::tensor_transaction::MAX_TENSOR_ROWS;
9use llama_cpp_4::context::{
10    CapturedTensorData, TensorAccess, TensorDataMut, TensorElementType, TensorRowMapping,
11    TensorSelector, TensorTransaction, TensorTransactionError, TensorTransactionHandler,
12    TensorTransactions, TensorWriteback, TransactionalTensorCapture,
13};
14use logit_loom::{
15    ActivationCapturePlanV1, ActivationCapturePositionsV1, ActivationCaptureReceiptV1,
16    ActivationCaptureRecordV1, ActivationCapturedDataV1, ActivationInvocationReceiptV1,
17    ActivationInvocationRowV1, ActivationOperatorV1, ActivationPhaseV1, ActivationProgramReceiptV1,
18    ActivationProgramV1, ActivationTelemetryDispositionV1, ActivationVectorBankV1,
19    ActivationVectorNormalizationV1, Digest, MAX_ACTIVATION_INVOCATION_RECEIPTS,
20    SpeculationTelemetryResolutionV1, TextTensorElementTypeV1, TextTensorSiteV1,
21    activation_f32_row_identity,
22};
23
24use crate::{Error, Model};
25
26/// Explicit llama.cpp graph-site compatibility profile.
27///
28/// Residual layer outputs and exact named sites require no additional layer
29/// declaration. `MoE` router and selected-expert sites are accepted only for
30/// the caller-declared layers in this profile; the profile identity remains
31/// bound to the model's exact selector implementation.
32#[derive(Clone, Debug, PartialEq, Eq)]
33pub struct LlamaCppTensorProfile {
34    selector_implementation: Digest,
35    moe_layers: Vec<u32>,
36    identity: Digest,
37}
38
39impl LlamaCppTensorProfile {
40    /// Constructs a profile with no architecture-specific `MoE` sites.
41    ///
42    /// Layer-output and exact named-site mechanics remain available.
43    ///
44    /// # Errors
45    ///
46    /// Returns an identity-encoding error.
47    pub fn layer_outputs(model: &Model) -> Result<Self, Error> {
48        Self::new(model, Vec::new())
49    }
50
51    /// Constructs a profile with canonically ordered, explicitly qualified
52    /// `MoE` layers.
53    ///
54    /// # Errors
55    ///
56    /// Returns an error for duplicate, unordered, or out-of-range layers, or
57    /// identity encoding failure.
58    pub fn new(model: &Model, moe_layers: Vec<u32>) -> Result<Self, Error> {
59        if moe_layers.windows(2).any(|window| window[0] >= window[1])
60            || moe_layers
61                .iter()
62                .any(|layer| *layer >= model.topology().layers)
63            || (!moe_layers.is_empty() && model.topology().experts.is_none())
64        {
65            return Err(Error::Invalid(
66                "MoE profile layers must be unique, ordered, in range, and expert-backed"
67                    .to_owned(),
68            ));
69        }
70        let selector_implementation = model.tensor_selector_implementation().clone();
71        let identity = Digest::of_serializable(
72            "llamacpp-tensor-profile-v1",
73            &(
74                model.topology().digest()?,
75                &selector_implementation,
76                &moe_layers,
77            ),
78        )?;
79        Ok(Self {
80            selector_implementation,
81            moe_layers,
82            identity,
83        })
84    }
85
86    /// Returns the exact compatibility-profile identity.
87    pub const fn identity(&self) -> &Digest {
88        &self.identity
89    }
90
91    fn supports(&self, site: &TextTensorSiteV1) -> bool {
92        match site {
93            TextTensorSiteV1::LayerOutput { .. } => true,
94            TextTensorSiteV1::RouterLogits { layer, selector }
95            | TextTensorSiteV1::RouterProbabilities { layer, selector }
96            | TextTensorSiteV1::SelectedExperts { layer, selector } => {
97                selector == &self.selector_implementation
98                    && self.moe_layers.binary_search(layer).is_ok()
99            }
100            TextTensorSiteV1::Named { selector, .. } => selector == &self.selector_implementation,
101        }
102    }
103}
104
105/// Validated activation program, vector banks, captures, and graph profile.
106#[derive(Clone, Debug, PartialEq)]
107pub struct ActivationConfiguration {
108    topology: logit_loom::TextModelTopologyV1,
109    program: ActivationProgramV1,
110    program_identity: Digest,
111    captures: Vec<(Digest, ActivationCapturePlanV1)>,
112    banks: BTreeMap<Digest, ActivationVectorBankV1>,
113    profile: LlamaCppTensorProfile,
114}
115
116impl ActivationConfiguration {
117    /// Validates one complete activation runtime before context allocation.
118    ///
119    /// Every vector reference must resolve exactly, every capture identity must
120    /// match the program's canonical observation list, and every selected
121    /// graph site must be present in the supplied compatibility profile.
122    ///
123    /// # Errors
124    ///
125    /// Returns an error for any topology, profile, vector, operation, capture,
126    /// ordering, or identity mismatch.
127    pub fn new(
128        model: &Model,
129        profile: LlamaCppTensorProfile,
130        program: ActivationProgramV1,
131        capture_plans: Vec<ActivationCapturePlanV1>,
132        vector_banks: Vec<ActivationVectorBankV1>,
133    ) -> Result<Self, Error> {
134        if profile.selector_implementation != *model.tensor_selector_implementation() {
135            return Err(Error::Incompatible(
136                "tensor profile belongs to another selector implementation".to_owned(),
137            ));
138        }
139        let topology = model.topology().clone();
140        let program_identity = program.digest_for(&topology)?;
141
142        let mut captures = Vec::with_capacity(capture_plans.len());
143        for plan in capture_plans {
144            let identity = plan.digest_for(&topology)?;
145            if let Some((prior, _)) = captures.last()
146                && prior >= &identity
147            {
148                return Err(Error::Invalid(
149                    "capture plans must have unique canonical identity order".to_owned(),
150                ));
151            }
152            captures.push((identity, plan));
153        }
154        let capture_identities = captures
155            .iter()
156            .map(|(identity, _)| identity.clone())
157            .collect::<Vec<_>>();
158        if capture_identities != program.observations {
159            return Err(Error::Incompatible(
160                "activation capture plans differ from program observations".to_owned(),
161            ));
162        }
163
164        let mut banks = BTreeMap::new();
165        for bank in vector_banks {
166            let identity = bank.digest_for(&topology)?;
167            if banks.insert(identity, bank).is_some() {
168                return Err(Error::Invalid(
169                    "activation vector-bank identities must be unique".to_owned(),
170                ));
171            }
172        }
173        let referenced = program
174            .operations
175            .iter()
176            .map(|operation| operation.vector_bank.clone())
177            .collect::<BTreeSet<_>>();
178        if referenced.len() != banks.len()
179            || referenced
180                .iter()
181                .any(|identity| !banks.contains_key(identity))
182        {
183            return Err(Error::Incompatible(
184                "activation vector banks must exactly resolve program references".to_owned(),
185            ));
186        }
187        for operation in &program.operations {
188            let bank = banks.get(&operation.vector_bank).ok_or_else(|| {
189                Error::Incompatible("activation vector bank is unavailable".to_owned())
190            })?;
191            if operation.operator == ActivationOperatorV1::ScaledProjectionRemoval
192                && bank.normalization != ActivationVectorNormalizationV1::UnitL2
193            {
194                return Err(Error::Invalid(
195                    "projection removal requires a unit-L2 vector bank".to_owned(),
196                ));
197            }
198            for site in &operation.sites {
199                validate_profile_site(&profile, site)?;
200                if bank.site_family != site.family()
201                    || bank.row_elements != site.row_elements(&topology)?
202                    || site.layer().and_then(|layer| bank.row(layer)).is_none()
203                {
204                    return Err(Error::Incompatible(
205                        "activation vector bank does not cover an operation site".to_owned(),
206                    ));
207                }
208            }
209        }
210        for (_, plan) in &captures {
211            for site in &plan.sites {
212                validate_profile_site(&profile, site)?;
213                if plan.retention == logit_loom::ActivationCaptureRetentionV1::Statistics
214                    && site.element_type() != TextTensorElementTypeV1::F32
215                {
216                    return Err(Error::Invalid(
217                        "statistics capture requires f32 tensor sites".to_owned(),
218                    ));
219                }
220            }
221        }
222        Ok(Self {
223            topology,
224            program,
225            program_identity,
226            captures,
227            banks,
228            profile,
229        })
230    }
231
232    /// Returns the exact activation-program identity.
233    pub const fn program_identity(&self) -> &Digest {
234        &self.program_identity
235    }
236
237    /// Returns the exact graph compatibility-profile identity.
238    pub const fn profile_identity(&self) -> &Digest {
239        self.profile.identity()
240    }
241
242    pub(crate) fn lower(
243        self,
244        maximum_rows: u32,
245    ) -> Result<(ActivationController, TensorTransactions), Error> {
246        let maximum_rows = usize::try_from(maximum_rows)
247            .map_err(|_| Error::Invalid("activation batch rows exceed usize".to_owned()))?;
248        if maximum_rows == 0 || maximum_rows > MAX_TENSOR_ROWS {
249            return Err(Error::Invalid(format!(
250                "activation batch rows must be between 1 and {MAX_TENSOR_ROWS}"
251            )));
252        }
253
254        let program = Arc::new(self.program);
255        let banks = Arc::new(self.banks);
256        let captures = Arc::new(self.captures);
257        let mut selected_sites = BTreeSet::new();
258        for operation in &program.operations {
259            selected_sites.extend(operation.sites.iter().cloned());
260        }
261        for (_, plan) in captures.iter() {
262            selected_sites.extend(plan.sites.iter().cloned());
263        }
264
265        let mutable_sites = program
266            .operations
267            .iter()
268            .flat_map(|operation| operation.sites.iter().cloned())
269            .collect::<BTreeSet<_>>();
270        let retained_sites = captures
271            .iter()
272            .flat_map(|(_, plan)| plan.sites.iter().cloned())
273            .collect::<BTreeSet<_>>();
274
275        let mut site_bindings = BTreeMap::new();
276        for site in selected_sites {
277            validate_profile_site(&self.profile, &site)?;
278            let name = native_tensor_name(&site);
279            let binding = SiteBinding {
280                site: site.clone(),
281                name: name.clone(),
282            };
283            if let Some(prior) = site_bindings.insert(name.clone(), binding)
284                && prior.site != site
285            {
286                return Err(Error::Incompatible(format!(
287                    "multiple tensor sites resolve to native graph node {name}"
288                )));
289            }
290        }
291        let site_bindings = Arc::new(site_bindings);
292        let mut selectors = Vec::with_capacity(site_bindings.len());
293        for binding in site_bindings.values() {
294            let site = &binding.site;
295            let element_type = match site.element_type() {
296                TextTensorElementTypeV1::F32 => TensorElementType::F32,
297                TextTensorElementTypeV1::I32 => TensorElementType::I32,
298            };
299            let access = if mutable_sites.contains(site) {
300                TensorAccess::ReadWriteF32
301            } else {
302                TensorAccess::ReadOnly
303            };
304            selectors.push(
305                TensorSelector::new(
306                    binding.name.clone(),
307                    element_type,
308                    usize::try_from(site.row_elements(&self.topology)?)
309                        .map_err(|_| Error::Invalid("tensor row width exceeds usize".to_owned()))?,
310                    maximum_rows,
311                    access,
312                    TensorRowMapping::BatchTokens,
313                    retained_sites.contains(site),
314                )
315                .map_err(|error| Error::Invalid(error.to_string()))?,
316            );
317        }
318
319        let state = Arc::new(Mutex::new(ActivationState::default()));
320        let controller = ActivationController {
321            topology: self.topology,
322            program: Arc::clone(&program),
323            program_identity: self.program_identity,
324            captures: Arc::clone(&captures),
325            site_bindings: Arc::clone(&site_bindings),
326            state: Arc::clone(&state),
327            captured_records: Vec::new(),
328            capture_aggregates: BTreeMap::new(),
329            provisional_captures: Vec::new(),
330            provisional_capture_aggregates: BTreeMap::new(),
331        };
332        let transactions = if mutable_sites.is_empty() {
333            TensorTransactions::capture(selectors)
334        } else {
335            TensorTransactions::new(
336                selectors,
337                ActivationHandler {
338                    program,
339                    banks,
340                    sites: site_bindings,
341                    state,
342                },
343            )
344        }
345        .map_err(|error| Error::Invalid(error.to_string()))?;
346        Ok((controller, transactions))
347    }
348}
349
350/// Captured records and per-plan aggregate receipts drained from a session.
351#[derive(Clone, Debug, PartialEq)]
352pub struct ActivationCaptureOutput {
353    /// Captured records in native execution order, then capture-plan order.
354    pub records: Vec<ActivationCaptureRecordV1>,
355    /// Aggregate receipts in canonical capture-plan identity order.
356    pub receipts: Vec<ActivationCaptureReceiptV1>,
357}
358
359/// Transaction records and aggregate program receipt drained from a session.
360#[derive(Clone, Debug, PartialEq, Eq)]
361pub struct ActivationProgramOutput {
362    /// Successful native write-back records in execution order.
363    pub invocations: Vec<ActivationInvocationReceiptV1>,
364    /// Aggregate activation-program receipt.
365    pub receipt: ActivationProgramReceiptV1,
366}
367
368#[derive(Clone, Debug)]
369struct SiteBinding {
370    site: TextTensorSiteV1,
371    name: String,
372}
373
374#[derive(Clone, Copy, Debug)]
375pub(crate) struct ActiveDecode {
376    phase: ActivationPhaseV1,
377    last_prefill: bool,
378    disposition: ActivationTelemetryDispositionV1,
379}
380
381#[derive(Debug, Default)]
382struct ActivationState {
383    active: Option<ActiveDecode>,
384    invocations: Vec<ActivationInvocationReceiptV1>,
385    invocation_identities: Vec<Digest>,
386    provisional_invocations: Vec<ProvisionalInvocation>,
387    elements_copied: u64,
388    write_backs: u64,
389}
390
391#[derive(Debug, Default)]
392struct CaptureAggregate {
393    records: Vec<Digest>,
394    elements: u64,
395    retained_bytes: u64,
396}
397
398#[derive(Clone, Debug)]
399struct ProvisionalCapture {
400    record: ActivationCaptureRecordV1,
401    identity: Digest,
402}
403
404#[derive(Clone, Debug)]
405struct ProvisionalInvocation {
406    invocation: ActivationInvocationReceiptV1,
407}
408
409#[derive(Debug)]
410pub(crate) struct ActivationController {
411    topology: logit_loom::TextModelTopologyV1,
412    program: Arc<ActivationProgramV1>,
413    program_identity: Digest,
414    captures: Arc<Vec<(Digest, ActivationCapturePlanV1)>>,
415    site_bindings: Arc<BTreeMap<String, SiteBinding>>,
416    state: Arc<Mutex<ActivationState>>,
417    captured_records: Vec<ActivationCaptureRecordV1>,
418    capture_aggregates: BTreeMap<Digest, CaptureAggregate>,
419    provisional_captures: Vec<ProvisionalCapture>,
420    provisional_capture_aggregates: BTreeMap<Digest, CaptureAggregate>,
421}
422
423impl ActivationController {
424    pub(crate) fn has_last_prefill_capture(&self) -> bool {
425        self.captures
426            .iter()
427            .any(|(_, plan)| plan.positions == ActivationCapturePositionsV1::LastPrefillToken)
428    }
429
430    pub(crate) fn begin_decode(
431        &self,
432        phase: ActivationPhaseV1,
433        last_prefill: bool,
434        disposition: ActivationTelemetryDispositionV1,
435    ) -> Result<(), Error> {
436        if !self.provisional_captures.is_empty() {
437            return Err(Error::Poisoned(
438                "provisional activation captures must be resolved before another decode".to_owned(),
439            ));
440        }
441        let mut state = lock_state(&self.state)?;
442        if state.active.is_some() {
443            return Err(Error::Poisoned(
444                "activation decode phase was already active".to_owned(),
445            ));
446        }
447        if !state.provisional_invocations.is_empty() {
448            return Err(Error::Poisoned(
449                "provisional activation invocations must be resolved before another decode"
450                    .to_owned(),
451            ));
452        }
453        state.active = Some(ActiveDecode {
454            phase,
455            last_prefill,
456            disposition,
457        });
458        Ok(())
459    }
460
461    pub(crate) fn end_decode(&self) -> Result<ActiveDecode, Error> {
462        lock_state(&self.state)?
463            .active
464            .take()
465            .ok_or_else(|| Error::Poisoned("activation decode phase was not active".to_owned()))
466    }
467
468    pub(crate) fn consume_captures(
469        &mut self,
470        native: Vec<TransactionalTensorCapture>,
471        decode: ActiveDecode,
472    ) -> Result<(), Error> {
473        let capture_plans = Arc::clone(&self.captures);
474        for capture in native {
475            let site = self
476                .site_bindings
477                .get(&capture.name)
478                .ok_or_else(|| {
479                    Error::Incompatible(format!(
480                        "native capture {} has no site binding",
481                        capture.name
482                    ))
483                })?
484                .site
485                .clone();
486            if capture.rows.len() != capture.shape.rows
487                || capture.shape.elements
488                    != capture
489                        .shape
490                        .row_elements
491                        .saturating_mul(capture.shape.rows)
492            {
493                return Err(Error::Incompatible(
494                    "native capture row accounting is inconsistent".to_owned(),
495                ));
496            }
497            for (row_index, row) in capture.rows.iter().enumerate() {
498                let position = u64::try_from(row.position).map_err(|_| {
499                    Error::Incompatible("native capture position is negative".to_owned())
500                })?;
501                let start = row_index
502                    .checked_mul(capture.shape.row_elements)
503                    .ok_or_else(|| Error::Invalid("capture row offset overflowed".to_owned()))?;
504                let end = start
505                    .checked_add(capture.shape.row_elements)
506                    .ok_or_else(|| Error::Invalid("capture row end overflowed".to_owned()))?;
507                for (plan_identity, plan) in capture_plans.iter() {
508                    if !plan.sites.contains(&site)
509                        || !capture_position_selected(plan, position, decode, capture.shape.rows)
510                    {
511                        continue;
512                    }
513                    let retained = match &capture.data {
514                        CapturedTensorData::F32(values) => {
515                            ActivationCapturedDataV1::from_f32(&values[start..end], plan.retention)?
516                        }
517                        CapturedTensorData::I32(values) => {
518                            ActivationCapturedDataV1::from_i32(&values[start..end], plan.retention)?
519                        }
520                    };
521                    let record = ActivationCaptureRecordV1 {
522                        plan: plan_identity.clone(),
523                        site: site.clone(),
524                        position,
525                        disposition: decode.disposition,
526                        retained,
527                    };
528                    let record_identity = record.digest_for(plan, &self.topology)?;
529                    if decode.disposition == ActivationTelemetryDispositionV1::Provisional {
530                        self.push_provisional_capture(
531                            plan_identity,
532                            plan,
533                            record,
534                            record_identity,
535                        )?;
536                    } else {
537                        self.commit_capture(plan_identity, plan, record, record_identity)?;
538                    }
539                }
540            }
541        }
542        Ok(())
543    }
544
545    fn push_provisional_capture(
546        &mut self,
547        plan_identity: &Digest,
548        plan: &ActivationCapturePlanV1,
549        record: ActivationCaptureRecordV1,
550        identity: Digest,
551    ) -> Result<(), Error> {
552        let elements = u64::from(record.retained.elements());
553        let bytes = retained_bytes(&record.retained);
554        let committed = self.capture_aggregates.get(plan_identity);
555        let pending = self.provisional_capture_aggregates.get(plan_identity);
556        let pending_elements = pending
557            .map_or(0, |aggregate| aggregate.elements)
558            .checked_add(elements)
559            .ok_or_else(|| Error::Invalid("capture element accounting overflowed".to_owned()))?;
560        let pending_bytes = pending
561            .map_or(0, |aggregate| aggregate.retained_bytes)
562            .checked_add(bytes)
563            .ok_or_else(|| {
564                Error::Invalid("capture retained-byte accounting overflowed".to_owned())
565            })?;
566        let total_elements = committed
567            .map_or(0, |aggregate| aggregate.elements)
568            .checked_add(pending_elements)
569            .ok_or_else(|| Error::Invalid("capture element accounting overflowed".to_owned()))?;
570        let total_bytes = committed
571            .map_or(0, |aggregate| aggregate.retained_bytes)
572            .checked_add(pending_bytes)
573            .ok_or_else(|| {
574                Error::Invalid("capture retained-byte accounting overflowed".to_owned())
575            })?;
576        if total_elements > plan.maximum_elements || total_bytes > plan.maximum_retained_bytes {
577            return Err(Error::Invalid(
578                "activation capture exceeded its declared aggregate bound".to_owned(),
579            ));
580        }
581        let aggregate = self
582            .provisional_capture_aggregates
583            .entry(plan_identity.clone())
584            .or_default();
585        aggregate.elements = pending_elements;
586        aggregate.retained_bytes = pending_bytes;
587        aggregate.records.push(identity.clone());
588        self.provisional_captures
589            .push(ProvisionalCapture { record, identity });
590        Ok(())
591    }
592
593    fn commit_capture(
594        &mut self,
595        plan_identity: &Digest,
596        plan: &ActivationCapturePlanV1,
597        record: ActivationCaptureRecordV1,
598        identity: Digest,
599    ) -> Result<(), Error> {
600        let aggregate = self
601            .capture_aggregates
602            .entry(plan_identity.clone())
603            .or_default();
604        let elements = u64::from(record.retained.elements());
605        let bytes = retained_bytes(&record.retained);
606        let next_elements = aggregate
607            .elements
608            .checked_add(elements)
609            .ok_or_else(|| Error::Invalid("capture element accounting overflowed".to_owned()))?;
610        let next_bytes = aggregate.retained_bytes.checked_add(bytes).ok_or_else(|| {
611            Error::Invalid("capture retained-byte accounting overflowed".to_owned())
612        })?;
613        if next_elements > plan.maximum_elements || next_bytes > plan.maximum_retained_bytes {
614            return Err(Error::Invalid(
615                "activation capture exceeded its declared aggregate bound".to_owned(),
616            ));
617        }
618        aggregate.elements = next_elements;
619        aggregate.retained_bytes = next_bytes;
620        aggregate.records.push(identity);
621        self.captured_records.push(record);
622        Ok(())
623    }
624
625    pub(crate) fn resolve_provisional(
626        &mut self,
627        admitted_position_exclusive: u64,
628    ) -> Result<Vec<SpeculationTelemetryResolutionV1>, Error> {
629        let capture_plans = Arc::clone(&self.captures);
630        let mut resolved_captures = Vec::with_capacity(self.provisional_captures.len());
631        for pending in &self.provisional_captures {
632            let (_, plan) = capture_plans
633                .iter()
634                .find(|(identity, _)| identity == &pending.record.plan)
635                .ok_or_else(|| Error::Incompatible("provisional capture has no plan".to_owned()))?;
636            let mut record = pending.record.clone();
637            record.disposition = if record.position < admitted_position_exclusive {
638                ActivationTelemetryDispositionV1::Admitted
639            } else {
640                ActivationTelemetryDispositionV1::Rejected
641            };
642            let identity = record.digest_for(plan, &self.topology)?;
643            let disposition = record.disposition;
644            resolved_captures.push((
645                record,
646                identity.clone(),
647                SpeculationTelemetryResolutionV1 {
648                    provisional: pending.identity.clone(),
649                    disposition,
650                    resolved: identity,
651                },
652            ));
653        }
654
655        let mut state = lock_state(&self.state)?;
656        if state.active.is_some() {
657            return Err(Error::Poisoned(
658                "cannot resolve activation telemetry during a decode".to_owned(),
659            ));
660        }
661        let mut resolved_invocations = Vec::with_capacity(state.provisional_invocations.len());
662        for pending in &state.provisional_invocations {
663            let mut invocation = pending.invocation.clone();
664            for row in &mut invocation.rows {
665                row.disposition = if row.position < admitted_position_exclusive {
666                    ActivationTelemetryDispositionV1::Admitted
667                } else {
668                    ActivationTelemetryDispositionV1::Rejected
669                };
670            }
671            let identity = invocation.digest()?;
672            resolved_invocations.push((invocation, identity));
673        }
674        let final_invocation_count = state
675            .invocation_identities
676            .len()
677            .checked_add(resolved_invocations.len())
678            .ok_or_else(|| Error::Invalid("activation invocation count overflowed".to_owned()))?;
679        if final_invocation_count > MAX_ACTIVATION_INVOCATION_RECEIPTS {
680            return Err(Error::Invalid(
681                "activation invocation count exceeded its supported bound".to_owned(),
682            ));
683        }
684        let added_elements =
685            resolved_invocations
686                .iter()
687                .try_fold(0_u64, |total, (invocation, _)| {
688                    total
689                        .checked_add(u64::from(invocation.elements))
690                        .ok_or_else(|| {
691                            Error::Invalid("activation element accounting overflowed".to_owned())
692                        })
693                })?;
694        let final_elements = state
695            .elements_copied
696            .checked_add(added_elements)
697            .ok_or_else(|| Error::Invalid("activation element accounting overflowed".to_owned()))?;
698        let added_write_backs = u64::try_from(resolved_invocations.len())
699            .map_err(|_| Error::Invalid("activation write-back count exceeds u64".to_owned()))?;
700        let final_write_backs = state
701            .write_backs
702            .checked_add(added_write_backs)
703            .ok_or_else(|| {
704                Error::Invalid("activation write-back accounting overflowed".to_owned())
705            })?;
706
707        let mut resolutions = Vec::with_capacity(resolved_captures.len());
708        for (invocation, identity) in resolved_invocations {
709            state.invocations.push(invocation);
710            state.invocation_identities.push(identity);
711        }
712        state.elements_copied = final_elements;
713        state.write_backs = final_write_backs;
714        state.provisional_invocations.clear();
715        drop(state);
716
717        for (record, identity, resolution) in resolved_captures {
718            let (_, plan) = capture_plans
719                .iter()
720                .find(|(plan_identity, _)| plan_identity == &record.plan)
721                .ok_or_else(|| Error::Incompatible("resolved capture has no plan".to_owned()))?;
722            self.commit_capture(&record.plan.clone(), plan, record, identity)?;
723            resolutions.push(resolution);
724        }
725        self.provisional_captures.clear();
726        self.provisional_capture_aggregates.clear();
727        Ok(resolutions)
728    }
729
730    pub(crate) fn take_capture_output(&mut self) -> Result<ActivationCaptureOutput, Error> {
731        if !self.provisional_captures.is_empty() {
732            return Err(Error::Invalid(
733                "cannot drain activation captures before provisional resolution".to_owned(),
734            ));
735        }
736        let mut receipts = Vec::with_capacity(self.capture_aggregates.len());
737        for (plan_identity, aggregate) in std::mem::take(&mut self.capture_aggregates) {
738            let (_, plan) = self
739                .captures
740                .iter()
741                .find(|(identity, _)| identity == &plan_identity)
742                .ok_or_else(|| Error::Incompatible("capture aggregate has no plan".to_owned()))?;
743            let receipt = ActivationCaptureReceiptV1 {
744                plan: plan_identity,
745                records: aggregate.records,
746                elements: aggregate.elements,
747                retained_bytes: aggregate.retained_bytes,
748            };
749            receipt.digest_for(plan)?;
750            receipts.push(receipt);
751        }
752        Ok(ActivationCaptureOutput {
753            records: std::mem::take(&mut self.captured_records),
754            receipts,
755        })
756    }
757
758    pub(crate) fn take_program_output(&self) -> Result<ActivationProgramOutput, Error> {
759        let mut state = lock_state(&self.state)?;
760        if state.active.is_some() {
761            return Err(Error::Invalid(
762                "cannot drain activation receipts during a decode".to_owned(),
763            ));
764        }
765        if !state.provisional_invocations.is_empty() {
766            return Err(Error::Invalid(
767                "cannot drain activation receipts before provisional resolution".to_owned(),
768            ));
769        }
770        let receipt = ActivationProgramReceiptV1 {
771            program: self.program_identity.clone(),
772            topology: self.topology.digest()?,
773            invocations: std::mem::take(&mut state.invocation_identities),
774            elements_copied: std::mem::take(&mut state.elements_copied),
775            write_backs: std::mem::take(&mut state.write_backs),
776        };
777        receipt.digest_for(&self.program)?;
778        Ok(ActivationProgramOutput {
779            invocations: std::mem::take(&mut state.invocations),
780            receipt,
781        })
782    }
783
784    pub(crate) fn reset(&mut self) -> Result<(), Error> {
785        if !self.provisional_captures.is_empty() {
786            return Err(Error::Poisoned(
787                "cannot reset unresolved provisional activation captures".to_owned(),
788            ));
789        }
790        self.captured_records.clear();
791        self.capture_aggregates.clear();
792        self.provisional_capture_aggregates.clear();
793        let mut state = lock_state(&self.state)?;
794        if state.active.is_some() {
795            return Err(Error::Poisoned(
796                "cannot reset activation evidence during a decode".to_owned(),
797            ));
798        }
799        if !state.provisional_invocations.is_empty() {
800            return Err(Error::Poisoned(
801                "cannot reset unresolved provisional activation invocations".to_owned(),
802            ));
803        }
804        state.invocations.clear();
805        state.invocation_identities.clear();
806        state.elements_copied = 0;
807        state.write_backs = 0;
808        Ok(())
809    }
810}
811
812struct ActivationHandler {
813    program: Arc<ActivationProgramV1>,
814    banks: Arc<BTreeMap<Digest, ActivationVectorBankV1>>,
815    sites: Arc<BTreeMap<String, SiteBinding>>,
816    state: Arc<Mutex<ActivationState>>,
817}
818
819impl TensorTransactionHandler for ActivationHandler {
820    #[allow(
821        clippy::too_many_lines,
822        reason = "the callback keeps validation, ordered row transforms, and receipt commit in one transactional scope"
823    )]
824    fn apply(
825        &mut self,
826        mut transaction: TensorTransaction<'_>,
827    ) -> Result<TensorWriteback, TensorTransactionError> {
828        let binding = self
829            .sites
830            .get(transaction.name)
831            .ok_or_else(|| TensorTransactionError::new("selected tensor has no site binding"))?;
832        let TensorDataMut::F32(values) = &mut transaction.data else {
833            return Err(TensorTransactionError::new(
834                "mutable activation transaction is not f32",
835            ));
836        };
837        let mut state = self
838            .state
839            .lock()
840            .map_err(|_| TensorTransactionError::new("activation state mutex was poisoned"))?;
841        let active = state.active.ok_or_else(|| {
842            TensorTransactionError::new("activation callback ran outside an active decode phase")
843        })?;
844        let input = activation_f32_row_identity(values)
845            .map_err(|error| TensorTransactionError::new(error.to_string()))?;
846        let mut changed_rows = Vec::new();
847        for (row_index, row) in transaction.rows.iter().enumerate() {
848            let position = u64::try_from(row.position)
849                .map_err(|_| TensorTransactionError::new("native causal position is negative"))?;
850            let start = row_index
851                .checked_mul(transaction.shape.row_elements)
852                .ok_or_else(|| TensorTransactionError::new("activation row offset overflowed"))?;
853            let end = start
854                .checked_add(transaction.shape.row_elements)
855                .ok_or_else(|| TensorTransactionError::new("activation row end overflowed"))?;
856            let row_values = &mut values[start..end];
857            let mut operations = Vec::new();
858            for (operation_index, operation) in self.program.operations.iter().enumerate() {
859                if operation.phase != active.phase
860                    || !operation.positions.contains(position)
861                    || !operation.sites.contains(&binding.site)
862                {
863                    continue;
864                }
865                let bank = self.banks.get(&operation.vector_bank).ok_or_else(|| {
866                    TensorTransactionError::new("activation vector bank is unavailable")
867                })?;
868                let layer = binding.site.layer().ok_or_else(|| {
869                    TensorTransactionError::new("mutable activation site has no layer")
870                })?;
871                let vector = bank.row(layer).ok_or_else(|| {
872                    TensorTransactionError::new("activation vector row is unavailable")
873                })?;
874                apply_operator(
875                    row_values,
876                    vector,
877                    operation.operator,
878                    operation
879                        .scale()
880                        .map_err(|error| TensorTransactionError::new(error.to_string()))?,
881                )?;
882                operations.push(u32::try_from(operation_index).map_err(|_| {
883                    TensorTransactionError::new("activation operation index exceeds u32")
884                })?);
885            }
886            if !operations.is_empty() {
887                let sequence_ids = row
888                    .sequence_ids
889                    .iter()
890                    .map(|sequence| {
891                        u32::try_from(*sequence).map_err(|_| {
892                            TensorTransactionError::new("native activation sequence id is negative")
893                        })
894                    })
895                    .collect::<Result<Vec<_>, _>>()?;
896                if sequence_ids.windows(2).any(|window| window[0] >= window[1]) {
897                    return Err(TensorTransactionError::new(
898                        "native activation sequence ids are not canonical",
899                    ));
900                }
901                changed_rows.push(ActivationInvocationRowV1 {
902                    batch_index: row.batch_index,
903                    operations,
904                    sequence_ids,
905                    position,
906                    disposition: active.disposition,
907                });
908            }
909        }
910        let output = activation_f32_row_identity(values)
911            .map_err(|error| TensorTransactionError::new(error.to_string()))?;
912        if changed_rows.is_empty() || input == output {
913            return Ok(TensorWriteback::Unchanged);
914        }
915        let elements = u32::try_from(transaction.shape.elements)
916            .map_err(|_| TensorTransactionError::new("tensor element count exceeds u32"))?;
917        let invocation = ActivationInvocationReceiptV1 {
918            site: binding.site.clone(),
919            phase: active.phase,
920            rows: changed_rows,
921            elements,
922            input,
923            output,
924        };
925        let identity = invocation
926            .digest()
927            .map_err(|error| TensorTransactionError::new(error.to_string()))?;
928        let invocation_count = state
929            .invocation_identities
930            .len()
931            .checked_add(state.provisional_invocations.len())
932            .ok_or_else(|| TensorTransactionError::new("activation invocation count overflowed"))?;
933        if invocation_count >= MAX_ACTIVATION_INVOCATION_RECEIPTS {
934            return Err(TensorTransactionError::new(
935                "activation invocation count exceeded its supported bound",
936            ));
937        }
938        if active.disposition == ActivationTelemetryDispositionV1::Provisional {
939            state
940                .provisional_invocations
941                .push(ProvisionalInvocation { invocation });
942            return Ok(TensorWriteback::Commit);
943        }
944        state.elements_copied = state
945            .elements_copied
946            .checked_add(u64::from(elements))
947            .ok_or_else(|| {
948                TensorTransactionError::new("activation element accounting overflowed")
949            })?;
950        state.write_backs = state.write_backs.checked_add(1).ok_or_else(|| {
951            TensorTransactionError::new("activation write-back accounting overflowed")
952        })?;
953        state.invocation_identities.push(identity);
954        state.invocations.push(invocation);
955        Ok(TensorWriteback::Commit)
956    }
957}
958
959fn validate_profile_site(
960    profile: &LlamaCppTensorProfile,
961    site: &TextTensorSiteV1,
962) -> Result<(), Error> {
963    if profile.supports(site) {
964        Ok(())
965    } else {
966        Err(Error::Incompatible(
967            "tensor site is absent from the exact llama.cpp compatibility profile".to_owned(),
968        ))
969    }
970}
971
972fn native_tensor_name(site: &TextTensorSiteV1) -> String {
973    match site {
974        TextTensorSiteV1::LayerOutput { layer } => format!("l_out-{layer}"),
975        TextTensorSiteV1::RouterLogits { layer, .. } => {
976            format!("ffn_moe_logits-{layer}")
977        }
978        TextTensorSiteV1::RouterProbabilities { layer, .. } => {
979            format!("ffn_moe_probs-{layer}")
980        }
981        TextTensorSiteV1::SelectedExperts { layer, .. } => {
982            format!("ffn_moe_topk-{layer}")
983        }
984        TextTensorSiteV1::Named { name, .. } => name.clone(),
985    }
986}
987
988fn capture_position_selected(
989    plan: &ActivationCapturePlanV1,
990    position: u64,
991    decode: ActiveDecode,
992    native_rows: usize,
993) -> bool {
994    match &plan.positions {
995        ActivationCapturePositionsV1::LastPrefillToken => {
996            decode.phase == ActivationPhaseV1::Prefill && decode.last_prefill && native_rows == 1
997        }
998        ActivationCapturePositionsV1::InclusiveRanges(_) => {
999            plan.positions.contains_explicit(position)
1000        }
1001    }
1002}
1003
1004fn retained_bytes(data: &ActivationCapturedDataV1) -> u64 {
1005    match data {
1006        ActivationCapturedDataV1::Digest { .. } => 0,
1007        ActivationCapturedDataV1::Statistics { .. } => 4 * 4,
1008        ActivationCapturedDataV1::F32Snapshot { values, .. } => u64::try_from(values.len())
1009            .unwrap_or(u64::MAX)
1010            .saturating_mul(4),
1011        ActivationCapturedDataV1::I32Snapshot { values, .. } => u64::try_from(values.len())
1012            .unwrap_or(u64::MAX)
1013            .saturating_mul(4),
1014    }
1015}
1016
1017fn lock_state(
1018    state: &Arc<Mutex<ActivationState>>,
1019) -> Result<MutexGuard<'_, ActivationState>, Error> {
1020    state
1021        .lock()
1022        .map_err(|_| Error::Poisoned("activation callback state was poisoned".to_owned()))
1023}
1024
1025fn apply_operator(
1026    values: &mut [f32],
1027    vector: &[f32],
1028    operator: ActivationOperatorV1,
1029    scale: f32,
1030) -> Result<(), TensorTransactionError> {
1031    if values.len() != vector.len() {
1032        return Err(TensorTransactionError::new(
1033            "activation vector width differs from tensor row",
1034        ));
1035    }
1036    match operator {
1037        ActivationOperatorV1::ScaledAdd => {
1038            for (value, direction) in values.iter_mut().zip(vector) {
1039                *value = finite_f32(f64::from(*value) + f64::from(scale) * f64::from(*direction))?;
1040            }
1041        }
1042        ActivationOperatorV1::ScaledProjectionRemoval => {
1043            let dot =
1044                values
1045                    .iter()
1046                    .zip(vector)
1047                    .try_fold(0.0_f64, |total, (value, direction)| {
1048                        let next = total + f64::from(*value) * f64::from(*direction);
1049                        next.is_finite().then_some(next).ok_or_else(|| {
1050                            TensorTransactionError::new(
1051                                "activation projection dot product overflowed",
1052                            )
1053                        })
1054                    })?;
1055            let factor = f64::from(scale) * dot;
1056            if !factor.is_finite() {
1057                return Err(TensorTransactionError::new(
1058                    "activation projection factor overflowed",
1059                ));
1060            }
1061            for (value, direction) in values.iter_mut().zip(vector) {
1062                *value = finite_f32(f64::from(*value) - factor * f64::from(*direction))?;
1063            }
1064        }
1065    }
1066    Ok(())
1067}
1068
1069#[allow(
1070    clippy::cast_possible_truncation,
1071    reason = "transaction results intentionally commit to the native f32 tensor contract"
1072)]
1073fn finite_f32(value: f64) -> Result<f32, TensorTransactionError> {
1074    let value = value as f32;
1075    value
1076        .is_finite()
1077        .then_some(value)
1078        .ok_or_else(|| TensorTransactionError::new("activation operator produced non-finite f32"))
1079}
1080
1081#[cfg(test)]
1082mod tests {
1083    use super::*;
1084    use logit_loom::{
1085        ActivationCaptureRetentionV1, TextModelTopologyV1, TextSpeculativeMechanismV1,
1086    };
1087
1088    #[test]
1089    fn scaled_add_is_ordered_and_finite() {
1090        let mut values = [1.0_f32, 2.0];
1091        apply_operator(
1092            &mut values,
1093            &[2.0, -1.0],
1094            ActivationOperatorV1::ScaledAdd,
1095            0.5,
1096        )
1097        .unwrap();
1098        assert_eq!(
1099            values.map(f32::to_bits),
1100            [2.0_f32.to_bits(), 1.5_f32.to_bits()]
1101        );
1102    }
1103
1104    #[test]
1105    fn projection_removal_uses_the_declared_scale() {
1106        let mut values = [2.0_f32, 3.0];
1107        apply_operator(
1108            &mut values,
1109            &[1.0, 0.0],
1110            ActivationOperatorV1::ScaledProjectionRemoval,
1111            0.5,
1112        )
1113        .unwrap();
1114        assert_eq!(
1115            values.map(f32::to_bits),
1116            [1.0_f32.to_bits(), 3.0_f32.to_bits()]
1117        );
1118    }
1119
1120    #[test]
1121    fn operator_failure_does_not_accept_nonfinite_output() {
1122        let mut values = [f32::MAX];
1123        assert!(
1124            apply_operator(
1125                &mut values,
1126                &[f32::MAX],
1127                ActivationOperatorV1::ScaledAdd,
1128                f32::MAX,
1129            )
1130            .is_err()
1131        );
1132    }
1133
1134    #[test]
1135    fn provisional_capture_rows_resolve_against_the_final_causal_prefix() {
1136        let topology = TextModelTopologyV1 {
1137            model: Digest::of_bytes("test-model", b"one"),
1138            backend: Digest::of_bytes("test-backend", b"one"),
1139            architecture_implementation: Digest::of_bytes("test-architecture", b"one"),
1140            layers: 2,
1141            embedding_width: 2,
1142            experts: None,
1143            experts_used: None,
1144            nextn_layers: 1,
1145            supported_speculation: vec![
1146                TextSpeculativeMechanismV1::Mtp,
1147                TextSpeculativeMechanismV1::Eagle3,
1148            ],
1149        };
1150        let site = TextTensorSiteV1::LayerOutput { layer: 0 };
1151        let plan = ActivationCapturePlanV1 {
1152            topology: topology.digest().unwrap(),
1153            sites: vec![site.clone()],
1154            positions: ActivationCapturePositionsV1::InclusiveRanges(vec![
1155                logit_loom::CausalPositionRangeV1 { start: 4, end: 5 },
1156            ]),
1157            retention: ActivationCaptureRetentionV1::Digest,
1158            maximum_elements: 4,
1159            maximum_retained_bytes: 0,
1160        };
1161        let plan_identity = plan.digest_for(&topology).unwrap();
1162        let program = ActivationProgramV1 {
1163            topology: topology.digest().unwrap(),
1164            operations: Vec::new(),
1165            observations: vec![plan_identity.clone()],
1166        };
1167        let program_identity = program.digest_for(&topology).unwrap();
1168        let captures = Arc::new(vec![(plan_identity.clone(), plan.clone())]);
1169        let pending = [4_u64, 5]
1170            .into_iter()
1171            .map(|position| {
1172                let record = ActivationCaptureRecordV1 {
1173                    plan: plan_identity.clone(),
1174                    site: site.clone(),
1175                    position,
1176                    disposition: ActivationTelemetryDispositionV1::Provisional,
1177                    retained: ActivationCapturedDataV1::from_f32(
1178                        &[1.0, 2.0],
1179                        ActivationCaptureRetentionV1::Digest,
1180                    )
1181                    .unwrap(),
1182                };
1183                let identity = record.digest_for(&plan, &topology).unwrap();
1184                ProvisionalCapture { record, identity }
1185            })
1186            .collect::<Vec<_>>();
1187        let mut provisional_aggregates = BTreeMap::new();
1188        provisional_aggregates.insert(
1189            plan_identity,
1190            CaptureAggregate {
1191                records: pending
1192                    .iter()
1193                    .map(|capture| capture.identity.clone())
1194                    .collect(),
1195                elements: 4,
1196                retained_bytes: 0,
1197            },
1198        );
1199        let mut controller = ActivationController {
1200            topology,
1201            program: Arc::new(program),
1202            program_identity,
1203            captures,
1204            site_bindings: Arc::new(BTreeMap::new()),
1205            state: Arc::new(Mutex::new(ActivationState::default())),
1206            captured_records: Vec::new(),
1207            capture_aggregates: BTreeMap::new(),
1208            provisional_captures: pending,
1209            provisional_capture_aggregates: provisional_aggregates,
1210        };
1211
1212        let resolutions = controller.resolve_provisional(5).unwrap();
1213        assert_eq!(resolutions.len(), 2);
1214        let output = controller.take_capture_output().unwrap();
1215        assert_eq!(
1216            output
1217                .records
1218                .iter()
1219                .map(|record| record.disposition)
1220                .collect::<Vec<_>>(),
1221            vec![
1222                ActivationTelemetryDispositionV1::Admitted,
1223                ActivationTelemetryDispositionV1::Rejected,
1224            ]
1225        );
1226        assert_eq!(output.receipts[0].records.len(), 2);
1227    }
1228}