1#![allow(clippy::type_complexity)]
4
5use std::{
6 collections::{BTreeMap, BTreeSet},
7 marker::PhantomData,
8 path::Path,
9};
10
11use eredu_core::cache::{
12 validate_prompt_cache_model_identity, PromptCacheDescriptor, PromptCacheError,
13 PromptCacheManifest, PromptCacheModelIdentity, PromptCacheOptions, PromptCacheTopology,
14};
15use eredu_core::{DistributedCommitEpoch, DistributedCommitOutcome, DistributedCommitPhase};
16use eredu_nn::{NeuralBackend, Tensor};
17
18mod control;
19pub use control::{ReplicatedTextControlState, ReplicatedTextSnapshotMechanisms};
20
21use crate::{
22 observe_model_logits, partitioned_replicated_text_materialization_tasks,
23 plan_local_replicated_text_materialization_tasks, replicated_text_materialization_tasks,
24 ActivationObserver, ArchitecturePartition, CommunicationManifest, ExecutionResidency,
25 ExpertPass, LayerWeightResidency, LayeredArchitecture, LayerwisePolicy, LayerwiseRuntime,
26 LayerwiseRuntimeError, ParameterGroupOwner, PartitionState, PreparedInputCacheIdentity,
27 ReplicatedTextArchitecture, ReplicatedTextMaterializationTask, ReplicatedTextOutputCompanion,
28 ReplicatedTextOutputSelection, ReplicatedTextParameterOwner, ReplicatedTextParameterPresence,
29 RoutedExpertProvider, RoutedLayeredArchitecture, RuntimeState,
30 SelectedReplicatedTextRealization, SelectedStateRealization, StateError, SubmissionBackend,
31 WeightLoweringKind,
32};
33
34pub trait ReplicatedTextSessionMechanisms<A, B>
42where
43 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
44 A: LayeredArchitecture<B, Self::State>,
45 Self::State: RuntimeState<B>,
46 Self::ResidentPolicy: LayerwisePolicy<B, A::Unit, Error = Self::PolicyError>,
47 Self::BoundedPolicy: LayerwisePolicy<B, A::Unit, Error = Self::PolicyError>,
48{
49 type State: RuntimeState<B>;
51 type PolicyError;
53 type ResidentPolicy: LayerwisePolicy<B, A::Unit, Error = Self::PolicyError>;
55 type BoundedPolicy: LayerwisePolicy<B, A::Unit, Error = Self::PolicyError>;
57 type StateCheckpoint;
59 type StateReport;
61 type ExecutionReport;
63 type Error;
65
66 fn take_materialization_report(
74 &mut self,
75 ) -> Result<Option<crate::WeightMaterializationReport>, Self::Error> {
76 Ok(None)
77 }
78
79 fn configure_partition(
82 &mut self,
83 _target_layout: crate::LocalModelLayout,
84 _source_layout: Option<crate::LocalModelLayout>,
85 _rank: eredu_core::cache::CacheRankIdentity,
86 _global_layer_start: usize,
87 ) {
88 }
89
90 #[allow(clippy::too_many_arguments)]
98 fn prepare_partition_materialization(
99 &mut self,
100 architecture: &mut A,
101 global_layout: &crate::ExecutionUnitLayout,
102 addresses: &[crate::ExecutionUnitAddress],
103 task_partition: &crate::ReplicatedTextMaterializationPartitionPlan,
104 units: &mut [A::Unit],
105 source_architecture: Option<&mut A>,
106 source_units: Option<&mut [A::Unit]>,
107 tasks: &[ReplicatedTextMaterializationTask],
108 addressable_parameters: &[String],
109 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
110 ) -> Result<(), Self::Error> {
111 let _ = (addresses, task_partition);
112 self.prepare_materialization(
113 architecture,
114 global_layout,
115 units,
116 source_architecture,
117 source_units,
118 tasks,
119 addressable_parameters,
120 context,
121 )
122 }
123
124 #[allow(clippy::too_many_arguments)]
126 fn prepare_materialization(
127 &mut self,
128 architecture: &mut A,
129 layout: &crate::ExecutionUnitLayout,
130 units: &mut [A::Unit],
131 source_architecture: Option<&mut A>,
132 source_units: Option<&mut [A::Unit]>,
133 tasks: &[ReplicatedTextMaterializationTask],
134 addressable_parameters: &[String],
135 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
136 ) -> Result<(), Self::Error>;
137
138 fn realize_state(
140 &mut self,
141 selected: &SelectedStateRealization,
142 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
143 ) -> Result<Self::State, Self::Error>;
144
145 fn resident_policy(
147 &mut self,
148 architecture: &mut A,
149 units: Vec<A::Unit>,
150 selected: &SelectedReplicatedTextRealization,
151 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
152 ) -> Result<Self::ResidentPolicy, Self::Error>;
153
154 fn bounded_policy(
156 &mut self,
157 architecture: &mut A,
158 selected: &SelectedReplicatedTextRealization,
159 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
160 ) -> Result<Self::BoundedPolicy, Self::Error>;
161
162 fn index_text_output(
164 &mut self,
165 output: B::Tensor,
166 sequence_index: i32,
167 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
168 ) -> Result<B::Tensor, Self::Error>;
169
170 fn checkpoint_state(
172 &mut self,
173 state: &Self::State,
174 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
175 ) -> Result<Self::StateCheckpoint, Self::Error>;
176
177 fn restore_state(
179 &mut self,
180 state: &mut Self::State,
181 checkpoint: Self::StateCheckpoint,
182 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
183 ) -> Result<(), Self::Error>;
184
185 fn fork_prediction_target_state(
192 &mut self,
193 state: &Self::State,
194 selected: &SelectedStateRealization,
195 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
196 ) -> Result<Self::State, Self::Error> {
197 let checkpoint = self.checkpoint_state(state, context)?;
198 let mut fork = self.realize_state(selected, context)?;
199 self.restore_state(&mut fork, checkpoint, context)?;
200 Ok(fork)
201 }
202
203 fn load_prompt_cache(
205 &mut self,
206 directory: &Path,
207 expected: &PromptCacheDescriptor,
208 identity: &PromptCacheModelIdentity,
209 prefix_token_ids: &[u32],
210 selected: &SelectedStateRealization,
211 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
212 ) -> Result<(Self::State, PromptCacheManifest), Self::Error>;
213
214 fn save_prompt_cache(
216 &mut self,
217 state: &mut Self::State,
218 destination: &Path,
219 descriptor: PromptCacheDescriptor,
220 prefix_token_ids: &[u32],
221 options: &PromptCacheOptions,
222 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
223 ) -> Result<PromptCacheManifest, Self::Error>;
224
225 fn state_report(&self, state: &Self::State) -> Result<Self::StateReport, Self::Error>;
227
228 fn execution_report(
230 &self,
231 residency: LayerWeightResidency,
232 bounded: Option<&Self::BoundedPolicy>,
233 ) -> Result<Self::ExecutionReport, Self::Error>;
234
235 fn complete(
237 &mut self,
238 output: &B::Tensor,
239 state: &Self::State,
240 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
241 ) -> Result<(), Self::Error>;
242}
243
244pub trait PredictionTargetOperation<A, B, S>
250where
251 B: NeuralBackend,
252 S: RuntimeState<B>,
253 A: LayeredArchitecture<B, S>,
254{
255 type Output;
257
258 fn apply(
260 self,
261 architecture: &mut A,
262 state: &mut S,
263 parallel: Option<&B::ParallelContext>,
264 context: &<B::Tensor as Tensor>::Context,
265 ) -> Result<Self::Output, A::Error>;
266}
267
268pub trait TransactionalPromptCacheMechanisms<A, B>: ReplicatedTextSessionMechanisms<A, B>
277where
278 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
279 A: LayeredArchitecture<B, Self::State>,
280 Self::State: RuntimeState<B>,
281 Self::ResidentPolicy: LayerwisePolicy<B, A::Unit, Error = Self::PolicyError>,
282 Self::BoundedPolicy: LayerwisePolicy<B, A::Unit, Error = Self::PolicyError>,
283{
284 type PromptCacheSaveTransaction;
286
287 #[allow(clippy::too_many_arguments)]
289 fn prepare_prompt_cache_save(
290 &mut self,
291 state: &mut Self::State,
292 destination: &Path,
293 descriptor: PromptCacheDescriptor,
294 prefix_token_ids: &[u32],
295 options: &PromptCacheOptions,
296 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
297 ) -> Result<Self::PromptCacheSaveTransaction, Self::Error>;
298
299 fn prepared_prompt_cache_manifest(
301 transaction: &Self::PromptCacheSaveTransaction,
302 ) -> &PromptCacheManifest;
303
304 fn publish_prompt_cache_save(
306 &mut self,
307 transaction: &mut Self::PromptCacheSaveTransaction,
308 ) -> Result<(), Self::Error>;
309
310 fn commit_prompt_cache_save(&mut self, transaction: Self::PromptCacheSaveTransaction);
312
313 fn rollback_prompt_cache_save(&mut self, transaction: Self::PromptCacheSaveTransaction);
315}
316
317enum ReplicatedTextRuntimeKind<A, B, S, R, P>
319where
320 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
321 S: RuntimeState<B>,
322 A: LayeredArchitecture<B, S>,
323 R: LayerwisePolicy<B, A::Unit>,
324 P: LayerwisePolicy<B, A::Unit, Error = R::Error>,
325{
326 Resident(LayerwiseRuntime<A, B, S, R>),
328 Bounded(LayerwiseRuntime<A, B, S, P>),
330}
331
332pub struct ReplicatedTextRuntime<A, B, S, R, P>
338where
339 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
340 S: RuntimeState<B>,
341 A: LayeredArchitecture<B, S>,
342 R: LayerwisePolicy<B, A::Unit>,
343 P: LayerwisePolicy<B, A::Unit, Error = R::Error>,
344{
345 kind: ReplicatedTextRuntimeKind<A, B, S, R, P>,
346}
347
348impl<A, B, S, R, P> ReplicatedTextRuntime<A, B, S, R, P>
349where
350 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
351 S: RuntimeState<B>,
352 A: LayeredArchitecture<B, S>,
353 R: LayerwisePolicy<B, A::Unit>,
354 P: LayerwisePolicy<B, A::Unit, Error = R::Error>,
355 A::Error: std::fmt::Display,
356 P::Error: std::fmt::Display,
357{
358 fn forward_with_observer<'a, O>(
359 &mut self,
360 input: A::Input<'a>,
361 state: &mut S,
362 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
363 observer: &mut O,
364 ) -> Result<
365 (B::Tensor, A::ForwardContext),
366 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
367 >
368 where
369 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
370 {
371 match &mut self.kind {
372 ReplicatedTextRuntimeKind::Resident(runtime) => runtime
373 .forward_with_observer_and_context(input, state, context, observer)
374 .map_err(map_layerwise_error),
375 ReplicatedTextRuntimeKind::Bounded(runtime) => runtime
376 .forward_with_observer_and_context(input, state, context, observer)
377 .map_err(map_layerwise_error),
378 }
379 }
380
381 fn forward_with_provider_and_observer<'a, Provider, Observer>(
382 &mut self,
383 input: A::Input<'a>,
384 state: &mut S,
385 pass: ExpertPass,
386 provider: &mut Provider,
387 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
388 observer: &mut Observer,
389 ) -> Result<
390 (B::Tensor, A::ForwardContext),
391 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
392 >
393 where
394 B: eredu_nn::GroupedNeuralBackend,
395 A: RoutedLayeredArchitecture<B, S>,
396 Provider: RoutedExpertProvider<B>,
397 Provider::Error: std::fmt::Display,
398 Observer: ActivationObserver<B::Tensor, A::Error> + ?Sized,
399 {
400 match &mut self.kind {
401 ReplicatedTextRuntimeKind::Resident(runtime) => runtime
402 .forward_with_provider_and_observer_and_context(
403 input, state, pass, provider, context, observer,
404 )
405 .map_err(map_layerwise_error),
406 ReplicatedTextRuntimeKind::Bounded(runtime) => runtime
407 .forward_with_provider_and_observer_and_context(
408 input, state, pass, provider, context, observer,
409 )
410 .map_err(map_layerwise_error),
411 }
412 }
413
414 fn bounded_policy(&self) -> Option<&P> {
415 match &self.kind {
416 ReplicatedTextRuntimeKind::Resident(_) => None,
417 ReplicatedTextRuntimeKind::Bounded(runtime) => Some(runtime.policy()),
418 }
419 }
420
421 fn prediction_target_capture(
422 &mut self,
423 forward: &A::ForwardContext,
424 _context: &<B::Tensor as Tensor>::Context,
425 ) -> Result<Option<B::Tensor>, A::Error> {
426 Ok(<A as crate::LayeredArchitecture<B, S>>::prediction_target_capture(forward).cloned())
427 }
428
429 fn apply_prediction_target_operation<O>(
430 &mut self,
431 state: &mut S,
432 operation: O,
433 context: &<B::Tensor as Tensor>::Context,
434 ) -> Result<O::Output, A::Error>
435 where
436 O: PredictionTargetOperation<A, B, S>,
437 {
438 match &mut self.kind {
439 ReplicatedTextRuntimeKind::Resident(runtime) => {
440 operation.apply(runtime.architecture_mut(), state, None, context)
441 }
442 ReplicatedTextRuntimeKind::Bounded(runtime) => {
443 operation.apply(runtime.architecture_mut(), state, None, context)
444 }
445 }
446 }
447}
448
449pub trait ReplicatedTextExecutionStrategy<A, B, S, R, P>
454where
455 B: NeuralBackend,
456 S: RuntimeState<B>,
457 A: LayeredArchitecture<B, S>,
458 R: LayerwisePolicy<B, A::Unit>,
459 P: LayerwisePolicy<B, A::Unit, Error = R::Error>,
460 A::Error: std::fmt::Display,
461 R::Error: std::fmt::Display,
462{
463 const PARTITIONED_SESSION: bool = false;
465 const DISTRIBUTED_PHASE_AGREEMENT: bool = false;
467
468 type Runtime;
470
471 fn bounded_policy(runtime: &Self::Runtime) -> Option<&P>;
473
474 fn execution_residency(
476 runtime: &Self::Runtime,
477 selected: &SelectedReplicatedTextRealization,
478 ) -> ExecutionResidency;
479
480 #[allow(clippy::too_many_arguments)]
482 fn forward_with_observer<'a, O>(
483 &mut self,
484 runtime: &mut Self::Runtime,
485 input: A::Input<'a>,
486 state: &mut S,
487 pass: ExpertPass,
488 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
489 observer: &mut O,
490 ) -> Result<
491 (B::Tensor, A::ForwardContext),
492 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
493 >
494 where
495 O: ActivationObserver<B::Tensor, A::Error> + ?Sized;
496
497 fn observe_output<O>(
501 _runtime: &mut Self::Runtime,
502 output: &B::Tensor,
503 observer: &mut O,
504 _context: &<B::Tensor as Tensor>::Context,
505 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>>
506 where
507 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
508 {
509 observe_model_logits(observer, output).map_err(ReplicatedTextSessionError::Architecture)
510 }
511
512 fn publish_observed_output(
515 _runtime: &mut Self::Runtime,
516 output: B::Tensor,
517 _context: &<B::Tensor as Tensor>::Context,
518 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>>
519 {
520 Ok(output)
521 }
522
523 fn prediction_target_capture(
529 _runtime: &mut Self::Runtime,
530 forward: &A::ForwardContext,
531 _context: &<B::Tensor as Tensor>::Context,
532 ) -> Result<
533 Option<B::Tensor>,
534 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
535 > {
536 Ok(<A as crate::LayeredArchitecture<B, S>>::prediction_target_capture(forward).cloned())
537 }
538
539 fn publish_prediction_target_capture(
541 _runtime: &mut Self::Runtime,
542 capture: B::Tensor,
543 _context: &<B::Tensor as Tensor>::Context,
544 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>>
545 {
546 Ok(capture)
547 }
548
549 fn apply_prediction_target_operation<O>(
551 _runtime: &mut Self::Runtime,
552 _state: &mut S,
553 _operation: O,
554 _context: &<B::Tensor as Tensor>::Context,
555 ) -> Result<
556 Option<O::Output>,
557 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
558 >
559 where
560 O: PredictionTargetOperation<A, B, S>,
561 {
562 Ok(None)
563 }
564
565 fn commit_after_completion(
568 _runtime: &mut Self::Runtime,
569 epoch: DistributedCommitEpoch,
570 _context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
571 ) -> DistributedCommitOutcome {
572 DistributedCommitOutcome::Committed(epoch)
573 }
574
575 fn agree_distributed_phase(
578 _runtime: &mut Self::Runtime,
579 _phase: crate::DistributedExecutionPhase,
580 local_success: bool,
581 _context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
582 ) -> Result<bool, ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>>
583 {
584 Ok(local_success)
585 }
586}
587
588pub trait ReplicatedRuntimeExecutionStrategy<A, B, S, R, P>:
594 ReplicatedTextExecutionStrategy<A, B, S, R, P, Runtime = ReplicatedTextRuntime<A, B, S, R, P>>
595where
596 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
597 S: RuntimeState<B>,
598 A: LayeredArchitecture<B, S>,
599 R: LayerwisePolicy<B, A::Unit>,
600 P: LayerwisePolicy<B, A::Unit, Error = R::Error>,
601 A::Error: std::fmt::Display,
602 R::Error: std::fmt::Display,
603{
604}
605
606#[derive(Debug, Default, Clone, Copy)]
608pub struct DirectReplicatedTextExecution;
609
610impl<A, B, S, R, P> ReplicatedTextExecutionStrategy<A, B, S, R, P> for DirectReplicatedTextExecution
611where
612 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
613 S: RuntimeState<B>,
614 A: LayeredArchitecture<B, S>,
615 R: LayerwisePolicy<B, A::Unit>,
616 P: LayerwisePolicy<B, A::Unit, Error = R::Error>,
617 A::Error: std::fmt::Display,
618 P::Error: std::fmt::Display,
619{
620 type Runtime = ReplicatedTextRuntime<A, B, S, R, P>;
621
622 fn bounded_policy(runtime: &Self::Runtime) -> Option<&P> {
623 runtime.bounded_policy()
624 }
625
626 fn execution_residency(
627 _runtime: &Self::Runtime,
628 selected: &SelectedReplicatedTextRealization,
629 ) -> ExecutionResidency {
630 selected.residency().execution_residency()
631 }
632
633 fn forward_with_observer<'a, O>(
634 &mut self,
635 runtime: &mut Self::Runtime,
636 input: A::Input<'a>,
637 state: &mut S,
638 _pass: ExpertPass,
639 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
640 observer: &mut O,
641 ) -> Result<
642 (B::Tensor, A::ForwardContext),
643 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
644 >
645 where
646 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
647 {
648 runtime.forward_with_observer(input, state, context, observer)
649 }
650
651 fn prediction_target_capture(
652 runtime: &mut Self::Runtime,
653 forward: &A::ForwardContext,
654 context: &<B::Tensor as Tensor>::Context,
655 ) -> Result<
656 Option<B::Tensor>,
657 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
658 > {
659 runtime
660 .prediction_target_capture(forward, context)
661 .map_err(ReplicatedTextSessionError::Architecture)
662 }
663
664 fn apply_prediction_target_operation<O>(
665 runtime: &mut Self::Runtime,
666 state: &mut S,
667 operation: O,
668 context: &<B::Tensor as Tensor>::Context,
669 ) -> Result<
670 Option<O::Output>,
671 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
672 >
673 where
674 O: PredictionTargetOperation<A, B, S>,
675 {
676 runtime
677 .apply_prediction_target_operation(state, operation, context)
678 .map(Some)
679 .map_err(ReplicatedTextSessionError::Architecture)
680 }
681}
682
683impl<A, B, S, R, P> ReplicatedRuntimeExecutionStrategy<A, B, S, R, P>
684 for DirectReplicatedTextExecution
685where
686 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
687 S: RuntimeState<B>,
688 A: LayeredArchitecture<B, S>,
689 R: LayerwisePolicy<B, A::Unit>,
690 P: LayerwisePolicy<B, A::Unit, Error = R::Error>,
691 A::Error: std::fmt::Display,
692 P::Error: std::fmt::Display,
693{
694}
695
696pub struct RoutedReplicatedTextExecution<P> {
698 provider: P,
699}
700
701impl<P> RoutedReplicatedTextExecution<P> {
702 pub const fn new(provider: P) -> Self {
704 Self { provider }
705 }
706
707 pub const fn provider(&self) -> &P {
709 &self.provider
710 }
711}
712
713impl<A, B, S, R, P, Provider> ReplicatedTextExecutionStrategy<A, B, S, R, P>
714 for RoutedReplicatedTextExecution<Provider>
715where
716 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>
717 + eredu_nn::GroupedNeuralBackend,
718 S: RuntimeState<B>,
719 A: LayeredArchitecture<B, S> + RoutedLayeredArchitecture<B, S>,
720 R: LayerwisePolicy<B, A::Unit>,
721 P: LayerwisePolicy<B, A::Unit, Error = R::Error>,
722 Provider: RoutedExpertProvider<B>,
723 Provider::Error: std::fmt::Display,
724 A::Error: std::fmt::Display,
725 P::Error: std::fmt::Display,
726{
727 type Runtime = ReplicatedTextRuntime<A, B, S, R, P>;
728
729 fn bounded_policy(runtime: &Self::Runtime) -> Option<&P> {
730 runtime.bounded_policy()
731 }
732
733 fn execution_residency(
734 _runtime: &Self::Runtime,
735 selected: &SelectedReplicatedTextRealization,
736 ) -> ExecutionResidency {
737 selected.residency().execution_residency()
738 }
739
740 fn forward_with_observer<'a, O>(
741 &mut self,
742 runtime: &mut Self::Runtime,
743 input: A::Input<'a>,
744 state: &mut S,
745 pass: ExpertPass,
746 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
747 observer: &mut O,
748 ) -> Result<
749 (B::Tensor, A::ForwardContext),
750 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
751 >
752 where
753 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
754 {
755 runtime.forward_with_provider_and_observer(
756 input,
757 state,
758 pass,
759 &mut self.provider,
760 context,
761 observer,
762 )
763 }
764
765 fn apply_prediction_target_operation<O>(
766 runtime: &mut Self::Runtime,
767 state: &mut S,
768 operation: O,
769 context: &<B::Tensor as Tensor>::Context,
770 ) -> Result<
771 Option<O::Output>,
772 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
773 >
774 where
775 O: PredictionTargetOperation<A, B, S>,
776 {
777 runtime
778 .apply_prediction_target_operation(state, operation, context)
779 .map(Some)
780 .map_err(ReplicatedTextSessionError::Architecture)
781 }
782}
783
784impl<A, B, S, R, P, Provider> ReplicatedRuntimeExecutionStrategy<A, B, S, R, P>
785 for RoutedReplicatedTextExecution<Provider>
786where
787 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>
788 + eredu_nn::GroupedNeuralBackend,
789 S: RuntimeState<B>,
790 A: LayeredArchitecture<B, S> + RoutedLayeredArchitecture<B, S>,
791 R: LayerwisePolicy<B, A::Unit>,
792 P: LayerwisePolicy<B, A::Unit, Error = R::Error>,
793 Provider: RoutedExpertProvider<B>,
794 Provider::Error: std::fmt::Display,
795 A::Error: std::fmt::Display,
796 P::Error: std::fmt::Display,
797{
798}
799
800fn record_successful_restoration<E>(
801 generation: &mut Option<u64>,
802 restored: Result<(), E>,
803) -> Result<(), E> {
804 restored?;
805 *generation = generation.and_then(|value| value.checked_add(1));
806 Ok(())
807}
808
809#[cfg(test)]
810mod restoration_witness_tests {
811 use super::record_successful_restoration;
812
813 #[test]
814 fn failed_restore_and_stale_snapshot_do_not_prove_new_restoration() {
815 let mut generation = Some(0);
816 let before = generation;
817 assert!(record_successful_restoration(&mut generation, Err("restore failed")).is_err());
818 assert_eq!(generation, before);
819 record_successful_restoration(&mut generation, Ok::<_, ()>(())).unwrap();
820 assert_eq!(generation, Some(1));
821 let prior_restore = generation;
822 assert!(
823 record_successful_restoration(&mut generation, Err("later restore failed")).is_err()
824 );
825 assert_eq!(generation, prior_restore);
826 }
827
828 #[test]
829 fn restoration_counter_overflow_permanently_disables_the_witness() {
830 let mut generation = Some(u64::MAX);
831 record_successful_restoration(&mut generation, Ok::<_, ()>(())).unwrap();
832 assert_eq!(generation, None);
833 record_successful_restoration(&mut generation, Ok::<_, ()>(())).unwrap();
834 assert_eq!(generation, None);
835 }
836}
837
838pub struct ReplicatedTextSession<A, B, M, D = DirectReplicatedTextExecution>
840where
841 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
842 M: ReplicatedTextSessionMechanisms<A, B>,
843 A: LayeredArchitecture<B, M::State>,
844 D: ReplicatedTextExecutionStrategy<A, B, M::State, M::ResidentPolicy, M::BoundedPolicy>,
845 A::Error: std::fmt::Display,
846 M::PolicyError: std::fmt::Display,
847{
848 selected: SelectedReplicatedTextRealization,
849 selected_state: SessionStateRealization,
850 execution: D::Runtime,
851 driver: D,
852 state: M::State,
853 mechanisms: M,
854 materialization_report: Option<crate::WeightMaterializationReport>,
855 prompt_cache_identity: Option<PromptCacheModelIdentity>,
856 committed_prompt_input_identity: Option<PreparedInputCacheIdentity>,
857 next_commit_epoch: DistributedCommitEpoch,
858 active_commit_epoch: Option<DistributedCommitEpoch>,
859 last_commit_outcome: Option<DistributedCommitOutcome>,
860 successful_state_restorations: Option<u64>,
861 control_identity: std::sync::Arc<()>,
862 control_fence: Option<crate::DistributedExecutionPhase>,
863 output_selection: ReplicatedTextOutputSelection,
864 backend: PhantomData<fn() -> B>,
865}
866
867#[derive(Debug, Clone, Eq, PartialEq)]
869pub enum SessionStateRealization {
870 Stateful(SelectedStateRealization),
872 Stateless,
874}
875
876pub struct PreparedPartitionedSessionRuntime<R, S> {
881 selected: SelectedReplicatedTextRealization,
882 runtime: R,
883 state: S,
884 selected_state: SessionStateRealization,
885 prompt_cache_identity: Option<PromptCacheModelIdentity>,
886 output_selection: ReplicatedTextOutputSelection,
887}
888
889impl<R, S> PreparedPartitionedSessionRuntime<R, S> {
890 pub const fn prompt_cache_identity(&self) -> Option<&PromptCacheModelIdentity> {
893 self.prompt_cache_identity.as_ref()
894 }
895}
896
897pub struct PartitionedSessionFactoryInput<A, G, W> {
905 architecture: A,
906 partition: ArchitecturePartition<G, W>,
907 communication: CommunicationManifest,
908 tasks: Vec<ReplicatedTextMaterializationTask>,
909}
910
911impl<A, G, W> PartitionedSessionFactoryInput<A, G, W> {
912 pub fn materialization_tasks(&self) -> &[ReplicatedTextMaterializationTask] {
914 &self.tasks
915 }
916
917 pub fn into_parts(
919 self,
920 ) -> (
921 A,
922 ArchitecturePartition<G, W>,
923 CommunicationManifest,
924 Vec<ReplicatedTextMaterializationTask>,
925 ) {
926 (
927 self.architecture,
928 self.partition,
929 self.communication,
930 self.tasks,
931 )
932 }
933}
934
935#[derive(Debug, Clone, Copy, Eq, PartialEq)]
937pub enum PartitionedUnitScope {
938 All,
940 Owned,
942}
943
944pub struct PreparedPartitionedRuntimeComponents<A, G, W, S, P> {
947 architecture: A,
948 partition: ArchitecturePartition<G, W>,
949 communication: CommunicationManifest,
950 execution_policy: P,
951 bounded_policy: Option<P>,
952 state: S,
953}
954
955impl<A, G, W, S, P> PreparedPartitionedRuntimeComponents<A, G, W, S, P> {
956 pub fn into_parts(
960 self,
961 ) -> (
962 A,
963 ArchitecturePartition<G, W>,
964 CommunicationManifest,
965 P,
966 Option<P>,
967 S,
968 ) {
969 (
970 self.architecture,
971 self.partition,
972 self.communication,
973 self.execution_policy,
974 self.bounded_policy,
975 self.state,
976 )
977 }
978}
979
980#[derive(Debug, thiserror::Error)]
982pub enum PartitionedRuntimeConstructionError {
983 #[error("partitioned runtime contract mismatch: {0}")]
985 Contract(String),
986 #[error("partitioned runtime architecture construction failed: {0}")]
988 Architecture(String),
989 #[error("partitioned runtime mechanism failed: {0}")]
991 Mechanism(String),
992}
993
994#[allow(clippy::too_many_arguments)]
1003pub fn prepare_default_partitioned_runtime<A, B, M, G, W, P>(
1004 input: PartitionedSessionFactoryInput<A, G, W>,
1005 mut source_architecture: Option<A>,
1006 target_parallel_layout: crate::LocalModelLayout,
1007 source_parallel_layout: Option<crate::LocalModelLayout>,
1008 selected: &SelectedReplicatedTextRealization,
1009 topology: &PromptCacheTopology,
1010 scope: PartitionedUnitScope,
1011 addressable_parameters: &[String],
1012 mechanisms: &mut M,
1013 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
1014) -> Result<
1015 PreparedPartitionedRuntimeComponents<A, G, W, M::State, P>,
1016 PartitionedRuntimeConstructionError,
1017>
1018where
1019 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
1020 M: ReplicatedTextSessionMechanisms<A, B, ResidentPolicy = P, BoundedPolicy = P>,
1021 A: LayeredArchitecture<B, M::State>,
1022 A::Error: std::fmt::Display,
1023 M::Error: std::fmt::Display,
1024 P: LayerwisePolicy<B, A::Unit, Error = M::PolicyError> + Clone,
1025{
1026 let (mut architecture, partition, communication, tasks) = input.into_parts();
1027 let global_layout = partition.unit_layout().clone();
1028 let addresses = match scope {
1029 PartitionedUnitScope::All => (0..global_layout.len())
1030 .map(|ordinal| {
1031 global_layout.address(ordinal).ok_or_else(|| {
1032 PartitionedRuntimeConstructionError::Contract(format!(
1033 "global unit ordinal {ordinal} has no canonical address"
1034 ))
1035 })
1036 })
1037 .collect::<Result<Vec<_>, _>>()?,
1038 PartitionedUnitScope::Owned => partition.units().collect::<Vec<_>>(),
1039 };
1040 if addresses.is_empty() {
1041 return Err(PartitionedRuntimeConstructionError::Contract(
1042 "partition owns no execution units".into(),
1043 ));
1044 }
1045 let task_partition =
1046 plan_local_replicated_text_materialization_tasks(&tasks, &global_layout, &addresses)
1047 .map_err(|error| PartitionedRuntimeConstructionError::Contract(error.to_string()))?;
1048 let mut units = addresses
1049 .iter()
1050 .map(|address| {
1051 architecture
1052 .build_unit(address.group(), address.index(), context)
1053 .map_err(|error| {
1054 PartitionedRuntimeConstructionError::Architecture(error.to_string())
1055 })
1056 })
1057 .collect::<Result<Vec<_>, _>>()?;
1058 let mut source_units = source_architecture
1059 .as_ref()
1060 .map(|source| {
1061 addresses
1062 .iter()
1063 .map(|address| {
1064 source
1065 .build_unit(address.group(), address.index(), context)
1066 .map_err(|error| {
1067 PartitionedRuntimeConstructionError::Architecture(error.to_string())
1068 })
1069 })
1070 .collect::<Result<Vec<_>, _>>()
1071 })
1072 .transpose()?;
1073 let local_state = partition.state().ok_or_else(|| {
1074 PartitionedRuntimeConstructionError::Contract(
1075 "partition owns no local mutable state".into(),
1076 )
1077 })?;
1078 let selected_state = selected
1079 .state()
1080 .for_partitioned_geometry(local_state)
1081 .map_err(|error| PartitionedRuntimeConstructionError::Contract(error.to_string()))?;
1082 let rank = eredu_core::cache::CacheRankIdentity::new(
1083 topology.stage().map(|(_, rank)| rank),
1084 topology.shard().map(|(_, rank)| rank),
1085 topology.addressable().map(|(_, rank)| rank),
1086 );
1087 mechanisms.configure_partition(
1088 target_parallel_layout,
1089 source_parallel_layout,
1090 rank,
1091 local_state.global_layer_offset(),
1092 );
1093 mechanisms
1094 .prepare_partition_materialization(
1095 &mut architecture,
1096 &global_layout,
1097 &addresses,
1098 &task_partition,
1099 &mut units,
1100 source_architecture.as_mut(),
1101 source_units.as_deref_mut(),
1102 &tasks,
1103 addressable_parameters,
1104 context,
1105 )
1106 .map_err(|error| PartitionedRuntimeConstructionError::Mechanism(error.to_string()))?;
1107 let state = mechanisms
1108 .realize_state(&selected_state, context)
1109 .map_err(|error| PartitionedRuntimeConstructionError::Mechanism(error.to_string()))?;
1110 if state.optional_layout() != Some(selected_state.layout()) {
1111 return Err(PartitionedRuntimeConstructionError::Contract(
1112 "realized partition state differs from selected local geometry".into(),
1113 ));
1114 }
1115 let (execution_policy, bounded_policy) = match selected.residency() {
1116 LayerWeightResidency::FullyResident => (
1117 mechanisms
1118 .resident_policy(&mut architecture, units, selected, context)
1119 .map_err(|error| {
1120 PartitionedRuntimeConstructionError::Mechanism(error.to_string())
1121 })?,
1122 None,
1123 ),
1124 LayerWeightResidency::LayerwiseHost(_) | LayerWeightResidency::DenseDiskStream(_) => {
1125 drop(units);
1126 let policy = mechanisms
1127 .bounded_policy(&mut architecture, selected, context)
1128 .map_err(|error| {
1129 PartitionedRuntimeConstructionError::Mechanism(error.to_string())
1130 })?;
1131 (policy.clone(), Some(policy))
1132 }
1133 };
1134 Ok(PreparedPartitionedRuntimeComponents {
1135 architecture,
1136 partition,
1137 communication,
1138 execution_policy,
1139 bounded_policy,
1140 state,
1141 })
1142}
1143
1144#[derive(Debug, thiserror::Error)]
1146pub enum PartitionedSessionPreparationError<E> {
1147 #[error("partitioned session authority mismatch: {0}")]
1149 Contract(String),
1150 #[error("partitioned session runtime factory failed: {0}")]
1152 Factory(E),
1153}
1154
1155#[allow(clippy::too_many_arguments)]
1163pub fn prepare_partitioned_session_runtime<A, B, R, S, G, W, E, F>(
1164 architecture: A,
1165 selected: SelectedReplicatedTextRealization,
1166 partition: ArchitecturePartition<G, W>,
1167 communication: CommunicationManifest,
1168 expected_tasks: Option<&[ReplicatedTextMaterializationTask]>,
1169 topology: PromptCacheTopology,
1170 output_selection: ReplicatedTextOutputSelection,
1171 context: &<B::Tensor as Tensor>::Context,
1172 factory: F,
1173) -> Result<PreparedPartitionedSessionRuntime<R, S>, PartitionedSessionPreparationError<E>>
1174where
1175 B: NeuralBackend,
1176 S: RuntimeState<B>,
1177 A: LayeredArchitecture<B, S>,
1178 A::Error: std::fmt::Display,
1179 F: FnOnce(
1180 PartitionedSessionFactoryInput<A, G, W>,
1181 &SelectedReplicatedTextRealization,
1182 &<B::Tensor as Tensor>::Context,
1183 ) -> Result<(R, S), E>,
1184{
1185 prepare_partitioned_session_runtime_with_exclusions(
1186 architecture,
1187 selected,
1188 partition,
1189 communication,
1190 expected_tasks,
1191 &std::collections::BTreeSet::new(),
1192 topology,
1193 output_selection,
1194 context,
1195 factory,
1196 )
1197}
1198
1199#[allow(clippy::too_many_arguments)]
1202pub fn prepare_partitioned_session_runtime_with_exclusions<A, B, R, S, G, W, E, F>(
1203 architecture: A,
1204 selected: SelectedReplicatedTextRealization,
1205 partition: ArchitecturePartition<G, W>,
1206 communication: CommunicationManifest,
1207 expected_tasks: Option<&[ReplicatedTextMaterializationTask]>,
1208 excluded_parameter_targets: &std::collections::BTreeSet<&str>,
1209 topology: PromptCacheTopology,
1210 output_selection: ReplicatedTextOutputSelection,
1211 context: &<B::Tensor as Tensor>::Context,
1212 factory: F,
1213) -> Result<PreparedPartitionedSessionRuntime<R, S>, PartitionedSessionPreparationError<E>>
1214where
1215 B: NeuralBackend,
1216 S: RuntimeState<B>,
1217 A: LayeredArchitecture<B, S>,
1218 A::Error: std::fmt::Display,
1219 F: FnOnce(
1220 PartitionedSessionFactoryInput<A, G, W>,
1221 &SelectedReplicatedTextRealization,
1222 &<B::Tensor as Tensor>::Context,
1223 ) -> Result<(R, S), E>,
1224{
1225 partition
1226 .validate_architecture::<B, S, A>(&architecture)
1227 .map_err(|error| PartitionedSessionPreparationError::Contract(error.to_string()))?;
1228 let parameters = architecture
1229 .parameter_description(context)
1230 .map_err(|error| PartitionedSessionPreparationError::Contract(error.to_string()))?;
1231 let mut tasks =
1232 partitioned_replicated_text_materialization_tasks(&selected, ¶meters, &partition)
1233 .map_err(|error| PartitionedSessionPreparationError::Contract(error.to_string()))?;
1234 tasks.retain(|task| !excluded_parameter_targets.contains(task.name()));
1235 if expected_tasks.is_some_and(|expected| expected != tasks) {
1236 let derived_names = tasks
1237 .iter()
1238 .map(ReplicatedTextMaterializationTask::name)
1239 .collect::<std::collections::BTreeSet<_>>();
1240 let first_missing = expected_tasks
1241 .expect("task proof was checked as present")
1242 .iter()
1243 .map(ReplicatedTextMaterializationTask::name)
1244 .find(|name| !derived_names.contains(name));
1245 let parameter_group = first_missing.and_then(|name| {
1246 parameters
1247 .groups()
1248 .iter()
1249 .find(|group| group.members().iter().any(|member| member.target() == name))
1250 });
1251 let partition_group = first_missing.and_then(|name| {
1252 partition
1253 .parameter_bindings()
1254 .iter()
1255 .find(|group| group.members().iter().any(|member| member.target() == name))
1256 });
1257 return Err(PartitionedSessionPreparationError::Contract(
1258 format!(
1259 "precomputed local materialization tasks differ from consumed partition authority: expected {:?}, derived {:?}, first missing current group {parameter_group:?}, admitted group {partition_group:?}",
1260 expected_tasks
1261 .expect("task proof was checked as present")
1262 .iter()
1263 .map(ReplicatedTextMaterializationTask::name)
1264 .collect::<Vec<_>>(),
1265 tasks
1266 .iter()
1267 .map(ReplicatedTextMaterializationTask::name)
1268 .collect::<Vec<_>>()
1269 ),
1270 ));
1271 }
1272 let partition_state = partition.state().cloned();
1273 let (selected_state, prompt_cache_identity) = match partition_state.as_ref() {
1274 Some(partition_state) => (
1275 SessionStateRealization::Stateful(
1276 selected
1277 .state()
1278 .for_partitioned_geometry(partition_state)
1279 .map_err(|error| {
1280 PartitionedSessionPreparationError::Contract(error.to_string())
1281 })?,
1282 ),
1283 Some(
1284 partition_state
1285 .prompt_cache_identity::<B, A>(&architecture, topology)
1286 .map_err(|error| {
1287 PartitionedSessionPreparationError::Contract(error.to_string())
1288 })?,
1289 ),
1290 ),
1291 None => (SessionStateRealization::Stateless, None),
1292 };
1293 let (runtime, state) = factory(
1294 PartitionedSessionFactoryInput {
1295 architecture,
1296 partition,
1297 communication,
1298 tasks,
1299 },
1300 &selected,
1301 context,
1302 )
1303 .map_err(PartitionedSessionPreparationError::Factory)?;
1304 match selected_state.state() {
1305 Some(local) if state.optional_layout() != Some(local.layout()) => {
1306 return Err(PartitionedSessionPreparationError::Contract(
1307 "partition runtime state differs from canonical local geometry".into(),
1308 ));
1309 }
1310 None if state.optional_layout().is_some() => {
1311 return Err(PartitionedSessionPreparationError::Contract(
1312 "stateless partition binding contains mutable state geometry".into(),
1313 ));
1314 }
1315 _ => {}
1316 }
1317 Ok(PreparedPartitionedSessionRuntime {
1318 selected,
1319 runtime,
1320 state,
1321 selected_state,
1322 prompt_cache_identity,
1323 output_selection,
1324 })
1325}
1326
1327impl SessionStateRealization {
1328 pub const fn state(&self) -> Option<&SelectedStateRealization> {
1330 match self {
1331 Self::Stateful(state) => Some(state),
1332 Self::Stateless => None,
1333 }
1334 }
1335}
1336
1337pub struct ReplicatedTextSessionCheckpoint<C> {
1339 state: C,
1340 prompt_input_identity: Option<PreparedInputCacheIdentity>,
1341 next_commit_epoch: DistributedCommitEpoch,
1342 last_commit_outcome: Option<DistributedCommitOutcome>,
1343}
1344
1345pub struct DistributedStateCheckpoint<C> {
1347 state: Option<C>,
1348}
1349
1350pub struct DistributedSessionCheckpoint<C> {
1352 state: Option<C>,
1353 prompt_input_identity: Option<PreparedInputCacheIdentity>,
1354 next_commit_epoch: DistributedCommitEpoch,
1355 last_commit_outcome: Option<DistributedCommitOutcome>,
1356}
1357
1358#[derive(Debug, thiserror::Error)]
1360pub enum ReplicatedTextSessionError<A, P, M>
1361where
1362 A: std::fmt::Display,
1363 P: std::fmt::Display,
1364 M: std::fmt::Display,
1365{
1366 #[error("replicated text contract mismatch: {0}")]
1368 Contract(String),
1369 #[error("replicated text architecture failed: {0}")]
1371 Architecture(A),
1372 #[error("replicated text residency failed: {0}")]
1374 Policy(P),
1375 #[error("replicated text mechanism failed: {0}")]
1377 Mechanism(M),
1378 #[error(transparent)]
1380 State(#[from] StateError),
1381 #[error(transparent)]
1383 PromptCache(#[from] PromptCacheError),
1384 #[error("distributed transaction epoch {epoch:?} was aborted")]
1386 CommitAborted {
1387 epoch: DistributedCommitEpoch,
1389 },
1390 #[error("distributed transaction epoch {epoch:?} is indeterminate at {phase:?}")]
1392 CommitIndeterminate {
1393 epoch: DistributedCommitEpoch,
1395 phase: DistributedCommitPhase,
1397 },
1398}
1399
1400#[derive(Debug, Clone, Eq, PartialEq)]
1402pub struct ReplicatedTextSessionReport<E, S> {
1403 execution: ExecutionResidency,
1404 execution_report: E,
1405 state_report: S,
1406 distributed_commit: Option<DistributedCommitOutcome>,
1407}
1408
1409pub struct PreparedReplicatedTextContract {
1414 selected: SelectedReplicatedTextRealization,
1415 tasks: Vec<ReplicatedTextMaterializationTask>,
1416 addressable_parameters: Vec<String>,
1417 prompt_cache_identity: PromptCacheModelIdentity,
1418 output_selection: ReplicatedTextOutputSelection,
1419}
1420
1421impl PreparedReplicatedTextContract {
1422 pub const fn selected(&self) -> &SelectedReplicatedTextRealization {
1424 &self.selected
1425 }
1426
1427 pub fn materialization_tasks(&self) -> &[ReplicatedTextMaterializationTask] {
1429 &self.tasks
1430 }
1431
1432 pub const fn prompt_cache_identity(&self) -> &PromptCacheModelIdentity {
1434 &self.prompt_cache_identity
1435 }
1436
1437 pub const fn output_selection(&self) -> ReplicatedTextOutputSelection {
1439 self.output_selection
1440 }
1441
1442 fn into_parts(
1443 self,
1444 ) -> (
1445 SelectedReplicatedTextRealization,
1446 Vec<ReplicatedTextMaterializationTask>,
1447 Vec<String>,
1448 PromptCacheModelIdentity,
1449 ReplicatedTextOutputSelection,
1450 ) {
1451 (
1452 self.selected,
1453 self.tasks,
1454 self.addressable_parameters,
1455 self.prompt_cache_identity,
1456 self.output_selection,
1457 )
1458 }
1459}
1460
1461pub fn prepare_replicated_text_contract<A, B, S>(
1464 architecture: &A,
1465 source_architecture: Option<&A>,
1466 selected: SelectedReplicatedTextRealization,
1467 expected_prompt_cache_architecture_identity: &str,
1468 context: &<B::Tensor as Tensor>::Context,
1469) -> Result<PreparedReplicatedTextContract, String>
1470where
1471 B: NeuralBackend,
1472 S: RuntimeState<B>,
1473 A: ReplicatedTextArchitecture<B, S>,
1474 A::Error: std::fmt::Display,
1475{
1476 prepare_replicated_text_contract_with_addressable_parameters::<A, B, S>(
1477 architecture,
1478 source_architecture,
1479 selected,
1480 expected_prompt_cache_architecture_identity,
1481 std::iter::empty::<&str>(),
1482 context,
1483 )
1484}
1485
1486pub fn prepare_replicated_text_contract_with_addressable_parameters<'a, A, B, S>(
1493 architecture: &A,
1494 source_architecture: Option<&A>,
1495 selected: SelectedReplicatedTextRealization,
1496 expected_prompt_cache_architecture_identity: &str,
1497 addressable_parameters: impl IntoIterator<Item = &'a str>,
1498 context: &<B::Tensor as Tensor>::Context,
1499) -> Result<PreparedReplicatedTextContract, String>
1500where
1501 B: NeuralBackend,
1502 S: RuntimeState<B>,
1503 A: ReplicatedTextArchitecture<B, S>,
1504 A::Error: std::fmt::Display,
1505{
1506 prepare_layered_text_contract_with_addressable_parameters::<A, B, S>(
1507 architecture,
1508 source_architecture,
1509 selected,
1510 expected_prompt_cache_architecture_identity,
1511 architecture.text_output_selection(),
1512 addressable_parameters,
1513 context,
1514 )
1515}
1516
1517pub fn prepare_layered_text_contract<A, B, S>(
1523 architecture: &A,
1524 source_architecture: Option<&A>,
1525 selected: SelectedReplicatedTextRealization,
1526 expected_prompt_cache_architecture_identity: &str,
1527 output_selection: ReplicatedTextOutputSelection,
1528 context: &<B::Tensor as Tensor>::Context,
1529) -> Result<PreparedReplicatedTextContract, String>
1530where
1531 B: NeuralBackend,
1532 S: RuntimeState<B>,
1533 A: LayeredArchitecture<B, S>,
1534 A::Error: std::fmt::Display,
1535{
1536 prepare_layered_text_contract_with_addressable_parameters::<A, B, S>(
1537 architecture,
1538 source_architecture,
1539 selected,
1540 expected_prompt_cache_architecture_identity,
1541 output_selection,
1542 std::iter::empty::<&str>(),
1543 context,
1544 )
1545}
1546
1547pub fn prepare_layered_text_contract_with_addressable_parameters<'a, A, B, S>(
1549 architecture: &A,
1550 source_architecture: Option<&A>,
1551 selected: SelectedReplicatedTextRealization,
1552 expected_prompt_cache_architecture_identity: &str,
1553 output_selection: ReplicatedTextOutputSelection,
1554 addressable_parameters: impl IntoIterator<Item = &'a str>,
1555 context: &<B::Tensor as Tensor>::Context,
1556) -> Result<PreparedReplicatedTextContract, String>
1557where
1558 B: NeuralBackend,
1559 S: RuntimeState<B>,
1560 A: LayeredArchitecture<B, S>,
1561 A::Error: std::fmt::Display,
1562{
1563 let mut addressable_parameters = addressable_parameters
1564 .into_iter()
1565 .map(str::to_owned)
1566 .collect::<BTreeSet<_>>();
1567 validate_selected_state(&selected)?;
1568 validate_architecture_geometry::<A, B, S>(architecture, &selected)?;
1569 if let Some(source) = source_architecture {
1570 validate_architecture_geometry::<A, B, S>(source, &selected)?;
1571 }
1572 let has_transform = selected.parameters().iter().any(|parameter| {
1573 matches!(
1574 parameter.lowering(),
1575 WeightLoweringKind::Transform | WeightLoweringKind::DerivedTransform
1576 )
1577 });
1578 if has_transform != source_architecture.is_some() {
1579 return Err(
1580 "selected transform tasks and source-format architecture ownership disagree".into(),
1581 );
1582 }
1583 if let Some(source) = source_architecture {
1584 validate_architecture_parameters::<A, B, S>(source, &selected, false, context)?;
1585 }
1586 let mut constructed_companions =
1587 validate_architecture_parameters::<A, B, S>(architecture, &selected, true, context)?;
1588 let mut tasks =
1589 replicated_text_materialization_tasks(&selected).map_err(|error| error.to_string())?;
1590 let selected_parameter_names = tasks
1591 .iter()
1592 .flat_map(|task| {
1593 std::iter::once(task.name().to_owned()).chain(
1594 task.output_companions()
1595 .iter()
1596 .map(|companion| companion.name().to_owned()),
1597 )
1598 })
1599 .collect::<BTreeSet<_>>();
1600 if !addressable_parameters.is_subset(&selected_parameter_names) {
1601 return Err(format!(
1602 "addressable parameter catalog contains unknown selected parameters: {:?}",
1603 addressable_parameters
1604 .difference(&selected_parameter_names)
1605 .collect::<Vec<_>>()
1606 ));
1607 }
1608 let addressable_companions = addressable_parameters
1609 .iter()
1610 .filter_map(|name| tasks.iter().find(|task| task.name() == name))
1611 .flat_map(|task| task.output_companions())
1612 .map(|companion| companion.name().to_owned())
1613 .collect::<Vec<_>>();
1614 addressable_parameters.extend(addressable_companions);
1615 for task in &tasks {
1616 let mut actual = constructed_companions
1617 .remove(task.name())
1618 .unwrap_or_default();
1619 actual.sort_by(|left, right| {
1620 left.role()
1621 .cmp(&right.role())
1622 .then_with(|| left.name().cmp(right.name()))
1623 });
1624 let expected = task.output_companions();
1625 let agrees = actual.len() == expected.len()
1626 && actual.iter().zip(expected).all(|(actual, expected)| {
1627 actual.name() == expected.name()
1628 && actual.role() == expected.role()
1629 && actual.logical_shape() == expected.logical_shape()
1630 && (actual.owner() == expected.owner()
1631 || matches!(
1632 (actual.owner(), expected.owner()),
1633 (
1634 ParameterGroupOwner::StaticAnyOf(actual_roles),
1635 ParameterGroupOwner::StaticRole(expected_role)
1636 ) if actual_roles.iter().any(|role| role == expected_role)
1637 ))
1638 });
1639 if !agrees {
1640 return Err(format!(
1641 "constructed output companions for {:?} differ from authoritative selection: lowering={:?}, executable={:?}, constructed={:?}, selected={:?}, retained={:?}",
1642 task.name(),
1643 task.lowering(),
1644 task.executable(),
1645 actual
1646 .iter()
1647 .map(|companion| (companion.name(), companion.role(), companion.logical_shape(), companion.owner()))
1648 .collect::<Vec<_>>(),
1649 expected
1650 .iter()
1651 .map(|companion| (companion.name(), companion.role(), companion.logical_shape(), companion.owner()))
1652 .collect::<Vec<_>>(),
1653 selected.requirements().parameters().iter().filter_map(|parameter| {
1654 parameter.linear_companion().filter(|(_, primary)| *primary == task.name()).map(|(role, primary)| (parameter.name(), role, primary))
1655 }).collect::<Vec<_>>(),
1656 ));
1657 }
1658 }
1659 if !constructed_companions.is_empty() {
1660 return Err(format!(
1661 "output companion catalog contains unknown materialization tasks: {:?}",
1662 constructed_companions.keys().collect::<Vec<_>>()
1663 ));
1664 }
1665 tasks.retain(|task| !addressable_parameters.contains(task.name()));
1666 let state = PartitionState::new(selected.state().layout().clone(), 0)
1667 .map_err(|error| error.to_string())?;
1668 let prompt_cache_identity = state
1669 .prompt_cache_identity::<B, A>(architecture, Default::default())
1670 .map_err(|error| error.to_string())?;
1671 if prompt_cache_identity.architecture_fingerprint()
1672 != expected_prompt_cache_architecture_identity
1673 || prompt_cache_identity.layer_count() != selected.state().layout().len()
1674 || prompt_cache_identity.global_layer_start() != 0
1675 || prompt_cache_identity.global_layer_end() != selected.state().layout().len()
1676 || prompt_cache_identity.topology() != &Default::default()
1677 {
1678 return Err("architecture prompt-cache identity differs from selection".into());
1679 }
1680 Ok(PreparedReplicatedTextContract {
1681 selected,
1682 tasks,
1683 addressable_parameters: addressable_parameters.into_iter().collect(),
1684 prompt_cache_identity,
1685 output_selection,
1686 })
1687}
1688
1689impl<E, S> ReplicatedTextSessionReport<E, S> {
1690 pub const fn execution(&self) -> ExecutionResidency {
1692 self.execution
1693 }
1694
1695 pub const fn execution_report(&self) -> &E {
1697 &self.execution_report
1698 }
1699
1700 pub const fn state_report(&self) -> &S {
1702 &self.state_report
1703 }
1704
1705 pub const fn distributed_commit(&self) -> Option<DistributedCommitOutcome> {
1707 self.distributed_commit
1708 }
1709}
1710
1711pub fn construct_replicated_text_session<A, B, M>(
1714 architecture: A,
1715 source_architecture: Option<A>,
1716 prepared: PreparedReplicatedTextContract,
1717 mechanisms: M,
1718 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
1719) -> Result<
1720 ReplicatedTextSession<A, B, M>,
1721 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
1722>
1723where
1724 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
1725 M: ReplicatedTextSessionMechanisms<A, B>,
1726 A: LayeredArchitecture<B, M::State>,
1727 A::Error: std::fmt::Display,
1728 M::PolicyError: std::fmt::Display,
1729 M::Error: std::fmt::Display,
1730{
1731 construct_replicated_text_session_with_execution(
1732 architecture,
1733 source_architecture,
1734 prepared,
1735 mechanisms,
1736 DirectReplicatedTextExecution,
1737 context,
1738 )
1739}
1740
1741pub fn construct_replicated_text_session_with_execution<A, B, M, D>(
1747 mut architecture: A,
1748 mut source_architecture: Option<A>,
1749 prepared: PreparedReplicatedTextContract,
1750 mut mechanisms: M,
1751 driver: D,
1752 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
1753) -> Result<
1754 ReplicatedTextSession<A, B, M, D>,
1755 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
1756>
1757where
1758 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
1759 M: ReplicatedTextSessionMechanisms<A, B>,
1760 A: LayeredArchitecture<B, M::State>,
1761 D: ReplicatedRuntimeExecutionStrategy<A, B, M::State, M::ResidentPolicy, M::BoundedPolicy>,
1762 A::Error: std::fmt::Display,
1763 M::PolicyError: std::fmt::Display,
1764 M::Error: std::fmt::Display,
1765{
1766 let (selected, tasks, addressable_parameters, prompt_cache_identity, output_selection) =
1767 prepared.into_parts();
1768 let mut units = construct_units::<A, B, M::State>(
1769 &architecture,
1770 selected.requirements().execution_units(),
1771 context,
1772 )
1773 .map_err(ReplicatedTextSessionError::Architecture)?;
1774 let mut source_units = source_architecture
1775 .as_ref()
1776 .map(|source| {
1777 construct_units::<A, B, M::State>(
1778 source,
1779 selected.requirements().execution_units(),
1780 context,
1781 )
1782 })
1783 .transpose()
1784 .map_err(ReplicatedTextSessionError::Architecture)?;
1785 mechanisms
1786 .prepare_materialization(
1787 &mut architecture,
1788 selected.requirements().execution_units(),
1789 &mut units,
1790 source_architecture.as_mut(),
1791 source_units.as_deref_mut(),
1792 &tasks,
1793 &addressable_parameters,
1794 context,
1795 )
1796 .map_err(ReplicatedTextSessionError::Mechanism)?;
1797 let materialization_report = mechanisms
1798 .take_materialization_report()
1799 .map_err(ReplicatedTextSessionError::Mechanism)?;
1800 let state = mechanisms
1801 .realize_state(selected.state(), context)
1802 .map_err(ReplicatedTextSessionError::Mechanism)?;
1803 validate_realized_state(&state, selected.state())?;
1804 let selected_state = SessionStateRealization::Stateful(selected.state().clone());
1805 let execution = match selected.residency() {
1806 LayerWeightResidency::FullyResident => {
1807 let policy = mechanisms
1808 .resident_policy(&mut architecture, units, &selected, context)
1809 .map_err(ReplicatedTextSessionError::Mechanism)?;
1810 ReplicatedTextRuntime {
1811 kind: ReplicatedTextRuntimeKind::Resident(LayerwiseRuntime::new(
1812 architecture,
1813 policy,
1814 )),
1815 }
1816 }
1817 LayerWeightResidency::LayerwiseHost(_) | LayerWeightResidency::DenseDiskStream(_) => {
1818 let policy = mechanisms
1819 .bounded_policy(&mut architecture, &selected, context)
1820 .map_err(ReplicatedTextSessionError::Mechanism)?;
1821 ReplicatedTextRuntime {
1822 kind: ReplicatedTextRuntimeKind::Bounded(LayerwiseRuntime::new(
1823 architecture,
1824 policy,
1825 )),
1826 }
1827 }
1828 };
1829 Ok(ReplicatedTextSession {
1830 selected,
1831 selected_state,
1832 execution,
1833 driver,
1834 state,
1835 mechanisms,
1836 materialization_report,
1837 prompt_cache_identity: Some(prompt_cache_identity),
1838 committed_prompt_input_identity: None,
1839 next_commit_epoch: DistributedCommitEpoch::FIRST,
1840 active_commit_epoch: None,
1841 last_commit_outcome: None,
1842 successful_state_restorations: Some(0),
1843 control_identity: std::sync::Arc::new(()),
1844 control_fence: None,
1845 output_selection,
1846 backend: PhantomData,
1847 })
1848}
1849
1850pub fn construct_replicated_text_session_with_runtime<A, B, M, D>(
1857 binding: PreparedPartitionedSessionRuntime<D::Runtime, M::State>,
1858 mut mechanisms: M,
1859 driver: D,
1860) -> Result<
1861 ReplicatedTextSession<A, B, M, D>,
1862 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
1863>
1864where
1865 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
1866 M: ReplicatedTextSessionMechanisms<A, B>,
1867 A: LayeredArchitecture<B, M::State>,
1868 D: ReplicatedTextExecutionStrategy<A, B, M::State, M::ResidentPolicy, M::BoundedPolicy>,
1869 A::Error: std::fmt::Display,
1870 M::PolicyError: std::fmt::Display,
1871 M::Error: std::fmt::Display,
1872{
1873 let PreparedPartitionedSessionRuntime {
1874 selected,
1875 runtime,
1876 state,
1877 selected_state,
1878 prompt_cache_identity,
1879 output_selection,
1880 } = binding;
1881 match selected_state.state() {
1882 Some(local) => {
1883 validate_realized_state(&state, local)?;
1884 let identity = prompt_cache_identity.as_ref().ok_or_else(|| {
1885 ReplicatedTextSessionError::Contract(
1886 "stateful partition is missing its rank-local cache identity".into(),
1887 )
1888 })?;
1889 let local_layers = identity
1890 .global_layer_end()
1891 .checked_sub(identity.global_layer_start());
1892 if identity.layer_count() != selected.state().layout().len()
1893 || local_layers != Some(local.layout().len())
1894 || identity.global_layer_end() > identity.layer_count()
1895 {
1896 return Err(ReplicatedTextSessionError::Contract(
1897 "rank-local cache identity differs from selected state geometry".into(),
1898 ));
1899 }
1900 let partition =
1901 PartitionState::new(local.layout().clone(), identity.global_layer_start())
1902 .map_err(|error| ReplicatedTextSessionError::Contract(error.to_string()))?;
1903 let expected = selected
1904 .state()
1905 .for_partitioned_geometry(&partition)
1906 .map_err(|error| ReplicatedTextSessionError::Contract(error.to_string()))?;
1907 if &expected != local {
1908 return Err(ReplicatedTextSessionError::Contract(
1909 "rank-local state realization is not the selected global interval".into(),
1910 ));
1911 }
1912 }
1913 None => {
1914 if prompt_cache_identity.is_some() || state.optional_layout().is_some() {
1915 return Err(ReplicatedTextSessionError::Contract(
1916 "stateless partition owns state or a prompt-cache shard identity".into(),
1917 ));
1918 }
1919 }
1920 }
1921 let materialization_report = mechanisms
1922 .take_materialization_report()
1923 .map_err(ReplicatedTextSessionError::Mechanism)?;
1924 Ok(ReplicatedTextSession {
1925 selected,
1926 selected_state,
1927 execution: runtime,
1928 driver,
1929 state,
1930 mechanisms,
1931 materialization_report,
1932 prompt_cache_identity,
1933 committed_prompt_input_identity: None,
1934 next_commit_epoch: DistributedCommitEpoch::FIRST,
1935 active_commit_epoch: None,
1936 last_commit_outcome: None,
1937 successful_state_restorations: Some(0),
1938 control_identity: std::sync::Arc::new(()),
1939 control_fence: None,
1940 output_selection,
1941 backend: PhantomData,
1942 })
1943}
1944
1945fn construct_units<A, B, S>(
1946 architecture: &A,
1947 layout: &crate::ExecutionUnitLayout,
1948 context: &<B::Tensor as Tensor>::Context,
1949) -> Result<Vec<A::Unit>, A::Error>
1950where
1951 B: NeuralBackend,
1952 S: RuntimeState<B>,
1953 A: LayeredArchitecture<B, S>,
1954{
1955 (0..layout.len())
1956 .map(|ordinal| {
1957 let address = layout
1958 .address(ordinal)
1959 .expect("validated replicated layout contains every ordinal");
1960 architecture.build_unit(address.group(), address.index(), context)
1961 })
1962 .collect()
1963}
1964
1965impl<A, B, M, D> ReplicatedTextSession<A, B, M, D>
1966where
1967 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
1968 M: ReplicatedTextSessionMechanisms<A, B>,
1969 A: LayeredArchitecture<B, M::State>,
1970 D: ReplicatedTextExecutionStrategy<A, B, M::State, M::ResidentPolicy, M::BoundedPolicy>,
1971 A::Error: std::fmt::Display,
1972 M::PolicyError: std::fmt::Display,
1973 M::Error: std::fmt::Display,
1974{
1975 pub const fn materialization_report(&self) -> Option<&crate::WeightMaterializationReport> {
1978 self.materialization_report.as_ref()
1979 }
1980
1981 pub const fn execution_strategy(&self) -> &D {
1983 &self.driver
1984 }
1985
1986 pub fn forward(
1988 &mut self,
1989 tokens: &B::Tensor,
1990 mask: Option<&B::Tensor>,
1991 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
1992 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
1993 where
1994 A: ReplicatedTextArchitecture<B, M::State>,
1995 {
1996 self.forward_with_observer(tokens, mask, context, &mut crate::NoopObserver)
1997 }
1998
1999 pub fn forward_with_observer<O>(
2001 &mut self,
2002 tokens: &B::Tensor,
2003 mask: Option<&B::Tensor>,
2004 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2005 observer: &mut O,
2006 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2007 where
2008 A: ReplicatedTextArchitecture<B, M::State>,
2009 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
2010 {
2011 let pass = tokens
2012 .shape()
2013 .last()
2014 .copied()
2015 .filter(|length| *length > 1)
2016 .map_or(ExpertPass::Decode, |_| ExpertPass::Prefill);
2017 let (output, checkpoint, forward_context) =
2018 self.execute_with_observer(tokens, mask, pass, context, observer)?;
2019 self.publish(output, checkpoint, forward_context, context)
2020 }
2021
2022 pub fn prefill(
2024 &mut self,
2025 tokens: &B::Tensor,
2026 mask: Option<&B::Tensor>,
2027 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2028 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2029 where
2030 A: ReplicatedTextArchitecture<B, M::State>,
2031 {
2032 self.prefill_with_observer(tokens, mask, context, &mut crate::NoopObserver)
2033 }
2034
2035 pub fn prefill_with_observer<O>(
2037 &mut self,
2038 tokens: &B::Tensor,
2039 mask: Option<&B::Tensor>,
2040 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2041 observer: &mut O,
2042 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2043 where
2044 A: ReplicatedTextArchitecture<B, M::State>,
2045 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
2046 {
2047 let input = A::text_input(tokens, mask);
2048 self.prefill_input_with_observer(input, context, observer)
2049 }
2050
2051 pub fn prefill_prediction_target(
2057 &mut self,
2058 tokens: &B::Tensor,
2059 mask: Option<&B::Tensor>,
2060 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2061 ) -> Result<
2062 (B::Tensor, B::Tensor),
2063 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2064 >
2065 where
2066 A: ReplicatedTextArchitecture<B, M::State>,
2067 {
2068 let input = A::text_input(tokens, mask);
2069 self.prefill_input_prediction_target(input, context)
2070 }
2071
2072 pub fn prefill_input_prediction_target<'a>(
2074 &mut self,
2075 input: A::Input<'a>,
2076 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2077 ) -> Result<
2078 (B::Tensor, B::Tensor),
2079 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2080 > {
2081 let (output, checkpoint, forward_context) = self.execute_input_before_publication(
2082 input,
2083 ExpertPass::Prefill,
2084 context,
2085 &mut crate::NoopObserver,
2086 )?;
2087 let capture = D::prediction_target_capture(&mut self.execution, &forward_context, context)
2088 .map_err(widen_infallible);
2089 let local_success = matches!(&capture, Ok(Some(_)));
2090 let agreed = match D::agree_distributed_phase(
2091 &mut self.execution,
2092 crate::DistributedExecutionPhase::PredictionTargetCapture,
2093 local_success,
2094 context,
2095 ) {
2096 Ok(agreed) => agreed,
2097 Err(error) => {
2098 return self.rollback_failure(checkpoint, widen_infallible(error), context)
2099 }
2100 };
2101 let capture = match capture {
2102 Ok(Some(capture)) if agreed => capture,
2103 Ok(Some(_)) => {
2104 return self.rollback_failure(
2105 checkpoint,
2106 ReplicatedTextSessionError::Contract(
2107 "another rank could not prepare the prediction target capture".into(),
2108 ),
2109 context,
2110 )
2111 }
2112 Ok(None) => {
2113 return self.rollback_failure(
2114 checkpoint,
2115 ReplicatedTextSessionError::Contract(
2116 "prediction target pass did not retain its declared hidden capture".into(),
2117 ),
2118 context,
2119 )
2120 }
2121 Err(error) => return self.rollback_failure(checkpoint, error, context),
2122 };
2123 let capture_publication =
2124 D::publish_prediction_target_capture(&mut self.execution, capture, context);
2125 let capture_publication_agreed = match D::agree_distributed_phase(
2126 &mut self.execution,
2127 crate::DistributedExecutionPhase::PredictionTargetCapturePublication,
2128 capture_publication.is_ok(),
2129 context,
2130 ) {
2131 Ok(agreed) => agreed,
2132 Err(error) => {
2133 return self.rollback_failure(checkpoint, widen_infallible(error), context)
2134 }
2135 };
2136 let capture = match capture_publication {
2137 Ok(capture) if capture_publication_agreed => capture,
2138 Ok(_) => {
2139 return self.rollback_failure(
2140 checkpoint,
2141 ReplicatedTextSessionError::Contract(
2142 "another rank failed to publish the prediction target capture".into(),
2143 ),
2144 context,
2145 )
2146 }
2147 Err(error) => {
2148 return self.rollback_failure(checkpoint, widen_infallible(error), context)
2149 }
2150 };
2151 let (output, checkpoint, forward_context) =
2152 self.publish_observed_output_transaction(output, checkpoint, forward_context, context)?;
2153 self.publish(output, checkpoint, forward_context, context)
2154 .map(|output| (output, capture))
2155 }
2156
2157 pub fn prefill_input<'a>(
2164 &mut self,
2165 input: A::Input<'a>,
2166 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2167 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2168 self.prefill_input_with_observer(input, context, &mut crate::NoopObserver)
2169 }
2170
2171 pub fn prefill_input_with_capture<'a, O, C, F>(
2178 &mut self,
2179 input: A::Input<'a>,
2180 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2181 observer: &mut O,
2182 capture: F,
2183 ) -> Result<(B::Tensor, C), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2184 where
2185 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
2186 F: FnOnce(&A::ForwardContext) -> Result<C, A::Error>,
2187 {
2188 if D::PARTITIONED_SESSION {
2189 return Err(ReplicatedTextSessionError::Contract(
2190 "partitioned prediction capture requires a selected bundle publication contract"
2191 .into(),
2192 ));
2193 }
2194 let (output, checkpoint, forward_context) =
2195 self.execute_input_before_publication(input, ExpertPass::Prefill, context, observer)?;
2196 let captured = match capture(&forward_context) {
2197 Ok(captured) => captured,
2198 Err(error) => {
2199 return self.rollback_failure(
2200 checkpoint,
2201 ReplicatedTextSessionError::Architecture(error),
2202 context,
2203 )
2204 }
2205 };
2206 let (output, checkpoint, forward_context) =
2207 self.publish_observed_output_transaction(output, checkpoint, forward_context, context)?;
2208 self.publish(output, checkpoint, forward_context, context)
2209 .map(|output| (output, captured))
2210 }
2211
2212 pub fn prefill_input_with_cache_identity<'a>(
2215 &mut self,
2216 input: A::Input<'a>,
2217 identity: PreparedInputCacheIdentity,
2218 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2219 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2220 self.prefill_input_with_observer_and_cache_identity(
2221 input,
2222 identity,
2223 context,
2224 &mut crate::NoopObserver,
2225 )
2226 }
2227
2228 pub fn prefill_input_with_observer_and_cache_identity<'a, O>(
2230 &mut self,
2231 input: A::Input<'a>,
2232 identity: PreparedInputCacheIdentity,
2233 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2234 observer: &mut O,
2235 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2236 where
2237 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
2238 {
2239 let output = self.prefill_input_with_observer(input, context, observer)?;
2240 self.committed_prompt_input_identity = Some(identity);
2241 Ok(output)
2242 }
2243
2244 pub fn prefill_input_with_observer<'a, O>(
2246 &mut self,
2247 input: A::Input<'a>,
2248 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2249 observer: &mut O,
2250 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2251 where
2252 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
2253 {
2254 let (output, checkpoint, forward_context) =
2255 self.execute_input_with_observer(input, ExpertPass::Prefill, context, observer)?;
2256 let sequence_index = self.output_selection.sequence_index();
2257 let output = match self
2258 .mechanisms
2259 .index_text_output(output, sequence_index, context)
2260 {
2261 Ok(output) => output,
2262 Err(error) => {
2263 return self.rollback_failure(
2264 checkpoint,
2265 ReplicatedTextSessionError::Mechanism(error),
2266 context,
2267 )
2268 }
2269 };
2270 self.publish(output, checkpoint, forward_context, context)
2271 }
2272
2273 pub fn decode_input<'a>(
2275 &mut self,
2276 input: A::Input<'a>,
2277 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2278 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2279 self.decode_input_with_observer(input, context, &mut crate::NoopObserver)
2280 }
2281
2282 pub fn decode_input_with_capture<'a, O, C, F>(
2284 &mut self,
2285 input: A::Input<'a>,
2286 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2287 observer: &mut O,
2288 capture: F,
2289 ) -> Result<(B::Tensor, C), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2290 where
2291 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
2292 F: FnOnce(&A::ForwardContext) -> Result<C, A::Error>,
2293 {
2294 if D::PARTITIONED_SESSION {
2295 return Err(ReplicatedTextSessionError::Contract(
2296 "partitioned prediction capture requires a selected bundle publication contract"
2297 .into(),
2298 ));
2299 }
2300 let (output, checkpoint, forward_context) =
2301 self.execute_input_before_publication(input, ExpertPass::Decode, context, observer)?;
2302 let captured = match capture(&forward_context) {
2303 Ok(captured) => captured,
2304 Err(error) => {
2305 return self.rollback_failure(
2306 checkpoint,
2307 ReplicatedTextSessionError::Architecture(error),
2308 context,
2309 )
2310 }
2311 };
2312 let (output, checkpoint, forward_context) =
2313 self.publish_observed_output_transaction(output, checkpoint, forward_context, context)?;
2314 self.publish(output, checkpoint, forward_context, context)
2315 .map(|output| (output, captured))
2316 }
2317
2318 pub fn decode_input_with_observer<'a, O>(
2320 &mut self,
2321 input: A::Input<'a>,
2322 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2323 observer: &mut O,
2324 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2325 where
2326 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
2327 {
2328 let (output, checkpoint, forward_context) =
2329 self.execute_input_with_observer(input, ExpertPass::Decode, context, observer)?;
2330 let sequence_index = self.output_selection.sequence_index();
2331 let output = match self
2332 .mechanisms
2333 .index_text_output(output, sequence_index, context)
2334 {
2335 Ok(output) => output,
2336 Err(error) => {
2337 return self.rollback_failure(
2338 checkpoint,
2339 ReplicatedTextSessionError::Mechanism(error),
2340 context,
2341 )
2342 }
2343 };
2344 self.publish(output, checkpoint, forward_context, context)
2345 }
2346
2347 pub fn decode(
2349 &mut self,
2350 tokens: &B::Tensor,
2351 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2352 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2353 where
2354 A: ReplicatedTextArchitecture<B, M::State>,
2355 {
2356 self.decode_with_observer(tokens, context, &mut crate::NoopObserver)
2357 }
2358
2359 pub fn decode_prediction_target(
2361 &mut self,
2362 tokens: &B::Tensor,
2363 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2364 ) -> Result<
2365 (B::Tensor, B::Tensor),
2366 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2367 >
2368 where
2369 A: ReplicatedTextArchitecture<B, M::State>,
2370 {
2371 let input = A::text_input(tokens, None);
2372 self.decode_input_prediction_target(input, context)
2373 }
2374
2375 pub fn decode_input_prediction_target<'a>(
2377 &mut self,
2378 input: A::Input<'a>,
2379 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2380 ) -> Result<
2381 (B::Tensor, B::Tensor),
2382 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2383 > {
2384 let (output, checkpoint, forward_context) = self.execute_input_before_publication(
2385 input,
2386 ExpertPass::Decode,
2387 context,
2388 &mut crate::NoopObserver,
2389 )?;
2390 let capture = D::prediction_target_capture(&mut self.execution, &forward_context, context)
2391 .map_err(widen_infallible);
2392 let local_success = matches!(&capture, Ok(Some(_)));
2393 let agreed = match D::agree_distributed_phase(
2394 &mut self.execution,
2395 crate::DistributedExecutionPhase::PredictionTargetCapture,
2396 local_success,
2397 context,
2398 ) {
2399 Ok(agreed) => agreed,
2400 Err(error) => {
2401 return self.rollback_failure(checkpoint, widen_infallible(error), context)
2402 }
2403 };
2404 let capture = match capture {
2405 Ok(Some(capture)) if agreed => capture,
2406 Ok(Some(_)) => {
2407 return self.rollback_failure(
2408 checkpoint,
2409 ReplicatedTextSessionError::Contract(
2410 "another rank could not prepare the prediction target capture".into(),
2411 ),
2412 context,
2413 )
2414 }
2415 Ok(None) => {
2416 return self.rollback_failure(
2417 checkpoint,
2418 ReplicatedTextSessionError::Contract(
2419 "prediction target pass did not retain its declared hidden capture".into(),
2420 ),
2421 context,
2422 )
2423 }
2424 Err(error) => return self.rollback_failure(checkpoint, error, context),
2425 };
2426 let capture_publication =
2427 D::publish_prediction_target_capture(&mut self.execution, capture, context);
2428 let capture_publication_agreed = match D::agree_distributed_phase(
2429 &mut self.execution,
2430 crate::DistributedExecutionPhase::PredictionTargetCapturePublication,
2431 capture_publication.is_ok(),
2432 context,
2433 ) {
2434 Ok(agreed) => agreed,
2435 Err(error) => {
2436 return self.rollback_failure(checkpoint, widen_infallible(error), context)
2437 }
2438 };
2439 let capture = match capture_publication {
2440 Ok(capture) if capture_publication_agreed => capture,
2441 Ok(_) => {
2442 return self.rollback_failure(
2443 checkpoint,
2444 ReplicatedTextSessionError::Contract(
2445 "another rank failed to publish the prediction target capture".into(),
2446 ),
2447 context,
2448 )
2449 }
2450 Err(error) => {
2451 return self.rollback_failure(checkpoint, widen_infallible(error), context)
2452 }
2453 };
2454 let (output, checkpoint, forward_context) =
2455 self.publish_observed_output_transaction(output, checkpoint, forward_context, context)?;
2456 self.publish(output, checkpoint, forward_context, context)
2457 .map(|output| (output, capture))
2458 }
2459
2460 pub fn decode_with_observer<O>(
2462 &mut self,
2463 tokens: &B::Tensor,
2464 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2465 observer: &mut O,
2466 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2467 where
2468 A: ReplicatedTextArchitecture<B, M::State>,
2469 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
2470 {
2471 let (output, checkpoint, forward_context) =
2472 self.execute_with_observer(tokens, None, ExpertPass::Decode, context, observer)?;
2473 let sequence_index = self.output_selection.sequence_index();
2474 let output = match self
2475 .mechanisms
2476 .index_text_output(output, sequence_index, context)
2477 {
2478 Ok(output) => output,
2479 Err(error) => {
2480 return self.rollback_failure(
2481 checkpoint,
2482 ReplicatedTextSessionError::Mechanism(error),
2483 context,
2484 )
2485 }
2486 };
2487 self.publish(output, checkpoint, forward_context, context)
2488 }
2489
2490 pub const fn successful_state_restoration_generation(&self) -> Option<u64> {
2496 self.successful_state_restorations
2497 }
2498
2499 pub fn checkpoint(
2501 &mut self,
2502 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2503 ) -> Result<M::StateCheckpoint, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2504 {
2505 self.ensure_commit_resolved()?;
2506 self.mechanisms
2507 .checkpoint_state(&self.state, context)
2508 .map_err(ReplicatedTextSessionError::Mechanism)
2509 }
2510
2511 pub fn exchange_prediction_target_state(
2518 &mut self,
2519 replacement: &mut M::State,
2520 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2521 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2522 self.ensure_commit_resolved()?;
2523 let validation = match self.selected_state.state() {
2524 Some(selected) => validate_realized_state(replacement, selected),
2525 None if replacement.optional_layout().is_none() => Ok(()),
2526 None => Err(ReplicatedTextSessionError::Contract(
2527 "stateless prediction target received stateful lane state".into(),
2528 )),
2529 };
2530 let phase = crate::DistributedExecutionPhase::PredictionTargetStatePreparation;
2531 let agreed =
2532 D::agree_distributed_phase(&mut self.execution, phase, validation.is_ok(), context)
2533 .map_err(widen_infallible)?;
2534 match validation {
2535 Ok(()) if agreed => {
2536 std::mem::swap(&mut self.state, replacement);
2537 Ok(())
2538 }
2539 Ok(()) => Err(ReplicatedTextSessionError::Contract(
2540 "another rank rejected its prediction target lane state".into(),
2541 )),
2542 Err(error) => Err(error),
2543 }
2544 }
2545
2546 pub fn recover_prediction_target_state_after_failure(
2555 &mut self,
2556 replacement: &mut M::State,
2557 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2558 let selected = self.selected_state.state().ok_or_else(|| {
2559 ReplicatedTextSessionError::Contract(
2560 "stateless prediction target cannot recover lane ownership".into(),
2561 )
2562 })?;
2563 validate_realized_state(&self.state, selected)?;
2564 validate_realized_state(replacement, selected)?;
2565 std::mem::swap(&mut self.state, replacement);
2566 Ok(())
2567 }
2568
2569 pub fn prepare_prediction_target_state(
2575 &mut self,
2576 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2577 ) -> Result<M::State, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2578 self.ensure_commit_resolved()?;
2579 let provisional = self.selected_state.state().map_or_else(
2580 || {
2581 Err(ReplicatedTextSessionError::Contract(
2582 "stateless session cannot prepare prediction target state".into(),
2583 ))
2584 },
2585 |selected| {
2586 self.mechanisms
2587 .fork_prediction_target_state(&self.state, selected, context)
2588 .map_err(ReplicatedTextSessionError::Mechanism)
2589 .and_then(|state| {
2590 validate_realized_state(&state, selected)?;
2591 Ok(state)
2592 })
2593 },
2594 );
2595 let phase = crate::DistributedExecutionPhase::PredictionTargetStatePreparation;
2596 let agreed =
2597 D::agree_distributed_phase(&mut self.execution, phase, provisional.is_ok(), context)
2598 .map_err(widen_infallible)?;
2599 match provisional {
2600 Ok(state) if agreed => Ok(state),
2601 Ok(_) => Err(ReplicatedTextSessionError::Contract(
2602 "another rank could not prepare prediction target lane state".into(),
2603 )),
2604 Err(error) => Err(error),
2605 }
2606 }
2607
2608 pub fn apply_prediction_target_operation<O>(
2613 &mut self,
2614 operation: O,
2615 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2616 ) -> Result<O::Output, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2617 where
2618 O: PredictionTargetOperation<A, B, M::State>,
2619 {
2620 self.ensure_commit_resolved()?;
2621 let checkpoint = self
2622 .mechanisms
2623 .checkpoint_state(&self.state, context)
2624 .map_err(ReplicatedTextSessionError::Mechanism)?;
2625 let execution = D::apply_prediction_target_operation(
2626 &mut self.execution,
2627 &mut self.state,
2628 operation,
2629 context,
2630 )
2631 .map_err(widen_infallible)
2632 .and_then(|output| {
2633 output.ok_or_else(|| {
2634 ReplicatedTextSessionError::Contract(
2635 "selected target execution has no typed prediction-extension handoff".into(),
2636 )
2637 })
2638 });
2639 let phase = crate::DistributedExecutionPhase::PredictionExtensionExecution;
2640 let agreed =
2641 D::agree_distributed_phase(&mut self.execution, phase, execution.is_ok(), context)
2642 .map_err(widen_infallible);
2643 match (execution, agreed) {
2644 (Ok(output), Ok(true)) => Ok(output),
2645 (execution, agreement) => {
2646 self.mechanisms
2647 .restore_state(&mut self.state, checkpoint, context)
2648 .map_err(ReplicatedTextSessionError::Mechanism)?;
2649 match (execution, agreement) {
2650 (_, Err(error)) => Err(error),
2651 (Err(error), _) => Err(error),
2652 (Ok(_), Ok(false)) => Err(ReplicatedTextSessionError::Contract(
2653 "another rank failed during prediction-extension execution".into(),
2654 )),
2655 (Ok(_), Ok(true)) => unreachable!("successful extension returned above"),
2656 }
2657 }
2658 }
2659 }
2660
2661 pub fn checkpoint_complete(
2663 &mut self,
2664 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2665 ) -> Result<
2666 ReplicatedTextSessionCheckpoint<M::StateCheckpoint>,
2667 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2668 > {
2669 Ok(ReplicatedTextSessionCheckpoint {
2670 state: self.checkpoint(context)?,
2671 prompt_input_identity: self.committed_prompt_input_identity.clone(),
2672 next_commit_epoch: self.next_commit_epoch,
2673 last_commit_outcome: self.last_commit_outcome,
2674 })
2675 }
2676
2677 pub fn checkpoint_distributed(
2679 &mut self,
2680 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2681 ) -> Result<
2682 DistributedStateCheckpoint<M::StateCheckpoint>,
2683 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2684 > {
2685 self.ensure_commit_resolved()?;
2686 self.require_cache_control_agreement()?;
2687 let checkpoint = self.selected_state.state().map(|_| {
2688 self.mechanisms
2689 .checkpoint_state(&self.state, context)
2690 .map_err(ReplicatedTextSessionError::Mechanism)
2691 });
2692 let success = checkpoint.as_ref().is_none_or(Result::is_ok);
2693 let phase = crate::DistributedExecutionPhase::SessionCheckpoint;
2694 let agreed = self.agree_cache_control_phase(phase, success, context)?;
2695 let state = match checkpoint {
2696 Some(Ok(checkpoint)) if agreed => Some(checkpoint),
2697 None if agreed => None,
2698 Some(Ok(_)) | None => return self.fence_remote_cache_control_failure(phase),
2699 Some(Err(error)) => {
2700 self.control_fence = Some(phase);
2701 return Err(error);
2702 }
2703 };
2704 Ok(DistributedStateCheckpoint { state })
2705 }
2706
2707 pub fn checkpoint_complete_distributed(
2709 &mut self,
2710 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2711 ) -> Result<
2712 DistributedSessionCheckpoint<M::StateCheckpoint>,
2713 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2714 > {
2715 let state = self.checkpoint_distributed(context)?.state;
2716 Ok(DistributedSessionCheckpoint {
2717 state,
2718 prompt_input_identity: self.committed_prompt_input_identity.clone(),
2719 next_commit_epoch: self.next_commit_epoch,
2720 last_commit_outcome: self.last_commit_outcome,
2721 })
2722 }
2723
2724 pub fn rollback(
2726 &mut self,
2727 checkpoint: M::StateCheckpoint,
2728 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2729 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2730 self.ensure_commit_resolved()?;
2731 self.mechanisms
2732 .restore_state(&mut self.state, checkpoint, context)
2733 .map_err(ReplicatedTextSessionError::Mechanism)?;
2734 self.committed_prompt_input_identity = None;
2736 Ok(())
2737 }
2738
2739 pub fn rollback_complete(
2741 &mut self,
2742 checkpoint: ReplicatedTextSessionCheckpoint<M::StateCheckpoint>,
2743 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2744 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2745 self.ensure_commit_resolved()?;
2746 self.mechanisms
2747 .restore_state(&mut self.state, checkpoint.state, context)
2748 .map_err(ReplicatedTextSessionError::Mechanism)?;
2749 self.committed_prompt_input_identity = checkpoint.prompt_input_identity;
2750 self.next_commit_epoch = checkpoint.next_commit_epoch;
2751 self.last_commit_outcome = checkpoint.last_commit_outcome;
2752 self.active_commit_epoch = None;
2753 Ok(())
2754 }
2755
2756 pub fn rollback_distributed(
2758 &mut self,
2759 checkpoint: DistributedStateCheckpoint<M::StateCheckpoint>,
2760 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2761 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2762 self.restore_distributed_state(checkpoint.state, None, context)
2763 }
2764
2765 pub fn rollback_complete_distributed(
2767 &mut self,
2768 checkpoint: DistributedSessionCheckpoint<M::StateCheckpoint>,
2769 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2770 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2771 self.restore_distributed_state(
2772 checkpoint.state,
2773 Some((
2774 checkpoint.prompt_input_identity,
2775 checkpoint.next_commit_epoch,
2776 checkpoint.last_commit_outcome,
2777 )),
2778 context,
2779 )
2780 }
2781
2782 pub fn reset_distributed(
2784 &mut self,
2785 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2786 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2787 self.ensure_commit_resolved()?;
2788 self.require_cache_control_agreement()?;
2789 let provisional = self.selected_state.state().map(|selected| {
2790 self.mechanisms
2791 .realize_state(selected, context)
2792 .map_err(ReplicatedTextSessionError::Mechanism)
2793 .and_then(|state| {
2794 validate_realized_state(&state, selected)?;
2795 Ok(state)
2796 })
2797 });
2798 let success = provisional.as_ref().is_none_or(Result::is_ok);
2799 let phase = crate::DistributedExecutionPhase::SessionResetPreparation;
2800 let agreed = self.agree_cache_control_phase(phase, success, context)?;
2801 match provisional {
2802 Some(Ok(state)) if agreed => self.state = state,
2803 None if agreed => {}
2804 Some(Ok(_)) | None => return self.fence_remote_cache_control_failure(phase),
2805 Some(Err(error)) => {
2806 self.control_fence = Some(phase);
2807 return Err(error);
2808 }
2809 }
2810 self.committed_prompt_input_identity = None;
2811 Ok(())
2812 }
2813
2814 pub fn reset(
2816 &mut self,
2817 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2818 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2819 self.ensure_commit_resolved()?;
2820 if let Some(selected_state) = self.selected_state.state() {
2821 let state = self
2822 .mechanisms
2823 .realize_state(selected_state, context)
2824 .map_err(ReplicatedTextSessionError::Mechanism)?;
2825 validate_realized_state(&state, selected_state)?;
2826 self.state = state;
2827 } else if self.state.optional_layout().is_some() {
2828 return Err(ReplicatedTextSessionError::Contract(
2829 "stateless session owns a stateful mechanism realization".into(),
2830 ));
2831 }
2832 self.committed_prompt_input_identity = None;
2833 Ok(())
2834 }
2835
2836 pub fn load_prompt_cache(
2838 &mut self,
2839 directory: &Path,
2840 expected: &PromptCacheDescriptor,
2841 prefix_token_ids: &[u32],
2842 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2843 ) -> Result<PromptCacheManifest, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2844 {
2845 self.ensure_control_unfenced()?;
2846 let identity = self.prompt_cache_identity()?.clone();
2847 validate_prompt_cache_model_identity(expected, &identity)?;
2848 let selected_state = self.selected_state.state().ok_or_else(|| {
2849 ReplicatedTextSessionError::Contract(
2850 "this partition rank owns no prompt-cache state shard".into(),
2851 )
2852 })?;
2853 let (state, manifest) = self
2854 .mechanisms
2855 .load_prompt_cache(
2856 directory,
2857 expected,
2858 &identity,
2859 prefix_token_ids,
2860 selected_state,
2861 context,
2862 )
2863 .map_err(ReplicatedTextSessionError::Mechanism)?;
2864 manifest.validate_compatibility(expected, prefix_token_ids)?;
2865 validate_realized_state(&state, selected_state)?;
2866 self.state = state;
2867 self.committed_prompt_input_identity = None;
2868 self.restore_distributed_commit(manifest.distributed_commit)?;
2869 Ok(manifest)
2870 }
2871
2872 pub fn load_prompt_cache_for_input(
2874 &mut self,
2875 directory: &Path,
2876 expected: &PromptCacheDescriptor,
2877 prefix_token_ids: &[u32],
2878 input_identity: PreparedInputCacheIdentity,
2879 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2880 ) -> Result<PromptCacheManifest, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2881 {
2882 self.validate_prompt_input_descriptor(expected, &input_identity)?;
2883 let manifest = self.load_prompt_cache(directory, expected, prefix_token_ids, context)?;
2884 self.committed_prompt_input_identity = Some(input_identity);
2885 Ok(manifest)
2886 }
2887
2888 pub fn load_prompt_cache_distributed(
2895 &mut self,
2896 directory: &Path,
2897 expected: &PromptCacheDescriptor,
2898 prefix_token_ids: &[u32],
2899 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2900 ) -> Result<
2901 Option<PromptCacheManifest>,
2902 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2903 > {
2904 self.load_prompt_cache_distributed_inner(
2905 directory,
2906 expected,
2907 prefix_token_ids,
2908 None,
2909 context,
2910 )
2911 }
2912
2913 pub fn load_prompt_cache_for_input_distributed(
2915 &mut self,
2916 directory: &Path,
2917 expected: &PromptCacheDescriptor,
2918 prefix_token_ids: &[u32],
2919 input_identity: PreparedInputCacheIdentity,
2920 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2921 ) -> Result<
2922 Option<PromptCacheManifest>,
2923 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2924 > {
2925 self.load_prompt_cache_distributed_inner(
2926 directory,
2927 expected,
2928 prefix_token_ids,
2929 Some(input_identity),
2930 context,
2931 )
2932 }
2933
2934 pub fn save_prompt_cache(
2936 &mut self,
2937 destination: &Path,
2938 descriptor: PromptCacheDescriptor,
2939 prefix_token_ids: &[u32],
2940 options: &PromptCacheOptions,
2941 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2942 ) -> Result<PromptCacheManifest, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2943 {
2944 self.ensure_control_unfenced()?;
2945 validate_prompt_cache_model_identity(&descriptor, self.prompt_cache_identity()?)?;
2946 let descriptor = descriptor.with_distributed_commit(self.last_commit_outcome);
2947 let manifest = self
2948 .mechanisms
2949 .save_prompt_cache(
2950 &mut self.state,
2951 destination,
2952 descriptor.clone(),
2953 prefix_token_ids,
2954 options,
2955 context,
2956 )
2957 .map_err(ReplicatedTextSessionError::Mechanism)?;
2958 manifest.validate_compatibility(&descriptor, prefix_token_ids)?;
2959 Ok(manifest)
2960 }
2961
2962 pub fn save_prompt_cache_for_input(
2964 &mut self,
2965 destination: &Path,
2966 descriptor: PromptCacheDescriptor,
2967 prefix_token_ids: &[u32],
2968 options: &PromptCacheOptions,
2969 input_identity: &PreparedInputCacheIdentity,
2970 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2971 ) -> Result<PromptCacheManifest, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2972 {
2973 self.validate_prompt_input_descriptor(&descriptor, input_identity)?;
2974 if self.committed_prompt_input_identity.as_ref() != Some(input_identity) {
2975 return Err(ReplicatedTextSessionError::Contract(
2976 "prompt-cache prepared-input identity differs from the committed prompt".into(),
2977 ));
2978 }
2979 self.save_prompt_cache(destination, descriptor, prefix_token_ids, options, context)
2980 }
2981
2982 fn load_prompt_cache_distributed_inner(
2983 &mut self,
2984 directory: &Path,
2985 expected: &PromptCacheDescriptor,
2986 prefix_token_ids: &[u32],
2987 input_identity: Option<PreparedInputCacheIdentity>,
2988 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2989 ) -> Result<
2990 Option<PromptCacheManifest>,
2991 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2992 > {
2993 self.ensure_commit_resolved()?;
2994 self.require_cache_control_agreement()?;
2995 let preflight = (|| {
2996 if let Some(input_identity) = input_identity.as_ref() {
2997 self.validate_prompt_input_descriptor(expected, input_identity)?;
2998 }
2999 match (
3000 self.selected_state.state(),
3001 self.prompt_cache_identity.as_ref(),
3002 ) {
3003 (Some(selected_state), Some(identity)) => {
3004 if !self.selected.prompt_cache() {
3005 return Err(ReplicatedTextSessionError::Contract(
3006 "prompt-cache persistence was not selected for this session".into(),
3007 ));
3008 }
3009 validate_prompt_cache_model_identity(expected, identity)?;
3010 Ok(Some((selected_state.clone(), identity.clone())))
3011 }
3012 (None, None) => Ok(None),
3013 _ => Err(ReplicatedTextSessionError::Contract(
3014 "partition cache state and rank-local identity ownership disagree".into(),
3015 )),
3016 }
3017 })();
3018 let phase = crate::DistributedExecutionPhase::PromptCacheLoadPreflight;
3019 let agreement = self.agree_cache_control_phase(phase, preflight.is_ok(), context);
3020 let local = match (preflight, agreement) {
3021 (Ok(local), Ok(true)) => local,
3022 (Ok(_), Ok(false)) => return self.fence_remote_cache_control_failure(phase),
3023 (Ok(_), Err(error)) => return Err(error),
3024 (Err(error), _) => {
3025 self.control_fence = Some(phase);
3026 return Err(error);
3027 }
3028 };
3029
3030 let provisional = local.map(|(selected_state, identity)| {
3031 self.mechanisms
3032 .load_prompt_cache(
3033 directory,
3034 expected,
3035 &identity,
3036 prefix_token_ids,
3037 &selected_state,
3038 context,
3039 )
3040 .map_err(ReplicatedTextSessionError::Mechanism)
3041 .and_then(|(state, manifest)| {
3042 manifest.validate_compatibility(expected, prefix_token_ids)?;
3043 validate_realized_state(&state, &selected_state)?;
3044 validate_distributed_commit_restore(manifest.distributed_commit)?;
3045 Ok((state, manifest))
3046 })
3047 });
3048 let local_success = provisional.as_ref().is_none_or(Result::is_ok);
3049 let phase = crate::DistributedExecutionPhase::PromptCacheLoadPreparation;
3050 let agreement = self.agree_cache_control_phase(phase, local_success, context);
3051 let provisional = match (provisional, agreement) {
3052 (Some(Ok(provisional)), Ok(true)) => Some(provisional),
3053 (None, Ok(true)) => None,
3054 (Some(Ok(_)) | None, Ok(false)) => {
3055 return self.fence_remote_cache_control_failure(phase);
3056 }
3057 (Some(Ok(_)) | None, Err(error)) => return Err(error),
3058 (Some(Err(error)), _) => {
3059 self.control_fence = Some(phase);
3060 return Err(error);
3061 }
3062 };
3063 let manifest = provisional.map(|(state, manifest)| {
3064 self.state = state;
3065 self.committed_prompt_input_identity = input_identity;
3066 self.active_commit_epoch = None;
3067 self.last_commit_outcome = manifest.distributed_commit;
3068 if let Some(outcome) = manifest.distributed_commit {
3069 self.next_commit_epoch = outcome.epoch().next().unwrap_or_else(|| {
3070 unreachable!("provisional commit epoch was validated before agreement")
3071 });
3072 }
3073 manifest
3074 });
3075 Ok(manifest)
3076 }
3077
3078 fn restore_distributed_state(
3079 &mut self,
3080 checkpoint: Option<M::StateCheckpoint>,
3081 metadata: Option<(
3082 Option<PreparedInputCacheIdentity>,
3083 DistributedCommitEpoch,
3084 Option<DistributedCommitOutcome>,
3085 )>,
3086 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3087 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3088 self.ensure_commit_resolved()?;
3089 self.require_cache_control_agreement()?;
3090 let provisional = match (self.selected_state.state(), checkpoint) {
3091 (Some(selected), Some(checkpoint)) => Some(
3092 self.mechanisms
3093 .realize_state(selected, context)
3094 .map_err(ReplicatedTextSessionError::Mechanism)
3095 .and_then(|mut state| {
3096 self.mechanisms
3097 .restore_state(&mut state, checkpoint, context)
3098 .map_err(ReplicatedTextSessionError::Mechanism)?;
3099 validate_realized_state(&state, selected)?;
3100 Ok(state)
3101 }),
3102 ),
3103 (None, None) => None,
3104 _ => Some(Err(ReplicatedTextSessionError::Contract(
3105 "distributed checkpoint presence differs from rank-local state ownership".into(),
3106 ))),
3107 };
3108 let metadata_valid = metadata.as_ref().is_none_or(|(_, next, outcome)| {
3109 outcome.is_none_or(|outcome| {
3110 outcome
3111 .epoch()
3112 .next()
3113 .is_some_and(|expected| expected == *next)
3114 })
3115 });
3116 let success = provisional.as_ref().is_none_or(Result::is_ok) && metadata_valid;
3117 let phase = crate::DistributedExecutionPhase::SessionRollbackPreparation;
3118 let agreed = self.agree_cache_control_phase(phase, success, context)?;
3119 let provisional = match provisional {
3120 Some(Ok(state)) if agreed => Some(state),
3121 None if agreed => None,
3122 Some(Ok(_)) | None => return self.fence_remote_cache_control_failure(phase),
3123 Some(Err(error)) => {
3124 self.control_fence = Some(phase);
3125 return Err(error);
3126 }
3127 };
3128 if !metadata_valid {
3129 self.control_fence = Some(phase);
3130 return Err(ReplicatedTextSessionError::Contract(
3131 "distributed checkpoint commit metadata is inconsistent".into(),
3132 ));
3133 }
3134 if let Some(state) = provisional {
3135 self.state = state;
3136 }
3137 match metadata {
3138 Some((identity, next, outcome)) => {
3139 self.committed_prompt_input_identity = identity;
3140 self.next_commit_epoch = next;
3141 self.last_commit_outcome = outcome;
3142 self.active_commit_epoch = None;
3143 }
3144 None => self.committed_prompt_input_identity = None,
3145 }
3146 Ok(())
3147 }
3148
3149 pub const fn committed_prompt_input_identity(&self) -> Option<&PreparedInputCacheIdentity> {
3151 self.committed_prompt_input_identity.as_ref()
3152 }
3153
3154 pub fn report(
3156 &self,
3157 ) -> Result<
3158 ReplicatedTextSessionReport<M::ExecutionReport, M::StateReport>,
3159 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3160 > {
3161 let execution_report = self
3162 .mechanisms
3163 .execution_report(
3164 self.selected.residency(),
3165 D::bounded_policy(&self.execution),
3166 )
3167 .map_err(ReplicatedTextSessionError::Mechanism)?;
3168 let state_report = self
3169 .mechanisms
3170 .state_report(&self.state)
3171 .map_err(ReplicatedTextSessionError::Mechanism)?;
3172 Ok(ReplicatedTextSessionReport {
3173 execution: D::execution_residency(&self.execution, &self.selected),
3174 execution_report,
3175 state_report,
3176 distributed_commit: self.last_commit_outcome,
3177 })
3178 }
3179
3180 fn execute_with_observer<O>(
3181 &mut self,
3182 tokens: &B::Tensor,
3183 mask: Option<&B::Tensor>,
3184 pass: ExpertPass,
3185 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3186 observer: &mut O,
3187 ) -> Result<
3188 (B::Tensor, M::StateCheckpoint, A::ForwardContext),
3189 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3190 >
3191 where
3192 A: ReplicatedTextArchitecture<B, M::State>,
3193 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
3194 {
3195 let input = A::text_input(tokens, mask);
3196 self.execute_input_with_observer(input, pass, context, observer)
3197 }
3198
3199 fn execute_input_with_observer<'a, O>(
3200 &mut self,
3201 input: A::Input<'a>,
3202 pass: ExpertPass,
3203 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3204 observer: &mut O,
3205 ) -> Result<
3206 (B::Tensor, M::StateCheckpoint, A::ForwardContext),
3207 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3208 >
3209 where
3210 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
3211 {
3212 let (output, checkpoint, forward_context) =
3213 self.execute_input_before_publication(input, pass, context, observer)?;
3214 self.publish_observed_output_transaction(output, checkpoint, forward_context, context)
3215 }
3216
3217 fn execute_input_before_publication<'a, O>(
3218 &mut self,
3219 input: A::Input<'a>,
3220 pass: ExpertPass,
3221 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3222 observer: &mut O,
3223 ) -> Result<
3224 (B::Tensor, M::StateCheckpoint, A::ForwardContext),
3225 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3226 >
3227 where
3228 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
3229 {
3230 self.begin_commit_epoch()?;
3231 let checkpoint = self.mechanisms.checkpoint_state(&self.state, context);
3232 let checkpoint_agreed = match D::agree_distributed_phase(
3233 &mut self.execution,
3234 crate::DistributedExecutionPhase::StateCheckpoint,
3235 checkpoint.is_ok(),
3236 context,
3237 ) {
3238 Ok(agreed) => agreed,
3239 Err(error) => return self.abort_without_rollback(widen_infallible(error)),
3240 };
3241 let checkpoint = match checkpoint {
3242 Ok(checkpoint) if checkpoint_agreed => checkpoint,
3243 Ok(_) => {
3244 return self.abort_without_rollback(ReplicatedTextSessionError::Contract(
3245 "another rank failed to capture its distributed state checkpoint".into(),
3246 ));
3247 }
3248 Err(error) => {
3249 return self.abort_without_rollback(ReplicatedTextSessionError::Mechanism(error));
3250 }
3251 };
3252 let execution = self
3253 .driver
3254 .forward_with_observer(
3255 &mut self.execution,
3256 input,
3257 &mut self.state,
3258 pass,
3259 context,
3260 observer,
3261 )
3262 .map_err(widen_infallible);
3263 let execution_agreed = match D::agree_distributed_phase(
3264 &mut self.execution,
3265 crate::DistributedExecutionPhase::Execution,
3266 execution.is_ok(),
3267 context,
3268 ) {
3269 Ok(agreed) => agreed,
3270 Err(error) => {
3271 let error = match execution {
3272 Err(local) => local,
3273 Ok(_) => widen_infallible(error),
3274 };
3275 return self.rollback_failure(checkpoint, error, context);
3276 }
3277 };
3278 let (output, forward_context) = match execution {
3279 Ok(output) if execution_agreed => output,
3280 Ok(_) => {
3281 return self.rollback_failure(
3282 checkpoint,
3283 ReplicatedTextSessionError::Contract(
3284 "another rank failed during distributed execution".into(),
3285 ),
3286 context,
3287 )
3288 }
3289 Err(error) => return self.rollback_failure(checkpoint, error, context),
3290 };
3291 let observation = D::observe_output(&mut self.execution, &output, observer, context);
3292 let observation_agreed = match D::agree_distributed_phase(
3293 &mut self.execution,
3294 crate::DistributedExecutionPhase::OutputObservation,
3295 observation.is_ok(),
3296 context,
3297 ) {
3298 Ok(agreed) => agreed,
3299 Err(error) => {
3300 return self.rollback_failure(checkpoint, widen_infallible(error), context)
3301 }
3302 };
3303 let output = match observation {
3304 Ok(output) if observation_agreed => output,
3305 Ok(_) => {
3306 return self.rollback_failure(
3307 checkpoint,
3308 ReplicatedTextSessionError::Contract(
3309 "another rank failed during distributed output observation".into(),
3310 ),
3311 context,
3312 )
3313 }
3314 Err(error) => {
3315 return self.rollback_failure(checkpoint, widen_infallible(error), context)
3316 }
3317 };
3318 Ok((output, checkpoint, forward_context))
3319 }
3320
3321 fn publish_observed_output_transaction(
3322 &mut self,
3323 output: B::Tensor,
3324 checkpoint: M::StateCheckpoint,
3325 forward_context: A::ForwardContext,
3326 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3327 ) -> Result<
3328 (B::Tensor, M::StateCheckpoint, A::ForwardContext),
3329 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3330 > {
3331 let publication = D::publish_observed_output(&mut self.execution, output, context);
3332 let publication_agreement = D::agree_distributed_phase(
3333 &mut self.execution,
3334 crate::DistributedExecutionPhase::OutputPublication,
3335 publication.is_ok(),
3336 context,
3337 );
3338 let output = match (publication, publication_agreement) {
3339 (Err(error), _) => {
3340 return self.rollback_failure(checkpoint, widen_infallible(error), context)
3341 }
3342 (Ok(_), Err(error)) => {
3343 return self.rollback_failure(checkpoint, widen_infallible(error), context)
3344 }
3345 (Ok(output), Ok(true)) => output,
3346 (Ok(_), Ok(false)) => {
3347 return self.rollback_failure(
3348 checkpoint,
3349 ReplicatedTextSessionError::Contract(
3350 "another rank failed during distributed output publication".into(),
3351 ),
3352 context,
3353 )
3354 }
3355 };
3356 Ok((output, checkpoint, forward_context))
3357 }
3358
3359 fn publish(
3360 &mut self,
3361 output: B::Tensor,
3362 checkpoint: M::StateCheckpoint,
3363 _forward_context: A::ForwardContext,
3364 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3365 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3366 let completion = self
3367 .selected
3368 .exact_completion()
3369 .then(|| self.mechanisms.complete(&output, &self.state, context))
3370 .transpose();
3371 let completion_agreed = match D::agree_distributed_phase(
3372 &mut self.execution,
3373 crate::DistributedExecutionPhase::MechanismCompletion,
3374 completion.is_ok(),
3375 context,
3376 ) {
3377 Ok(agreed) => agreed,
3378 Err(error) => {
3379 return self.rollback_failure(checkpoint, widen_infallible(error), context)
3380 }
3381 };
3382 match completion {
3383 Ok(_) if completion_agreed => {}
3384 Ok(_) => {
3385 return self.rollback_failure(
3386 checkpoint,
3387 ReplicatedTextSessionError::Contract(
3388 "another rank failed during distributed mechanism completion".into(),
3389 ),
3390 context,
3391 )
3392 }
3393 Err(error) => {
3394 return self.rollback_failure(
3395 checkpoint,
3396 ReplicatedTextSessionError::Mechanism(error),
3397 context,
3398 )
3399 }
3400 }
3401 let epoch = self.active_commit_epoch.ok_or_else(|| {
3402 ReplicatedTextSessionError::Contract(
3403 "distributed transaction lost its active commit epoch".into(),
3404 )
3405 })?;
3406 match D::commit_after_completion(&mut self.execution, epoch, context) {
3407 DistributedCommitOutcome::Committed(committed) if committed == epoch => {
3408 self.last_commit_outcome = Some(DistributedCommitOutcome::Committed(epoch));
3409 self.active_commit_epoch = None;
3410 }
3411 DistributedCommitOutcome::Aborted(aborted) if aborted == epoch => {
3412 self.last_commit_outcome = Some(DistributedCommitOutcome::Aborted(epoch));
3413 self.active_commit_epoch = None;
3414 let restored = self
3415 .mechanisms
3416 .restore_state(&mut self.state, checkpoint, context)
3417 .map_err(ReplicatedTextSessionError::Mechanism);
3418 record_successful_restoration(&mut self.successful_state_restorations, restored)?;
3419 return Err(ReplicatedTextSessionError::CommitAborted { epoch });
3420 }
3421 DistributedCommitOutcome::Indeterminate {
3422 epoch: uncertain,
3423 phase,
3424 } if uncertain == epoch => {
3425 self.last_commit_outcome =
3426 Some(DistributedCommitOutcome::Indeterminate { epoch, phase });
3427 self.active_commit_epoch = None;
3428 return Err(ReplicatedTextSessionError::CommitIndeterminate { epoch, phase });
3429 }
3430 outcome => {
3431 return self.rollback_failure(
3432 checkpoint,
3433 ReplicatedTextSessionError::Contract(format!(
3434 "distributed commit returned epoch {} for active epoch {}",
3435 outcome.epoch().value(),
3436 epoch.value()
3437 )),
3438 context,
3439 );
3440 }
3441 }
3442 Ok(output)
3443 }
3444
3445 fn rollback_failure<T>(
3446 &mut self,
3447 checkpoint: M::StateCheckpoint,
3448 error: ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3449 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3450 ) -> Result<T, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3451 let restored = self
3452 .mechanisms
3453 .restore_state(&mut self.state, checkpoint, context)
3454 .map_err(ReplicatedTextSessionError::Mechanism);
3455 if let Some(epoch) = self.active_commit_epoch.take() {
3456 self.last_commit_outcome = Some(DistributedCommitOutcome::Aborted(epoch));
3457 }
3458 record_successful_restoration(&mut self.successful_state_restorations, restored)?;
3459 Err(error)
3460 }
3461
3462 fn begin_commit_epoch(
3463 &mut self,
3464 ) -> Result<
3465 DistributedCommitEpoch,
3466 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3467 > {
3468 self.ensure_commit_resolved()?;
3469 if self.active_commit_epoch.is_some() {
3470 return Err(ReplicatedTextSessionError::Contract(
3471 "distributed transaction already has an active commit epoch".into(),
3472 ));
3473 }
3474 let epoch = self.next_commit_epoch;
3475 self.next_commit_epoch = epoch.next().ok_or_else(|| {
3476 ReplicatedTextSessionError::Contract("distributed commit epoch overflow".into())
3477 })?;
3478 self.active_commit_epoch = Some(epoch);
3479 Ok(epoch)
3480 }
3481
3482 fn ensure_commit_resolved(
3483 &self,
3484 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3485 self.ensure_control_unfenced()?;
3486 if let Some(DistributedCommitOutcome::Indeterminate { epoch, phase }) =
3487 self.last_commit_outcome
3488 {
3489 return Err(ReplicatedTextSessionError::CommitIndeterminate { epoch, phase });
3490 }
3491 Ok(())
3492 }
3493
3494 fn ensure_control_unfenced(
3495 &self,
3496 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3497 if let Some(phase) = self.control_fence {
3498 return Err(ReplicatedTextSessionError::Contract(format!(
3499 "distributed session is fenced after failed cache control at {phase:?}"
3500 )));
3501 }
3502 Ok(())
3503 }
3504
3505 fn agree_cache_control_phase(
3506 &mut self,
3507 phase: crate::DistributedExecutionPhase,
3508 local_success: bool,
3509 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3510 ) -> Result<bool, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3511 D::agree_distributed_phase(&mut self.execution, phase, local_success, context)
3512 .map_err(widen_infallible)
3513 .inspect_err(|_| self.control_fence = Some(phase))
3514 }
3515
3516 fn require_cache_control_agreement(
3517 &self,
3518 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3519 if D::PARTITIONED_SESSION && !D::DISTRIBUTED_PHASE_AGREEMENT {
3520 return Err(ReplicatedTextSessionError::Contract(
3521 "partitioned cache control requires the selected bounded failure agreement".into(),
3522 ));
3523 }
3524 Ok(())
3525 }
3526
3527 fn fence_remote_cache_control_failure<T>(
3528 &mut self,
3529 phase: crate::DistributedExecutionPhase,
3530 ) -> Result<T, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3531 self.control_fence = Some(phase);
3532 Err(ReplicatedTextSessionError::Contract(format!(
3533 "another rank failed distributed cache control at {phase:?}"
3534 )))
3535 }
3536
3537 fn abort_without_rollback<T>(
3538 &mut self,
3539 error: ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3540 ) -> Result<T, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3541 if let Some(epoch) = self.active_commit_epoch.take() {
3542 self.last_commit_outcome = Some(DistributedCommitOutcome::Aborted(epoch));
3543 }
3544 Err(error)
3545 }
3546
3547 fn restore_distributed_commit(
3548 &mut self,
3549 outcome: Option<DistributedCommitOutcome>,
3550 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3551 self.active_commit_epoch = None;
3552 self.last_commit_outcome = outcome;
3553 if let Some(outcome) = outcome {
3554 self.next_commit_epoch = outcome.epoch().next().ok_or_else(|| {
3555 ReplicatedTextSessionError::Contract("distributed commit epoch overflow".into())
3556 })?;
3557 }
3558 Ok(())
3559 }
3560
3561 fn validate_prompt_input_descriptor(
3562 &self,
3563 descriptor: &PromptCacheDescriptor,
3564 input_identity: &PreparedInputCacheIdentity,
3565 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3566 if descriptor.prefix_content_fingerprint() != input_identity.prefix_content_fingerprint() {
3567 return Err(ReplicatedTextSessionError::Contract(
3568 "prompt-cache content identity differs from the prepared input".into(),
3569 ));
3570 }
3571 Ok(())
3572 }
3573
3574 fn prompt_cache_identity(
3575 &self,
3576 ) -> Result<
3577 &PromptCacheModelIdentity,
3578 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3579 > {
3580 if !self.selected.prompt_cache() {
3581 return Err(ReplicatedTextSessionError::Contract(
3582 "prompt-cache persistence was not selected for this session".into(),
3583 ));
3584 }
3585 self.prompt_cache_identity.as_ref().ok_or_else(|| {
3586 ReplicatedTextSessionError::Contract(
3587 "this partition rank owns no prompt-cache model identity".into(),
3588 )
3589 })
3590 }
3591}
3592
3593impl<A, B, M, D> ReplicatedTextSession<A, B, M, D>
3594where
3595 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
3596 M: TransactionalPromptCacheMechanisms<A, B>,
3597 A: LayeredArchitecture<B, M::State>,
3598 D: ReplicatedTextExecutionStrategy<A, B, M::State, M::ResidentPolicy, M::BoundedPolicy>,
3599 A::Error: std::fmt::Display,
3600 M::PolicyError: std::fmt::Display,
3601 M::Error: std::fmt::Display,
3602{
3603 pub fn save_prompt_cache_distributed(
3605 &mut self,
3606 destination: &Path,
3607 descriptor: PromptCacheDescriptor,
3608 prefix_token_ids: &[u32],
3609 options: &PromptCacheOptions,
3610 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3611 ) -> Result<
3612 Option<PromptCacheManifest>,
3613 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3614 > {
3615 self.save_prompt_cache_distributed_inner(
3616 destination,
3617 descriptor,
3618 prefix_token_ids,
3619 options,
3620 None,
3621 context,
3622 )
3623 }
3624
3625 pub fn save_prompt_cache_for_input_distributed(
3627 &mut self,
3628 destination: &Path,
3629 descriptor: PromptCacheDescriptor,
3630 prefix_token_ids: &[u32],
3631 options: &PromptCacheOptions,
3632 input_identity: &PreparedInputCacheIdentity,
3633 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3634 ) -> Result<
3635 Option<PromptCacheManifest>,
3636 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3637 > {
3638 self.save_prompt_cache_distributed_inner(
3639 destination,
3640 descriptor,
3641 prefix_token_ids,
3642 options,
3643 Some(input_identity),
3644 context,
3645 )
3646 }
3647
3648 #[allow(clippy::too_many_arguments)]
3649 fn save_prompt_cache_distributed_inner(
3650 &mut self,
3651 destination: &Path,
3652 descriptor: PromptCacheDescriptor,
3653 prefix_token_ids: &[u32],
3654 options: &PromptCacheOptions,
3655 input_identity: Option<&PreparedInputCacheIdentity>,
3656 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3657 ) -> Result<
3658 Option<PromptCacheManifest>,
3659 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3660 > {
3661 self.ensure_commit_resolved()?;
3662 self.require_cache_control_agreement()?;
3663 let preflight = (|| {
3664 if let Some(input_identity) = input_identity {
3665 self.validate_prompt_input_descriptor(&descriptor, input_identity)?;
3666 if self.committed_prompt_input_identity.as_ref() != Some(input_identity) {
3667 return Err(ReplicatedTextSessionError::Contract(
3668 "prompt-cache prepared-input identity differs from the committed prompt"
3669 .into(),
3670 ));
3671 }
3672 }
3673 match (
3674 self.selected_state.state(),
3675 self.prompt_cache_identity.as_ref(),
3676 ) {
3677 (Some(_), Some(identity)) => {
3678 if !self.selected.prompt_cache() {
3679 return Err(ReplicatedTextSessionError::Contract(
3680 "prompt-cache persistence was not selected for this session".into(),
3681 ));
3682 }
3683 validate_prompt_cache_model_identity(&descriptor, identity)?;
3684 Ok(Some(
3685 descriptor
3686 .clone()
3687 .with_distributed_commit(self.last_commit_outcome),
3688 ))
3689 }
3690 (None, None) => Ok(None),
3691 _ => Err(ReplicatedTextSessionError::Contract(
3692 "partition cache state and rank-local identity ownership disagree".into(),
3693 )),
3694 }
3695 })();
3696 let phase = crate::DistributedExecutionPhase::PromptCacheSavePreflight;
3697 let agreement = self.agree_cache_control_phase(phase, preflight.is_ok(), context);
3698 let local_descriptor = match (preflight, agreement) {
3699 (Ok(local), Ok(true)) => local,
3700 (Ok(_), Ok(false)) => return self.fence_remote_cache_control_failure(phase),
3701 (Ok(_), Err(error)) => return Err(error),
3702 (Err(error), _) => {
3703 self.control_fence = Some(phase);
3704 return Err(error);
3705 }
3706 };
3707
3708 let mut transaction = local_descriptor.map(|descriptor| {
3709 self.mechanisms
3710 .prepare_prompt_cache_save(
3711 &mut self.state,
3712 destination,
3713 descriptor.clone(),
3714 prefix_token_ids,
3715 options,
3716 context,
3717 )
3718 .map_err(ReplicatedTextSessionError::Mechanism)
3719 .and_then(|transaction| {
3720 M::prepared_prompt_cache_manifest(&transaction)
3721 .validate_compatibility(&descriptor, prefix_token_ids)?;
3722 Ok(transaction)
3723 })
3724 });
3725 let local_success = transaction.as_ref().is_none_or(Result::is_ok);
3726 let phase = crate::DistributedExecutionPhase::PromptCacheSavePreparation;
3727 let agreement = self.agree_cache_control_phase(phase, local_success, context);
3728 let mut transaction = match (transaction.take(), agreement) {
3729 (Some(Ok(transaction)), Ok(true)) => Some(transaction),
3730 (None, Ok(true)) => None,
3731 (Some(Ok(transaction)), Ok(false)) => {
3732 self.mechanisms.rollback_prompt_cache_save(transaction);
3733 return self.fence_remote_cache_control_failure(phase);
3734 }
3735 (None, Ok(false)) => return self.fence_remote_cache_control_failure(phase),
3736 (Some(Ok(transaction)), Err(error)) => {
3737 self.mechanisms.rollback_prompt_cache_save(transaction);
3738 return Err(error);
3739 }
3740 (None, Err(error)) => return Err(error),
3741 (Some(Err(error)), _) => {
3742 self.control_fence = Some(phase);
3743 return Err(error);
3744 }
3745 };
3746
3747 let publication = transaction
3748 .as_mut()
3749 .map(|transaction| self.mechanisms.publish_prompt_cache_save(transaction));
3750 let local_success = publication.as_ref().is_none_or(Result::is_ok);
3751 let phase = crate::DistributedExecutionPhase::PromptCacheSavePublication;
3752 let agreement = self.agree_cache_control_phase(phase, local_success, context);
3753 let agreed = match (publication, agreement) {
3754 (Some(Err(error)), _) => {
3755 self.mechanisms.rollback_prompt_cache_save(
3756 transaction
3757 .take()
3758 .expect("failed publication retains its transaction"),
3759 );
3760 self.control_fence = Some(phase);
3761 return Err(ReplicatedTextSessionError::Mechanism(error));
3762 }
3763 (Some(Ok(())) | None, Err(error)) => {
3764 if let Some(transaction) = transaction.take() {
3765 self.mechanisms.rollback_prompt_cache_save(transaction);
3766 }
3767 return Err(error);
3768 }
3769 (Some(Ok(())) | None, Ok(agreed)) => agreed,
3770 };
3771 if !agreed {
3772 if let Some(transaction) = transaction {
3773 self.mechanisms.rollback_prompt_cache_save(transaction);
3774 }
3775 return self.fence_remote_cache_control_failure(phase);
3776 }
3777 Ok(transaction.map(|transaction| {
3778 let manifest = M::prepared_prompt_cache_manifest(&transaction).clone();
3779 self.mechanisms.commit_prompt_cache_save(transaction);
3780 manifest
3781 }))
3782 }
3783}
3784
3785fn validate_distributed_commit_restore<A, P, M>(
3786 outcome: Option<DistributedCommitOutcome>,
3787) -> Result<(), ReplicatedTextSessionError<A, P, M>>
3788where
3789 A: std::fmt::Display,
3790 P: std::fmt::Display,
3791 M: std::fmt::Display,
3792{
3793 if outcome.is_some_and(|outcome| outcome.epoch().next().is_none()) {
3794 return Err(ReplicatedTextSessionError::Contract(
3795 "distributed commit epoch overflow".into(),
3796 ));
3797 }
3798 Ok(())
3799}
3800
3801fn validate_architecture_geometry<A, B, S>(
3802 architecture: &A,
3803 selected: &SelectedReplicatedTextRealization,
3804) -> Result<(), String>
3805where
3806 B: NeuralBackend,
3807 S: RuntimeState<B>,
3808 A: LayeredArchitecture<B, S>,
3809 A::Error: std::fmt::Display,
3810{
3811 let requirements = selected.requirements();
3812 let graph = architecture
3813 .execution_graph()
3814 .map_err(|error| error.to_string())?;
3815 if &graph != requirements.execution_graph() {
3816 return Err("architecture execution graph differs from selection".into());
3817 }
3818 if requirements.execution_units().group_count() != graph.groups().len() {
3819 return Err("selected execution-unit groups differ from architecture graph".into());
3820 }
3821 for group in 0..graph.groups().len() {
3822 let actual = architecture
3823 .group_unit_count(group)
3824 .map_err(|error| error.to_string())?;
3825 let expected = requirements
3826 .execution_units()
3827 .group_range(group)
3828 .expect("validated requirement exposes every graph group")
3829 .len();
3830 if actual != expected
3831 || architecture.group_transport(group) != requirements.group_transports()[group]
3832 {
3833 return Err(format!(
3834 "architecture execution group {group} differs from selection"
3835 ));
3836 }
3837 }
3838 let layout = architecture
3839 .state_layout()
3840 .map_err(|error| error.to_string())?;
3841 if &layout != selected.state().layout() {
3842 return Err("architecture state layout differs from selection".into());
3843 }
3844 Ok(())
3845}
3846
3847fn validate_architecture_parameters<A, B, S>(
3848 architecture: &A,
3849 selected: &SelectedReplicatedTextRealization,
3850 selected_formats: bool,
3851 context: &<B::Tensor as Tensor>::Context,
3852) -> Result<BTreeMap<String, Vec<ReplicatedTextOutputCompanion>>, String>
3853where
3854 B: NeuralBackend,
3855 S: RuntimeState<B>,
3856 A: LayeredArchitecture<B, S>,
3857 A::Error: std::fmt::Display,
3858{
3859 let requirements = selected.requirements();
3860 let description = architecture
3861 .parameter_description(context)
3862 .map_err(|error| error.to_string())?;
3863 let mut actual = BTreeMap::<String, (Vec<usize>, ParameterGroupOwner)>::new();
3864 let mut companions = Vec::new();
3865 for group in description.groups() {
3866 for member in group.group().members() {
3867 match (member.linear_companion(), member.linear_companion_of()) {
3868 (None, None) => {
3869 if actual
3870 .insert(
3871 member.target().to_owned(),
3872 (member.global_shape().to_vec(), group.owner().clone()),
3873 )
3874 .is_some()
3875 {
3876 return Err(format!(
3877 "constructed architecture repeats primary parameter {:?}",
3878 member.target()
3879 ));
3880 }
3881 }
3882 (Some(role), Some(primary)) => companions.push((
3883 member.target().to_owned(),
3884 member.global_shape().to_vec(),
3885 group.owner().clone(),
3886 role,
3887 primary.to_owned(),
3888 )),
3889 _ => {
3890 return Err(format!(
3891 "constructed parameter {:?} has incomplete linear companion metadata",
3892 member.target()
3893 ));
3894 }
3895 }
3896 }
3897 }
3898 let expected = requirements
3899 .parameters()
3900 .iter()
3901 .filter(|parameter| {
3902 !matches!(
3903 parameter.presence(),
3904 ReplicatedTextParameterPresence::OptionalAbsent
3905 | ReplicatedTextParameterPresence::Tied { .. }
3906 ) && parameter.role() != crate::ReplicatedTextParameterRole::FormatCompanion
3907 })
3908 .map(|parameter| parameter.name())
3909 .collect::<BTreeSet<_>>();
3910 for (name, shape, owner, _, _) in &companions {
3911 if expected.contains(name.as_str())
3912 && actual
3913 .insert(name.clone(), (shape.clone(), owner.clone()))
3914 .is_some()
3915 {
3916 return Err(format!(
3917 "constructed architecture repeats selected parameter {name:?}"
3918 ));
3919 }
3920 }
3921 let actual_names = actual.keys().map(String::as_str).collect::<BTreeSet<_>>();
3922 if expected != actual_names {
3923 return Err(format!(
3924 "selected parameter catalog differs from constructed architecture: missing {:?}, unexpected {:?}",
3925 expected.difference(&actual_names).collect::<Vec<_>>(),
3926 actual_names.difference(&expected).collect::<Vec<_>>()
3927 ));
3928 }
3929 for parameter in requirements.parameters().iter().filter(|parameter| {
3930 !matches!(
3931 parameter.presence(),
3932 ReplicatedTextParameterPresence::OptionalAbsent
3933 | ReplicatedTextParameterPresence::Tied { .. }
3934 ) && parameter.role() != crate::ReplicatedTextParameterRole::FormatCompanion
3935 }) {
3936 let (shape, owner) = actual
3937 .get(parameter.name())
3938 .expect("equal parameter-name sets contain every requirement");
3939 let owner_matches = match (owner, parameter.owner()) {
3940 (
3941 ParameterGroupOwner::StaticRole(actual),
3942 ReplicatedTextParameterOwner::StaticRole(expected),
3943 ) => actual == expected,
3944 (
3945 ParameterGroupOwner::StaticAnyOf(actual),
3946 ReplicatedTextParameterOwner::StaticRole(expected),
3947 ) => actual.iter().any(|role| role == expected),
3948 (
3949 ParameterGroupOwner::ExecutionUnit {
3950 group, global_unit, ..
3951 },
3952 ReplicatedTextParameterOwner::ExecutionUnit {
3953 group: expected_group,
3954 unit: expected_unit,
3955 },
3956 ) => group.as_str() == expected_group && global_unit == expected_unit,
3957 _ => false,
3958 };
3959 let mut expected_shape = parameter.logical_shape().to_vec();
3960 let realization = selected_formats
3961 .then(|| {
3962 selected
3963 .parameters()
3964 .iter()
3965 .find(|realization| realization.name() == parameter.name())
3966 })
3967 .flatten();
3968 let executable = realization.map_or(parameter.native_executable(), |realization| {
3969 realization.executable()
3970 });
3971 if (!selected_formats
3972 || realization.is_some_and(|realization| {
3973 matches!(
3974 realization.lowering(),
3975 crate::WeightLoweringKind::Direct | crate::WeightLoweringKind::Derived
3976 )
3977 }))
3978 && executable == eredu_checkpoint::LinearFormat::MxFp4
3979 && matches!(
3980 parameter.source_encoding(),
3981 Some(eredu_checkpoint::SourceTensorEncoding::Safetensors(
3982 eredu_checkpoint::StoredDtype::U8
3983 ))
3984 )
3985 {
3986 expected_shape = parameter
3987 .physical_shape()
3988 .expect("direct native realization has admitted physical geometry")
3989 .to_vec();
3990 }
3991 let selected_lowering = realization.map_or(crate::WeightLoweringKind::Direct, |selected| {
3997 selected.lowering()
3998 });
3999 let selected_executable_shape = Some((|| {
4000 let descriptor = parameter
4001 .lowering_descriptor(executable)
4002 .map_err(|error| error.to_string())?;
4003 let mut packed = descriptor.logical_shape().to_vec();
4004 let Some(axis) = descriptor.packed_axis() else {
4005 return Ok(packed);
4006 };
4007 let bits = match executable {
4008 eredu_checkpoint::LinearFormat::Affine(config) => usize::try_from(config.bits)
4009 .map_err(|_| {
4010 format!(
4011 "selected parameter {:?} has invalid affine packing bits",
4012 parameter.name()
4013 )
4014 })?,
4015 eredu_checkpoint::LinearFormat::MxFp4 => 4,
4016 eredu_checkpoint::LinearFormat::Dense
4017 | eredu_checkpoint::LinearFormat::E4M3BlockFp8(_) => return Ok(packed),
4018 eredu_checkpoint::LinearFormat::GgufIQuant { ggml_type, .. } => {
4019 if matches!(
4020 selected_lowering,
4021 crate::WeightLoweringKind::Transform
4022 | crate::WeightLoweringKind::DerivedTransform
4023 ) {
4024 return Err(format!(
4025 "selected parameter {:?} cannot apply a load-time GGUF transform",
4026 parameter.name()
4027 ));
4028 }
4029 let (block, bytes) = ggml_type
4030 .block_and_bytes()
4031 .map_err(|error| error.to_string())?;
4032 let block = usize::try_from(block)
4033 .map_err(|_| "GGUF block width is not representable".to_owned())?;
4034 let bytes = usize::try_from(bytes)
4035 .map_err(|_| "GGUF block bytes are not representable".to_owned())?;
4036 let packed_bytes = packed[axis]
4037 .checked_mul(bytes)
4038 .ok_or_else(|| "GGUF executable geometry overflowed".to_owned())?;
4039 if !packed_bytes.is_multiple_of(block) {
4040 return Err(format!(
4041 "selected parameter {:?} GGUF executable geometry is not block aligned",
4042 parameter.name()
4043 ));
4044 }
4045 packed[axis] = packed_bytes / block;
4046 return Ok(packed);
4047 }
4048 };
4049 let packed_bits = packed[axis].checked_mul(bits).ok_or_else(|| {
4050 format!(
4051 "selected parameter {:?} executable geometry overflowed",
4052 parameter.name()
4053 )
4054 })?;
4055 if !packed_bits.is_multiple_of(32) {
4056 return Err(format!(
4057 "selected parameter {:?} executable geometry {}x{} bits is not U32 aligned (logical {:?}, physical {:?})",
4058 parameter.name(),
4059 packed[axis],
4060 bits,
4061 descriptor.logical_shape(),
4062 descriptor.physical_shape()
4063 ));
4064 }
4065 packed[axis] = packed_bits / 32;
4066 Ok(packed)
4067 })()?);
4068 let shape_matches = shape == &expected_shape
4069 || selected_executable_shape
4070 .as_ref()
4071 .is_some_and(|selected| shape == selected);
4072 if !shape_matches || !owner_matches {
4073 return Err(format!(
4074 "selected parameter {:?} expects logical shape {expected_shape:?} or selected executable shape {selected_executable_shape:?} and owner {:?}, constructed shape {shape:?} and owner {owner:?}",
4075 parameter.name(),
4076 parameter.owner()
4077 ));
4078 }
4079 }
4080
4081 let mut output_companions = BTreeMap::<String, Vec<ReplicatedTextOutputCompanion>>::new();
4082 for (name, shape, owner, role, primary) in companions {
4083 if name == primary || !expected.contains(primary.as_str()) {
4084 return Err(format!(
4085 "constructed companion {name:?} names unselected primary {primary:?}"
4086 ));
4087 }
4088 if selected_formats {
4089 let recipe = selected.requirements().derived_recipes().get(&name);
4090 let output = selected.requirements().derived_recipe_outputs().get(&name);
4091 let companion = match (recipe, output) {
4092 (Some(recipe), Some(output)) => {
4093 ReplicatedTextOutputCompanion::new(name, role, shape, owner).map(|companion| {
4094 companion.with_derived_recipe(recipe.clone(), output.clone())
4095 })
4096 }
4097 (None, None) => ReplicatedTextOutputCompanion::new(name, role, shape, owner),
4098 _ => {
4099 return Err(format!(
4100 "constructed companion {name:?} has incomplete derived metadata"
4101 ))
4102 }
4103 }
4104 .map_err(|error| error.to_string())?;
4105 let companion = match requirements
4106 .parameters()
4107 .iter()
4108 .find(|parameter| parameter.name() == companion.name())
4109 {
4110 Some(parameter)
4111 if matches!(
4112 parameter.source_encoding(),
4113 Some(eredu_checkpoint::SourceTensorEncoding::Gguf { .. })
4114 ) =>
4115 {
4116 let [source] = parameter.physical_sources() else {
4117 return Err(format!(
4118 "translated catalog companion {:?} has ambiguous provenance",
4119 companion.name()
4120 ));
4121 };
4122 companion.with_catalog_source(source.clone())
4123 }
4124 _ => companion,
4125 };
4126 output_companions
4127 .entry(primary)
4128 .or_default()
4129 .push(companion);
4130 }
4131 }
4132 Ok(output_companions)
4133}
4134
4135fn validate_selected_state(selected: &SelectedReplicatedTextRealization) -> Result<(), String> {
4136 use eredu_core::cache::StateResidencyClass;
4137
4138 if !selected.topology().is_replicated() {
4139 return Err("selected replicated-text topology is not replicated".into());
4140 }
4141 if selected.state().layout() != selected.requirements().state_layout()
4142 || selected.state().access() != selected.requirements().state_access()
4143 {
4144 return Err("selected state contract differs from architecture requirements".into());
4145 }
4146 let mut cursor = 0;
4147 for layer in 0..selected.state().layout().len() {
4148 for expected in selected
4149 .state()
4150 .layout()
4151 .components(layer)
4152 .expect("validated state layout exposes every layer")
4153 {
4154 let component = selected.state().components().get(cursor).ok_or_else(|| {
4155 format!("selected state omits component {cursor} at layer {layer}")
4156 })?;
4157 if component.layer() != layer || component.component() != expected {
4158 return Err(format!(
4159 "selected state component {cursor} differs from layer {layer} requirements"
4160 ));
4161 }
4162 let expected_placement = match selected.state().policy() {
4163 crate::CacheResidencyPolicy::Device => crate::StateComponentPlacement::Device,
4164 crate::CacheResidencyPolicy::Paged(_) => match expected.residency() {
4165 StateResidencyClass::SealablePaged => crate::StateComponentPlacement::Paged,
4166 StateResidencyClass::AlwaysDeviceMutable
4167 | StateResidencyClass::LayerScopedOffloadable => {
4168 crate::StateComponentPlacement::Device
4169 }
4170 },
4171 };
4172 if component.placement() != expected_placement {
4173 return Err(format!(
4174 "selected state component {cursor} has {:?} placement, expected {expected_placement:?}",
4175 component.placement()
4176 ));
4177 }
4178 cursor += 1;
4179 }
4180 }
4181 if cursor != selected.state().components().len() {
4182 return Err("selected state contains components beyond its architecture layout".into());
4183 }
4184 if !selected.state().checkpoint() || !selected.state().rollback() || !selected.state().reset() {
4185 return Err("selected state omits a required transactional lifecycle facility".into());
4186 }
4187 if selected.state().prompt_cache() != selected.prompt_cache()
4188 || selected.state().observation_retention()
4189 != (selected.session().output_observation()
4190 || selected.session().activation_inspection())
4191 {
4192 return Err("selected state lifecycle differs from selected session facilities".into());
4193 }
4194 if selected.grouped_operations() != selected.requirements().grouped_operations() {
4195 return Err("selected grouped operations differ from architecture requirements".into());
4196 }
4197 Ok(())
4198}
4199
4200fn validate_realized_state<A, P, M, B, S>(
4201 state: &S,
4202 selected: &SelectedStateRealization,
4203) -> Result<(), ReplicatedTextSessionError<A, P, M>>
4204where
4205 A: std::fmt::Display,
4206 P: std::fmt::Display,
4207 M: std::fmt::Display,
4208 B: NeuralBackend,
4209 S: RuntimeState<B>,
4210{
4211 if state.layout() != selected.layout() {
4212 return Err(ReplicatedTextSessionError::Contract(
4213 "realized state layout differs from selection".into(),
4214 ));
4215 }
4216 Ok(())
4217}
4218
4219fn map_layerwise_error<A, P>(
4220 error: LayerwiseRuntimeError<A, P>,
4221) -> ReplicatedTextSessionError<A, P, std::convert::Infallible>
4222where
4223 A: std::fmt::Display,
4224 P: std::fmt::Display,
4225{
4226 match error {
4227 LayerwiseRuntimeError::Architecture(error) => {
4228 ReplicatedTextSessionError::Architecture(error)
4229 }
4230 LayerwiseRuntimeError::State(error) => ReplicatedTextSessionError::State(error),
4231 LayerwiseRuntimeError::Layout(error) => {
4232 ReplicatedTextSessionError::Contract(error.to_string())
4233 }
4234 LayerwiseRuntimeError::Policy(error) => ReplicatedTextSessionError::Policy(error),
4235 LayerwiseRuntimeError::Submission(error) => ReplicatedTextSessionError::Contract(error),
4236 }
4237}
4238
4239fn widen_infallible<A, P, M>(
4240 error: ReplicatedTextSessionError<A, P, std::convert::Infallible>,
4241) -> ReplicatedTextSessionError<A, P, M>
4242where
4243 A: std::fmt::Display,
4244 P: std::fmt::Display,
4245 M: std::fmt::Display,
4246{
4247 match error {
4248 ReplicatedTextSessionError::Contract(error) => ReplicatedTextSessionError::Contract(error),
4249 ReplicatedTextSessionError::Architecture(error) => {
4250 ReplicatedTextSessionError::Architecture(error)
4251 }
4252 ReplicatedTextSessionError::Policy(error) => ReplicatedTextSessionError::Policy(error),
4253 ReplicatedTextSessionError::Mechanism(error) => match error {},
4254 ReplicatedTextSessionError::State(error) => ReplicatedTextSessionError::State(error),
4255 ReplicatedTextSessionError::PromptCache(error) => {
4256 ReplicatedTextSessionError::PromptCache(error)
4257 }
4258 ReplicatedTextSessionError::CommitAborted { epoch } => {
4259 ReplicatedTextSessionError::CommitAborted { epoch }
4260 }
4261 ReplicatedTextSessionError::CommitIndeterminate { epoch, phase } => {
4262 ReplicatedTextSessionError::CommitIndeterminate { epoch, phase }
4263 }
4264 }
4265}