1#![allow(clippy::type_complexity)]
4
5use std::{
6 collections::{BTreeMap, BTreeSet},
7 marker::PhantomData,
8 path::Path,
9};
10
11use eredu_core::cache::{
12 validate_prompt_cache_model_identity, PromptCacheDescriptor, PromptCacheError,
13 PromptCacheManifest, PromptCacheModelIdentity, PromptCacheOptions, PromptCacheTopology,
14};
15use eredu_core::{DistributedCommitEpoch, DistributedCommitOutcome, DistributedCommitPhase};
16use eredu_nn::{NeuralBackend, Tensor};
17
18use crate::{
19 observe_model_logits, partitioned_replicated_text_materialization_tasks,
20 plan_local_replicated_text_materialization_tasks, replicated_text_materialization_tasks,
21 ActivationObserver, ArchitecturePartition, CommunicationManifest, ExecutionResidency,
22 ExpertPass, LayerWeightResidency, LayeredArchitecture, LayerwisePolicy, LayerwiseRuntime,
23 LayerwiseRuntimeError, ParameterGroupOwner, PartitionState, PreparedInputCacheIdentity,
24 ReplicatedTextArchitecture, ReplicatedTextMaterializationTask, ReplicatedTextOutputCompanion,
25 ReplicatedTextOutputSelection, ReplicatedTextParameterOwner, ReplicatedTextParameterPresence,
26 RoutedExpertProvider, RoutedLayeredArchitecture, RuntimeState,
27 SelectedReplicatedTextRealization, SelectedStateRealization, StateError, SubmissionBackend,
28 WeightLoweringKind,
29};
30
31pub trait ReplicatedTextSessionMechanisms<A, B>
39where
40 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
41 A: LayeredArchitecture<B, Self::State>,
42 Self::State: RuntimeState<B>,
43 Self::ResidentPolicy: LayerwisePolicy<B, A::Unit, Error = Self::PolicyError>,
44 Self::BoundedPolicy: LayerwisePolicy<B, A::Unit, Error = Self::PolicyError>,
45{
46 type State: RuntimeState<B>;
48 type PolicyError;
50 type ResidentPolicy: LayerwisePolicy<B, A::Unit, Error = Self::PolicyError>;
52 type BoundedPolicy: LayerwisePolicy<B, A::Unit, Error = Self::PolicyError>;
54 type StateCheckpoint;
56 type StateReport;
58 type ExecutionReport;
60 type Error;
62
63 fn take_materialization_report(
71 &mut self,
72 ) -> Result<Option<crate::WeightMaterializationReport>, Self::Error> {
73 Ok(None)
74 }
75
76 fn configure_partition(
79 &mut self,
80 _target_layout: crate::LocalModelLayout,
81 _source_layout: Option<crate::LocalModelLayout>,
82 _rank: eredu_core::cache::CacheRankIdentity,
83 _global_layer_start: usize,
84 ) {
85 }
86
87 #[allow(clippy::too_many_arguments)]
95 fn prepare_partition_materialization(
96 &mut self,
97 architecture: &mut A,
98 global_layout: &crate::ExecutionUnitLayout,
99 addresses: &[crate::ExecutionUnitAddress],
100 task_partition: &crate::ReplicatedTextMaterializationPartitionPlan,
101 units: &mut [A::Unit],
102 source_architecture: Option<&mut A>,
103 source_units: Option<&mut [A::Unit]>,
104 tasks: &[ReplicatedTextMaterializationTask],
105 addressable_parameters: &[String],
106 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
107 ) -> Result<(), Self::Error> {
108 let _ = (addresses, task_partition);
109 self.prepare_materialization(
110 architecture,
111 global_layout,
112 units,
113 source_architecture,
114 source_units,
115 tasks,
116 addressable_parameters,
117 context,
118 )
119 }
120
121 #[allow(clippy::too_many_arguments)]
123 fn prepare_materialization(
124 &mut self,
125 architecture: &mut A,
126 layout: &crate::ExecutionUnitLayout,
127 units: &mut [A::Unit],
128 source_architecture: Option<&mut A>,
129 source_units: Option<&mut [A::Unit]>,
130 tasks: &[ReplicatedTextMaterializationTask],
131 addressable_parameters: &[String],
132 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
133 ) -> Result<(), Self::Error>;
134
135 fn realize_state(
137 &mut self,
138 selected: &SelectedStateRealization,
139 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
140 ) -> Result<Self::State, Self::Error>;
141
142 fn resident_policy(
144 &mut self,
145 architecture: &mut A,
146 units: Vec<A::Unit>,
147 selected: &SelectedReplicatedTextRealization,
148 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
149 ) -> Result<Self::ResidentPolicy, Self::Error>;
150
151 fn bounded_policy(
153 &mut self,
154 architecture: &mut A,
155 selected: &SelectedReplicatedTextRealization,
156 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
157 ) -> Result<Self::BoundedPolicy, Self::Error>;
158
159 fn index_text_output(
161 &mut self,
162 output: B::Tensor,
163 sequence_index: i32,
164 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
165 ) -> Result<B::Tensor, Self::Error>;
166
167 fn checkpoint_state(
169 &mut self,
170 state: &Self::State,
171 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
172 ) -> Result<Self::StateCheckpoint, Self::Error>;
173
174 fn restore_state(
176 &mut self,
177 state: &mut Self::State,
178 checkpoint: Self::StateCheckpoint,
179 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
180 ) -> Result<(), Self::Error>;
181
182 fn fork_prediction_target_state(
189 &mut self,
190 state: &Self::State,
191 selected: &SelectedStateRealization,
192 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
193 ) -> Result<Self::State, Self::Error> {
194 let checkpoint = self.checkpoint_state(state, context)?;
195 let mut fork = self.realize_state(selected, context)?;
196 self.restore_state(&mut fork, checkpoint, context)?;
197 Ok(fork)
198 }
199
200 fn load_prompt_cache(
202 &mut self,
203 directory: &Path,
204 expected: &PromptCacheDescriptor,
205 identity: &PromptCacheModelIdentity,
206 prefix_token_ids: &[u32],
207 selected: &SelectedStateRealization,
208 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
209 ) -> Result<(Self::State, PromptCacheManifest), Self::Error>;
210
211 fn save_prompt_cache(
213 &mut self,
214 state: &mut Self::State,
215 destination: &Path,
216 descriptor: PromptCacheDescriptor,
217 prefix_token_ids: &[u32],
218 options: &PromptCacheOptions,
219 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
220 ) -> Result<PromptCacheManifest, Self::Error>;
221
222 fn state_report(&self, state: &Self::State) -> Result<Self::StateReport, Self::Error>;
224
225 fn execution_report(
227 &self,
228 residency: LayerWeightResidency,
229 bounded: Option<&Self::BoundedPolicy>,
230 ) -> Result<Self::ExecutionReport, Self::Error>;
231
232 fn complete(
234 &mut self,
235 output: &B::Tensor,
236 state: &Self::State,
237 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
238 ) -> Result<(), Self::Error>;
239}
240
241pub trait PredictionTargetOperation<A, B, S>
247where
248 B: NeuralBackend,
249 S: RuntimeState<B>,
250 A: LayeredArchitecture<B, S>,
251{
252 type Output;
254
255 fn apply(
257 self,
258 architecture: &mut A,
259 state: &mut S,
260 parallel: Option<&B::ParallelContext>,
261 context: &<B::Tensor as Tensor>::Context,
262 ) -> Result<Self::Output, A::Error>;
263}
264
265pub trait TransactionalPromptCacheMechanisms<A, B>: ReplicatedTextSessionMechanisms<A, B>
274where
275 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
276 A: LayeredArchitecture<B, Self::State>,
277 Self::State: RuntimeState<B>,
278 Self::ResidentPolicy: LayerwisePolicy<B, A::Unit, Error = Self::PolicyError>,
279 Self::BoundedPolicy: LayerwisePolicy<B, A::Unit, Error = Self::PolicyError>,
280{
281 type PromptCacheSaveTransaction;
283
284 #[allow(clippy::too_many_arguments)]
286 fn prepare_prompt_cache_save(
287 &mut self,
288 state: &mut Self::State,
289 destination: &Path,
290 descriptor: PromptCacheDescriptor,
291 prefix_token_ids: &[u32],
292 options: &PromptCacheOptions,
293 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
294 ) -> Result<Self::PromptCacheSaveTransaction, Self::Error>;
295
296 fn prepared_prompt_cache_manifest(
298 transaction: &Self::PromptCacheSaveTransaction,
299 ) -> &PromptCacheManifest;
300
301 fn publish_prompt_cache_save(
303 &mut self,
304 transaction: &mut Self::PromptCacheSaveTransaction,
305 ) -> Result<(), Self::Error>;
306
307 fn commit_prompt_cache_save(&mut self, transaction: Self::PromptCacheSaveTransaction);
309
310 fn rollback_prompt_cache_save(&mut self, transaction: Self::PromptCacheSaveTransaction);
312}
313
314enum ReplicatedTextRuntimeKind<A, B, S, R, P>
316where
317 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
318 S: RuntimeState<B>,
319 A: LayeredArchitecture<B, S>,
320 R: LayerwisePolicy<B, A::Unit>,
321 P: LayerwisePolicy<B, A::Unit, Error = R::Error>,
322{
323 Resident(LayerwiseRuntime<A, B, S, R>),
325 Bounded(LayerwiseRuntime<A, B, S, P>),
327}
328
329pub struct ReplicatedTextRuntime<A, B, S, R, P>
335where
336 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
337 S: RuntimeState<B>,
338 A: LayeredArchitecture<B, S>,
339 R: LayerwisePolicy<B, A::Unit>,
340 P: LayerwisePolicy<B, A::Unit, Error = R::Error>,
341{
342 kind: ReplicatedTextRuntimeKind<A, B, S, R, P>,
343}
344
345impl<A, B, S, R, P> ReplicatedTextRuntime<A, B, S, R, P>
346where
347 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
348 S: RuntimeState<B>,
349 A: LayeredArchitecture<B, S>,
350 R: LayerwisePolicy<B, A::Unit>,
351 P: LayerwisePolicy<B, A::Unit, Error = R::Error>,
352 A::Error: std::fmt::Display,
353 P::Error: std::fmt::Display,
354{
355 fn forward_with_observer<'a, O>(
356 &mut self,
357 input: A::Input<'a>,
358 state: &mut S,
359 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
360 observer: &mut O,
361 ) -> Result<
362 (B::Tensor, A::ForwardContext),
363 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
364 >
365 where
366 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
367 {
368 match &mut self.kind {
369 ReplicatedTextRuntimeKind::Resident(runtime) => runtime
370 .forward_with_observer_and_context(input, state, context, observer)
371 .map_err(map_layerwise_error),
372 ReplicatedTextRuntimeKind::Bounded(runtime) => runtime
373 .forward_with_observer_and_context(input, state, context, observer)
374 .map_err(map_layerwise_error),
375 }
376 }
377
378 fn forward_with_provider_and_observer<'a, Provider, Observer>(
379 &mut self,
380 input: A::Input<'a>,
381 state: &mut S,
382 pass: ExpertPass,
383 provider: &mut Provider,
384 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
385 observer: &mut Observer,
386 ) -> Result<
387 (B::Tensor, A::ForwardContext),
388 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
389 >
390 where
391 B: eredu_nn::GroupedNeuralBackend,
392 A: RoutedLayeredArchitecture<B, S>,
393 Provider: RoutedExpertProvider<B>,
394 Provider::Error: std::fmt::Display,
395 Observer: ActivationObserver<B::Tensor, A::Error> + ?Sized,
396 {
397 match &mut self.kind {
398 ReplicatedTextRuntimeKind::Resident(runtime) => runtime
399 .forward_with_provider_and_observer_and_context(
400 input, state, pass, provider, context, observer,
401 )
402 .map_err(map_layerwise_error),
403 ReplicatedTextRuntimeKind::Bounded(runtime) => runtime
404 .forward_with_provider_and_observer_and_context(
405 input, state, pass, provider, context, observer,
406 )
407 .map_err(map_layerwise_error),
408 }
409 }
410
411 fn bounded_policy(&self) -> Option<&P> {
412 match &self.kind {
413 ReplicatedTextRuntimeKind::Resident(_) => None,
414 ReplicatedTextRuntimeKind::Bounded(runtime) => Some(runtime.policy()),
415 }
416 }
417
418 fn prediction_target_capture(
419 &mut self,
420 forward: &A::ForwardContext,
421 _context: &<B::Tensor as Tensor>::Context,
422 ) -> Result<Option<B::Tensor>, A::Error> {
423 Ok(<A as crate::LayeredArchitecture<B, S>>::prediction_target_capture(forward).cloned())
424 }
425
426 fn apply_prediction_target_operation<O>(
427 &mut self,
428 state: &mut S,
429 operation: O,
430 context: &<B::Tensor as Tensor>::Context,
431 ) -> Result<O::Output, A::Error>
432 where
433 O: PredictionTargetOperation<A, B, S>,
434 {
435 match &mut self.kind {
436 ReplicatedTextRuntimeKind::Resident(runtime) => {
437 operation.apply(runtime.architecture_mut(), state, None, context)
438 }
439 ReplicatedTextRuntimeKind::Bounded(runtime) => {
440 operation.apply(runtime.architecture_mut(), state, None, context)
441 }
442 }
443 }
444}
445
446pub trait ReplicatedTextExecutionStrategy<A, B, S, R, P>
451where
452 B: NeuralBackend,
453 S: RuntimeState<B>,
454 A: LayeredArchitecture<B, S>,
455 R: LayerwisePolicy<B, A::Unit>,
456 P: LayerwisePolicy<B, A::Unit, Error = R::Error>,
457 A::Error: std::fmt::Display,
458 R::Error: std::fmt::Display,
459{
460 const PARTITIONED_SESSION: bool = false;
462 const DISTRIBUTED_PHASE_AGREEMENT: bool = false;
464
465 type Runtime;
467
468 fn bounded_policy(runtime: &Self::Runtime) -> Option<&P>;
470
471 fn execution_residency(
473 runtime: &Self::Runtime,
474 selected: &SelectedReplicatedTextRealization,
475 ) -> ExecutionResidency;
476
477 #[allow(clippy::too_many_arguments)]
479 fn forward_with_observer<'a, O>(
480 &mut self,
481 runtime: &mut Self::Runtime,
482 input: A::Input<'a>,
483 state: &mut S,
484 pass: ExpertPass,
485 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
486 observer: &mut O,
487 ) -> Result<
488 (B::Tensor, A::ForwardContext),
489 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
490 >
491 where
492 O: ActivationObserver<B::Tensor, A::Error> + ?Sized;
493
494 fn observe_output<O>(
498 _runtime: &mut Self::Runtime,
499 output: &B::Tensor,
500 observer: &mut O,
501 _context: &<B::Tensor as Tensor>::Context,
502 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>>
503 where
504 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
505 {
506 observe_model_logits(observer, output).map_err(ReplicatedTextSessionError::Architecture)
507 }
508
509 fn publish_observed_output(
512 _runtime: &mut Self::Runtime,
513 output: B::Tensor,
514 _context: &<B::Tensor as Tensor>::Context,
515 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>>
516 {
517 Ok(output)
518 }
519
520 fn prediction_target_capture(
526 _runtime: &mut Self::Runtime,
527 forward: &A::ForwardContext,
528 _context: &<B::Tensor as Tensor>::Context,
529 ) -> Result<
530 Option<B::Tensor>,
531 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
532 > {
533 Ok(<A as crate::LayeredArchitecture<B, S>>::prediction_target_capture(forward).cloned())
534 }
535
536 fn publish_prediction_target_capture(
538 _runtime: &mut Self::Runtime,
539 capture: B::Tensor,
540 _context: &<B::Tensor as Tensor>::Context,
541 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>>
542 {
543 Ok(capture)
544 }
545
546 fn apply_prediction_target_operation<O>(
548 _runtime: &mut Self::Runtime,
549 _state: &mut S,
550 _operation: O,
551 _context: &<B::Tensor as Tensor>::Context,
552 ) -> Result<
553 Option<O::Output>,
554 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
555 >
556 where
557 O: PredictionTargetOperation<A, B, S>,
558 {
559 Ok(None)
560 }
561
562 fn commit_after_completion(
565 _runtime: &mut Self::Runtime,
566 epoch: DistributedCommitEpoch,
567 _context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
568 ) -> DistributedCommitOutcome {
569 DistributedCommitOutcome::Committed(epoch)
570 }
571
572 fn agree_distributed_phase(
575 _runtime: &mut Self::Runtime,
576 _phase: crate::DistributedExecutionPhase,
577 local_success: bool,
578 _context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
579 ) -> Result<bool, ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>>
580 {
581 Ok(local_success)
582 }
583}
584
585pub trait ReplicatedRuntimeExecutionStrategy<A, B, S, R, P>:
591 ReplicatedTextExecutionStrategy<A, B, S, R, P, Runtime = ReplicatedTextRuntime<A, B, S, R, P>>
592where
593 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
594 S: RuntimeState<B>,
595 A: LayeredArchitecture<B, S>,
596 R: LayerwisePolicy<B, A::Unit>,
597 P: LayerwisePolicy<B, A::Unit, Error = R::Error>,
598 A::Error: std::fmt::Display,
599 R::Error: std::fmt::Display,
600{
601}
602
603#[derive(Debug, Default, Clone, Copy)]
605pub struct DirectReplicatedTextExecution;
606
607impl<A, B, S, R, P> ReplicatedTextExecutionStrategy<A, B, S, R, P> for DirectReplicatedTextExecution
608where
609 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
610 S: RuntimeState<B>,
611 A: LayeredArchitecture<B, S>,
612 R: LayerwisePolicy<B, A::Unit>,
613 P: LayerwisePolicy<B, A::Unit, Error = R::Error>,
614 A::Error: std::fmt::Display,
615 P::Error: std::fmt::Display,
616{
617 type Runtime = ReplicatedTextRuntime<A, B, S, R, P>;
618
619 fn bounded_policy(runtime: &Self::Runtime) -> Option<&P> {
620 runtime.bounded_policy()
621 }
622
623 fn execution_residency(
624 _runtime: &Self::Runtime,
625 selected: &SelectedReplicatedTextRealization,
626 ) -> ExecutionResidency {
627 selected.residency().execution_residency()
628 }
629
630 fn forward_with_observer<'a, O>(
631 &mut self,
632 runtime: &mut Self::Runtime,
633 input: A::Input<'a>,
634 state: &mut S,
635 _pass: ExpertPass,
636 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
637 observer: &mut O,
638 ) -> Result<
639 (B::Tensor, A::ForwardContext),
640 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
641 >
642 where
643 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
644 {
645 runtime.forward_with_observer(input, state, context, observer)
646 }
647
648 fn prediction_target_capture(
649 runtime: &mut Self::Runtime,
650 forward: &A::ForwardContext,
651 context: &<B::Tensor as Tensor>::Context,
652 ) -> Result<
653 Option<B::Tensor>,
654 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
655 > {
656 runtime
657 .prediction_target_capture(forward, context)
658 .map_err(ReplicatedTextSessionError::Architecture)
659 }
660
661 fn apply_prediction_target_operation<O>(
662 runtime: &mut Self::Runtime,
663 state: &mut S,
664 operation: O,
665 context: &<B::Tensor as Tensor>::Context,
666 ) -> Result<
667 Option<O::Output>,
668 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
669 >
670 where
671 O: PredictionTargetOperation<A, B, S>,
672 {
673 runtime
674 .apply_prediction_target_operation(state, operation, context)
675 .map(Some)
676 .map_err(ReplicatedTextSessionError::Architecture)
677 }
678}
679
680impl<A, B, S, R, P> ReplicatedRuntimeExecutionStrategy<A, B, S, R, P>
681 for DirectReplicatedTextExecution
682where
683 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
684 S: RuntimeState<B>,
685 A: LayeredArchitecture<B, S>,
686 R: LayerwisePolicy<B, A::Unit>,
687 P: LayerwisePolicy<B, A::Unit, Error = R::Error>,
688 A::Error: std::fmt::Display,
689 P::Error: std::fmt::Display,
690{
691}
692
693pub struct RoutedReplicatedTextExecution<P> {
695 provider: P,
696}
697
698impl<P> RoutedReplicatedTextExecution<P> {
699 pub const fn new(provider: P) -> Self {
701 Self { provider }
702 }
703
704 pub const fn provider(&self) -> &P {
706 &self.provider
707 }
708}
709
710impl<A, B, S, R, P, Provider> ReplicatedTextExecutionStrategy<A, B, S, R, P>
711 for RoutedReplicatedTextExecution<Provider>
712where
713 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>
714 + eredu_nn::GroupedNeuralBackend,
715 S: RuntimeState<B>,
716 A: LayeredArchitecture<B, S> + RoutedLayeredArchitecture<B, S>,
717 R: LayerwisePolicy<B, A::Unit>,
718 P: LayerwisePolicy<B, A::Unit, Error = R::Error>,
719 Provider: RoutedExpertProvider<B>,
720 Provider::Error: std::fmt::Display,
721 A::Error: std::fmt::Display,
722 P::Error: std::fmt::Display,
723{
724 type Runtime = ReplicatedTextRuntime<A, B, S, R, P>;
725
726 fn bounded_policy(runtime: &Self::Runtime) -> Option<&P> {
727 runtime.bounded_policy()
728 }
729
730 fn execution_residency(
731 _runtime: &Self::Runtime,
732 selected: &SelectedReplicatedTextRealization,
733 ) -> ExecutionResidency {
734 selected.residency().execution_residency()
735 }
736
737 fn forward_with_observer<'a, O>(
738 &mut self,
739 runtime: &mut Self::Runtime,
740 input: A::Input<'a>,
741 state: &mut S,
742 pass: ExpertPass,
743 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
744 observer: &mut O,
745 ) -> Result<
746 (B::Tensor, A::ForwardContext),
747 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
748 >
749 where
750 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
751 {
752 runtime.forward_with_provider_and_observer(
753 input,
754 state,
755 pass,
756 &mut self.provider,
757 context,
758 observer,
759 )
760 }
761
762 fn apply_prediction_target_operation<O>(
763 runtime: &mut Self::Runtime,
764 state: &mut S,
765 operation: O,
766 context: &<B::Tensor as Tensor>::Context,
767 ) -> Result<
768 Option<O::Output>,
769 ReplicatedTextSessionError<A::Error, R::Error, std::convert::Infallible>,
770 >
771 where
772 O: PredictionTargetOperation<A, B, S>,
773 {
774 runtime
775 .apply_prediction_target_operation(state, operation, context)
776 .map(Some)
777 .map_err(ReplicatedTextSessionError::Architecture)
778 }
779}
780
781impl<A, B, S, R, P, Provider> ReplicatedRuntimeExecutionStrategy<A, B, S, R, P>
782 for RoutedReplicatedTextExecution<Provider>
783where
784 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>
785 + eredu_nn::GroupedNeuralBackend,
786 S: RuntimeState<B>,
787 A: LayeredArchitecture<B, S> + RoutedLayeredArchitecture<B, S>,
788 R: LayerwisePolicy<B, A::Unit>,
789 P: LayerwisePolicy<B, A::Unit, Error = R::Error>,
790 Provider: RoutedExpertProvider<B>,
791 Provider::Error: std::fmt::Display,
792 A::Error: std::fmt::Display,
793 P::Error: std::fmt::Display,
794{
795}
796
797fn record_successful_restoration<E>(
798 generation: &mut Option<u64>,
799 restored: Result<(), E>,
800) -> Result<(), E> {
801 restored?;
802 *generation = generation.and_then(|value| value.checked_add(1));
803 Ok(())
804}
805
806#[cfg(test)]
807mod restoration_witness_tests {
808 use super::record_successful_restoration;
809
810 #[test]
811 fn failed_restore_and_stale_snapshot_do_not_prove_new_restoration() {
812 let mut generation = Some(0);
813 let before = generation;
814 assert!(record_successful_restoration(&mut generation, Err("restore failed")).is_err());
815 assert_eq!(generation, before);
816 record_successful_restoration(&mut generation, Ok::<_, ()>(())).unwrap();
817 assert_eq!(generation, Some(1));
818 let prior_restore = generation;
819 assert!(
820 record_successful_restoration(&mut generation, Err("later restore failed")).is_err()
821 );
822 assert_eq!(generation, prior_restore);
823 }
824
825 #[test]
826 fn restoration_counter_overflow_permanently_disables_the_witness() {
827 let mut generation = Some(u64::MAX);
828 record_successful_restoration(&mut generation, Ok::<_, ()>(())).unwrap();
829 assert_eq!(generation, None);
830 record_successful_restoration(&mut generation, Ok::<_, ()>(())).unwrap();
831 assert_eq!(generation, None);
832 }
833}
834
835pub struct ReplicatedTextSession<A, B, M, D = DirectReplicatedTextExecution>
837where
838 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
839 M: ReplicatedTextSessionMechanisms<A, B>,
840 A: LayeredArchitecture<B, M::State>,
841 D: ReplicatedTextExecutionStrategy<A, B, M::State, M::ResidentPolicy, M::BoundedPolicy>,
842 A::Error: std::fmt::Display,
843 M::PolicyError: std::fmt::Display,
844{
845 selected: SelectedReplicatedTextRealization,
846 selected_state: SessionStateRealization,
847 execution: D::Runtime,
848 driver: D,
849 state: M::State,
850 mechanisms: M,
851 materialization_report: Option<crate::WeightMaterializationReport>,
852 prompt_cache_identity: Option<PromptCacheModelIdentity>,
853 committed_prompt_input_identity: Option<PreparedInputCacheIdentity>,
854 next_commit_epoch: DistributedCommitEpoch,
855 active_commit_epoch: Option<DistributedCommitEpoch>,
856 last_commit_outcome: Option<DistributedCommitOutcome>,
857 successful_state_restorations: Option<u64>,
858 control_fence: Option<crate::DistributedExecutionPhase>,
859 output_selection: ReplicatedTextOutputSelection,
860 backend: PhantomData<fn() -> B>,
861}
862
863#[derive(Debug, Clone, Eq, PartialEq)]
865pub enum SessionStateRealization {
866 Stateful(SelectedStateRealization),
868 Stateless,
870}
871
872pub struct PreparedPartitionedSessionRuntime<R, S> {
877 selected: SelectedReplicatedTextRealization,
878 runtime: R,
879 state: S,
880 selected_state: SessionStateRealization,
881 prompt_cache_identity: Option<PromptCacheModelIdentity>,
882 output_selection: ReplicatedTextOutputSelection,
883}
884
885impl<R, S> PreparedPartitionedSessionRuntime<R, S> {
886 pub const fn prompt_cache_identity(&self) -> Option<&PromptCacheModelIdentity> {
889 self.prompt_cache_identity.as_ref()
890 }
891}
892
893pub struct PartitionedSessionFactoryInput<A, G, W> {
901 architecture: A,
902 partition: ArchitecturePartition<G, W>,
903 communication: CommunicationManifest,
904 tasks: Vec<ReplicatedTextMaterializationTask>,
905}
906
907impl<A, G, W> PartitionedSessionFactoryInput<A, G, W> {
908 pub fn materialization_tasks(&self) -> &[ReplicatedTextMaterializationTask] {
910 &self.tasks
911 }
912
913 pub fn into_parts(
915 self,
916 ) -> (
917 A,
918 ArchitecturePartition<G, W>,
919 CommunicationManifest,
920 Vec<ReplicatedTextMaterializationTask>,
921 ) {
922 (
923 self.architecture,
924 self.partition,
925 self.communication,
926 self.tasks,
927 )
928 }
929}
930
931#[derive(Debug, Clone, Copy, Eq, PartialEq)]
933pub enum PartitionedUnitScope {
934 All,
936 Owned,
938}
939
940pub struct PreparedPartitionedRuntimeComponents<A, G, W, S, P> {
943 architecture: A,
944 partition: ArchitecturePartition<G, W>,
945 communication: CommunicationManifest,
946 execution_policy: P,
947 bounded_policy: Option<P>,
948 state: S,
949}
950
951impl<A, G, W, S, P> PreparedPartitionedRuntimeComponents<A, G, W, S, P> {
952 pub fn into_parts(
956 self,
957 ) -> (
958 A,
959 ArchitecturePartition<G, W>,
960 CommunicationManifest,
961 P,
962 Option<P>,
963 S,
964 ) {
965 (
966 self.architecture,
967 self.partition,
968 self.communication,
969 self.execution_policy,
970 self.bounded_policy,
971 self.state,
972 )
973 }
974}
975
976#[derive(Debug, thiserror::Error)]
978pub enum PartitionedRuntimeConstructionError {
979 #[error("partitioned runtime contract mismatch: {0}")]
981 Contract(String),
982 #[error("partitioned runtime architecture construction failed: {0}")]
984 Architecture(String),
985 #[error("partitioned runtime mechanism failed: {0}")]
987 Mechanism(String),
988}
989
990#[allow(clippy::too_many_arguments)]
999pub fn prepare_default_partitioned_runtime<A, B, M, G, W, P>(
1000 input: PartitionedSessionFactoryInput<A, G, W>,
1001 mut source_architecture: Option<A>,
1002 target_parallel_layout: crate::LocalModelLayout,
1003 source_parallel_layout: Option<crate::LocalModelLayout>,
1004 selected: &SelectedReplicatedTextRealization,
1005 topology: &PromptCacheTopology,
1006 scope: PartitionedUnitScope,
1007 addressable_parameters: &[String],
1008 mechanisms: &mut M,
1009 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
1010) -> Result<
1011 PreparedPartitionedRuntimeComponents<A, G, W, M::State, P>,
1012 PartitionedRuntimeConstructionError,
1013>
1014where
1015 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
1016 M: ReplicatedTextSessionMechanisms<A, B, ResidentPolicy = P, BoundedPolicy = P>,
1017 A: LayeredArchitecture<B, M::State>,
1018 A::Error: std::fmt::Display,
1019 M::Error: std::fmt::Display,
1020 P: LayerwisePolicy<B, A::Unit, Error = M::PolicyError> + Clone,
1021{
1022 let (mut architecture, partition, communication, tasks) = input.into_parts();
1023 let global_layout = partition.unit_layout().clone();
1024 let addresses = match scope {
1025 PartitionedUnitScope::All => (0..global_layout.len())
1026 .map(|ordinal| {
1027 global_layout.address(ordinal).ok_or_else(|| {
1028 PartitionedRuntimeConstructionError::Contract(format!(
1029 "global unit ordinal {ordinal} has no canonical address"
1030 ))
1031 })
1032 })
1033 .collect::<Result<Vec<_>, _>>()?,
1034 PartitionedUnitScope::Owned => partition.units().collect::<Vec<_>>(),
1035 };
1036 if addresses.is_empty() {
1037 return Err(PartitionedRuntimeConstructionError::Contract(
1038 "partition owns no execution units".into(),
1039 ));
1040 }
1041 let task_partition =
1042 plan_local_replicated_text_materialization_tasks(&tasks, &global_layout, &addresses)
1043 .map_err(|error| PartitionedRuntimeConstructionError::Contract(error.to_string()))?;
1044 let mut units = addresses
1045 .iter()
1046 .map(|address| {
1047 architecture
1048 .build_unit(address.group(), address.index(), context)
1049 .map_err(|error| {
1050 PartitionedRuntimeConstructionError::Architecture(error.to_string())
1051 })
1052 })
1053 .collect::<Result<Vec<_>, _>>()?;
1054 let mut source_units = source_architecture
1055 .as_ref()
1056 .map(|source| {
1057 addresses
1058 .iter()
1059 .map(|address| {
1060 source
1061 .build_unit(address.group(), address.index(), context)
1062 .map_err(|error| {
1063 PartitionedRuntimeConstructionError::Architecture(error.to_string())
1064 })
1065 })
1066 .collect::<Result<Vec<_>, _>>()
1067 })
1068 .transpose()?;
1069 let local_state = partition.state().ok_or_else(|| {
1070 PartitionedRuntimeConstructionError::Contract(
1071 "partition owns no local mutable state".into(),
1072 )
1073 })?;
1074 let selected_state = selected
1075 .state()
1076 .for_partitioned_geometry(local_state)
1077 .map_err(|error| PartitionedRuntimeConstructionError::Contract(error.to_string()))?;
1078 let rank = eredu_core::cache::CacheRankIdentity::new(
1079 topology.stage().map(|(_, rank)| rank),
1080 topology.shard().map(|(_, rank)| rank),
1081 topology.addressable().map(|(_, rank)| rank),
1082 );
1083 mechanisms.configure_partition(
1084 target_parallel_layout,
1085 source_parallel_layout,
1086 rank,
1087 local_state.global_layer_offset(),
1088 );
1089 mechanisms
1090 .prepare_partition_materialization(
1091 &mut architecture,
1092 &global_layout,
1093 &addresses,
1094 &task_partition,
1095 &mut units,
1096 source_architecture.as_mut(),
1097 source_units.as_deref_mut(),
1098 &tasks,
1099 addressable_parameters,
1100 context,
1101 )
1102 .map_err(|error| PartitionedRuntimeConstructionError::Mechanism(error.to_string()))?;
1103 let state = mechanisms
1104 .realize_state(&selected_state, context)
1105 .map_err(|error| PartitionedRuntimeConstructionError::Mechanism(error.to_string()))?;
1106 if state.optional_layout() != Some(selected_state.layout()) {
1107 return Err(PartitionedRuntimeConstructionError::Contract(
1108 "realized partition state differs from selected local geometry".into(),
1109 ));
1110 }
1111 let (execution_policy, bounded_policy) = match selected.residency() {
1112 LayerWeightResidency::FullyResident => (
1113 mechanisms
1114 .resident_policy(&mut architecture, units, selected, context)
1115 .map_err(|error| {
1116 PartitionedRuntimeConstructionError::Mechanism(error.to_string())
1117 })?,
1118 None,
1119 ),
1120 LayerWeightResidency::LayerwiseHost(_) | LayerWeightResidency::DenseDiskStream(_) => {
1121 drop(units);
1122 let policy = mechanisms
1123 .bounded_policy(&mut architecture, selected, context)
1124 .map_err(|error| {
1125 PartitionedRuntimeConstructionError::Mechanism(error.to_string())
1126 })?;
1127 (policy.clone(), Some(policy))
1128 }
1129 };
1130 Ok(PreparedPartitionedRuntimeComponents {
1131 architecture,
1132 partition,
1133 communication,
1134 execution_policy,
1135 bounded_policy,
1136 state,
1137 })
1138}
1139
1140#[derive(Debug, thiserror::Error)]
1142pub enum PartitionedSessionPreparationError<E> {
1143 #[error("partitioned session authority mismatch: {0}")]
1145 Contract(String),
1146 #[error("partitioned session runtime factory failed: {0}")]
1148 Factory(E),
1149}
1150
1151#[allow(clippy::too_many_arguments)]
1159pub fn prepare_partitioned_session_runtime<A, B, R, S, G, W, E, F>(
1160 architecture: A,
1161 selected: SelectedReplicatedTextRealization,
1162 partition: ArchitecturePartition<G, W>,
1163 communication: CommunicationManifest,
1164 expected_tasks: Option<&[ReplicatedTextMaterializationTask]>,
1165 topology: PromptCacheTopology,
1166 output_selection: ReplicatedTextOutputSelection,
1167 context: &<B::Tensor as Tensor>::Context,
1168 factory: F,
1169) -> Result<PreparedPartitionedSessionRuntime<R, S>, PartitionedSessionPreparationError<E>>
1170where
1171 B: NeuralBackend,
1172 S: RuntimeState<B>,
1173 A: LayeredArchitecture<B, S>,
1174 A::Error: std::fmt::Display,
1175 F: FnOnce(
1176 PartitionedSessionFactoryInput<A, G, W>,
1177 &SelectedReplicatedTextRealization,
1178 &<B::Tensor as Tensor>::Context,
1179 ) -> Result<(R, S), E>,
1180{
1181 prepare_partitioned_session_runtime_with_exclusions(
1182 architecture,
1183 selected,
1184 partition,
1185 communication,
1186 expected_tasks,
1187 &std::collections::BTreeSet::new(),
1188 topology,
1189 output_selection,
1190 context,
1191 factory,
1192 )
1193}
1194
1195#[allow(clippy::too_many_arguments)]
1198pub fn prepare_partitioned_session_runtime_with_exclusions<A, B, R, S, G, W, E, F>(
1199 architecture: A,
1200 selected: SelectedReplicatedTextRealization,
1201 partition: ArchitecturePartition<G, W>,
1202 communication: CommunicationManifest,
1203 expected_tasks: Option<&[ReplicatedTextMaterializationTask]>,
1204 excluded_parameter_targets: &std::collections::BTreeSet<&str>,
1205 topology: PromptCacheTopology,
1206 output_selection: ReplicatedTextOutputSelection,
1207 context: &<B::Tensor as Tensor>::Context,
1208 factory: F,
1209) -> Result<PreparedPartitionedSessionRuntime<R, S>, PartitionedSessionPreparationError<E>>
1210where
1211 B: NeuralBackend,
1212 S: RuntimeState<B>,
1213 A: LayeredArchitecture<B, S>,
1214 A::Error: std::fmt::Display,
1215 F: FnOnce(
1216 PartitionedSessionFactoryInput<A, G, W>,
1217 &SelectedReplicatedTextRealization,
1218 &<B::Tensor as Tensor>::Context,
1219 ) -> Result<(R, S), E>,
1220{
1221 partition
1222 .validate_architecture::<B, S, A>(&architecture)
1223 .map_err(|error| PartitionedSessionPreparationError::Contract(error.to_string()))?;
1224 let parameters = architecture
1225 .parameter_description(context)
1226 .map_err(|error| PartitionedSessionPreparationError::Contract(error.to_string()))?;
1227 let mut tasks =
1228 partitioned_replicated_text_materialization_tasks(&selected, ¶meters, &partition)
1229 .map_err(|error| PartitionedSessionPreparationError::Contract(error.to_string()))?;
1230 tasks.retain(|task| !excluded_parameter_targets.contains(task.name()));
1231 if expected_tasks.is_some_and(|expected| expected != tasks) {
1232 let derived_names = tasks
1233 .iter()
1234 .map(ReplicatedTextMaterializationTask::name)
1235 .collect::<std::collections::BTreeSet<_>>();
1236 let first_missing = expected_tasks
1237 .expect("task proof was checked as present")
1238 .iter()
1239 .map(ReplicatedTextMaterializationTask::name)
1240 .find(|name| !derived_names.contains(name));
1241 let parameter_group = first_missing.and_then(|name| {
1242 parameters
1243 .groups()
1244 .iter()
1245 .find(|group| group.members().iter().any(|member| member.target() == name))
1246 });
1247 let partition_group = first_missing.and_then(|name| {
1248 partition
1249 .parameter_bindings()
1250 .iter()
1251 .find(|group| group.members().iter().any(|member| member.target() == name))
1252 });
1253 return Err(PartitionedSessionPreparationError::Contract(
1254 format!(
1255 "precomputed local materialization tasks differ from consumed partition authority: expected {:?}, derived {:?}, first missing current group {parameter_group:?}, admitted group {partition_group:?}",
1256 expected_tasks
1257 .expect("task proof was checked as present")
1258 .iter()
1259 .map(ReplicatedTextMaterializationTask::name)
1260 .collect::<Vec<_>>(),
1261 tasks
1262 .iter()
1263 .map(ReplicatedTextMaterializationTask::name)
1264 .collect::<Vec<_>>()
1265 ),
1266 ));
1267 }
1268 let partition_state = partition.state().cloned();
1269 let (selected_state, prompt_cache_identity) = match partition_state.as_ref() {
1270 Some(partition_state) => (
1271 SessionStateRealization::Stateful(
1272 selected
1273 .state()
1274 .for_partitioned_geometry(partition_state)
1275 .map_err(|error| {
1276 PartitionedSessionPreparationError::Contract(error.to_string())
1277 })?,
1278 ),
1279 Some(
1280 partition_state
1281 .prompt_cache_identity::<B, A>(&architecture, topology)
1282 .map_err(|error| {
1283 PartitionedSessionPreparationError::Contract(error.to_string())
1284 })?,
1285 ),
1286 ),
1287 None => (SessionStateRealization::Stateless, None),
1288 };
1289 let (runtime, state) = factory(
1290 PartitionedSessionFactoryInput {
1291 architecture,
1292 partition,
1293 communication,
1294 tasks,
1295 },
1296 &selected,
1297 context,
1298 )
1299 .map_err(PartitionedSessionPreparationError::Factory)?;
1300 match selected_state.state() {
1301 Some(local) if state.optional_layout() != Some(local.layout()) => {
1302 return Err(PartitionedSessionPreparationError::Contract(
1303 "partition runtime state differs from canonical local geometry".into(),
1304 ));
1305 }
1306 None if state.optional_layout().is_some() => {
1307 return Err(PartitionedSessionPreparationError::Contract(
1308 "stateless partition binding contains mutable state geometry".into(),
1309 ));
1310 }
1311 _ => {}
1312 }
1313 Ok(PreparedPartitionedSessionRuntime {
1314 selected,
1315 runtime,
1316 state,
1317 selected_state,
1318 prompt_cache_identity,
1319 output_selection,
1320 })
1321}
1322
1323impl SessionStateRealization {
1324 pub const fn state(&self) -> Option<&SelectedStateRealization> {
1326 match self {
1327 Self::Stateful(state) => Some(state),
1328 Self::Stateless => None,
1329 }
1330 }
1331}
1332
1333pub struct ReplicatedTextSessionCheckpoint<C> {
1335 state: C,
1336 prompt_input_identity: Option<PreparedInputCacheIdentity>,
1337 next_commit_epoch: DistributedCommitEpoch,
1338 last_commit_outcome: Option<DistributedCommitOutcome>,
1339}
1340
1341pub struct DistributedStateCheckpoint<C> {
1343 state: Option<C>,
1344}
1345
1346pub struct DistributedSessionCheckpoint<C> {
1348 state: Option<C>,
1349 prompt_input_identity: Option<PreparedInputCacheIdentity>,
1350 next_commit_epoch: DistributedCommitEpoch,
1351 last_commit_outcome: Option<DistributedCommitOutcome>,
1352}
1353
1354#[derive(Debug, thiserror::Error)]
1356pub enum ReplicatedTextSessionError<A, P, M>
1357where
1358 A: std::fmt::Display,
1359 P: std::fmt::Display,
1360 M: std::fmt::Display,
1361{
1362 #[error("replicated text contract mismatch: {0}")]
1364 Contract(String),
1365 #[error("replicated text architecture failed: {0}")]
1367 Architecture(A),
1368 #[error("replicated text residency failed: {0}")]
1370 Policy(P),
1371 #[error("replicated text mechanism failed: {0}")]
1373 Mechanism(M),
1374 #[error(transparent)]
1376 State(#[from] StateError),
1377 #[error(transparent)]
1379 PromptCache(#[from] PromptCacheError),
1380 #[error("distributed transaction epoch {epoch:?} was aborted")]
1382 CommitAborted {
1383 epoch: DistributedCommitEpoch,
1385 },
1386 #[error("distributed transaction epoch {epoch:?} is indeterminate at {phase:?}")]
1388 CommitIndeterminate {
1389 epoch: DistributedCommitEpoch,
1391 phase: DistributedCommitPhase,
1393 },
1394}
1395
1396#[derive(Debug, Clone, Eq, PartialEq)]
1398pub struct ReplicatedTextSessionReport<E, S> {
1399 execution: ExecutionResidency,
1400 execution_report: E,
1401 state_report: S,
1402 distributed_commit: Option<DistributedCommitOutcome>,
1403}
1404
1405pub struct PreparedReplicatedTextContract {
1410 selected: SelectedReplicatedTextRealization,
1411 tasks: Vec<ReplicatedTextMaterializationTask>,
1412 addressable_parameters: Vec<String>,
1413 prompt_cache_identity: PromptCacheModelIdentity,
1414 output_selection: ReplicatedTextOutputSelection,
1415}
1416
1417impl PreparedReplicatedTextContract {
1418 pub const fn selected(&self) -> &SelectedReplicatedTextRealization {
1420 &self.selected
1421 }
1422
1423 pub fn materialization_tasks(&self) -> &[ReplicatedTextMaterializationTask] {
1425 &self.tasks
1426 }
1427
1428 pub const fn prompt_cache_identity(&self) -> &PromptCacheModelIdentity {
1430 &self.prompt_cache_identity
1431 }
1432
1433 pub const fn output_selection(&self) -> ReplicatedTextOutputSelection {
1435 self.output_selection
1436 }
1437
1438 fn into_parts(
1439 self,
1440 ) -> (
1441 SelectedReplicatedTextRealization,
1442 Vec<ReplicatedTextMaterializationTask>,
1443 Vec<String>,
1444 PromptCacheModelIdentity,
1445 ReplicatedTextOutputSelection,
1446 ) {
1447 (
1448 self.selected,
1449 self.tasks,
1450 self.addressable_parameters,
1451 self.prompt_cache_identity,
1452 self.output_selection,
1453 )
1454 }
1455}
1456
1457pub fn prepare_replicated_text_contract<A, B, S>(
1460 architecture: &A,
1461 source_architecture: Option<&A>,
1462 selected: SelectedReplicatedTextRealization,
1463 expected_prompt_cache_architecture_identity: &str,
1464 context: &<B::Tensor as Tensor>::Context,
1465) -> Result<PreparedReplicatedTextContract, String>
1466where
1467 B: NeuralBackend,
1468 S: RuntimeState<B>,
1469 A: ReplicatedTextArchitecture<B, S>,
1470 A::Error: std::fmt::Display,
1471{
1472 prepare_replicated_text_contract_with_addressable_parameters::<A, B, S>(
1473 architecture,
1474 source_architecture,
1475 selected,
1476 expected_prompt_cache_architecture_identity,
1477 std::iter::empty::<&str>(),
1478 context,
1479 )
1480}
1481
1482pub fn prepare_replicated_text_contract_with_addressable_parameters<'a, A, B, S>(
1489 architecture: &A,
1490 source_architecture: Option<&A>,
1491 selected: SelectedReplicatedTextRealization,
1492 expected_prompt_cache_architecture_identity: &str,
1493 addressable_parameters: impl IntoIterator<Item = &'a str>,
1494 context: &<B::Tensor as Tensor>::Context,
1495) -> Result<PreparedReplicatedTextContract, String>
1496where
1497 B: NeuralBackend,
1498 S: RuntimeState<B>,
1499 A: ReplicatedTextArchitecture<B, S>,
1500 A::Error: std::fmt::Display,
1501{
1502 prepare_layered_text_contract_with_addressable_parameters::<A, B, S>(
1503 architecture,
1504 source_architecture,
1505 selected,
1506 expected_prompt_cache_architecture_identity,
1507 architecture.text_output_selection(),
1508 addressable_parameters,
1509 context,
1510 )
1511}
1512
1513pub fn prepare_layered_text_contract<A, B, S>(
1519 architecture: &A,
1520 source_architecture: Option<&A>,
1521 selected: SelectedReplicatedTextRealization,
1522 expected_prompt_cache_architecture_identity: &str,
1523 output_selection: ReplicatedTextOutputSelection,
1524 context: &<B::Tensor as Tensor>::Context,
1525) -> Result<PreparedReplicatedTextContract, String>
1526where
1527 B: NeuralBackend,
1528 S: RuntimeState<B>,
1529 A: LayeredArchitecture<B, S>,
1530 A::Error: std::fmt::Display,
1531{
1532 prepare_layered_text_contract_with_addressable_parameters::<A, B, S>(
1533 architecture,
1534 source_architecture,
1535 selected,
1536 expected_prompt_cache_architecture_identity,
1537 output_selection,
1538 std::iter::empty::<&str>(),
1539 context,
1540 )
1541}
1542
1543pub fn prepare_layered_text_contract_with_addressable_parameters<'a, A, B, S>(
1545 architecture: &A,
1546 source_architecture: Option<&A>,
1547 selected: SelectedReplicatedTextRealization,
1548 expected_prompt_cache_architecture_identity: &str,
1549 output_selection: ReplicatedTextOutputSelection,
1550 addressable_parameters: impl IntoIterator<Item = &'a str>,
1551 context: &<B::Tensor as Tensor>::Context,
1552) -> Result<PreparedReplicatedTextContract, String>
1553where
1554 B: NeuralBackend,
1555 S: RuntimeState<B>,
1556 A: LayeredArchitecture<B, S>,
1557 A::Error: std::fmt::Display,
1558{
1559 let mut addressable_parameters = addressable_parameters
1560 .into_iter()
1561 .map(str::to_owned)
1562 .collect::<BTreeSet<_>>();
1563 validate_selected_state(&selected)?;
1564 validate_architecture_geometry::<A, B, S>(architecture, &selected)?;
1565 if let Some(source) = source_architecture {
1566 validate_architecture_geometry::<A, B, S>(source, &selected)?;
1567 }
1568 let has_transform = selected.parameters().iter().any(|parameter| {
1569 matches!(
1570 parameter.lowering(),
1571 WeightLoweringKind::Transform | WeightLoweringKind::DerivedTransform
1572 )
1573 });
1574 if has_transform != source_architecture.is_some() {
1575 return Err(
1576 "selected transform tasks and source-format architecture ownership disagree".into(),
1577 );
1578 }
1579 if let Some(source) = source_architecture {
1580 validate_architecture_parameters::<A, B, S>(source, &selected, false, context)?;
1581 }
1582 let mut constructed_companions =
1583 validate_architecture_parameters::<A, B, S>(architecture, &selected, true, context)?;
1584 let mut tasks =
1585 replicated_text_materialization_tasks(&selected).map_err(|error| error.to_string())?;
1586 let selected_parameter_names = tasks
1587 .iter()
1588 .flat_map(|task| {
1589 std::iter::once(task.name().to_owned()).chain(
1590 task.output_companions()
1591 .iter()
1592 .map(|companion| companion.name().to_owned()),
1593 )
1594 })
1595 .collect::<BTreeSet<_>>();
1596 if !addressable_parameters.is_subset(&selected_parameter_names) {
1597 return Err(format!(
1598 "addressable parameter catalog contains unknown selected parameters: {:?}",
1599 addressable_parameters
1600 .difference(&selected_parameter_names)
1601 .collect::<Vec<_>>()
1602 ));
1603 }
1604 let addressable_companions = addressable_parameters
1605 .iter()
1606 .filter_map(|name| tasks.iter().find(|task| task.name() == name))
1607 .flat_map(|task| task.output_companions())
1608 .map(|companion| companion.name().to_owned())
1609 .collect::<Vec<_>>();
1610 addressable_parameters.extend(addressable_companions);
1611 for task in &tasks {
1612 let mut actual = constructed_companions
1613 .remove(task.name())
1614 .unwrap_or_default();
1615 actual.sort_by(|left, right| {
1616 left.role()
1617 .cmp(&right.role())
1618 .then_with(|| left.name().cmp(right.name()))
1619 });
1620 let expected = task.output_companions();
1621 let agrees = actual.len() == expected.len()
1622 && actual.iter().zip(expected).all(|(actual, expected)| {
1623 actual.name() == expected.name()
1624 && actual.role() == expected.role()
1625 && actual.logical_shape() == expected.logical_shape()
1626 && (actual.owner() == expected.owner()
1627 || matches!(
1628 (actual.owner(), expected.owner()),
1629 (
1630 ParameterGroupOwner::StaticAnyOf(actual_roles),
1631 ParameterGroupOwner::StaticRole(expected_role)
1632 ) if actual_roles.iter().any(|role| role == expected_role)
1633 ))
1634 });
1635 if !agrees {
1636 return Err(format!(
1637 "constructed output companions for {:?} differ from authoritative selection: lowering={:?}, executable={:?}, constructed={:?}, selected={:?}, retained={:?}",
1638 task.name(),
1639 task.lowering(),
1640 task.executable(),
1641 actual
1642 .iter()
1643 .map(|companion| (companion.name(), companion.role(), companion.logical_shape(), companion.owner()))
1644 .collect::<Vec<_>>(),
1645 expected
1646 .iter()
1647 .map(|companion| (companion.name(), companion.role(), companion.logical_shape(), companion.owner()))
1648 .collect::<Vec<_>>(),
1649 selected.requirements().parameters().iter().filter_map(|parameter| {
1650 parameter.linear_companion().filter(|(_, primary)| *primary == task.name()).map(|(role, primary)| (parameter.name(), role, primary))
1651 }).collect::<Vec<_>>(),
1652 ));
1653 }
1654 }
1655 if !constructed_companions.is_empty() {
1656 return Err(format!(
1657 "output companion catalog contains unknown materialization tasks: {:?}",
1658 constructed_companions.keys().collect::<Vec<_>>()
1659 ));
1660 }
1661 tasks.retain(|task| !addressable_parameters.contains(task.name()));
1662 let state = PartitionState::new(selected.state().layout().clone(), 0)
1663 .map_err(|error| error.to_string())?;
1664 let prompt_cache_identity = state
1665 .prompt_cache_identity::<B, A>(architecture, Default::default())
1666 .map_err(|error| error.to_string())?;
1667 if prompt_cache_identity.architecture_fingerprint()
1668 != expected_prompt_cache_architecture_identity
1669 || prompt_cache_identity.layer_count() != selected.state().layout().len()
1670 || prompt_cache_identity.global_layer_start() != 0
1671 || prompt_cache_identity.global_layer_end() != selected.state().layout().len()
1672 || prompt_cache_identity.topology() != &Default::default()
1673 {
1674 return Err("architecture prompt-cache identity differs from selection".into());
1675 }
1676 Ok(PreparedReplicatedTextContract {
1677 selected,
1678 tasks,
1679 addressable_parameters: addressable_parameters.into_iter().collect(),
1680 prompt_cache_identity,
1681 output_selection,
1682 })
1683}
1684
1685impl<E, S> ReplicatedTextSessionReport<E, S> {
1686 pub const fn execution(&self) -> ExecutionResidency {
1688 self.execution
1689 }
1690
1691 pub const fn execution_report(&self) -> &E {
1693 &self.execution_report
1694 }
1695
1696 pub const fn state_report(&self) -> &S {
1698 &self.state_report
1699 }
1700
1701 pub const fn distributed_commit(&self) -> Option<DistributedCommitOutcome> {
1703 self.distributed_commit
1704 }
1705}
1706
1707pub fn construct_replicated_text_session<A, B, M>(
1710 architecture: A,
1711 source_architecture: Option<A>,
1712 prepared: PreparedReplicatedTextContract,
1713 mechanisms: M,
1714 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
1715) -> Result<
1716 ReplicatedTextSession<A, B, M>,
1717 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
1718>
1719where
1720 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
1721 M: ReplicatedTextSessionMechanisms<A, B>,
1722 A: LayeredArchitecture<B, M::State>,
1723 A::Error: std::fmt::Display,
1724 M::PolicyError: std::fmt::Display,
1725 M::Error: std::fmt::Display,
1726{
1727 construct_replicated_text_session_with_execution(
1728 architecture,
1729 source_architecture,
1730 prepared,
1731 mechanisms,
1732 DirectReplicatedTextExecution,
1733 context,
1734 )
1735}
1736
1737pub fn construct_replicated_text_session_with_execution<A, B, M, D>(
1743 mut architecture: A,
1744 mut source_architecture: Option<A>,
1745 prepared: PreparedReplicatedTextContract,
1746 mut mechanisms: M,
1747 driver: D,
1748 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
1749) -> Result<
1750 ReplicatedTextSession<A, B, M, D>,
1751 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
1752>
1753where
1754 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
1755 M: ReplicatedTextSessionMechanisms<A, B>,
1756 A: LayeredArchitecture<B, M::State>,
1757 D: ReplicatedRuntimeExecutionStrategy<A, B, M::State, M::ResidentPolicy, M::BoundedPolicy>,
1758 A::Error: std::fmt::Display,
1759 M::PolicyError: std::fmt::Display,
1760 M::Error: std::fmt::Display,
1761{
1762 let (selected, tasks, addressable_parameters, prompt_cache_identity, output_selection) =
1763 prepared.into_parts();
1764 let mut units = construct_units::<A, B, M::State>(
1765 &architecture,
1766 selected.requirements().execution_units(),
1767 context,
1768 )
1769 .map_err(ReplicatedTextSessionError::Architecture)?;
1770 let mut source_units = source_architecture
1771 .as_ref()
1772 .map(|source| {
1773 construct_units::<A, B, M::State>(
1774 source,
1775 selected.requirements().execution_units(),
1776 context,
1777 )
1778 })
1779 .transpose()
1780 .map_err(ReplicatedTextSessionError::Architecture)?;
1781 mechanisms
1782 .prepare_materialization(
1783 &mut architecture,
1784 selected.requirements().execution_units(),
1785 &mut units,
1786 source_architecture.as_mut(),
1787 source_units.as_deref_mut(),
1788 &tasks,
1789 &addressable_parameters,
1790 context,
1791 )
1792 .map_err(ReplicatedTextSessionError::Mechanism)?;
1793 let materialization_report = mechanisms
1794 .take_materialization_report()
1795 .map_err(ReplicatedTextSessionError::Mechanism)?;
1796 let state = mechanisms
1797 .realize_state(selected.state(), context)
1798 .map_err(ReplicatedTextSessionError::Mechanism)?;
1799 validate_realized_state(&state, selected.state())?;
1800 let selected_state = SessionStateRealization::Stateful(selected.state().clone());
1801 let execution = match selected.residency() {
1802 LayerWeightResidency::FullyResident => {
1803 let policy = mechanisms
1804 .resident_policy(&mut architecture, units, &selected, context)
1805 .map_err(ReplicatedTextSessionError::Mechanism)?;
1806 ReplicatedTextRuntime {
1807 kind: ReplicatedTextRuntimeKind::Resident(LayerwiseRuntime::new(
1808 architecture,
1809 policy,
1810 )),
1811 }
1812 }
1813 LayerWeightResidency::LayerwiseHost(_) | LayerWeightResidency::DenseDiskStream(_) => {
1814 let policy = mechanisms
1815 .bounded_policy(&mut architecture, &selected, context)
1816 .map_err(ReplicatedTextSessionError::Mechanism)?;
1817 ReplicatedTextRuntime {
1818 kind: ReplicatedTextRuntimeKind::Bounded(LayerwiseRuntime::new(
1819 architecture,
1820 policy,
1821 )),
1822 }
1823 }
1824 };
1825 Ok(ReplicatedTextSession {
1826 selected,
1827 selected_state,
1828 execution,
1829 driver,
1830 state,
1831 mechanisms,
1832 materialization_report,
1833 prompt_cache_identity: Some(prompt_cache_identity),
1834 committed_prompt_input_identity: None,
1835 next_commit_epoch: DistributedCommitEpoch::FIRST,
1836 active_commit_epoch: None,
1837 last_commit_outcome: None,
1838 successful_state_restorations: Some(0),
1839 control_fence: None,
1840 output_selection,
1841 backend: PhantomData,
1842 })
1843}
1844
1845pub fn construct_replicated_text_session_with_runtime<A, B, M, D>(
1852 binding: PreparedPartitionedSessionRuntime<D::Runtime, M::State>,
1853 mut mechanisms: M,
1854 driver: D,
1855) -> Result<
1856 ReplicatedTextSession<A, B, M, D>,
1857 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
1858>
1859where
1860 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
1861 M: ReplicatedTextSessionMechanisms<A, B>,
1862 A: LayeredArchitecture<B, M::State>,
1863 D: ReplicatedTextExecutionStrategy<A, B, M::State, M::ResidentPolicy, M::BoundedPolicy>,
1864 A::Error: std::fmt::Display,
1865 M::PolicyError: std::fmt::Display,
1866 M::Error: std::fmt::Display,
1867{
1868 let PreparedPartitionedSessionRuntime {
1869 selected,
1870 runtime,
1871 state,
1872 selected_state,
1873 prompt_cache_identity,
1874 output_selection,
1875 } = binding;
1876 match selected_state.state() {
1877 Some(local) => {
1878 validate_realized_state(&state, local)?;
1879 let identity = prompt_cache_identity.as_ref().ok_or_else(|| {
1880 ReplicatedTextSessionError::Contract(
1881 "stateful partition is missing its rank-local cache identity".into(),
1882 )
1883 })?;
1884 let local_layers = identity
1885 .global_layer_end()
1886 .checked_sub(identity.global_layer_start());
1887 if identity.layer_count() != selected.state().layout().len()
1888 || local_layers != Some(local.layout().len())
1889 || identity.global_layer_end() > identity.layer_count()
1890 {
1891 return Err(ReplicatedTextSessionError::Contract(
1892 "rank-local cache identity differs from selected state geometry".into(),
1893 ));
1894 }
1895 let partition =
1896 PartitionState::new(local.layout().clone(), identity.global_layer_start())
1897 .map_err(|error| ReplicatedTextSessionError::Contract(error.to_string()))?;
1898 let expected = selected
1899 .state()
1900 .for_partitioned_geometry(&partition)
1901 .map_err(|error| ReplicatedTextSessionError::Contract(error.to_string()))?;
1902 if &expected != local {
1903 return Err(ReplicatedTextSessionError::Contract(
1904 "rank-local state realization is not the selected global interval".into(),
1905 ));
1906 }
1907 }
1908 None => {
1909 if prompt_cache_identity.is_some() || state.optional_layout().is_some() {
1910 return Err(ReplicatedTextSessionError::Contract(
1911 "stateless partition owns state or a prompt-cache shard identity".into(),
1912 ));
1913 }
1914 }
1915 }
1916 let materialization_report = mechanisms
1917 .take_materialization_report()
1918 .map_err(ReplicatedTextSessionError::Mechanism)?;
1919 Ok(ReplicatedTextSession {
1920 selected,
1921 selected_state,
1922 execution: runtime,
1923 driver,
1924 state,
1925 mechanisms,
1926 materialization_report,
1927 prompt_cache_identity,
1928 committed_prompt_input_identity: None,
1929 next_commit_epoch: DistributedCommitEpoch::FIRST,
1930 active_commit_epoch: None,
1931 last_commit_outcome: None,
1932 successful_state_restorations: Some(0),
1933 control_fence: None,
1934 output_selection,
1935 backend: PhantomData,
1936 })
1937}
1938
1939fn construct_units<A, B, S>(
1940 architecture: &A,
1941 layout: &crate::ExecutionUnitLayout,
1942 context: &<B::Tensor as Tensor>::Context,
1943) -> Result<Vec<A::Unit>, A::Error>
1944where
1945 B: NeuralBackend,
1946 S: RuntimeState<B>,
1947 A: LayeredArchitecture<B, S>,
1948{
1949 (0..layout.len())
1950 .map(|ordinal| {
1951 let address = layout
1952 .address(ordinal)
1953 .expect("validated replicated layout contains every ordinal");
1954 architecture.build_unit(address.group(), address.index(), context)
1955 })
1956 .collect()
1957}
1958
1959impl<A, B, M, D> ReplicatedTextSession<A, B, M, D>
1960where
1961 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
1962 M: ReplicatedTextSessionMechanisms<A, B>,
1963 A: LayeredArchitecture<B, M::State>,
1964 D: ReplicatedTextExecutionStrategy<A, B, M::State, M::ResidentPolicy, M::BoundedPolicy>,
1965 A::Error: std::fmt::Display,
1966 M::PolicyError: std::fmt::Display,
1967 M::Error: std::fmt::Display,
1968{
1969 pub const fn materialization_report(&self) -> Option<&crate::WeightMaterializationReport> {
1972 self.materialization_report.as_ref()
1973 }
1974
1975 pub const fn execution_strategy(&self) -> &D {
1977 &self.driver
1978 }
1979
1980 pub fn forward(
1982 &mut self,
1983 tokens: &B::Tensor,
1984 mask: Option<&B::Tensor>,
1985 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
1986 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
1987 where
1988 A: ReplicatedTextArchitecture<B, M::State>,
1989 {
1990 self.forward_with_observer(tokens, mask, context, &mut crate::NoopObserver)
1991 }
1992
1993 pub fn forward_with_observer<O>(
1995 &mut self,
1996 tokens: &B::Tensor,
1997 mask: Option<&B::Tensor>,
1998 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
1999 observer: &mut O,
2000 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2001 where
2002 A: ReplicatedTextArchitecture<B, M::State>,
2003 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
2004 {
2005 let pass = tokens
2006 .shape()
2007 .last()
2008 .copied()
2009 .filter(|length| *length > 1)
2010 .map_or(ExpertPass::Decode, |_| ExpertPass::Prefill);
2011 let (output, checkpoint, forward_context) =
2012 self.execute_with_observer(tokens, mask, pass, context, observer)?;
2013 self.publish(output, checkpoint, forward_context, context)
2014 }
2015
2016 pub fn prefill(
2018 &mut self,
2019 tokens: &B::Tensor,
2020 mask: Option<&B::Tensor>,
2021 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2022 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2023 where
2024 A: ReplicatedTextArchitecture<B, M::State>,
2025 {
2026 self.prefill_with_observer(tokens, mask, context, &mut crate::NoopObserver)
2027 }
2028
2029 pub fn prefill_with_observer<O>(
2031 &mut self,
2032 tokens: &B::Tensor,
2033 mask: Option<&B::Tensor>,
2034 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2035 observer: &mut O,
2036 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2037 where
2038 A: ReplicatedTextArchitecture<B, M::State>,
2039 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
2040 {
2041 let input = A::text_input(tokens, mask);
2042 self.prefill_input_with_observer(input, context, observer)
2043 }
2044
2045 pub fn prefill_prediction_target(
2051 &mut self,
2052 tokens: &B::Tensor,
2053 mask: Option<&B::Tensor>,
2054 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2055 ) -> Result<
2056 (B::Tensor, B::Tensor),
2057 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2058 >
2059 where
2060 A: ReplicatedTextArchitecture<B, M::State>,
2061 {
2062 let input = A::text_input(tokens, mask);
2063 self.prefill_input_prediction_target(input, context)
2064 }
2065
2066 pub fn prefill_input_prediction_target<'a>(
2068 &mut self,
2069 input: A::Input<'a>,
2070 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2071 ) -> Result<
2072 (B::Tensor, B::Tensor),
2073 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2074 > {
2075 let (output, checkpoint, forward_context) = self.execute_input_before_publication(
2076 input,
2077 ExpertPass::Prefill,
2078 context,
2079 &mut crate::NoopObserver,
2080 )?;
2081 let capture = D::prediction_target_capture(&mut self.execution, &forward_context, context)
2082 .map_err(widen_infallible);
2083 let local_success = matches!(&capture, Ok(Some(_)));
2084 let agreed = match D::agree_distributed_phase(
2085 &mut self.execution,
2086 crate::DistributedExecutionPhase::PredictionTargetCapture,
2087 local_success,
2088 context,
2089 ) {
2090 Ok(agreed) => agreed,
2091 Err(error) => {
2092 return self.rollback_failure(checkpoint, widen_infallible(error), context)
2093 }
2094 };
2095 let capture = match capture {
2096 Ok(Some(capture)) if agreed => capture,
2097 Ok(Some(_)) => {
2098 return self.rollback_failure(
2099 checkpoint,
2100 ReplicatedTextSessionError::Contract(
2101 "another rank could not prepare the prediction target capture".into(),
2102 ),
2103 context,
2104 )
2105 }
2106 Ok(None) => {
2107 return self.rollback_failure(
2108 checkpoint,
2109 ReplicatedTextSessionError::Contract(
2110 "prediction target pass did not retain its declared hidden capture".into(),
2111 ),
2112 context,
2113 )
2114 }
2115 Err(error) => return self.rollback_failure(checkpoint, error, context),
2116 };
2117 let capture_publication =
2118 D::publish_prediction_target_capture(&mut self.execution, capture, context);
2119 let capture_publication_agreed = match D::agree_distributed_phase(
2120 &mut self.execution,
2121 crate::DistributedExecutionPhase::PredictionTargetCapturePublication,
2122 capture_publication.is_ok(),
2123 context,
2124 ) {
2125 Ok(agreed) => agreed,
2126 Err(error) => {
2127 return self.rollback_failure(checkpoint, widen_infallible(error), context)
2128 }
2129 };
2130 let capture = match capture_publication {
2131 Ok(capture) if capture_publication_agreed => capture,
2132 Ok(_) => {
2133 return self.rollback_failure(
2134 checkpoint,
2135 ReplicatedTextSessionError::Contract(
2136 "another rank failed to publish the prediction target capture".into(),
2137 ),
2138 context,
2139 )
2140 }
2141 Err(error) => {
2142 return self.rollback_failure(checkpoint, widen_infallible(error), context)
2143 }
2144 };
2145 let (output, checkpoint, forward_context) =
2146 self.publish_observed_output_transaction(output, checkpoint, forward_context, context)?;
2147 self.publish(output, checkpoint, forward_context, context)
2148 .map(|output| (output, capture))
2149 }
2150
2151 pub fn prefill_input<'a>(
2158 &mut self,
2159 input: A::Input<'a>,
2160 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2161 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2162 self.prefill_input_with_observer(input, context, &mut crate::NoopObserver)
2163 }
2164
2165 pub fn prefill_input_with_capture<'a, O, C, F>(
2172 &mut self,
2173 input: A::Input<'a>,
2174 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2175 observer: &mut O,
2176 capture: F,
2177 ) -> Result<(B::Tensor, C), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2178 where
2179 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
2180 F: FnOnce(&A::ForwardContext) -> Result<C, A::Error>,
2181 {
2182 if D::PARTITIONED_SESSION {
2183 return Err(ReplicatedTextSessionError::Contract(
2184 "partitioned prediction capture requires a selected bundle publication contract"
2185 .into(),
2186 ));
2187 }
2188 let (output, checkpoint, forward_context) =
2189 self.execute_input_before_publication(input, ExpertPass::Prefill, context, observer)?;
2190 let captured = match capture(&forward_context) {
2191 Ok(captured) => captured,
2192 Err(error) => {
2193 return self.rollback_failure(
2194 checkpoint,
2195 ReplicatedTextSessionError::Architecture(error),
2196 context,
2197 )
2198 }
2199 };
2200 let (output, checkpoint, forward_context) =
2201 self.publish_observed_output_transaction(output, checkpoint, forward_context, context)?;
2202 self.publish(output, checkpoint, forward_context, context)
2203 .map(|output| (output, captured))
2204 }
2205
2206 pub fn prefill_input_with_cache_identity<'a>(
2209 &mut self,
2210 input: A::Input<'a>,
2211 identity: PreparedInputCacheIdentity,
2212 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2213 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2214 self.prefill_input_with_observer_and_cache_identity(
2215 input,
2216 identity,
2217 context,
2218 &mut crate::NoopObserver,
2219 )
2220 }
2221
2222 pub fn prefill_input_with_observer_and_cache_identity<'a, O>(
2224 &mut self,
2225 input: A::Input<'a>,
2226 identity: PreparedInputCacheIdentity,
2227 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2228 observer: &mut O,
2229 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2230 where
2231 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
2232 {
2233 let output = self.prefill_input_with_observer(input, context, observer)?;
2234 self.committed_prompt_input_identity = Some(identity);
2235 Ok(output)
2236 }
2237
2238 pub fn prefill_input_with_observer<'a, O>(
2240 &mut self,
2241 input: A::Input<'a>,
2242 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2243 observer: &mut O,
2244 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2245 where
2246 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
2247 {
2248 let (output, checkpoint, forward_context) =
2249 self.execute_input_with_observer(input, ExpertPass::Prefill, context, observer)?;
2250 let sequence_index = self.output_selection.sequence_index();
2251 let output = match self
2252 .mechanisms
2253 .index_text_output(output, sequence_index, context)
2254 {
2255 Ok(output) => output,
2256 Err(error) => {
2257 return self.rollback_failure(
2258 checkpoint,
2259 ReplicatedTextSessionError::Mechanism(error),
2260 context,
2261 )
2262 }
2263 };
2264 self.publish(output, checkpoint, forward_context, context)
2265 }
2266
2267 pub fn decode_input<'a>(
2269 &mut self,
2270 input: A::Input<'a>,
2271 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2272 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2273 self.decode_input_with_observer(input, context, &mut crate::NoopObserver)
2274 }
2275
2276 pub fn decode_input_with_capture<'a, O, C, F>(
2278 &mut self,
2279 input: A::Input<'a>,
2280 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2281 observer: &mut O,
2282 capture: F,
2283 ) -> Result<(B::Tensor, C), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2284 where
2285 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
2286 F: FnOnce(&A::ForwardContext) -> Result<C, A::Error>,
2287 {
2288 if D::PARTITIONED_SESSION {
2289 return Err(ReplicatedTextSessionError::Contract(
2290 "partitioned prediction capture requires a selected bundle publication contract"
2291 .into(),
2292 ));
2293 }
2294 let (output, checkpoint, forward_context) =
2295 self.execute_input_before_publication(input, ExpertPass::Decode, context, observer)?;
2296 let captured = match capture(&forward_context) {
2297 Ok(captured) => captured,
2298 Err(error) => {
2299 return self.rollback_failure(
2300 checkpoint,
2301 ReplicatedTextSessionError::Architecture(error),
2302 context,
2303 )
2304 }
2305 };
2306 let (output, checkpoint, forward_context) =
2307 self.publish_observed_output_transaction(output, checkpoint, forward_context, context)?;
2308 self.publish(output, checkpoint, forward_context, context)
2309 .map(|output| (output, captured))
2310 }
2311
2312 pub fn decode_input_with_observer<'a, O>(
2314 &mut self,
2315 input: A::Input<'a>,
2316 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2317 observer: &mut O,
2318 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2319 where
2320 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
2321 {
2322 let (output, checkpoint, forward_context) =
2323 self.execute_input_with_observer(input, ExpertPass::Decode, context, observer)?;
2324 let sequence_index = self.output_selection.sequence_index();
2325 let output = match self
2326 .mechanisms
2327 .index_text_output(output, sequence_index, context)
2328 {
2329 Ok(output) => output,
2330 Err(error) => {
2331 return self.rollback_failure(
2332 checkpoint,
2333 ReplicatedTextSessionError::Mechanism(error),
2334 context,
2335 )
2336 }
2337 };
2338 self.publish(output, checkpoint, forward_context, context)
2339 }
2340
2341 pub fn decode(
2343 &mut self,
2344 tokens: &B::Tensor,
2345 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2346 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2347 where
2348 A: ReplicatedTextArchitecture<B, M::State>,
2349 {
2350 self.decode_with_observer(tokens, context, &mut crate::NoopObserver)
2351 }
2352
2353 pub fn decode_prediction_target(
2355 &mut self,
2356 tokens: &B::Tensor,
2357 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2358 ) -> Result<
2359 (B::Tensor, B::Tensor),
2360 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2361 >
2362 where
2363 A: ReplicatedTextArchitecture<B, M::State>,
2364 {
2365 let input = A::text_input(tokens, None);
2366 self.decode_input_prediction_target(input, context)
2367 }
2368
2369 pub fn decode_input_prediction_target<'a>(
2371 &mut self,
2372 input: A::Input<'a>,
2373 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2374 ) -> Result<
2375 (B::Tensor, B::Tensor),
2376 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2377 > {
2378 let (output, checkpoint, forward_context) = self.execute_input_before_publication(
2379 input,
2380 ExpertPass::Decode,
2381 context,
2382 &mut crate::NoopObserver,
2383 )?;
2384 let capture = D::prediction_target_capture(&mut self.execution, &forward_context, context)
2385 .map_err(widen_infallible);
2386 let local_success = matches!(&capture, Ok(Some(_)));
2387 let agreed = match D::agree_distributed_phase(
2388 &mut self.execution,
2389 crate::DistributedExecutionPhase::PredictionTargetCapture,
2390 local_success,
2391 context,
2392 ) {
2393 Ok(agreed) => agreed,
2394 Err(error) => {
2395 return self.rollback_failure(checkpoint, widen_infallible(error), context)
2396 }
2397 };
2398 let capture = match capture {
2399 Ok(Some(capture)) if agreed => capture,
2400 Ok(Some(_)) => {
2401 return self.rollback_failure(
2402 checkpoint,
2403 ReplicatedTextSessionError::Contract(
2404 "another rank could not prepare the prediction target capture".into(),
2405 ),
2406 context,
2407 )
2408 }
2409 Ok(None) => {
2410 return self.rollback_failure(
2411 checkpoint,
2412 ReplicatedTextSessionError::Contract(
2413 "prediction target pass did not retain its declared hidden capture".into(),
2414 ),
2415 context,
2416 )
2417 }
2418 Err(error) => return self.rollback_failure(checkpoint, error, context),
2419 };
2420 let capture_publication =
2421 D::publish_prediction_target_capture(&mut self.execution, capture, context);
2422 let capture_publication_agreed = match D::agree_distributed_phase(
2423 &mut self.execution,
2424 crate::DistributedExecutionPhase::PredictionTargetCapturePublication,
2425 capture_publication.is_ok(),
2426 context,
2427 ) {
2428 Ok(agreed) => agreed,
2429 Err(error) => {
2430 return self.rollback_failure(checkpoint, widen_infallible(error), context)
2431 }
2432 };
2433 let capture = match capture_publication {
2434 Ok(capture) if capture_publication_agreed => capture,
2435 Ok(_) => {
2436 return self.rollback_failure(
2437 checkpoint,
2438 ReplicatedTextSessionError::Contract(
2439 "another rank failed to publish the prediction target capture".into(),
2440 ),
2441 context,
2442 )
2443 }
2444 Err(error) => {
2445 return self.rollback_failure(checkpoint, widen_infallible(error), context)
2446 }
2447 };
2448 let (output, checkpoint, forward_context) =
2449 self.publish_observed_output_transaction(output, checkpoint, forward_context, context)?;
2450 self.publish(output, checkpoint, forward_context, context)
2451 .map(|output| (output, capture))
2452 }
2453
2454 pub fn decode_with_observer<O>(
2456 &mut self,
2457 tokens: &B::Tensor,
2458 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2459 observer: &mut O,
2460 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2461 where
2462 A: ReplicatedTextArchitecture<B, M::State>,
2463 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
2464 {
2465 let (output, checkpoint, forward_context) =
2466 self.execute_with_observer(tokens, None, ExpertPass::Decode, context, observer)?;
2467 let sequence_index = self.output_selection.sequence_index();
2468 let output = match self
2469 .mechanisms
2470 .index_text_output(output, sequence_index, context)
2471 {
2472 Ok(output) => output,
2473 Err(error) => {
2474 return self.rollback_failure(
2475 checkpoint,
2476 ReplicatedTextSessionError::Mechanism(error),
2477 context,
2478 )
2479 }
2480 };
2481 self.publish(output, checkpoint, forward_context, context)
2482 }
2483
2484 pub const fn successful_state_restoration_generation(&self) -> Option<u64> {
2490 self.successful_state_restorations
2491 }
2492
2493 pub fn checkpoint(
2495 &mut self,
2496 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2497 ) -> Result<M::StateCheckpoint, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2498 {
2499 self.ensure_commit_resolved()?;
2500 self.mechanisms
2501 .checkpoint_state(&self.state, context)
2502 .map_err(ReplicatedTextSessionError::Mechanism)
2503 }
2504
2505 pub fn exchange_prediction_target_state(
2512 &mut self,
2513 replacement: &mut M::State,
2514 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2515 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2516 self.ensure_commit_resolved()?;
2517 let validation = match self.selected_state.state() {
2518 Some(selected) => validate_realized_state(replacement, selected),
2519 None if replacement.optional_layout().is_none() => Ok(()),
2520 None => Err(ReplicatedTextSessionError::Contract(
2521 "stateless prediction target received stateful lane state".into(),
2522 )),
2523 };
2524 let phase = crate::DistributedExecutionPhase::PredictionTargetStatePreparation;
2525 let agreed =
2526 D::agree_distributed_phase(&mut self.execution, phase, validation.is_ok(), context)
2527 .map_err(widen_infallible)?;
2528 match validation {
2529 Ok(()) if agreed => {
2530 std::mem::swap(&mut self.state, replacement);
2531 Ok(())
2532 }
2533 Ok(()) => Err(ReplicatedTextSessionError::Contract(
2534 "another rank rejected its prediction target lane state".into(),
2535 )),
2536 Err(error) => Err(error),
2537 }
2538 }
2539
2540 pub fn recover_prediction_target_state_after_failure(
2549 &mut self,
2550 replacement: &mut M::State,
2551 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2552 let selected = self.selected_state.state().ok_or_else(|| {
2553 ReplicatedTextSessionError::Contract(
2554 "stateless prediction target cannot recover lane ownership".into(),
2555 )
2556 })?;
2557 validate_realized_state(&self.state, selected)?;
2558 validate_realized_state(replacement, selected)?;
2559 std::mem::swap(&mut self.state, replacement);
2560 Ok(())
2561 }
2562
2563 pub fn prepare_prediction_target_state(
2569 &mut self,
2570 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2571 ) -> Result<M::State, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2572 self.ensure_commit_resolved()?;
2573 let provisional = self.selected_state.state().map_or_else(
2574 || {
2575 Err(ReplicatedTextSessionError::Contract(
2576 "stateless session cannot prepare prediction target state".into(),
2577 ))
2578 },
2579 |selected| {
2580 self.mechanisms
2581 .fork_prediction_target_state(&self.state, selected, context)
2582 .map_err(ReplicatedTextSessionError::Mechanism)
2583 .and_then(|state| {
2584 validate_realized_state(&state, selected)?;
2585 Ok(state)
2586 })
2587 },
2588 );
2589 let phase = crate::DistributedExecutionPhase::PredictionTargetStatePreparation;
2590 let agreed =
2591 D::agree_distributed_phase(&mut self.execution, phase, provisional.is_ok(), context)
2592 .map_err(widen_infallible)?;
2593 match provisional {
2594 Ok(state) if agreed => Ok(state),
2595 Ok(_) => Err(ReplicatedTextSessionError::Contract(
2596 "another rank could not prepare prediction target lane state".into(),
2597 )),
2598 Err(error) => Err(error),
2599 }
2600 }
2601
2602 pub fn apply_prediction_target_operation<O>(
2607 &mut self,
2608 operation: O,
2609 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2610 ) -> Result<O::Output, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2611 where
2612 O: PredictionTargetOperation<A, B, M::State>,
2613 {
2614 self.ensure_commit_resolved()?;
2615 let checkpoint = self
2616 .mechanisms
2617 .checkpoint_state(&self.state, context)
2618 .map_err(ReplicatedTextSessionError::Mechanism)?;
2619 let execution = D::apply_prediction_target_operation(
2620 &mut self.execution,
2621 &mut self.state,
2622 operation,
2623 context,
2624 )
2625 .map_err(widen_infallible)
2626 .and_then(|output| {
2627 output.ok_or_else(|| {
2628 ReplicatedTextSessionError::Contract(
2629 "selected target execution has no typed prediction-extension handoff".into(),
2630 )
2631 })
2632 });
2633 let phase = crate::DistributedExecutionPhase::PredictionExtensionExecution;
2634 let agreed =
2635 D::agree_distributed_phase(&mut self.execution, phase, execution.is_ok(), context)
2636 .map_err(widen_infallible);
2637 match (execution, agreed) {
2638 (Ok(output), Ok(true)) => Ok(output),
2639 (execution, agreement) => {
2640 self.mechanisms
2641 .restore_state(&mut self.state, checkpoint, context)
2642 .map_err(ReplicatedTextSessionError::Mechanism)?;
2643 match (execution, agreement) {
2644 (_, Err(error)) => Err(error),
2645 (Err(error), _) => Err(error),
2646 (Ok(_), Ok(false)) => Err(ReplicatedTextSessionError::Contract(
2647 "another rank failed during prediction-extension execution".into(),
2648 )),
2649 (Ok(_), Ok(true)) => unreachable!("successful extension returned above"),
2650 }
2651 }
2652 }
2653 }
2654
2655 pub fn checkpoint_complete(
2657 &mut self,
2658 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2659 ) -> Result<
2660 ReplicatedTextSessionCheckpoint<M::StateCheckpoint>,
2661 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2662 > {
2663 Ok(ReplicatedTextSessionCheckpoint {
2664 state: self.checkpoint(context)?,
2665 prompt_input_identity: self.committed_prompt_input_identity.clone(),
2666 next_commit_epoch: self.next_commit_epoch,
2667 last_commit_outcome: self.last_commit_outcome,
2668 })
2669 }
2670
2671 pub fn checkpoint_distributed(
2673 &mut self,
2674 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2675 ) -> Result<
2676 DistributedStateCheckpoint<M::StateCheckpoint>,
2677 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2678 > {
2679 self.ensure_commit_resolved()?;
2680 self.require_cache_control_agreement()?;
2681 let checkpoint = self.selected_state.state().map(|_| {
2682 self.mechanisms
2683 .checkpoint_state(&self.state, context)
2684 .map_err(ReplicatedTextSessionError::Mechanism)
2685 });
2686 let success = checkpoint.as_ref().is_none_or(Result::is_ok);
2687 let phase = crate::DistributedExecutionPhase::SessionCheckpoint;
2688 let agreed = self.agree_cache_control_phase(phase, success, context)?;
2689 let state = match checkpoint {
2690 Some(Ok(checkpoint)) if agreed => Some(checkpoint),
2691 None if agreed => None,
2692 Some(Ok(_)) | None => return self.fence_remote_cache_control_failure(phase),
2693 Some(Err(error)) => {
2694 self.control_fence = Some(phase);
2695 return Err(error);
2696 }
2697 };
2698 Ok(DistributedStateCheckpoint { state })
2699 }
2700
2701 pub fn checkpoint_complete_distributed(
2703 &mut self,
2704 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2705 ) -> Result<
2706 DistributedSessionCheckpoint<M::StateCheckpoint>,
2707 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2708 > {
2709 let state = self.checkpoint_distributed(context)?.state;
2710 Ok(DistributedSessionCheckpoint {
2711 state,
2712 prompt_input_identity: self.committed_prompt_input_identity.clone(),
2713 next_commit_epoch: self.next_commit_epoch,
2714 last_commit_outcome: self.last_commit_outcome,
2715 })
2716 }
2717
2718 pub fn rollback(
2720 &mut self,
2721 checkpoint: M::StateCheckpoint,
2722 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2723 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2724 self.ensure_commit_resolved()?;
2725 self.mechanisms
2726 .restore_state(&mut self.state, checkpoint, context)
2727 .map_err(ReplicatedTextSessionError::Mechanism)?;
2728 self.committed_prompt_input_identity = None;
2730 Ok(())
2731 }
2732
2733 pub fn rollback_complete(
2735 &mut self,
2736 checkpoint: ReplicatedTextSessionCheckpoint<M::StateCheckpoint>,
2737 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2738 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2739 self.ensure_commit_resolved()?;
2740 self.mechanisms
2741 .restore_state(&mut self.state, checkpoint.state, context)
2742 .map_err(ReplicatedTextSessionError::Mechanism)?;
2743 self.committed_prompt_input_identity = checkpoint.prompt_input_identity;
2744 self.next_commit_epoch = checkpoint.next_commit_epoch;
2745 self.last_commit_outcome = checkpoint.last_commit_outcome;
2746 self.active_commit_epoch = None;
2747 Ok(())
2748 }
2749
2750 pub fn rollback_distributed(
2752 &mut self,
2753 checkpoint: DistributedStateCheckpoint<M::StateCheckpoint>,
2754 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2755 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2756 self.restore_distributed_state(checkpoint.state, None, context)
2757 }
2758
2759 pub fn rollback_complete_distributed(
2761 &mut self,
2762 checkpoint: DistributedSessionCheckpoint<M::StateCheckpoint>,
2763 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2764 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2765 self.restore_distributed_state(
2766 checkpoint.state,
2767 Some((
2768 checkpoint.prompt_input_identity,
2769 checkpoint.next_commit_epoch,
2770 checkpoint.last_commit_outcome,
2771 )),
2772 context,
2773 )
2774 }
2775
2776 pub fn reset_distributed(
2778 &mut self,
2779 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2780 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2781 self.ensure_commit_resolved()?;
2782 self.require_cache_control_agreement()?;
2783 let provisional = self.selected_state.state().map(|selected| {
2784 self.mechanisms
2785 .realize_state(selected, context)
2786 .map_err(ReplicatedTextSessionError::Mechanism)
2787 .and_then(|state| {
2788 validate_realized_state(&state, selected)?;
2789 Ok(state)
2790 })
2791 });
2792 let success = provisional.as_ref().is_none_or(Result::is_ok);
2793 let phase = crate::DistributedExecutionPhase::SessionResetPreparation;
2794 let agreed = self.agree_cache_control_phase(phase, success, context)?;
2795 match provisional {
2796 Some(Ok(state)) if agreed => self.state = state,
2797 None if agreed => {}
2798 Some(Ok(_)) | None => return self.fence_remote_cache_control_failure(phase),
2799 Some(Err(error)) => {
2800 self.control_fence = Some(phase);
2801 return Err(error);
2802 }
2803 }
2804 self.committed_prompt_input_identity = None;
2805 Ok(())
2806 }
2807
2808 pub fn reset(
2810 &mut self,
2811 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2812 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
2813 self.ensure_commit_resolved()?;
2814 if let Some(selected_state) = self.selected_state.state() {
2815 let state = self
2816 .mechanisms
2817 .realize_state(selected_state, context)
2818 .map_err(ReplicatedTextSessionError::Mechanism)?;
2819 validate_realized_state(&state, selected_state)?;
2820 self.state = state;
2821 } else if self.state.optional_layout().is_some() {
2822 return Err(ReplicatedTextSessionError::Contract(
2823 "stateless session owns a stateful mechanism realization".into(),
2824 ));
2825 }
2826 self.committed_prompt_input_identity = None;
2827 Ok(())
2828 }
2829
2830 pub fn load_prompt_cache(
2832 &mut self,
2833 directory: &Path,
2834 expected: &PromptCacheDescriptor,
2835 prefix_token_ids: &[u32],
2836 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2837 ) -> Result<PromptCacheManifest, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2838 {
2839 self.ensure_control_unfenced()?;
2840 let identity = self.prompt_cache_identity()?.clone();
2841 validate_prompt_cache_model_identity(expected, &identity)?;
2842 let selected_state = self.selected_state.state().ok_or_else(|| {
2843 ReplicatedTextSessionError::Contract(
2844 "this partition rank owns no prompt-cache state shard".into(),
2845 )
2846 })?;
2847 let (state, manifest) = self
2848 .mechanisms
2849 .load_prompt_cache(
2850 directory,
2851 expected,
2852 &identity,
2853 prefix_token_ids,
2854 selected_state,
2855 context,
2856 )
2857 .map_err(ReplicatedTextSessionError::Mechanism)?;
2858 manifest.validate_compatibility(expected, prefix_token_ids)?;
2859 validate_realized_state(&state, selected_state)?;
2860 self.state = state;
2861 self.committed_prompt_input_identity = None;
2862 self.restore_distributed_commit(manifest.distributed_commit)?;
2863 Ok(manifest)
2864 }
2865
2866 pub fn load_prompt_cache_for_input(
2868 &mut self,
2869 directory: &Path,
2870 expected: &PromptCacheDescriptor,
2871 prefix_token_ids: &[u32],
2872 input_identity: PreparedInputCacheIdentity,
2873 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2874 ) -> Result<PromptCacheManifest, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2875 {
2876 self.validate_prompt_input_descriptor(expected, &input_identity)?;
2877 let manifest = self.load_prompt_cache(directory, expected, prefix_token_ids, context)?;
2878 self.committed_prompt_input_identity = Some(input_identity);
2879 Ok(manifest)
2880 }
2881
2882 pub fn load_prompt_cache_distributed(
2889 &mut self,
2890 directory: &Path,
2891 expected: &PromptCacheDescriptor,
2892 prefix_token_ids: &[u32],
2893 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2894 ) -> Result<
2895 Option<PromptCacheManifest>,
2896 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2897 > {
2898 self.load_prompt_cache_distributed_inner(
2899 directory,
2900 expected,
2901 prefix_token_ids,
2902 None,
2903 context,
2904 )
2905 }
2906
2907 pub fn load_prompt_cache_for_input_distributed(
2909 &mut self,
2910 directory: &Path,
2911 expected: &PromptCacheDescriptor,
2912 prefix_token_ids: &[u32],
2913 input_identity: PreparedInputCacheIdentity,
2914 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2915 ) -> Result<
2916 Option<PromptCacheManifest>,
2917 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2918 > {
2919 self.load_prompt_cache_distributed_inner(
2920 directory,
2921 expected,
2922 prefix_token_ids,
2923 Some(input_identity),
2924 context,
2925 )
2926 }
2927
2928 pub fn save_prompt_cache(
2930 &mut self,
2931 destination: &Path,
2932 descriptor: PromptCacheDescriptor,
2933 prefix_token_ids: &[u32],
2934 options: &PromptCacheOptions,
2935 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2936 ) -> Result<PromptCacheManifest, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2937 {
2938 self.ensure_control_unfenced()?;
2939 validate_prompt_cache_model_identity(&descriptor, self.prompt_cache_identity()?)?;
2940 let descriptor = descriptor.with_distributed_commit(self.last_commit_outcome);
2941 let manifest = self
2942 .mechanisms
2943 .save_prompt_cache(
2944 &mut self.state,
2945 destination,
2946 descriptor.clone(),
2947 prefix_token_ids,
2948 options,
2949 context,
2950 )
2951 .map_err(ReplicatedTextSessionError::Mechanism)?;
2952 manifest.validate_compatibility(&descriptor, prefix_token_ids)?;
2953 Ok(manifest)
2954 }
2955
2956 pub fn save_prompt_cache_for_input(
2958 &mut self,
2959 destination: &Path,
2960 descriptor: PromptCacheDescriptor,
2961 prefix_token_ids: &[u32],
2962 options: &PromptCacheOptions,
2963 input_identity: &PreparedInputCacheIdentity,
2964 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2965 ) -> Result<PromptCacheManifest, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>>
2966 {
2967 self.validate_prompt_input_descriptor(&descriptor, input_identity)?;
2968 if self.committed_prompt_input_identity.as_ref() != Some(input_identity) {
2969 return Err(ReplicatedTextSessionError::Contract(
2970 "prompt-cache prepared-input identity differs from the committed prompt".into(),
2971 ));
2972 }
2973 self.save_prompt_cache(destination, descriptor, prefix_token_ids, options, context)
2974 }
2975
2976 fn load_prompt_cache_distributed_inner(
2977 &mut self,
2978 directory: &Path,
2979 expected: &PromptCacheDescriptor,
2980 prefix_token_ids: &[u32],
2981 input_identity: Option<PreparedInputCacheIdentity>,
2982 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
2983 ) -> Result<
2984 Option<PromptCacheManifest>,
2985 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
2986 > {
2987 self.ensure_commit_resolved()?;
2988 self.require_cache_control_agreement()?;
2989 let preflight = (|| {
2990 if let Some(input_identity) = input_identity.as_ref() {
2991 self.validate_prompt_input_descriptor(expected, input_identity)?;
2992 }
2993 match (
2994 self.selected_state.state(),
2995 self.prompt_cache_identity.as_ref(),
2996 ) {
2997 (Some(selected_state), Some(identity)) => {
2998 if !self.selected.prompt_cache() {
2999 return Err(ReplicatedTextSessionError::Contract(
3000 "prompt-cache persistence was not selected for this session".into(),
3001 ));
3002 }
3003 validate_prompt_cache_model_identity(expected, identity)?;
3004 Ok(Some((selected_state.clone(), identity.clone())))
3005 }
3006 (None, None) => Ok(None),
3007 _ => Err(ReplicatedTextSessionError::Contract(
3008 "partition cache state and rank-local identity ownership disagree".into(),
3009 )),
3010 }
3011 })();
3012 let phase = crate::DistributedExecutionPhase::PromptCacheLoadPreflight;
3013 let agreement = self.agree_cache_control_phase(phase, preflight.is_ok(), context);
3014 let local = match (preflight, agreement) {
3015 (Ok(local), Ok(true)) => local,
3016 (Ok(_), Ok(false)) => return self.fence_remote_cache_control_failure(phase),
3017 (Ok(_), Err(error)) => return Err(error),
3018 (Err(error), _) => {
3019 self.control_fence = Some(phase);
3020 return Err(error);
3021 }
3022 };
3023
3024 let provisional = local.map(|(selected_state, identity)| {
3025 self.mechanisms
3026 .load_prompt_cache(
3027 directory,
3028 expected,
3029 &identity,
3030 prefix_token_ids,
3031 &selected_state,
3032 context,
3033 )
3034 .map_err(ReplicatedTextSessionError::Mechanism)
3035 .and_then(|(state, manifest)| {
3036 manifest.validate_compatibility(expected, prefix_token_ids)?;
3037 validate_realized_state(&state, &selected_state)?;
3038 validate_distributed_commit_restore(manifest.distributed_commit)?;
3039 Ok((state, manifest))
3040 })
3041 });
3042 let local_success = provisional.as_ref().is_none_or(Result::is_ok);
3043 let phase = crate::DistributedExecutionPhase::PromptCacheLoadPreparation;
3044 let agreement = self.agree_cache_control_phase(phase, local_success, context);
3045 let provisional = match (provisional, agreement) {
3046 (Some(Ok(provisional)), Ok(true)) => Some(provisional),
3047 (None, Ok(true)) => None,
3048 (Some(Ok(_)) | None, Ok(false)) => {
3049 return self.fence_remote_cache_control_failure(phase);
3050 }
3051 (Some(Ok(_)) | None, Err(error)) => return Err(error),
3052 (Some(Err(error)), _) => {
3053 self.control_fence = Some(phase);
3054 return Err(error);
3055 }
3056 };
3057 let manifest = provisional.map(|(state, manifest)| {
3058 self.state = state;
3059 self.committed_prompt_input_identity = input_identity;
3060 self.active_commit_epoch = None;
3061 self.last_commit_outcome = manifest.distributed_commit;
3062 if let Some(outcome) = manifest.distributed_commit {
3063 self.next_commit_epoch = outcome.epoch().next().unwrap_or_else(|| {
3064 unreachable!("provisional commit epoch was validated before agreement")
3065 });
3066 }
3067 manifest
3068 });
3069 Ok(manifest)
3070 }
3071
3072 fn restore_distributed_state(
3073 &mut self,
3074 checkpoint: Option<M::StateCheckpoint>,
3075 metadata: Option<(
3076 Option<PreparedInputCacheIdentity>,
3077 DistributedCommitEpoch,
3078 Option<DistributedCommitOutcome>,
3079 )>,
3080 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3081 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3082 self.ensure_commit_resolved()?;
3083 self.require_cache_control_agreement()?;
3084 let provisional = match (self.selected_state.state(), checkpoint) {
3085 (Some(selected), Some(checkpoint)) => Some(
3086 self.mechanisms
3087 .realize_state(selected, context)
3088 .map_err(ReplicatedTextSessionError::Mechanism)
3089 .and_then(|mut state| {
3090 self.mechanisms
3091 .restore_state(&mut state, checkpoint, context)
3092 .map_err(ReplicatedTextSessionError::Mechanism)?;
3093 validate_realized_state(&state, selected)?;
3094 Ok(state)
3095 }),
3096 ),
3097 (None, None) => None,
3098 _ => Some(Err(ReplicatedTextSessionError::Contract(
3099 "distributed checkpoint presence differs from rank-local state ownership".into(),
3100 ))),
3101 };
3102 let metadata_valid = metadata.as_ref().is_none_or(|(_, next, outcome)| {
3103 outcome.is_none_or(|outcome| {
3104 outcome
3105 .epoch()
3106 .next()
3107 .is_some_and(|expected| expected == *next)
3108 })
3109 });
3110 let success = provisional.as_ref().is_none_or(Result::is_ok) && metadata_valid;
3111 let phase = crate::DistributedExecutionPhase::SessionRollbackPreparation;
3112 let agreed = self.agree_cache_control_phase(phase, success, context)?;
3113 let provisional = match provisional {
3114 Some(Ok(state)) if agreed => Some(state),
3115 None if agreed => None,
3116 Some(Ok(_)) | None => return self.fence_remote_cache_control_failure(phase),
3117 Some(Err(error)) => {
3118 self.control_fence = Some(phase);
3119 return Err(error);
3120 }
3121 };
3122 if !metadata_valid {
3123 self.control_fence = Some(phase);
3124 return Err(ReplicatedTextSessionError::Contract(
3125 "distributed checkpoint commit metadata is inconsistent".into(),
3126 ));
3127 }
3128 if let Some(state) = provisional {
3129 self.state = state;
3130 }
3131 match metadata {
3132 Some((identity, next, outcome)) => {
3133 self.committed_prompt_input_identity = identity;
3134 self.next_commit_epoch = next;
3135 self.last_commit_outcome = outcome;
3136 self.active_commit_epoch = None;
3137 }
3138 None => self.committed_prompt_input_identity = None,
3139 }
3140 Ok(())
3141 }
3142
3143 pub const fn committed_prompt_input_identity(&self) -> Option<&PreparedInputCacheIdentity> {
3145 self.committed_prompt_input_identity.as_ref()
3146 }
3147
3148 pub fn report(
3150 &self,
3151 ) -> Result<
3152 ReplicatedTextSessionReport<M::ExecutionReport, M::StateReport>,
3153 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3154 > {
3155 let execution_report = self
3156 .mechanisms
3157 .execution_report(
3158 self.selected.residency(),
3159 D::bounded_policy(&self.execution),
3160 )
3161 .map_err(ReplicatedTextSessionError::Mechanism)?;
3162 let state_report = self
3163 .mechanisms
3164 .state_report(&self.state)
3165 .map_err(ReplicatedTextSessionError::Mechanism)?;
3166 Ok(ReplicatedTextSessionReport {
3167 execution: D::execution_residency(&self.execution, &self.selected),
3168 execution_report,
3169 state_report,
3170 distributed_commit: self.last_commit_outcome,
3171 })
3172 }
3173
3174 fn execute_with_observer<O>(
3175 &mut self,
3176 tokens: &B::Tensor,
3177 mask: Option<&B::Tensor>,
3178 pass: ExpertPass,
3179 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3180 observer: &mut O,
3181 ) -> Result<
3182 (B::Tensor, M::StateCheckpoint, A::ForwardContext),
3183 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3184 >
3185 where
3186 A: ReplicatedTextArchitecture<B, M::State>,
3187 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
3188 {
3189 let input = A::text_input(tokens, mask);
3190 self.execute_input_with_observer(input, pass, context, observer)
3191 }
3192
3193 fn execute_input_with_observer<'a, O>(
3194 &mut self,
3195 input: A::Input<'a>,
3196 pass: ExpertPass,
3197 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3198 observer: &mut O,
3199 ) -> Result<
3200 (B::Tensor, M::StateCheckpoint, A::ForwardContext),
3201 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3202 >
3203 where
3204 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
3205 {
3206 let (output, checkpoint, forward_context) =
3207 self.execute_input_before_publication(input, pass, context, observer)?;
3208 self.publish_observed_output_transaction(output, checkpoint, forward_context, context)
3209 }
3210
3211 fn execute_input_before_publication<'a, O>(
3212 &mut self,
3213 input: A::Input<'a>,
3214 pass: ExpertPass,
3215 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3216 observer: &mut O,
3217 ) -> Result<
3218 (B::Tensor, M::StateCheckpoint, A::ForwardContext),
3219 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3220 >
3221 where
3222 O: ActivationObserver<B::Tensor, A::Error> + ?Sized,
3223 {
3224 self.begin_commit_epoch()?;
3225 let checkpoint = self.mechanisms.checkpoint_state(&self.state, context);
3226 let checkpoint_agreed = match D::agree_distributed_phase(
3227 &mut self.execution,
3228 crate::DistributedExecutionPhase::StateCheckpoint,
3229 checkpoint.is_ok(),
3230 context,
3231 ) {
3232 Ok(agreed) => agreed,
3233 Err(error) => return self.abort_without_rollback(widen_infallible(error)),
3234 };
3235 let checkpoint = match checkpoint {
3236 Ok(checkpoint) if checkpoint_agreed => checkpoint,
3237 Ok(_) => {
3238 return self.abort_without_rollback(ReplicatedTextSessionError::Contract(
3239 "another rank failed to capture its distributed state checkpoint".into(),
3240 ));
3241 }
3242 Err(error) => {
3243 return self.abort_without_rollback(ReplicatedTextSessionError::Mechanism(error));
3244 }
3245 };
3246 let execution = self
3247 .driver
3248 .forward_with_observer(
3249 &mut self.execution,
3250 input,
3251 &mut self.state,
3252 pass,
3253 context,
3254 observer,
3255 )
3256 .map_err(widen_infallible);
3257 let execution_agreed = match D::agree_distributed_phase(
3258 &mut self.execution,
3259 crate::DistributedExecutionPhase::Execution,
3260 execution.is_ok(),
3261 context,
3262 ) {
3263 Ok(agreed) => agreed,
3264 Err(error) => {
3265 let error = match execution {
3266 Err(local) => local,
3267 Ok(_) => widen_infallible(error),
3268 };
3269 return self.rollback_failure(checkpoint, error, context);
3270 }
3271 };
3272 let (output, forward_context) = match execution {
3273 Ok(output) if execution_agreed => output,
3274 Ok(_) => {
3275 return self.rollback_failure(
3276 checkpoint,
3277 ReplicatedTextSessionError::Contract(
3278 "another rank failed during distributed execution".into(),
3279 ),
3280 context,
3281 )
3282 }
3283 Err(error) => return self.rollback_failure(checkpoint, error, context),
3284 };
3285 let observation = D::observe_output(&mut self.execution, &output, observer, context);
3286 let observation_agreed = match D::agree_distributed_phase(
3287 &mut self.execution,
3288 crate::DistributedExecutionPhase::OutputObservation,
3289 observation.is_ok(),
3290 context,
3291 ) {
3292 Ok(agreed) => agreed,
3293 Err(error) => {
3294 return self.rollback_failure(checkpoint, widen_infallible(error), context)
3295 }
3296 };
3297 let output = match observation {
3298 Ok(output) if observation_agreed => output,
3299 Ok(_) => {
3300 return self.rollback_failure(
3301 checkpoint,
3302 ReplicatedTextSessionError::Contract(
3303 "another rank failed during distributed output observation".into(),
3304 ),
3305 context,
3306 )
3307 }
3308 Err(error) => {
3309 return self.rollback_failure(checkpoint, widen_infallible(error), context)
3310 }
3311 };
3312 Ok((output, checkpoint, forward_context))
3313 }
3314
3315 fn publish_observed_output_transaction(
3316 &mut self,
3317 output: B::Tensor,
3318 checkpoint: M::StateCheckpoint,
3319 forward_context: A::ForwardContext,
3320 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3321 ) -> Result<
3322 (B::Tensor, M::StateCheckpoint, A::ForwardContext),
3323 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3324 > {
3325 let publication = D::publish_observed_output(&mut self.execution, output, context);
3326 let publication_agreement = D::agree_distributed_phase(
3327 &mut self.execution,
3328 crate::DistributedExecutionPhase::OutputPublication,
3329 publication.is_ok(),
3330 context,
3331 );
3332 let output = match (publication, publication_agreement) {
3333 (Err(error), _) => {
3334 return self.rollback_failure(checkpoint, widen_infallible(error), context)
3335 }
3336 (Ok(_), Err(error)) => {
3337 return self.rollback_failure(checkpoint, widen_infallible(error), context)
3338 }
3339 (Ok(output), Ok(true)) => output,
3340 (Ok(_), Ok(false)) => {
3341 return self.rollback_failure(
3342 checkpoint,
3343 ReplicatedTextSessionError::Contract(
3344 "another rank failed during distributed output publication".into(),
3345 ),
3346 context,
3347 )
3348 }
3349 };
3350 Ok((output, checkpoint, forward_context))
3351 }
3352
3353 fn publish(
3354 &mut self,
3355 output: B::Tensor,
3356 checkpoint: M::StateCheckpoint,
3357 _forward_context: A::ForwardContext,
3358 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3359 ) -> Result<B::Tensor, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3360 let completion = self
3361 .selected
3362 .exact_completion()
3363 .then(|| self.mechanisms.complete(&output, &self.state, context))
3364 .transpose();
3365 let completion_agreed = match D::agree_distributed_phase(
3366 &mut self.execution,
3367 crate::DistributedExecutionPhase::MechanismCompletion,
3368 completion.is_ok(),
3369 context,
3370 ) {
3371 Ok(agreed) => agreed,
3372 Err(error) => {
3373 return self.rollback_failure(checkpoint, widen_infallible(error), context)
3374 }
3375 };
3376 match completion {
3377 Ok(_) if completion_agreed => {}
3378 Ok(_) => {
3379 return self.rollback_failure(
3380 checkpoint,
3381 ReplicatedTextSessionError::Contract(
3382 "another rank failed during distributed mechanism completion".into(),
3383 ),
3384 context,
3385 )
3386 }
3387 Err(error) => {
3388 return self.rollback_failure(
3389 checkpoint,
3390 ReplicatedTextSessionError::Mechanism(error),
3391 context,
3392 )
3393 }
3394 }
3395 let epoch = self.active_commit_epoch.ok_or_else(|| {
3396 ReplicatedTextSessionError::Contract(
3397 "distributed transaction lost its active commit epoch".into(),
3398 )
3399 })?;
3400 match D::commit_after_completion(&mut self.execution, epoch, context) {
3401 DistributedCommitOutcome::Committed(committed) if committed == epoch => {
3402 self.last_commit_outcome = Some(DistributedCommitOutcome::Committed(epoch));
3403 self.active_commit_epoch = None;
3404 }
3405 DistributedCommitOutcome::Aborted(aborted) if aborted == epoch => {
3406 self.last_commit_outcome = Some(DistributedCommitOutcome::Aborted(epoch));
3407 self.active_commit_epoch = None;
3408 let restored = self
3409 .mechanisms
3410 .restore_state(&mut self.state, checkpoint, context)
3411 .map_err(ReplicatedTextSessionError::Mechanism);
3412 record_successful_restoration(&mut self.successful_state_restorations, restored)?;
3413 return Err(ReplicatedTextSessionError::CommitAborted { epoch });
3414 }
3415 DistributedCommitOutcome::Indeterminate {
3416 epoch: uncertain,
3417 phase,
3418 } if uncertain == epoch => {
3419 self.last_commit_outcome =
3420 Some(DistributedCommitOutcome::Indeterminate { epoch, phase });
3421 self.active_commit_epoch = None;
3422 return Err(ReplicatedTextSessionError::CommitIndeterminate { epoch, phase });
3423 }
3424 outcome => {
3425 return self.rollback_failure(
3426 checkpoint,
3427 ReplicatedTextSessionError::Contract(format!(
3428 "distributed commit returned epoch {} for active epoch {}",
3429 outcome.epoch().value(),
3430 epoch.value()
3431 )),
3432 context,
3433 );
3434 }
3435 }
3436 Ok(output)
3437 }
3438
3439 fn rollback_failure<T>(
3440 &mut self,
3441 checkpoint: M::StateCheckpoint,
3442 error: ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3443 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3444 ) -> Result<T, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3445 let restored = self
3446 .mechanisms
3447 .restore_state(&mut self.state, checkpoint, context)
3448 .map_err(ReplicatedTextSessionError::Mechanism);
3449 if let Some(epoch) = self.active_commit_epoch.take() {
3450 self.last_commit_outcome = Some(DistributedCommitOutcome::Aborted(epoch));
3451 }
3452 record_successful_restoration(&mut self.successful_state_restorations, restored)?;
3453 Err(error)
3454 }
3455
3456 fn begin_commit_epoch(
3457 &mut self,
3458 ) -> Result<
3459 DistributedCommitEpoch,
3460 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3461 > {
3462 self.ensure_commit_resolved()?;
3463 if self.active_commit_epoch.is_some() {
3464 return Err(ReplicatedTextSessionError::Contract(
3465 "distributed transaction already has an active commit epoch".into(),
3466 ));
3467 }
3468 let epoch = self.next_commit_epoch;
3469 self.next_commit_epoch = epoch.next().ok_or_else(|| {
3470 ReplicatedTextSessionError::Contract("distributed commit epoch overflow".into())
3471 })?;
3472 self.active_commit_epoch = Some(epoch);
3473 Ok(epoch)
3474 }
3475
3476 fn ensure_commit_resolved(
3477 &self,
3478 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3479 self.ensure_control_unfenced()?;
3480 if let Some(DistributedCommitOutcome::Indeterminate { epoch, phase }) =
3481 self.last_commit_outcome
3482 {
3483 return Err(ReplicatedTextSessionError::CommitIndeterminate { epoch, phase });
3484 }
3485 Ok(())
3486 }
3487
3488 fn ensure_control_unfenced(
3489 &self,
3490 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3491 if let Some(phase) = self.control_fence {
3492 return Err(ReplicatedTextSessionError::Contract(format!(
3493 "distributed session is fenced after failed cache control at {phase:?}"
3494 )));
3495 }
3496 Ok(())
3497 }
3498
3499 fn agree_cache_control_phase(
3500 &mut self,
3501 phase: crate::DistributedExecutionPhase,
3502 local_success: bool,
3503 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3504 ) -> Result<bool, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3505 D::agree_distributed_phase(&mut self.execution, phase, local_success, context)
3506 .map_err(widen_infallible)
3507 .inspect_err(|_| self.control_fence = Some(phase))
3508 }
3509
3510 fn require_cache_control_agreement(
3511 &self,
3512 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3513 if D::PARTITIONED_SESSION && !D::DISTRIBUTED_PHASE_AGREEMENT {
3514 return Err(ReplicatedTextSessionError::Contract(
3515 "partitioned cache control requires the selected bounded failure agreement".into(),
3516 ));
3517 }
3518 Ok(())
3519 }
3520
3521 fn fence_remote_cache_control_failure<T>(
3522 &mut self,
3523 phase: crate::DistributedExecutionPhase,
3524 ) -> Result<T, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3525 self.control_fence = Some(phase);
3526 Err(ReplicatedTextSessionError::Contract(format!(
3527 "another rank failed distributed cache control at {phase:?}"
3528 )))
3529 }
3530
3531 fn abort_without_rollback<T>(
3532 &mut self,
3533 error: ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3534 ) -> Result<T, ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3535 if let Some(epoch) = self.active_commit_epoch.take() {
3536 self.last_commit_outcome = Some(DistributedCommitOutcome::Aborted(epoch));
3537 }
3538 Err(error)
3539 }
3540
3541 fn restore_distributed_commit(
3542 &mut self,
3543 outcome: Option<DistributedCommitOutcome>,
3544 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3545 self.active_commit_epoch = None;
3546 self.last_commit_outcome = outcome;
3547 if let Some(outcome) = outcome {
3548 self.next_commit_epoch = outcome.epoch().next().ok_or_else(|| {
3549 ReplicatedTextSessionError::Contract("distributed commit epoch overflow".into())
3550 })?;
3551 }
3552 Ok(())
3553 }
3554
3555 fn validate_prompt_input_descriptor(
3556 &self,
3557 descriptor: &PromptCacheDescriptor,
3558 input_identity: &PreparedInputCacheIdentity,
3559 ) -> Result<(), ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>> {
3560 if descriptor.prefix_content_fingerprint() != input_identity.prefix_content_fingerprint() {
3561 return Err(ReplicatedTextSessionError::Contract(
3562 "prompt-cache content identity differs from the prepared input".into(),
3563 ));
3564 }
3565 Ok(())
3566 }
3567
3568 fn prompt_cache_identity(
3569 &self,
3570 ) -> Result<
3571 &PromptCacheModelIdentity,
3572 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3573 > {
3574 if !self.selected.prompt_cache() {
3575 return Err(ReplicatedTextSessionError::Contract(
3576 "prompt-cache persistence was not selected for this session".into(),
3577 ));
3578 }
3579 self.prompt_cache_identity.as_ref().ok_or_else(|| {
3580 ReplicatedTextSessionError::Contract(
3581 "this partition rank owns no prompt-cache model identity".into(),
3582 )
3583 })
3584 }
3585}
3586
3587impl<A, B, M, D> ReplicatedTextSession<A, B, M, D>
3588where
3589 B: SubmissionBackend<Executor = <<B as NeuralBackend>::Tensor as Tensor>::Context>,
3590 M: TransactionalPromptCacheMechanisms<A, B>,
3591 A: LayeredArchitecture<B, M::State>,
3592 D: ReplicatedTextExecutionStrategy<A, B, M::State, M::ResidentPolicy, M::BoundedPolicy>,
3593 A::Error: std::fmt::Display,
3594 M::PolicyError: std::fmt::Display,
3595 M::Error: std::fmt::Display,
3596{
3597 pub fn save_prompt_cache_distributed(
3599 &mut self,
3600 destination: &Path,
3601 descriptor: PromptCacheDescriptor,
3602 prefix_token_ids: &[u32],
3603 options: &PromptCacheOptions,
3604 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3605 ) -> Result<
3606 Option<PromptCacheManifest>,
3607 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3608 > {
3609 self.save_prompt_cache_distributed_inner(
3610 destination,
3611 descriptor,
3612 prefix_token_ids,
3613 options,
3614 None,
3615 context,
3616 )
3617 }
3618
3619 pub fn save_prompt_cache_for_input_distributed(
3621 &mut self,
3622 destination: &Path,
3623 descriptor: PromptCacheDescriptor,
3624 prefix_token_ids: &[u32],
3625 options: &PromptCacheOptions,
3626 input_identity: &PreparedInputCacheIdentity,
3627 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3628 ) -> Result<
3629 Option<PromptCacheManifest>,
3630 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3631 > {
3632 self.save_prompt_cache_distributed_inner(
3633 destination,
3634 descriptor,
3635 prefix_token_ids,
3636 options,
3637 Some(input_identity),
3638 context,
3639 )
3640 }
3641
3642 #[allow(clippy::too_many_arguments)]
3643 fn save_prompt_cache_distributed_inner(
3644 &mut self,
3645 destination: &Path,
3646 descriptor: PromptCacheDescriptor,
3647 prefix_token_ids: &[u32],
3648 options: &PromptCacheOptions,
3649 input_identity: Option<&PreparedInputCacheIdentity>,
3650 context: &<<B as NeuralBackend>::Tensor as Tensor>::Context,
3651 ) -> Result<
3652 Option<PromptCacheManifest>,
3653 ReplicatedTextSessionError<A::Error, M::PolicyError, M::Error>,
3654 > {
3655 self.ensure_commit_resolved()?;
3656 self.require_cache_control_agreement()?;
3657 let preflight = (|| {
3658 if let Some(input_identity) = input_identity {
3659 self.validate_prompt_input_descriptor(&descriptor, input_identity)?;
3660 if self.committed_prompt_input_identity.as_ref() != Some(input_identity) {
3661 return Err(ReplicatedTextSessionError::Contract(
3662 "prompt-cache prepared-input identity differs from the committed prompt"
3663 .into(),
3664 ));
3665 }
3666 }
3667 match (
3668 self.selected_state.state(),
3669 self.prompt_cache_identity.as_ref(),
3670 ) {
3671 (Some(_), Some(identity)) => {
3672 if !self.selected.prompt_cache() {
3673 return Err(ReplicatedTextSessionError::Contract(
3674 "prompt-cache persistence was not selected for this session".into(),
3675 ));
3676 }
3677 validate_prompt_cache_model_identity(&descriptor, identity)?;
3678 Ok(Some(
3679 descriptor
3680 .clone()
3681 .with_distributed_commit(self.last_commit_outcome),
3682 ))
3683 }
3684 (None, None) => Ok(None),
3685 _ => Err(ReplicatedTextSessionError::Contract(
3686 "partition cache state and rank-local identity ownership disagree".into(),
3687 )),
3688 }
3689 })();
3690 let phase = crate::DistributedExecutionPhase::PromptCacheSavePreflight;
3691 let agreement = self.agree_cache_control_phase(phase, preflight.is_ok(), context);
3692 let local_descriptor = match (preflight, agreement) {
3693 (Ok(local), Ok(true)) => local,
3694 (Ok(_), Ok(false)) => return self.fence_remote_cache_control_failure(phase),
3695 (Ok(_), Err(error)) => return Err(error),
3696 (Err(error), _) => {
3697 self.control_fence = Some(phase);
3698 return Err(error);
3699 }
3700 };
3701
3702 let mut transaction = local_descriptor.map(|descriptor| {
3703 self.mechanisms
3704 .prepare_prompt_cache_save(
3705 &mut self.state,
3706 destination,
3707 descriptor.clone(),
3708 prefix_token_ids,
3709 options,
3710 context,
3711 )
3712 .map_err(ReplicatedTextSessionError::Mechanism)
3713 .and_then(|transaction| {
3714 M::prepared_prompt_cache_manifest(&transaction)
3715 .validate_compatibility(&descriptor, prefix_token_ids)?;
3716 Ok(transaction)
3717 })
3718 });
3719 let local_success = transaction.as_ref().is_none_or(Result::is_ok);
3720 let phase = crate::DistributedExecutionPhase::PromptCacheSavePreparation;
3721 let agreement = self.agree_cache_control_phase(phase, local_success, context);
3722 let mut transaction = match (transaction.take(), agreement) {
3723 (Some(Ok(transaction)), Ok(true)) => Some(transaction),
3724 (None, Ok(true)) => None,
3725 (Some(Ok(transaction)), Ok(false)) => {
3726 self.mechanisms.rollback_prompt_cache_save(transaction);
3727 return self.fence_remote_cache_control_failure(phase);
3728 }
3729 (None, Ok(false)) => return self.fence_remote_cache_control_failure(phase),
3730 (Some(Ok(transaction)), Err(error)) => {
3731 self.mechanisms.rollback_prompt_cache_save(transaction);
3732 return Err(error);
3733 }
3734 (None, Err(error)) => return Err(error),
3735 (Some(Err(error)), _) => {
3736 self.control_fence = Some(phase);
3737 return Err(error);
3738 }
3739 };
3740
3741 let publication = transaction
3742 .as_mut()
3743 .map(|transaction| self.mechanisms.publish_prompt_cache_save(transaction));
3744 let local_success = publication.as_ref().is_none_or(Result::is_ok);
3745 let phase = crate::DistributedExecutionPhase::PromptCacheSavePublication;
3746 let agreement = self.agree_cache_control_phase(phase, local_success, context);
3747 let agreed = match (publication, agreement) {
3748 (Some(Err(error)), _) => {
3749 self.mechanisms.rollback_prompt_cache_save(
3750 transaction
3751 .take()
3752 .expect("failed publication retains its transaction"),
3753 );
3754 self.control_fence = Some(phase);
3755 return Err(ReplicatedTextSessionError::Mechanism(error));
3756 }
3757 (Some(Ok(())) | None, Err(error)) => {
3758 if let Some(transaction) = transaction.take() {
3759 self.mechanisms.rollback_prompt_cache_save(transaction);
3760 }
3761 return Err(error);
3762 }
3763 (Some(Ok(())) | None, Ok(agreed)) => agreed,
3764 };
3765 if !agreed {
3766 if let Some(transaction) = transaction {
3767 self.mechanisms.rollback_prompt_cache_save(transaction);
3768 }
3769 return self.fence_remote_cache_control_failure(phase);
3770 }
3771 Ok(transaction.map(|transaction| {
3772 let manifest = M::prepared_prompt_cache_manifest(&transaction).clone();
3773 self.mechanisms.commit_prompt_cache_save(transaction);
3774 manifest
3775 }))
3776 }
3777}
3778
3779fn validate_distributed_commit_restore<A, P, M>(
3780 outcome: Option<DistributedCommitOutcome>,
3781) -> Result<(), ReplicatedTextSessionError<A, P, M>>
3782where
3783 A: std::fmt::Display,
3784 P: std::fmt::Display,
3785 M: std::fmt::Display,
3786{
3787 if outcome.is_some_and(|outcome| outcome.epoch().next().is_none()) {
3788 return Err(ReplicatedTextSessionError::Contract(
3789 "distributed commit epoch overflow".into(),
3790 ));
3791 }
3792 Ok(())
3793}
3794
3795fn validate_architecture_geometry<A, B, S>(
3796 architecture: &A,
3797 selected: &SelectedReplicatedTextRealization,
3798) -> Result<(), String>
3799where
3800 B: NeuralBackend,
3801 S: RuntimeState<B>,
3802 A: LayeredArchitecture<B, S>,
3803 A::Error: std::fmt::Display,
3804{
3805 let requirements = selected.requirements();
3806 let graph = architecture
3807 .execution_graph()
3808 .map_err(|error| error.to_string())?;
3809 if &graph != requirements.execution_graph() {
3810 return Err("architecture execution graph differs from selection".into());
3811 }
3812 if requirements.execution_units().group_count() != graph.groups().len() {
3813 return Err("selected execution-unit groups differ from architecture graph".into());
3814 }
3815 for group in 0..graph.groups().len() {
3816 let actual = architecture
3817 .group_unit_count(group)
3818 .map_err(|error| error.to_string())?;
3819 let expected = requirements
3820 .execution_units()
3821 .group_range(group)
3822 .expect("validated requirement exposes every graph group")
3823 .len();
3824 if actual != expected
3825 || architecture.group_transport(group) != requirements.group_transports()[group]
3826 {
3827 return Err(format!(
3828 "architecture execution group {group} differs from selection"
3829 ));
3830 }
3831 }
3832 let layout = architecture
3833 .state_layout()
3834 .map_err(|error| error.to_string())?;
3835 if &layout != selected.state().layout() {
3836 return Err("architecture state layout differs from selection".into());
3837 }
3838 Ok(())
3839}
3840
3841fn validate_architecture_parameters<A, B, S>(
3842 architecture: &A,
3843 selected: &SelectedReplicatedTextRealization,
3844 selected_formats: bool,
3845 context: &<B::Tensor as Tensor>::Context,
3846) -> Result<BTreeMap<String, Vec<ReplicatedTextOutputCompanion>>, String>
3847where
3848 B: NeuralBackend,
3849 S: RuntimeState<B>,
3850 A: LayeredArchitecture<B, S>,
3851 A::Error: std::fmt::Display,
3852{
3853 let requirements = selected.requirements();
3854 let description = architecture
3855 .parameter_description(context)
3856 .map_err(|error| error.to_string())?;
3857 let mut actual = BTreeMap::<String, (Vec<usize>, ParameterGroupOwner)>::new();
3858 let mut companions = Vec::new();
3859 for group in description.groups() {
3860 for member in group.group().members() {
3861 match (member.linear_companion(), member.linear_companion_of()) {
3862 (None, None) => {
3863 if actual
3864 .insert(
3865 member.target().to_owned(),
3866 (member.global_shape().to_vec(), group.owner().clone()),
3867 )
3868 .is_some()
3869 {
3870 return Err(format!(
3871 "constructed architecture repeats primary parameter {:?}",
3872 member.target()
3873 ));
3874 }
3875 }
3876 (Some(role), Some(primary)) => companions.push((
3877 member.target().to_owned(),
3878 member.global_shape().to_vec(),
3879 group.owner().clone(),
3880 role,
3881 primary.to_owned(),
3882 )),
3883 _ => {
3884 return Err(format!(
3885 "constructed parameter {:?} has incomplete linear companion metadata",
3886 member.target()
3887 ));
3888 }
3889 }
3890 }
3891 }
3892 let expected = requirements
3893 .parameters()
3894 .iter()
3895 .filter(|parameter| {
3896 !matches!(
3897 parameter.presence(),
3898 ReplicatedTextParameterPresence::OptionalAbsent
3899 | ReplicatedTextParameterPresence::Tied { .. }
3900 ) && parameter.role() != crate::ReplicatedTextParameterRole::FormatCompanion
3901 })
3902 .map(|parameter| parameter.name())
3903 .collect::<BTreeSet<_>>();
3904 for (name, shape, owner, _, _) in &companions {
3905 if expected.contains(name.as_str())
3906 && actual
3907 .insert(name.clone(), (shape.clone(), owner.clone()))
3908 .is_some()
3909 {
3910 return Err(format!(
3911 "constructed architecture repeats selected parameter {name:?}"
3912 ));
3913 }
3914 }
3915 let actual_names = actual.keys().map(String::as_str).collect::<BTreeSet<_>>();
3916 if expected != actual_names {
3917 return Err(format!(
3918 "selected parameter catalog differs from constructed architecture: missing {:?}, unexpected {:?}",
3919 expected.difference(&actual_names).collect::<Vec<_>>(),
3920 actual_names.difference(&expected).collect::<Vec<_>>()
3921 ));
3922 }
3923 for parameter in requirements.parameters().iter().filter(|parameter| {
3924 !matches!(
3925 parameter.presence(),
3926 ReplicatedTextParameterPresence::OptionalAbsent
3927 | ReplicatedTextParameterPresence::Tied { .. }
3928 ) && parameter.role() != crate::ReplicatedTextParameterRole::FormatCompanion
3929 }) {
3930 let (shape, owner) = actual
3931 .get(parameter.name())
3932 .expect("equal parameter-name sets contain every requirement");
3933 let owner_matches = match (owner, parameter.owner()) {
3934 (
3935 ParameterGroupOwner::StaticRole(actual),
3936 ReplicatedTextParameterOwner::StaticRole(expected),
3937 ) => actual == expected,
3938 (
3939 ParameterGroupOwner::StaticAnyOf(actual),
3940 ReplicatedTextParameterOwner::StaticRole(expected),
3941 ) => actual.iter().any(|role| role == expected),
3942 (
3943 ParameterGroupOwner::ExecutionUnit {
3944 group, global_unit, ..
3945 },
3946 ReplicatedTextParameterOwner::ExecutionUnit {
3947 group: expected_group,
3948 unit: expected_unit,
3949 },
3950 ) => group.as_str() == expected_group && global_unit == expected_unit,
3951 _ => false,
3952 };
3953 let mut expected_shape = parameter.logical_shape().to_vec();
3954 let realization = selected_formats
3955 .then(|| {
3956 selected
3957 .parameters()
3958 .iter()
3959 .find(|realization| realization.name() == parameter.name())
3960 })
3961 .flatten();
3962 let executable = realization.map_or(parameter.native_executable(), |realization| {
3963 realization.executable()
3964 });
3965 if (!selected_formats
3966 || realization.is_some_and(|realization| {
3967 matches!(
3968 realization.lowering(),
3969 crate::WeightLoweringKind::Direct | crate::WeightLoweringKind::Derived
3970 )
3971 }))
3972 && executable == eredu_checkpoint::LinearFormat::MxFp4
3973 && matches!(
3974 parameter.source_encoding(),
3975 Some(eredu_checkpoint::SourceTensorEncoding::Safetensors(
3976 eredu_checkpoint::StoredDtype::U8
3977 ))
3978 )
3979 {
3980 expected_shape = parameter
3981 .physical_shape()
3982 .expect("direct native realization has admitted physical geometry")
3983 .to_vec();
3984 }
3985 let selected_lowering = realization.map_or(crate::WeightLoweringKind::Direct, |selected| {
3991 selected.lowering()
3992 });
3993 let selected_executable_shape = Some((|| {
3994 let descriptor = parameter
3995 .lowering_descriptor(executable)
3996 .map_err(|error| error.to_string())?;
3997 let mut packed = descriptor.logical_shape().to_vec();
3998 let Some(axis) = descriptor.packed_axis() else {
3999 return Ok(packed);
4000 };
4001 let bits = match executable {
4002 eredu_checkpoint::LinearFormat::Affine(config) => usize::try_from(config.bits)
4003 .map_err(|_| {
4004 format!(
4005 "selected parameter {:?} has invalid affine packing bits",
4006 parameter.name()
4007 )
4008 })?,
4009 eredu_checkpoint::LinearFormat::MxFp4 => 4,
4010 eredu_checkpoint::LinearFormat::Dense
4011 | eredu_checkpoint::LinearFormat::E4M3BlockFp8(_) => return Ok(packed),
4012 eredu_checkpoint::LinearFormat::GgufIQuant { ggml_type, .. } => {
4013 if matches!(
4014 selected_lowering,
4015 crate::WeightLoweringKind::Transform
4016 | crate::WeightLoweringKind::DerivedTransform
4017 ) {
4018 return Err(format!(
4019 "selected parameter {:?} cannot apply a load-time GGUF transform",
4020 parameter.name()
4021 ));
4022 }
4023 let (block, bytes) = ggml_type
4024 .block_and_bytes()
4025 .map_err(|error| error.to_string())?;
4026 let block = usize::try_from(block)
4027 .map_err(|_| "GGUF block width is not representable".to_owned())?;
4028 let bytes = usize::try_from(bytes)
4029 .map_err(|_| "GGUF block bytes are not representable".to_owned())?;
4030 let packed_bytes = packed[axis]
4031 .checked_mul(bytes)
4032 .ok_or_else(|| "GGUF executable geometry overflowed".to_owned())?;
4033 if !packed_bytes.is_multiple_of(block) {
4034 return Err(format!(
4035 "selected parameter {:?} GGUF executable geometry is not block aligned",
4036 parameter.name()
4037 ));
4038 }
4039 packed[axis] = packed_bytes / block;
4040 return Ok(packed);
4041 }
4042 };
4043 let packed_bits = packed[axis].checked_mul(bits).ok_or_else(|| {
4044 format!(
4045 "selected parameter {:?} executable geometry overflowed",
4046 parameter.name()
4047 )
4048 })?;
4049 if !packed_bits.is_multiple_of(32) {
4050 return Err(format!(
4051 "selected parameter {:?} executable geometry {}x{} bits is not U32 aligned (logical {:?}, physical {:?})",
4052 parameter.name(),
4053 packed[axis],
4054 bits,
4055 descriptor.logical_shape(),
4056 descriptor.physical_shape()
4057 ));
4058 }
4059 packed[axis] = packed_bits / 32;
4060 Ok(packed)
4061 })()?);
4062 let shape_matches = shape == &expected_shape
4063 || selected_executable_shape
4064 .as_ref()
4065 .is_some_and(|selected| shape == selected);
4066 if !shape_matches || !owner_matches {
4067 return Err(format!(
4068 "selected parameter {:?} expects logical shape {expected_shape:?} or selected executable shape {selected_executable_shape:?} and owner {:?}, constructed shape {shape:?} and owner {owner:?}",
4069 parameter.name(),
4070 parameter.owner()
4071 ));
4072 }
4073 }
4074
4075 let mut output_companions = BTreeMap::<String, Vec<ReplicatedTextOutputCompanion>>::new();
4076 for (name, shape, owner, role, primary) in companions {
4077 if name == primary || !expected.contains(primary.as_str()) {
4078 return Err(format!(
4079 "constructed companion {name:?} names unselected primary {primary:?}"
4080 ));
4081 }
4082 if selected_formats {
4083 let recipe = selected.requirements().derived_recipes().get(&name);
4084 let output = selected.requirements().derived_recipe_outputs().get(&name);
4085 let companion = match (recipe, output) {
4086 (Some(recipe), Some(output)) => {
4087 ReplicatedTextOutputCompanion::new(name, role, shape, owner).map(|companion| {
4088 companion.with_derived_recipe(recipe.clone(), output.clone())
4089 })
4090 }
4091 (None, None) => ReplicatedTextOutputCompanion::new(name, role, shape, owner),
4092 _ => {
4093 return Err(format!(
4094 "constructed companion {name:?} has incomplete derived metadata"
4095 ))
4096 }
4097 }
4098 .map_err(|error| error.to_string())?;
4099 let companion = match requirements
4100 .parameters()
4101 .iter()
4102 .find(|parameter| parameter.name() == companion.name())
4103 {
4104 Some(parameter)
4105 if matches!(
4106 parameter.source_encoding(),
4107 Some(eredu_checkpoint::SourceTensorEncoding::Gguf { .. })
4108 ) =>
4109 {
4110 let [source] = parameter.physical_sources() else {
4111 return Err(format!(
4112 "translated catalog companion {:?} has ambiguous provenance",
4113 companion.name()
4114 ));
4115 };
4116 companion.with_catalog_source(source.clone())
4117 }
4118 _ => companion,
4119 };
4120 output_companions
4121 .entry(primary)
4122 .or_default()
4123 .push(companion);
4124 }
4125 }
4126 Ok(output_companions)
4127}
4128
4129fn validate_selected_state(selected: &SelectedReplicatedTextRealization) -> Result<(), String> {
4130 use eredu_core::cache::StateResidencyClass;
4131
4132 if !selected.topology().is_replicated() {
4133 return Err("selected replicated-text topology is not replicated".into());
4134 }
4135 if selected.state().layout() != selected.requirements().state_layout()
4136 || selected.state().access() != selected.requirements().state_access()
4137 {
4138 return Err("selected state contract differs from architecture requirements".into());
4139 }
4140 let mut cursor = 0;
4141 for layer in 0..selected.state().layout().len() {
4142 for expected in selected
4143 .state()
4144 .layout()
4145 .components(layer)
4146 .expect("validated state layout exposes every layer")
4147 {
4148 let component = selected.state().components().get(cursor).ok_or_else(|| {
4149 format!("selected state omits component {cursor} at layer {layer}")
4150 })?;
4151 if component.layer() != layer || component.component() != expected {
4152 return Err(format!(
4153 "selected state component {cursor} differs from layer {layer} requirements"
4154 ));
4155 }
4156 let expected_placement = match selected.state().policy() {
4157 crate::CacheResidencyPolicy::Device => crate::StateComponentPlacement::Device,
4158 crate::CacheResidencyPolicy::Paged(_) => match expected.residency() {
4159 StateResidencyClass::SealablePaged => crate::StateComponentPlacement::Paged,
4160 StateResidencyClass::AlwaysDeviceMutable
4161 | StateResidencyClass::LayerScopedOffloadable => {
4162 crate::StateComponentPlacement::Device
4163 }
4164 },
4165 };
4166 if component.placement() != expected_placement {
4167 return Err(format!(
4168 "selected state component {cursor} has {:?} placement, expected {expected_placement:?}",
4169 component.placement()
4170 ));
4171 }
4172 cursor += 1;
4173 }
4174 }
4175 if cursor != selected.state().components().len() {
4176 return Err("selected state contains components beyond its architecture layout".into());
4177 }
4178 if !selected.state().checkpoint() || !selected.state().rollback() || !selected.state().reset() {
4179 return Err("selected state omits a required transactional lifecycle facility".into());
4180 }
4181 if selected.state().prompt_cache() != selected.prompt_cache()
4182 || selected.state().observation_retention()
4183 != (selected.session().output_observation()
4184 || selected.session().activation_inspection())
4185 {
4186 return Err("selected state lifecycle differs from selected session facilities".into());
4187 }
4188 if selected.grouped_operations() != selected.requirements().grouped_operations() {
4189 return Err("selected grouped operations differ from architecture requirements".into());
4190 }
4191 Ok(())
4192}
4193
4194fn validate_realized_state<A, P, M, B, S>(
4195 state: &S,
4196 selected: &SelectedStateRealization,
4197) -> Result<(), ReplicatedTextSessionError<A, P, M>>
4198where
4199 A: std::fmt::Display,
4200 P: std::fmt::Display,
4201 M: std::fmt::Display,
4202 B: NeuralBackend,
4203 S: RuntimeState<B>,
4204{
4205 if state.layout() != selected.layout() {
4206 return Err(ReplicatedTextSessionError::Contract(
4207 "realized state layout differs from selection".into(),
4208 ));
4209 }
4210 Ok(())
4211}
4212
4213fn map_layerwise_error<A, P>(
4214 error: LayerwiseRuntimeError<A, P>,
4215) -> ReplicatedTextSessionError<A, P, std::convert::Infallible>
4216where
4217 A: std::fmt::Display,
4218 P: std::fmt::Display,
4219{
4220 match error {
4221 LayerwiseRuntimeError::Architecture(error) => {
4222 ReplicatedTextSessionError::Architecture(error)
4223 }
4224 LayerwiseRuntimeError::State(error) => ReplicatedTextSessionError::State(error),
4225 LayerwiseRuntimeError::Layout(error) => {
4226 ReplicatedTextSessionError::Contract(error.to_string())
4227 }
4228 LayerwiseRuntimeError::Policy(error) => ReplicatedTextSessionError::Policy(error),
4229 LayerwiseRuntimeError::Submission(error) => ReplicatedTextSessionError::Contract(error),
4230 }
4231}
4232
4233fn widen_infallible<A, P, M>(
4234 error: ReplicatedTextSessionError<A, P, std::convert::Infallible>,
4235) -> ReplicatedTextSessionError<A, P, M>
4236where
4237 A: std::fmt::Display,
4238 P: std::fmt::Display,
4239 M: std::fmt::Display,
4240{
4241 match error {
4242 ReplicatedTextSessionError::Contract(error) => ReplicatedTextSessionError::Contract(error),
4243 ReplicatedTextSessionError::Architecture(error) => {
4244 ReplicatedTextSessionError::Architecture(error)
4245 }
4246 ReplicatedTextSessionError::Policy(error) => ReplicatedTextSessionError::Policy(error),
4247 ReplicatedTextSessionError::Mechanism(error) => match error {},
4248 ReplicatedTextSessionError::State(error) => ReplicatedTextSessionError::State(error),
4249 ReplicatedTextSessionError::PromptCache(error) => {
4250 ReplicatedTextSessionError::PromptCache(error)
4251 }
4252 ReplicatedTextSessionError::CommitAborted { epoch } => {
4253 ReplicatedTextSessionError::CommitAborted { epoch }
4254 }
4255 ReplicatedTextSessionError::CommitIndeterminate { epoch, phase } => {
4256 ReplicatedTextSessionError::CommitIndeterminate { epoch, phase }
4257 }
4258 }
4259}