1use crate::bin;
93use crate::coordinator;
94use crate::coordinator::forced_gap_handler::{ForcedGapGraph, ForcedGapProblem};
95use crate::coordinator::loss_handler::{RawLossSite, apply_loss_random_imputation, has_loss_model};
96use crate::coordinator::reweight_handler::{
97 ProjectedErrors, apply_reweights, correction_weights, decode_projected, deduplicate_decoder_input,
98 hard_decoding_hypergraph, ignore_edge_isolated_history_vertices, load_projected_decoder, prepare_decoder,
99 probability_reweights,
100};
101use crate::coordinator::{
102 DecoderCacheKey, DecoderReweighting, FingerprintSource, LoadedDecoder, LossHandler, LossStrategy,
103 build_modifier_fingerprints,
104};
105use crate::decoder::DynDecoder;
106use crate::decoder::blackbox_decoder::{self, DecodingHypergraph, EdgeReweight, Hyperedge, ParityFactor};
107use crate::decoder::blackbox_util::{assert_parity_factor, is_parity_factor};
108use crate::jit::loss_compiler::{GadgetLoss, build_cross_gadget_loss_sites, build_cross_gadget_output_links};
109use crate::misc::bit_vector::{self, flip_bit, get_bit, set_bit};
110use crate::misc::fastrace::{Event, Span, SpanContext};
111use crate::misc::index::{ErrorIndex, WILDCARD};
112use crate::misc::pauli_frame_symbolic_propagator::{CorrectionBasis, PauliFrameSymbolicPropagator};
113use crate::misc::pauli_frame_tracker::PauliFrameTracker;
114use crate::misc::relative_program::{self, RelativeMapping, RelativeProgram};
115use crate::misc::sync::{TaskCounter, check_or_receiver, get_or_receiver};
116use crate::misc::validation::{
117 apply_check_model_reroutes, apply_error_model_reroutes, validate_outcomes, validate_probability_modifier,
118};
119use crate::util::BitVector;
120use binar::{BitVec, BitwiseMut};
121use futures_util::future::{join_all, try_join_all};
122use hashbrown::{HashMap, HashSet};
123use prost::Message;
124use serde::{Deserialize, Serialize};
125use std::collections::VecDeque;
126use std::sync::Arc;
127#[cfg(feature = "cli")]
128use structdoc::StructDoc;
129use tokio::sync::{Mutex, RwLock, watch};
130use tokio::task::JoinHandle;
131use tokio_util::sync::CancellationToken;
132use tonic::{Request, Response, Status};
133
134pub mod trace {
135 include!("../proto/deq.coordinator.window_coordinator.rs");
136}
137
138#[derive(Debug, Clone, Serialize, Deserialize)]
139#[serde(deny_unknown_fields)]
140#[cfg_attr(feature = "cli", derive(StructDoc))]
141pub struct WindowCoordinatorConfig {
142 #[serde(default)]
145 pub assert_parity_factor: bool,
146 #[serde(default = "default_true")]
151 pub merge_hyperedges: bool,
152 #[serde(default = "default_true")]
157 pub persistent_decoder: bool,
158 #[serde(default)]
163 pub decoder_reweighting: DecoderReweighting,
164 #[serde(default)]
168 pub forced_gap: bool,
169 #[serde(default)]
172 pub frame_probabilities: bool,
173 #[serde(default)]
176 pub forced_gap_strategy: ForcedGapStrategy,
177 #[serde(default)]
180 pub window_parallelism: WindowParallelism,
181 #[serde(default = "default_buffer_radius")]
186 pub buffer_radius: usize,
187 #[serde(default, alias = "center_radius")]
196 pub lookahead_radius: Option<usize>,
197 #[serde(default)]
200 pub trace_filepath: Option<String>,
201 #[serde(default = "default_true")]
205 pub loss_random_imputation: bool,
206 #[serde(default)]
209 pub loss_random_imputation_seed: Option<u64>,
210 #[serde(default)]
216 pub loss_strategy: LossStrategy,
217 #[serde(default)]
221 #[cfg_attr(feature = "cli", structdoc(leaf = "JSON object"))]
222 pub loss_config: serde_json::Value,
223}
224
225impl WindowCoordinatorConfig {
226 pub fn lookahead_radius(&self) -> usize {
229 if self.buffer_radius == 0 {
230 return 0;
231 }
232 self.lookahead_radius.unwrap_or(self.buffer_radius)
233 }
234
235 pub fn effective_window_radius(&self) -> usize {
238 self.buffer_radius + self.lookahead_radius()
239 }
240}
241
242fn default_true() -> bool {
243 true
244}
245
246fn default_buffer_radius() -> usize {
247 1
248}
249
250pub struct WindowCoordinator {
253 pub config: WindowCoordinatorConfig,
254 pub port_types: RwLock<HashMap<u64, Arc<bin::PortType>>>,
256 pub gadget_types: RwLock<HashMap<u64, Arc<bin::GadgetType>>>,
257 pub check_model_types: Arc<RwLock<HashMap<u64, Arc<bin::CheckModelType>>>>,
258 pub error_model_types: RwLock<HashMap<u64, Arc<bin::ErrorModelType>>>,
259 pub gadgets: Arc<RwLock<HashMap<u64, Gadget>>>,
261 pub check_models: Arc<RwLock<HashMap<u64, CheckModel>>>,
262 pub error_models: Arc<RwLock<HashMap<u64, ErrorModel>>>,
263 pending_referring_by_gid: Mutex<HashMap<u64, Vec<u64>>>,
267 pending_referring_by_cid: Mutex<HashMap<u64, Vec<u64>>>,
269 pending_referring_by_port: Mutex<HashMap<(u64, u64), Vec<PendingPortReferral>>>,
273 pub next_gid: Mutex<u64>,
275 pub next_cid: Mutex<u64>,
276 pub next_eid: Mutex<u64>,
277 pub loaded_decoders: RwLock<HashMap<DecoderCacheKey, LoadedDecoder>>,
286 pub decoder: DynDecoder,
288 gap_decoder: Option<DynDecoder>,
289 pub pauli_frame_tracker: Mutex<PauliFrameTracker>,
291 forced_gap_state: Option<RwLock<ForcedGapState>>,
293 pub cancellation: RwLock<CancellationToken>,
296 pub task_counter: Arc<TaskCounter>,
298 loss_imputation_seed: Option<u64>,
300 pub loss_handler: LossHandler,
303 pub use_loaded_reweights: bool,
305 pub trace_shot: Arc<Mutex<trace::Shot>>,
307 pub trace: Mutex<trace::WindowCoordinatorTrace>,
309}
310
311#[derive(Clone, Debug, Default, PartialEq, Eq)]
315pub struct GadgetState {
316 pub committed: bool,
318 pub reserved_by: Option<u64>,
320}
321
322impl GadgetState {
323 fn release_buffer(&self, leader_gid: u64) -> Option<Self> {
324 (self.reserved_by == Some(leader_gid)).then_some(Self {
325 committed: self.committed,
326 reserved_by: None,
327 })
328 }
329}
330
331pub struct Gadget {
332 pub instance: bin::Gadget,
333 pub outcomes: watch::Sender<Option<BitVector>>,
334 pub probability_modifiers: Vec<(u64, bin::ProbabilityModifier)>,
335 pub loss_mask: Option<BitVector>,
336 pub decoder_seed: Option<u64>,
338 pub binding_cid: Option<u64>,
340 pub outputs: Vec<watch::Sender<Option<bin::gadget::Connector>>>,
342 pub pauli_frame: watch::Sender<Option<BitVector>>,
344 pub correction_count: u64,
345 pub correction_weight: f64,
346 pub is_free_hop: bool,
351 pub state: watch::Sender<GadgetState>,
353}
354
355pub struct CheckModel {
356 pub instance: bin::CheckModel,
357 pub attaching_eid_vec: Vec<u64>,
359 pub error_model_ready: watch::Sender<Option<()>>,
360 pub modified_remote_gadgets: Arc<Vec<Option<bin::check_model_type::RemoteGadget>>>,
362 pub expanded_remote_gadgets: Option<Vec<Option<u64>>>,
364 pub syndrome: watch::Sender<Option<BitVector>>,
366 pub syndrome_count: u64,
367 pub referring_eids: Vec<u64>,
372}
373
374#[derive(Clone)]
375pub struct ErrorModel {
376 pub instance: bin::ErrorModel,
377 pub modified_remote_check_models: Arc<Vec<Option<bin::error_model_type::RemoteCheckModel>>>,
379}
380
381impl FingerprintSource for ErrorModel {
384 fn instance(&self) -> &bin::ErrorModel {
385 &self.instance
386 }
387 fn modified_remote_check_models(&self) -> &Arc<Vec<Option<bin::error_model_type::RemoteCheckModel>>> {
388 &self.modified_remote_check_models
389 }
390}
391
392fn committing_local_cids_sorted(committing_cids: &HashSet<u64>, mapping: &RelativeMapping) -> Vec<u32> {
398 let mut out: Vec<u32> = committing_cids
399 .iter()
400 .filter_map(|gcid| mapping.local_cid_of.get(gcid).map(|&lcid| lcid as u32))
401 .collect();
402 out.sort_unstable();
403 out
404}
405
406fn logical_flip_signature(logical_targets: &[CorrectionBasis], mapping: &RelativeMapping) -> Vec<u64> {
407 let mut signature = Vec::with_capacity(logical_targets.len() * 3);
408 for &target in logical_targets {
409 let (kind, gid, index) = match target {
410 CorrectionBasis::Readout { gid, index } => (0, gid, index),
411 CorrectionBasis::Residual { gid, index } => (1, gid, index),
412 };
413 signature.extend([
414 kind,
415 u64::try_from(mapping.local_gid_of[&gid]).unwrap(),
416 u64::try_from(index).unwrap(),
417 ]);
418 }
419 signature
420}
421
422struct PendingPortReferral {
426 eid: u64,
427 ri: usize,
429 owner_gid: u64,
430 modified_remote: Arc<Vec<Option<bin::error_model_type::RemoteCheckModel>>>,
431}
432
433#[derive(Clone, Debug, PartialEq, Eq)]
449pub enum ExplorePhase {
450 MandatoryZone,
451 LookaheadZone,
452}
453
454#[derive(Debug)]
467pub struct ExploredWindow {
468 pub center_gid: u64,
470 pub gadgets: HashSet<u64>,
472 pub center_distance: HashMap<u64, usize>,
474 pub phase: HashMap<u64, ExplorePhase>,
476 pub frontier: VecDeque<u64>,
478 pub commit_region: HashSet<u64>,
480 pub committing_cids: HashSet<u64>,
482 pub decoder_window: HashSet<u64>,
484}
485
486#[derive(Debug)]
488pub struct CommitRegionDecoder {
489 graph: Arc<ForcedGapGraph>,
490 local_edge_of: Vec<Option<u64>>,
491}
492
493impl CommitRegionDecoder {
494 fn new(
495 hypergraph: &DecodingHypergraph,
496 logical_flips: &[Vec<u64>],
497 retained_edges: &[bool],
498 target_count: usize,
499 persistent: bool,
500 ) -> Self {
501 assert_eq!(retained_edges.len(), hypergraph.hyperedges.len());
502 assert_eq!(logical_flips.len(), hypergraph.hyperedges.len());
503 let mut hyperedges = vec![];
504 let mut retained_logical_flips = vec![];
505 let local_edge_of = retained_edges
506 .iter()
507 .enumerate()
508 .map(|(edge, &retained)| {
509 retained.then(|| {
510 let local_edge = u64::try_from(hyperedges.len()).unwrap();
511 hyperedges.push(hypergraph.hyperedges[edge].clone());
512 retained_logical_flips.push(logical_flips[edge].clone());
513 local_edge
514 })
515 })
516 .collect();
517 Self {
518 graph: Arc::new(ForcedGapGraph::new(
519 Arc::new(DecodingHypergraph {
520 vertex_num: hypergraph.vertex_num,
521 hyperedges,
522 }),
523 Arc::new(retained_logical_flips),
524 target_count,
525 persistent,
526 )),
527 local_edge_of,
528 }
529 }
530
531 fn project(
532 &self,
533 hypergraph: &DecodingHypergraph,
534 mut syndrome: BitVector,
535 baseline: &ParityFactor,
536 reweights: Vec<EdgeReweight>,
537 ) -> Result<(BitVector, ParityFactor, Vec<EdgeReweight>), Status> {
538 if !is_parity_factor(hypergraph, baseline, &syndrome) {
539 return Err(Status::internal("forced-gap baseline does not satisfy the syndrome"));
540 }
541 let subgraph = baseline
542 .subgraph
543 .iter()
544 .filter_map(|&edge| {
545 let edge = usize::try_from(edge).unwrap();
546 if let Some(local_edge) = self.local_edge_of[edge] {
547 Some(local_edge)
548 } else {
549 for &vertex in &hypergraph.hyperedges[edge].vertices {
550 flip_bit(&mut syndrome, vertex);
551 }
552 None
553 }
554 })
555 .collect();
556 let reweights = reweights
557 .into_iter()
558 .filter_map(|reweight| {
559 match usize::try_from(reweight.edge)
560 .ok()
561 .and_then(|edge| self.local_edge_of.get(edge))
562 {
563 Some(Some(edge)) => Some(Ok(EdgeReweight {
564 edge: *edge,
565 probability: reweight.probability,
566 })),
567 Some(None) => None,
568 None => Some(Err(Status::invalid_argument("reweighted edge is outside the window graph"))),
569 }
570 })
571 .collect::<Result<_, _>>()?;
572 Ok((syndrome, ParityFactor { subgraph }, reweights))
573 }
574
575 #[allow(clippy::too_many_arguments)]
576 fn problem(
577 &self,
578 decoder: DynDecoder,
579 hypergraph: &DecodingHypergraph,
580 syndrome: BitVector,
581 decoder_seed: Option<u64>,
582 baseline: &ParityFactor,
583 reweights: Vec<EdgeReweight>,
584 use_loaded_reweights: bool,
585 ) -> Result<ForcedGapProblem, Status> {
586 let (syndrome, baseline, reweights) = self.project(hypergraph, syndrome, baseline, reweights)?;
587 Ok(self
588 .graph
589 .problem(decoder, syndrome, decoder_seed, baseline, reweights, use_loaded_reweights))
590 }
591}
592
593#[derive(Clone)]
594struct RecordedGapEdge {
595 gid: u64,
596 checks: Vec<(u64, u64)>,
597 probability: f64,
598 residual: Vec<u64>,
599 readouts: Vec<u64>,
600 selected: bool,
601}
602
603struct CausalGapSnapshot {
604 hypergraph: DecodingHypergraph,
605 syndrome: BitVector,
606 baseline: ParityFactor,
607 edges: Vec<RecordedGapEdge>,
608}
609
610impl CausalGapSnapshot {
611 fn restore_history(&mut self, history: impl IntoIterator<Item = RecordedGapEdge>, vertices: &HashMap<(u64, u64), u64>) {
612 for edge in history {
613 let detectors: Vec<_> = edge.checks.iter().filter_map(|check| vertices.get(check).copied()).collect();
614 if edge.selected {
615 self.baseline.subgraph.push(u64::try_from(self.edges.len()).unwrap());
616 for &vertex in &detectors {
617 flip_bit(&mut self.syndrome, vertex);
618 }
619 }
620 self.hypergraph.hyperedges.push(Hyperedge {
621 observable_flips: None,
622 vertices: detectors,
623 probability: edge.probability,
624 });
625 self.edges.push(edge);
626 }
627 }
628}
629
630struct CausalGapProblem {
631 problems: Vec<Arc<ForcedGapProblem>>,
632}
633
634fn original_scoring_baseline(original: &[ErrorIndex], projected: &ProjectedErrors, baseline: &ParityFactor) -> ParityFactor {
635 let indices: HashMap<_, _> = original
636 .iter()
637 .enumerate()
638 .map(|(index, error)| ((error.eid, error.error_index), u64::try_from(index).unwrap()))
639 .collect();
640 ParityFactor {
641 subgraph: baseline
642 .subgraph
643 .iter()
644 .map(|&index| {
645 let error = &projected[usize::try_from(index).unwrap()];
646 indices[&(error.eid, error.error_index)]
647 })
648 .collect(),
649 }
650}
651
652impl CausalGapProblem {
653 async fn probability(&self, target: usize) -> Result<f64, Status> {
654 self.problems[target].probability(target).await
655 }
656}
657
658#[derive(Clone)]
659enum ForcedGapScore {
660 Ready(Result<f64, Status>),
661 Deferred { problem: Arc<CausalGapProblem>, target: usize },
662}
663
664impl ForcedGapScore {
665 async fn probability(&self) -> Result<f64, Status> {
666 match self {
667 Self::Ready(probability) => probability.clone(),
668 Self::Deferred { problem, target } => problem.probability(*target).await,
669 }
670 }
671}
672
673struct ForcedGapState {
674 symbolic: PauliFrameSymbolicPropagator,
675 scores: HashMap<CorrectionBasis, ForcedGapScore>,
676 history: HashMap<u64, Vec<RecordedGapEdge>>,
677}
678
679impl ForcedGapState {
680 fn new() -> Self {
681 Self {
682 symbolic: PauliFrameSymbolicPropagator::new(),
683 scores: HashMap::new(),
684 history: HashMap::new(),
685 }
686 }
687
688 fn reset(&mut self) {
689 self.symbolic.reset();
690 self.scores.clear();
691 self.history.clear();
692 }
693
694 fn register(&mut self, targets: &[CorrectionBasis], scores: Vec<ForcedGapScore>) {
695 assert_eq!(targets.len(), scores.len());
696 for (&target, score) in targets.iter().zip(scores) {
697 assert!(
698 self.scores.insert(target, score).is_none(),
699 "forced-gap target registered more than once for {target:?}"
700 );
701 }
702 }
703
704 fn readout_scores(&self, gid: u64) -> Vec<Vec<ForcedGapScore>> {
705 self.symbolic
706 .readout_components(gid)
707 .into_iter()
708 .map(|components| {
709 components
710 .into_iter()
711 .filter_map(|target| self.scores.get(&target).cloned())
712 .collect()
713 })
714 .collect()
715 }
716
717 fn output_scores(&self, gid: u64) -> Vec<Vec<ForcedGapScore>> {
718 let components = self.symbolic.output_components(gid);
719 if (0..components.len()).any(|index| !self.scores.contains_key(&CorrectionBasis::Residual { gid, index })) {
720 return vec![];
721 }
722 components
723 .into_iter()
724 .map(|components| {
725 components
726 .into_iter()
727 .filter_map(|target| self.scores.get(&target).cloned())
728 .collect()
729 })
730 .collect()
731 }
732}
733
734#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
735#[cfg_attr(feature = "cli", derive(StructDoc))]
736#[serde(rename_all = "snake_case")]
737pub enum WindowParallelism {
738 #[default]
740 Sliding,
741 FullyParallel,
743 Serial,
745}
746
747#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
748#[cfg_attr(feature = "cli", derive(StructDoc))]
749#[serde(rename_all = "snake_case")]
750pub enum ForcedGapStrategy {
751 Eager,
753 #[default]
755 Lazy,
756}
757
758impl WindowCoordinator {
759 pub fn new(config: serde_json::Value, decoder: DynDecoder) -> Self {
760 Self::with_gap_decoder(config, decoder, None)
761 }
762
763 #[must_use]
769 pub fn with_gap_decoder(config: serde_json::Value, decoder: DynDecoder, gap_decoder: Option<DynDecoder>) -> Self {
770 let config: WindowCoordinatorConfig = serde_json::from_value(config).unwrap();
771 assert!(
772 !config.frame_probabilities || config.forced_gap,
773 "frame_probabilities requires forced_gap"
774 );
775 let use_loaded_reweights = config
776 .decoder_reweighting
777 .use_loaded(config.persistent_decoder, decoder.features())
778 .unwrap_or_else(|error| panic!("invalid decoder reweighting configuration: {error}"));
779 config
780 .decoder_reweighting
781 .use_loaded(config.persistent_decoder, gap_decoder.as_ref().unwrap_or(&decoder).features())
782 .unwrap_or_else(|error| panic!("invalid gap decoder reweighting configuration: {error}"));
783 let loss_imputation_seed = if config.loss_random_imputation {
784 use rand::Rng;
785 Some(config.loss_random_imputation_seed.unwrap_or_else(|| rand::rng().next_u64()))
786 } else {
787 None
788 };
789 let loss_handler = LossHandler::new(config.loss_strategy, config.loss_config.clone())
790 .unwrap_or_else(|error| panic!("invalid loss configuration: {error}"));
791 let forced_gap_state = config.forced_gap.then(|| RwLock::new(ForcedGapState::new()));
792 assert!(
793 !config.forced_gap || !loss_handler.hands_off_to_decoder(),
794 "forced_gap does not support loss_strategy \"handoff\"; use \"reweight\" or \"ignore\""
795 );
796 Self {
797 config,
798 port_types: Default::default(),
799 gadget_types: Default::default(),
800 check_model_types: Default::default(),
801 error_model_types: Default::default(),
802 gadgets: Default::default(),
803 check_models: Default::default(),
804 error_models: Default::default(),
805 pending_referring_by_gid: Default::default(),
806 pending_referring_by_cid: Mutex::default(),
807 pending_referring_by_port: Default::default(),
808 next_gid: Mutex::new(1),
809 next_cid: Mutex::new(1),
810 next_eid: Mutex::new(1),
811 loaded_decoders: Default::default(),
812 decoder,
813 gap_decoder,
814 pauli_frame_tracker: Default::default(),
815 forced_gap_state,
816 cancellation: RwLock::default(),
817 task_counter: TaskCounter::new(),
818 loss_imputation_seed,
819 loss_handler,
820 use_loaded_reweights,
821 trace_shot: Arc::new(Mutex::new(trace::Shot::default())),
822 trace: Mutex::new(trace::WindowCoordinatorTrace::default()),
823 }
824 }
825
826 fn gap_decoder(&self) -> &DynDecoder {
827 self.gap_decoder.as_ref().unwrap_or(&self.decoder)
828 }
829
830 pub async fn cancel_pending(&self) {
837 let token = self.cancellation.read().await;
838 token.cancel();
839 }
840
841 async fn record_event(&self, event: trace::event::Event) {
842 if self.config.trace_filepath.is_some() {
843 self.trace_shot.lock().await.events.push(trace::Event {
844 timestamp_ns: crate::misc::util::timestamp_ns(),
845 event: Some(event),
846 });
847 }
848 }
849
850 async fn wait_for_pauli_frame(&self, gid: u64) -> Result<Response<coordinator::Readouts>, Status> {
852 let token = self.cancellation.read().await.clone();
853 let gadgets = self.gadgets.read().await;
854 let gadget = gadgets.get(&gid).ok_or_else(|| Status::not_found(format!("gid={gid}")))?;
855 let pauli_frame = get_or_receiver(&gadget.pauli_frame, token.clone());
856 drop(gadgets);
857 let readouts = match pauli_frame {
858 Ok(pf) => Some(pf),
859 Err(handle) => handle.await.unwrap_or(None),
860 }
861 .ok_or_else(|| Status::cancelled("decode cancelled"))?;
862 let (syndrome_count, correction_count, correction_weight) = {
863 let gadgets = self.gadgets.read().await;
864 let gadget = gadgets
865 .get(&gid)
866 .ok_or_else(|| Status::cancelled("decode cancelled by reset"))?;
867 let check_models = self.check_models.read().await;
868 let syndrome_count = gadget.binding_cid.map_or(0, |cid| check_models[&cid].syndrome_count);
869 (syndrome_count, gadget.correction_count, gadget.correction_weight)
870 };
871 let probabilities = if self.config.forced_gap {
872 self.wait_for_forced_gap_scores(gid, token.clone()).await?
873 } else {
874 vec![]
875 };
876 let frame_probabilities = if self.config.frame_probabilities {
877 let scores = self.forced_gap_state.as_ref().unwrap().read().await.output_scores(gid);
878 Self::resolve_forced_gap_scores(scores, token).await?
879 } else {
880 vec![]
881 };
882 Ok((coordinator::Readouts {
883 gid,
884 readouts: Some(readouts),
885 probabilities,
886 syndrome_count,
887 correction_count,
888 correction_weight,
889 frame_probabilities,
890 })
891 .into())
892 }
893
894 async fn wait_for_history_commitment(&self, gid: u64) -> Result<(), Status> {
895 if self.config.window_parallelism == WindowParallelism::FullyParallel {
896 return Ok(());
897 }
898 let token = self.cancellation.read().await.clone();
899 let mut watchers = vec![];
900 if self.config.window_parallelism == WindowParallelism::Serial {
901 let gadgets = self.gadgets.read().await;
902 watchers.extend(
903 gadgets
904 .iter()
905 .filter(|(other_gid, gadget)| {
906 **other_gid < gid
907 && (self.config.buffer_radius == 0 || !gadget.is_free_hop)
908 && !gadget.state.borrow().committed
909 })
910 .map(|(_, gadget)| gadget.state.subscribe()),
911 );
912 } else {
913 let gadgets = self.gadgets.read().await;
914 let mut pending = vec![gid];
915 let mut visited = HashSet::new();
916 while let Some(current) = pending.pop() {
917 if !visited.insert(current) {
918 continue;
919 }
920 let gadget = gadgets
921 .get(¤t)
922 .ok_or_else(|| Status::not_found(format!("gid={current}")))?;
923 if current != gid && !gadget.is_free_hop && !gadget.state.borrow().committed {
924 watchers.push(gadget.state.subscribe());
925 }
926 pending.extend(gadget.instance.connectors.iter().map(|input| input.gid));
927 if let Some(remote) = gadget
928 .instance
929 .modifier
930 .as_ref()
931 .and_then(|modifier| modifier.remote_conditional_correction.as_ref())
932 {
933 pending.extend(remote.remote_readouts.iter().map(|readout| readout.gid));
934 }
935 }
936 }
937 for mut watcher in watchers {
938 tokio::select! {
939 result = watcher.wait_for(|state| state.committed) => {
940 result.map_err(|_| Status::cancelled("history predecessor removed"))?;
941 }
942 () = token.cancelled() => return Err(Status::cancelled("history commitment wait cancelled")),
943 }
944 }
945 Ok(())
946 }
947
948 async fn wait_for_forced_gap_scores(&self, gid: u64, token: CancellationToken) -> Result<Vec<f64>, Status> {
949 let state = self.forced_gap_state.as_ref().unwrap();
950 let readouts = state.read().await.readout_scores(gid);
951 Self::resolve_forced_gap_scores(readouts, token).await
952 }
953
954 async fn resolve_forced_gap_scores(
955 readouts: Vec<Vec<ForcedGapScore>>,
956 token: CancellationToken,
957 ) -> Result<Vec<f64>, Status> {
958 let solve = try_join_all(readouts.into_iter().map(|scores| async move {
959 let probabilities = try_join_all(scores.iter().map(ForcedGapScore::probability)).await?;
960 Ok::<_, Status>(probabilities.into_iter().fold(0.0, f64::max))
961 }));
962 tokio::select! {
963 results = solve => results,
964 () = token.cancelled() => {
965 Err(Status::cancelled("forced-gap scoring cancelled by reset"))
966 }
967 }
968 }
969
970 async fn register_forced_gap_scores(
971 &self,
972 logical_targets: &[CorrectionBasis],
973 problem: &Result<Arc<CausalGapProblem>, Status>,
974 ) {
975 let scores = (0..logical_targets.len())
976 .map(|target| match problem {
977 Ok(problem) => ForcedGapScore::Deferred {
978 problem: Arc::clone(problem),
979 target,
980 },
981 Err(error) => ForcedGapScore::Ready(Err(error.clone())),
982 })
983 .collect();
984 self.forced_gap_state
985 .as_ref()
986 .unwrap()
987 .write()
988 .await
989 .register(logical_targets, scores);
990 }
991
992 async fn complete_eager_scores(&self, logical_targets: &[CorrectionBasis], problem: &CausalGapProblem) {
993 let probabilities = join_all((0..logical_targets.len()).map(|target| problem.probability(target))).await;
994 let mut state = self.forced_gap_state.as_ref().unwrap().write().await;
995 for (&target, probability) in logical_targets.iter().zip(probabilities) {
996 *state.scores.get_mut(&target).unwrap() = ForcedGapScore::Ready(probability);
997 }
998 }
999
1000 async fn explore_mandatory_zone(&self, center_gid: u64) -> Option<ExploredWindow> {
1052 let buffer_radius = self.config.buffer_radius;
1053 let token = self.cancellation.read().await.clone();
1054
1055 let mut explored = ExploredWindow {
1056 center_gid,
1057 gadgets: HashSet::from([center_gid]),
1058 center_distance: HashMap::from([(center_gid, 0)]),
1059 phase: HashMap::from([(center_gid, ExplorePhase::MandatoryZone)]),
1060 frontier: VecDeque::from([center_gid]),
1061 commit_region: HashSet::new(),
1062 committing_cids: HashSet::new(),
1063 decoder_window: HashSet::new(),
1064 };
1065
1066 if buffer_radius == 0 {
1067 explored.frontier.clear();
1068 return Some(explored);
1069 }
1070
1071 while let Some(fgid) = explored.frontier.pop_front() {
1072 if token.is_cancelled() {
1073 return None;
1074 }
1075 let my_dist = explored.center_distance[&fgid];
1076 let mut sync_neighbors: Vec<(u64, bool)> = vec![];
1077 let mut async_handles: Vec<JoinHandle<Option<bin::gadget::Connector>>> = vec![];
1078
1079 {
1080 let gadgets = self.gadgets.read().await;
1081 let gadget = gadgets.get(&fgid)?;
1082
1083 for connector in &gadget.instance.connectors {
1085 if explored.gadgets.contains(&connector.gid) {
1086 continue;
1087 }
1088 let peer = &gadgets[&connector.gid];
1089 let peer_dist = if peer.is_free_hop { my_dist } else { my_dist + 1 };
1090 if peer_dist <= buffer_radius {
1091 explored.gadgets.insert(connector.gid);
1092 explored.center_distance.insert(connector.gid, peer_dist);
1093 explored.phase.insert(connector.gid, ExplorePhase::MandatoryZone);
1094 sync_neighbors.push((connector.gid, peer.is_free_hop));
1095 }
1096 }
1097
1098 for sender in &gadget.outputs {
1106 match get_or_receiver(sender, token.clone()) {
1107 Ok(connector) => {
1108 if explored.gadgets.contains(&connector.gid) {
1109 continue;
1110 }
1111 let peer = gadgets.get(&connector.gid)?;
1112 let peer_dist = if peer.is_free_hop { my_dist } else { my_dist + 1 };
1113 if peer_dist <= buffer_radius {
1114 explored.gadgets.insert(connector.gid);
1115 explored.center_distance.insert(connector.gid, peer_dist);
1116 explored.phase.insert(connector.gid, ExplorePhase::MandatoryZone);
1117 sync_neighbors.push((connector.gid, peer.is_free_hop));
1118 }
1119 }
1120 Err(handle) => {
1121 if my_dist < buffer_radius {
1122 async_handles.push(handle);
1123 }
1124 }
1125 }
1126 }
1127 }
1128
1129 for (gid, is_free_hop) in sync_neighbors {
1131 let peer_dist = explored.center_distance[&gid];
1132 if peer_dist < buffer_radius {
1133 if is_free_hop {
1134 explored.frontier.push_front(gid);
1135 } else {
1136 explored.frontier.push_back(gid);
1137 }
1138 }
1139 }
1140
1141 for handle in async_handles {
1143 if let Some(connector) = handle.await.unwrap_or(None) {
1144 if explored.gadgets.contains(&connector.gid) {
1145 continue;
1146 }
1147 let gadgets = self.gadgets.read().await;
1148 let peer = &gadgets[&connector.gid];
1149 let peer_dist = if peer.is_free_hop { my_dist } else { my_dist + 1 };
1150 if peer_dist <= buffer_radius {
1151 explored.gadgets.insert(connector.gid);
1152 explored.center_distance.insert(connector.gid, peer_dist);
1153 explored.phase.insert(connector.gid, ExplorePhase::MandatoryZone);
1154 if peer_dist < buffer_radius {
1155 if peer.is_free_hop {
1156 explored.frontier.push_front(connector.gid);
1157 } else {
1158 explored.frontier.push_back(connector.gid);
1159 }
1160 }
1161 }
1162 }
1163 }
1164 }
1165
1166 explored.frontier.clear();
1170 for &gid in &explored.gadgets {
1171 if explored.center_distance[&gid] == buffer_radius {
1172 explored.frontier.push_back(gid);
1173 } else {
1174 }
1179 }
1180
1181 Some(explored)
1182 }
1183
1184 async fn await_mandatory_zone_syndrome(&self, explored: &ExploredWindow) -> Option<()> {
1201 let token = self.cancellation.read().await.clone();
1202
1203 let mandatory_cids: Vec<u64> = {
1205 let gadgets = self.gadgets.read().await;
1206 explored
1207 .gadgets
1208 .iter()
1209 .filter_map(|&gid| gadgets.get(&gid)?.binding_cid)
1210 .collect()
1211 };
1212
1213 let mut handles: Vec<JoinHandle<bool>> = vec![];
1219 {
1220 let check_models = self.check_models.read().await;
1221 for cid in &mandatory_cids {
1222 if let Some(check_model) = check_models.get(cid)
1223 && let Err(handle) = check_or_receiver(&check_model.syndrome, token.clone())
1224 {
1225 handles.push(handle);
1226 }
1227 }
1228 }
1229 futures_util::future::join_all(handles).await;
1230 if token.is_cancelled() {
1231 return None;
1232 }
1233
1234 Some(())
1235 }
1236
1237 async fn explore_lookahead_zone(&self, explored: &mut ExploredWindow) -> Option<()> {
1268 let lookahead_radius = self.config.lookahead_radius();
1269 if lookahead_radius == 0 {
1270 return Some(()); }
1272
1273 let buffer_radius = self.config.buffer_radius;
1274 let window_radius = buffer_radius + lookahead_radius;
1275 let token = self.cancellation.read().await.clone();
1276
1277 while let Some(fgid) = explored.frontier.pop_front() {
1278 if token.is_cancelled() {
1279 return None;
1280 }
1281 let my_dist = explored.center_distance[&fgid];
1282 let mut sync_neighbors: Vec<(u64, bool)> = vec![];
1283
1284 {
1285 let gadgets = self.gadgets.read().await;
1286 let check_models = self.check_models.read().await;
1287 let gadget = gadgets.get(&fgid)?;
1288
1289 let syndrome_ready = |peer: &Gadget| -> bool {
1295 if peer.is_free_hop {
1296 return true;
1297 }
1298 match peer.binding_cid {
1299 None => true,
1300 Some(cid) => check_models.get(&cid).is_some_and(|cm| cm.syndrome.borrow().is_some()),
1301 }
1302 };
1303
1304 for connector in &gadget.instance.connectors {
1306 if explored.gadgets.contains(&connector.gid) {
1307 continue;
1308 }
1309 let peer = &gadgets[&connector.gid];
1310 let peer_dist = if peer.is_free_hop { my_dist } else { my_dist + 1 };
1311 if peer_dist <= window_radius && syndrome_ready(peer) {
1312 explored.gadgets.insert(connector.gid);
1313 explored.center_distance.insert(connector.gid, peer_dist);
1314 explored.phase.insert(connector.gid, ExplorePhase::LookaheadZone);
1315 sync_neighbors.push((connector.gid, peer.is_free_hop));
1316 }
1317 }
1318
1319 for sender in &gadget.outputs {
1321 if let Some(connector) = *sender.borrow() {
1322 if explored.gadgets.contains(&connector.gid) {
1323 continue;
1324 }
1325 if let Some(peer) = gadgets.get(&connector.gid) {
1326 let peer_dist = if peer.is_free_hop { my_dist } else { my_dist + 1 };
1327 if peer_dist <= window_radius && syndrome_ready(peer) {
1328 explored.gadgets.insert(connector.gid);
1329 explored.center_distance.insert(connector.gid, peer_dist);
1330 explored.phase.insert(connector.gid, ExplorePhase::LookaheadZone);
1331 sync_neighbors.push((connector.gid, peer.is_free_hop));
1332 }
1333 }
1334 }
1335 }
1337 }
1338
1339 for (gid, is_free_hop) in sync_neighbors {
1341 let peer_dist = explored.center_distance[&gid];
1342 if peer_dist < window_radius {
1343 if is_free_hop {
1344 explored.frontier.push_front(gid);
1345 } else {
1346 explored.frontier.push_back(gid);
1347 }
1348 }
1349 }
1350 }
1351
1352 Some(())
1353 }
1354
1355 fn select_commit_region(&self, explored: &mut ExploredWindow, gadgets: &HashMap<u64, Gadget>) {
1380 let buffer_radius = self.config.buffer_radius;
1381
1382 if buffer_radius == 0 {
1386 explored.commit_region.clear();
1387 explored.commit_region.insert(explored.center_gid);
1388 explored.committing_cids.clear();
1389 if let Some(cid) = gadgets[&explored.center_gid].binding_cid {
1390 explored.committing_cids.insert(cid);
1391 }
1392 return;
1393 }
1394
1395 let mut boundary_dist: HashMap<u64, usize> = HashMap::new();
1401 let mut deque: VecDeque<u64> = VecDeque::new();
1402
1403 for &gid in &explored.gadgets {
1404 let gadget = &gadgets[&gid];
1405 let mut has_open_boundary = false;
1406
1407 for connector in &gadget.instance.connectors {
1409 if !explored.gadgets.contains(&connector.gid) {
1410 has_open_boundary = true;
1411 break;
1412 }
1413 }
1414
1415 if !has_open_boundary {
1417 for sender in &gadget.outputs {
1418 match sender.borrow().as_ref() {
1419 Some(conn) => {
1420 if !explored.gadgets.contains(&conn.gid) {
1421 has_open_boundary = true;
1422 break;
1423 }
1424 }
1425 None => {
1426 has_open_boundary = true;
1428 break;
1429 }
1430 }
1431 }
1432 }
1433
1434 if has_open_boundary {
1435 boundary_dist.insert(gid, 0);
1436 deque.push_back(gid);
1437 }
1438 }
1439
1440 while let Some(fgid) = deque.pop_front() {
1448 let my_bdist = boundary_dist[&fgid];
1449 let gadget = &gadgets[&fgid];
1450 let step = if gadget.is_free_hop { 0 } else { 1 };
1451
1452 let mut visit = |peer_gid: u64| {
1453 if !explored.gadgets.contains(&peer_gid) {
1454 return;
1455 }
1456 let new_dist = my_bdist + step;
1457 let entry = boundary_dist.entry(peer_gid).or_insert(usize::MAX);
1458 if new_dist < *entry {
1459 *entry = new_dist;
1460 let peer = &gadgets[&peer_gid];
1461 if peer.is_free_hop {
1462 deque.push_front(peer_gid);
1463 } else {
1464 deque.push_back(peer_gid);
1465 }
1466 }
1467 };
1468
1469 for connector in &gadget.instance.connectors {
1470 visit(connector.gid);
1471 }
1472 for sender in &gadget.outputs {
1473 if let Some(conn) = sender.borrow().as_ref() {
1474 visit(conn.gid);
1475 }
1476 }
1477 }
1478
1479 for &gid in &explored.gadgets {
1481 boundary_dist.entry(gid).or_insert(usize::MAX);
1482 }
1483
1484 explored.commit_region = HashSet::new();
1489
1490 for &gid in &explored.gadgets {
1491 let gadget = &gadgets[&gid];
1492 if gadget.is_free_hop {
1493 continue; }
1495 let state = gadget.state.borrow();
1496 if state.committed || state.reserved_by.is_some() {
1497 continue; }
1499 let bdist = boundary_dist.get(&gid).copied().unwrap_or(0);
1500 if bdist >= buffer_radius {
1501 explored.commit_region.insert(gid);
1502 }
1503 }
1504
1505 loop {
1509 let mut changed = false;
1510 for &gid in &explored.gadgets {
1511 if explored.commit_region.contains(&gid) {
1512 continue;
1513 }
1514 let g = &gadgets[&gid];
1515 let state = g.state.borrow();
1516 if !g.is_free_hop || state.committed || state.reserved_by.is_some() {
1517 continue;
1518 }
1519 drop(state);
1520 let in_region = |ngid: u64| {
1521 explored.commit_region.contains(&ngid) || gadgets.get(&ngid).is_some_and(|p| p.state.borrow().committed)
1522 };
1523 let adjacent = g.instance.connectors.iter().any(|c| in_region(c.gid))
1524 || g.outputs
1525 .iter()
1526 .any(|s| s.borrow().as_ref().is_some_and(|c| in_region(c.gid)));
1527 if adjacent {
1528 explored.commit_region.insert(gid);
1529 changed = true;
1530 }
1531 }
1532 if !changed {
1533 break;
1534 }
1535 }
1536
1537 debug_assert!(explored.commit_region.iter().all(|g| explored.gadgets.contains(g)));
1539 debug_assert!(
1540 explored.commit_region.contains(&explored.center_gid),
1541 "center gadget {} must be in commit region (boundary_dist={:?}, buffer_radius={})",
1542 explored.center_gid,
1543 boundary_dist.get(&explored.center_gid),
1544 buffer_radius,
1545 );
1546
1547 explored.committing_cids.clear();
1548 for &gid in &explored.commit_region {
1549 let gadget = &gadgets[&gid];
1550 if let Some(cid) = gadget.binding_cid {
1551 explored.committing_cids.insert(cid);
1552 }
1553 }
1554 }
1555
1556 fn shrink_window(&self, explored: &mut ExploredWindow, gadgets: &HashMap<u64, Gadget>) {
1570 let buffer_radius = self.config.buffer_radius;
1571 explored.decoder_window.clear();
1572
1573 let mut dist: HashMap<u64, usize> = HashMap::new();
1576 let mut deque: VecDeque<u64> = VecDeque::new();
1577
1578 for &cgid in &explored.commit_region {
1579 dist.insert(cgid, 0);
1580 deque.push_back(cgid);
1581 explored.decoder_window.insert(cgid);
1582 }
1583
1584 while let Some(fgid) = deque.pop_front() {
1585 let my_dist = dist[&fgid];
1586 let gadget = &gadgets[&fgid];
1587
1588 let mut visit = |peer_gid: u64| {
1589 if !explored.gadgets.contains(&peer_gid) {
1590 return;
1591 }
1592 let peer = &gadgets[&peer_gid];
1593 let step = if peer.is_free_hop { 0 } else { 1 };
1594 let new_dist = my_dist + step;
1595 if new_dist > buffer_radius {
1596 return;
1597 }
1598 let entry = dist.entry(peer_gid).or_insert(usize::MAX);
1599 if new_dist < *entry {
1600 *entry = new_dist;
1601 explored.decoder_window.insert(peer_gid);
1602 if new_dist < buffer_radius {
1603 if peer.is_free_hop {
1604 deque.push_front(peer_gid);
1605 } else {
1606 deque.push_back(peer_gid);
1607 }
1608 }
1609 }
1610 };
1611
1612 for connector in &gadget.instance.connectors {
1613 visit(connector.gid);
1614 }
1615 for sender in &gadget.outputs {
1616 if let Some(conn) = sender.borrow().as_ref() {
1617 visit(conn.gid);
1618 }
1619 }
1620 }
1621 }
1622
1623 async fn decode_and_commit(
1631 &self,
1632 center_gid: u64,
1633 commit_region: &HashSet<u64>,
1634 committing_cids: &HashSet<u64>,
1635 window: &HashSet<u64>,
1636 ) -> Result<(), Status> {
1637 let cancelled = || Status::cancelled("decode cancelled");
1638 let missing = |entity: &'static str, id: u64| {
1639 move || Status::internal(format!("window centered at gid={center_gid}: missing {entity} for id={id}"))
1640 };
1641 let span = Span::root("decode_window", SpanContext::random());
1642 span.add_property(|| ("center_gid", format!("{center_gid}")));
1643 span.add_property(|| ("commit_region", format!("{:?}", commit_region)));
1644 span.add_property(|| ("window", format!("{:?}", window)));
1645
1646 let token = self.cancellation.read().await.clone();
1649 let mut handles: Vec<JoinHandle<bool>> = vec![];
1650 {
1651 let gadgets = self.gadgets.read().await;
1652 for &window_gid in window {
1653 let gadget = gadgets.get(&window_gid).ok_or_else(missing("gadget", window_gid))?;
1654 if let Err(handle) = check_or_receiver(&gadget.outcomes, token.clone()) {
1655 handles.push(handle);
1656 }
1657 }
1658 for &cgid in commit_region {
1660 if !window.contains(&cgid)
1661 && let Some(gadget) = gadgets.get(&cgid)
1662 && let Err(handle) = check_or_receiver(&gadget.outcomes, token.clone())
1663 {
1664 handles.push(handle);
1665 }
1666 }
1667 }
1668 futures_util::future::join_all(handles).await;
1669 if token.is_cancelled() {
1670 return Err(cancelled());
1671 }
1672 span.add_event(Event::new("outcomes_ready"));
1673
1674 span.add_property(|| ("committing_cids", format!("{:?}", committing_cids)));
1675
1676 self.record_event(trace::event::Event::Decode(trace::DecodeEvent {
1677 gid: center_gid,
1678 is_leader: true,
1679 leader_gid: center_gid,
1680 window: {
1681 let mut v: Vec<_> = window.iter().copied().collect();
1682 v.sort();
1683 v
1684 },
1685 committing_gids: {
1686 let mut v: Vec<_> = commit_region.iter().copied().collect();
1687 v.sort();
1688 v
1689 },
1690 committing_cids: {
1691 let mut v: Vec<_> = committing_cids.iter().copied().collect();
1692 v.sort();
1693 v
1694 },
1695 }))
1696 .await;
1697
1698 if committing_cids.is_empty() {
1700 let gadgets = self.gadgets.write().await;
1701 let mut tracker = self.pauli_frame_tracker.lock().await;
1702 for &cgid in commit_region {
1703 let pauli_frame_gadget = tracker.gadgets.get(&cgid).ok_or_else(missing("pauli-frame gadget", cgid))?;
1704 let residual = BitVec::zeros(pauli_frame_gadget.num_output_observables());
1705 let readout_flips = BitVec::zeros(pauli_frame_gadget.num_readouts());
1706 for (update_gid, pauli_frame) in tracker.load_correction(cgid, residual, readout_flips) {
1707 let update_gadget = gadgets.get(&update_gid).ok_or_else(missing("gadget", update_gid))?;
1708 debug_assert!(update_gadget.pauli_frame.borrow().is_none(), "bug");
1709 update_gadget.pauli_frame.send_replace(Some(pauli_frame));
1710 }
1711 }
1712 for &commit_gid in commit_region {
1714 let gadget = gadgets.get(&commit_gid).ok_or_else(missing("gadget", commit_gid))?;
1715 gadget.state.send_replace(GadgetState {
1716 committed: true,
1717 reserved_by: None,
1718 });
1719 }
1720 for &wgid in window {
1722 let Some(g) = gadgets.get(&wgid) else { continue };
1723 let released = g.state.borrow().release_buffer(center_gid);
1724 if let Some(state) = released {
1725 g.state.send_replace(state);
1726 }
1727 }
1728 span.add_property(|| ("cid", "None"));
1729 drop(tracker);
1730 drop(gadgets);
1731 return Ok(());
1732 }
1733
1734 let window_cids = {
1736 let gadgets = self.gadgets.read().await;
1737 let mut window_cids: HashSet<u64> = HashSet::new();
1738 for &window_gid in window {
1739 let gadget = gadgets.get(&window_gid).ok_or_else(missing("gadget", window_gid))?;
1740 if let Some(cid) = gadget.binding_cid {
1741 window_cids.insert(cid);
1742 }
1743 }
1744 window_cids
1745 };
1746 let mut handles: Vec<JoinHandle<bool>> = vec![];
1749 {
1750 let check_models = self.check_models.read().await;
1751 for &window_cid in &window_cids {
1752 let check_model = check_models.get(&window_cid).ok_or_else(missing("check model", window_cid))?;
1753 if let Err(handle) = check_or_receiver(&check_model.syndrome, token.clone()) {
1754 handles.push(handle);
1755 }
1756 }
1757 }
1758 futures_util::future::join_all(handles).await;
1759 if token.is_cancelled() {
1760 return Err(cancelled());
1761 }
1762 span.add_property(|| ("window_cids", format!("{:?}", window_cids)));
1763
1764 let terminal_gids = self.await_error_models(window).await?;
1770 let mut selected_error_models = HashMap::new();
1771 let mut expanded_gadgets: Vec<relative_program::ExpandedGadget> = vec![];
1772 {
1773 let check_model_types = self.check_model_types.read().await;
1774 let error_model_types = self.error_model_types.read().await;
1775 let gadgets = self.gadgets.read().await;
1776 let check_models = self.check_models.read().await;
1777 let error_models = self.error_models.read().await;
1778 let mut gid_vec: Vec<_> = window.iter().copied().collect();
1779 gid_vec.sort_unstable();
1780
1781 for &gid in gid_vec.iter() {
1787 let gadget = gadgets.get(&gid).ok_or_else(missing("gadget", gid))?;
1788 let inputs: Vec<_> = gadget.instance.connectors.iter().cloned().map(Some).collect();
1789 let outputs: Vec<_> = gadget.outputs.iter().map(|v| *v.borrow()).collect();
1790 let gtype = gadget.instance.gtype;
1791 let cid = gadget.binding_cid;
1792 let is_committed = gadget.state.borrow().committed;
1793 let (check_model, error_models) = if let Some(cid) = cid {
1794 let check_model = check_models.get(&cid).ok_or_else(missing("check model", cid))?;
1795 let remote_gadgets = check_model
1796 .expanded_remote_gadgets
1797 .clone()
1798 .ok_or_else(missing("expanded remote gadgets for check model", cid))?;
1799 let expanded_check_model = relative_program::ExpandedCheckModel {
1800 cid,
1801 ctype: check_model.instance.ctype,
1802 remote_gadgets,
1803 count_checks: check_model_types
1804 .get(&check_model.instance.ctype)
1805 .ok_or_else(missing("check model type", check_model.instance.ctype))?
1806 .checks
1807 .len(),
1808 };
1809 let expanded_error_models = if is_committed {
1810 vec![]
1813 } else {
1814 let mut ems = vec![];
1815 if check_model.attaching_eid_vec.is_empty()
1816 && terminal_gids.contains(&gid)
1817 && let Some(terminal) = &check_model.instance.terminal_error_model
1818 {
1819 let terminal_type = &error_model_types[&terminal.etype];
1820 let model = ErrorModel {
1821 instance: terminal.clone(),
1822 modified_remote_check_models: Arc::new(
1823 terminal_type.remote_check_models.iter().cloned().map(Some).collect(),
1824 ),
1825 };
1826 let remote_check_models =
1827 Self::expand_remote_check_models_in_window(gid, &model, &gadgets, window);
1828 selected_error_models.insert(terminal.eid, model);
1829 ems.push(relative_program::ExpandedErrorModel {
1830 eid: terminal.eid,
1831 etype: terminal.etype,
1832 remote_check_models,
1833 });
1834 }
1835 for &eid in check_model.attaching_eid_vec.iter() {
1836 let error_model = error_models.get(&eid).ok_or_else(missing("error model", eid))?;
1837 selected_error_models.insert(eid, error_model.clone());
1838 let remote_check_models =
1839 Self::expand_remote_check_models_in_window(gid, error_model, &gadgets, window);
1840 ems.push(relative_program::ExpandedErrorModel {
1841 eid,
1842 etype: error_model.instance.etype,
1843 remote_check_models,
1844 });
1845 }
1846 ems
1847 };
1848 (Some(expanded_check_model), expanded_error_models)
1849 } else {
1850 (None, vec![])
1851 };
1852 expanded_gadgets.push(relative_program::ExpandedGadget {
1853 gid,
1854 gtype,
1855 inputs,
1856 outputs,
1857 check_model,
1858 error_models,
1859 });
1860 }
1861
1862 let mut outside_gadget_eids: HashMap<u64, Vec<u64>> = HashMap::new();
1866 let mut processed_eids: HashSet<u64> = HashSet::new();
1867 if self.config.buffer_radius > 0 {
1869 for &gid in gid_vec.iter() {
1870 let gadget = gadgets.get(&gid).ok_or_else(missing("gadget", gid))?;
1871 let Some(cid) = gadget.binding_cid else { continue };
1872 let check_model = check_models.get(&cid).ok_or_else(missing("check model", cid))?;
1873 for &referring_eid in check_model.referring_eids.iter() {
1874 if !processed_eids.insert(referring_eid) {
1875 continue;
1876 }
1877 let error_model = error_models
1878 .get(&referring_eid)
1879 .ok_or_else(missing("error model", referring_eid))?;
1880 let owner_cid = error_model.instance.cid;
1881 let owner_cm = check_models.get(&owner_cid).ok_or_else(missing("check model", owner_cid))?;
1882 let owner_gid = owner_cm.instance.gid;
1883 if window.contains(&owner_gid) {
1884 continue; }
1886 let owner_gadget = gadgets.get(&owner_gid).ok_or_else(missing("gadget", owner_gid))?;
1887 if owner_gadget.state.borrow().committed {
1888 continue; }
1890 outside_gadget_eids.entry(owner_gid).or_default().push(referring_eid);
1891 }
1892 }
1893 }
1894 let mut outside_gids: Vec<_> = outside_gadget_eids.keys().cloned().collect();
1895 outside_gids.sort();
1896 for &gid in outside_gids.iter() {
1897 let gadget = gadgets.get(&gid).ok_or_else(missing("gadget", gid))?;
1898 let inputs: Vec<_> = gadget.instance.connectors.iter().cloned().map(Some).collect();
1899 let outputs: Vec<_> = gadget.outputs.iter().map(|v| *v.borrow()).collect();
1901 let eids = &outside_gadget_eids[&gid];
1902 let mut expanded_error_models = vec![];
1903 for &eid in eids {
1904 let error_model = error_models.get(&eid).ok_or_else(missing("error model", eid))?;
1905 selected_error_models.insert(eid, error_model.clone());
1906 let remote_check_models = Self::expand_remote_check_models_in_window(gid, error_model, &gadgets, window);
1907 expanded_error_models.push(relative_program::ExpandedErrorModel {
1908 eid,
1909 etype: error_model.instance.etype,
1910 remote_check_models,
1911 });
1912 }
1913 expanded_gadgets.push(relative_program::ExpandedGadget {
1914 gid,
1915 gtype: gadget.instance.gtype,
1916 inputs,
1917 outputs,
1918 check_model: None, error_models: expanded_error_models,
1920 });
1921 }
1922 }
1923 let (relative_program, mapping) = RelativeProgram::new(&expanded_gadgets);
1924 let logical_targets = if self.config.forced_gap {
1925 let gadgets = self.gadgets.read().await;
1926 let state = self.forced_gap_state.as_ref().unwrap().read().await;
1927 let mut targets = state.symbolic.readout_targets(
1928 mapping
1929 .global_gid_of
1930 .iter()
1931 .filter(|gid| commit_region.contains(*gid))
1932 .copied(),
1933 );
1934 let mut boundaries = vec![];
1935 for &gid in &mapping.global_gid_of {
1936 if !commit_region.contains(&gid) {
1937 continue;
1938 }
1939 for (port, output) in gadgets[&gid].outputs.iter().enumerate() {
1940 if output.borrow().as_ref().is_none_or(|peer| !commit_region.contains(&peer.gid)) {
1941 boundaries.push((gid, u64::try_from(port).unwrap()));
1942 }
1943 }
1944 }
1945 targets.extend(state.symbolic.boundary_targets(&boundaries));
1946 targets
1947 } else {
1948 vec![]
1949 };
1950 span.add_event(Event::new("relative_program"));
1951 span.add_event(Event::new("committing"));
1952
1953 let (parity_factor, errors, correction_weights, forced_gap_problem) = self
1954 .decode_parity_factor(
1955 committing_cids,
1956 &logical_targets,
1957 window,
1958 &relative_program,
1959 &mapping,
1960 &span,
1961 &selected_error_models,
1962 )
1963 .await?;
1964 let forced_gap_problem = forced_gap_problem.map(|problem| problem.map(Arc::new));
1965 if let Some(problem) = &forced_gap_problem {
1966 self.register_forced_gap_scores(&logical_targets, problem).await;
1967 }
1968 span.add_event(Event::new("decoded"));
1969
1970 let selected_errors = Self::global_subgraph_of(&mapping, &errors, &parity_factor.subgraph);
1971 self.record_event(trace::event::Event::DecodeFinished(trace::DecodeFinishedEvent {
1972 leader_gid: center_gid,
1973 }))
1974 .await;
1975 span.add_property(|| ("parity_factor", format!("{selected_errors:?}")));
1976
1977 self.update_pauli_frame(
1979 center_gid,
1980 commit_region,
1981 committing_cids,
1982 window,
1983 &parity_factor,
1984 &errors,
1985 &correction_weights,
1986 &relative_program,
1987 &mapping,
1988 &selected_error_models,
1989 )
1990 .await;
1991 span.add_event(Event::new("pauli_frame_updated"));
1992
1993 if self.config.forced_gap_strategy == ForcedGapStrategy::Eager
1994 && let Some(Ok(problem)) = forced_gap_problem
1995 {
1996 self.complete_eager_scores(&logical_targets, &problem).await;
1997 }
1998 Ok(())
1999 }
2000
2001 #[allow(clippy::too_many_arguments)]
2002 async fn update_pauli_frame(
2003 &self,
2004 center_gid: u64,
2005 commit_region: &HashSet<u64>,
2006 committing_cids: &HashSet<u64>,
2007 window: &HashSet<u64>,
2008 parity_factor: &blackbox_decoder::ParityFactor,
2009 errors: &ProjectedErrors,
2010 correction_weights: &[f64],
2011 relative_program: &RelativeProgram,
2012 mapping: &RelativeMapping,
2013 error_models: &HashMap<u64, ErrorModel>,
2014 ) {
2015 let error_model_types = self.error_model_types.read().await;
2016 let mut gadgets = self.gadgets.write().await;
2017 let mut check_models = self.check_models.write().await;
2018 let mut tracker = self.pauli_frame_tracker.lock().await;
2019
2020 let mut gadget_residuals: HashMap<u64, BitVec> = HashMap::new();
2022 let mut gadget_readout_flips: HashMap<u64, BitVec> = HashMap::new();
2023 for &commit_gid in commit_region {
2024 let pauli_frame_gadget = tracker.gadgets.get(&commit_gid).unwrap();
2025 gadget_residuals.insert(commit_gid, BitVec::zeros(pauli_frame_gadget.num_output_observables()));
2026 gadget_readout_flips.insert(commit_gid, BitVec::zeros(pauli_frame_gadget.num_readouts()));
2027 }
2028
2029 let mut syndrome_flips: HashMap<u64, HashSet<u64>> = HashMap::new();
2031 assert_eq!(parity_factor.subgraph.len(), correction_weights.len());
2032 for (&ei, &weight) in parity_factor.subgraph.iter().zip(correction_weights) {
2033 let local_error = &errors[ei as usize];
2034 let local_eid = local_error.eid;
2035 let eid = mapping.global_eid_of[local_eid];
2036 let error_index = local_error.error_index;
2037 let error_model = error_models.get(&eid).unwrap();
2038 if !committing_cids.contains(&error_model.instance.cid) {
2039 continue; }
2041 let error_model_type = error_model_types.get(&error_model.instance.etype).unwrap();
2042 let error = &error_model_type.errors[error_index];
2043
2044 let error_gadget_gid = check_models.get(&error_model.instance.cid).unwrap().instance.gid;
2046 let gadget = gadgets.get_mut(&error_gadget_gid).unwrap();
2047 gadget.correction_count += 1;
2048 gadget.correction_weight += weight;
2049 let residual = gadget_residuals.get_mut(&error_gadget_gid).unwrap();
2050 let readout_flips = gadget_readout_flips.get_mut(&error_gadget_gid).unwrap();
2051
2052 for &ri in error.residual.iter() {
2054 residual.negate_index(ri as usize);
2055 }
2056 for &ri in error.readout_flips.iter() {
2057 readout_flips.negate_index(ri as usize);
2058 }
2059 let local_gid = *mapping.local_gid_of.get(&error_gadget_gid).unwrap();
2061 let local_eid_bias = mapping.local_eid_bias[local_gid];
2062 let expanded_gadget = &relative_program.local_gadgets[local_gid];
2063 let expanded_error_model = &expanded_gadget.error_models[local_eid - local_eid_bias];
2064 assert!(mapping.global_eid_of[expanded_error_model.eid as usize] == eid);
2065 for check in error.checks.iter() {
2066 let (cid, check_index) = if let Some(ri) = check.remote_check_model {
2067 let remote_local_cid = expanded_error_model.remote_check_models[ri as usize]
2068 .expect("committed errors must have all checks inside the window");
2069 let remote_cid = mapping.global_cid_of[remote_local_cid as usize];
2070 let check_index = check.check_index
2071 + error_model.modified_remote_check_models[ri as usize]
2072 .as_ref()
2073 .unwrap()
2074 .check_bias;
2075 (remote_cid, check_index)
2076 } else {
2077 (error_model.instance.cid, check.check_index)
2078 };
2079 if !syndrome_flips.contains_key(&cid) {
2080 syndrome_flips.insert(cid, HashSet::new());
2081 }
2082 let flips = syndrome_flips.get_mut(&cid).unwrap();
2083 if flips.contains(&check_index) {
2084 flips.remove(&check_index);
2085 } else {
2086 flips.insert(check_index);
2087 }
2088 }
2089 }
2090 for (cid, flips) in syndrome_flips.iter() {
2091 let check_model = check_models.get_mut(cid).unwrap();
2092 let mut syndrome = check_model.syndrome.borrow().clone().unwrap();
2093 for &check_index in flips.iter() {
2094 flip_bit(&mut syndrome, check_index);
2095 }
2096 check_model.syndrome.send_replace(Some(syndrome));
2097 }
2098
2099 for &commit_gid in commit_region {
2101 let residual = gadget_residuals.remove(&commit_gid).unwrap();
2102 let readout_flips = gadget_readout_flips.remove(&commit_gid).unwrap();
2103 let updates = tracker.load_correction(commit_gid, residual, readout_flips);
2104 for (update_gid, pauli_frame) in updates {
2105 let update_gadget = gadgets.get(&update_gid).unwrap();
2106 debug_assert!(update_gadget.pauli_frame.borrow().is_none(), "bug");
2107 update_gadget.pauli_frame.send_replace(Some(pauli_frame));
2108 }
2109 }
2110
2111 for &commit_gid in commit_region {
2113 let gadget = gadgets.get_mut(&commit_gid).unwrap();
2114 gadget.state.send_replace(GadgetState {
2115 committed: true,
2116 reserved_by: None,
2117 });
2118 }
2119
2120 for &wgid in window {
2122 let Some(g) = gadgets.get_mut(&wgid) else { continue };
2123 let released = g.state.borrow().release_buffer(center_gid);
2124 if let Some(state) = released {
2125 g.state.send_replace(state);
2126 }
2127 }
2128 }
2129
2130 fn global_subgraph_of(mapping: &RelativeMapping, errors: &ProjectedErrors, subgraph: &[u64]) -> Vec<(u64, u64)> {
2131 subgraph
2132 .iter()
2133 .map(|&ei| {
2134 assert!(
2135 (ei as usize) < errors.len(),
2136 "decoder returned subgraph edge index {ei} but cached errors has only {} entries — \
2137 stale `loaded_decoders` cache (key/value mismatch)",
2138 errors.len(),
2139 );
2140 let local_error = &errors[ei as usize];
2141 let local_eid = local_error.eid;
2142 let eid = mapping.global_eid_of[local_eid];
2143 let error_index = local_error.error_index;
2144 (eid, error_index as u64)
2145 })
2146 .collect()
2147 }
2148
2149 async fn build_loss_sites(&self, relative_program: &RelativeProgram, mapping: &RelativeMapping) -> Vec<RawLossSite> {
2161 let mut loss_sites = Vec::new();
2162 if !self.loss_handler.tracks_losses() {
2163 return loss_sites;
2164 }
2165 let port_types = self.port_types.read().await;
2166 let gadget_types = self.gadget_types.read().await;
2167 let gadgets = self.gadgets.read().await;
2168 if !mapping
2169 .global_gid_of
2170 .iter()
2171 .any(|gid| gadgets.get(gid).is_some_and(|gadget| gadget.loss_mask.is_some()))
2172 {
2173 return loss_sites;
2174 }
2175 let mut gid_of_index: Vec<u64> = Vec::new();
2180 let mut index_of_gid: HashMap<u64, usize> = HashMap::new();
2181 for &gid in &mapping.global_gid_of {
2182 let Some(gadget) = gadgets.get(&gid) else { continue };
2183 if !has_loss_model(&gadget_types, gadget.instance.gtype) {
2184 continue;
2185 }
2186 index_of_gid.insert(gid, gid_of_index.len());
2187 gid_of_index.push(gid);
2188 }
2189 if gid_of_index.is_empty() {
2190 return loss_sites;
2191 }
2192
2193 let local_eid_of_index: Vec<Option<usize>> = gid_of_index
2196 .iter()
2197 .map(|&gid| {
2198 let local_gid = *mapping.local_gid_of.get(&gid)?;
2199 relative_program.local_gadgets[local_gid]
2200 .error_models
2201 .first()
2202 .map(|model| model.eid as usize)
2203 })
2204 .collect();
2205
2206 let loss_masks: Vec<BitVector> = gid_of_index
2207 .iter()
2208 .map(|&gid| gadgets.get(&gid).and_then(|g| g.loss_mask.clone()).unwrap_or_default())
2209 .collect();
2210
2211 let gadget_instances: Vec<&bin::Gadget> =
2216 gid_of_index.iter().map(|&gid| &gadgets.get(&gid).unwrap().instance).collect();
2217 let output_links_vec = build_cross_gadget_output_links(&gadget_instances, &index_of_gid, &gadget_types, &port_types);
2218
2219 let gadget_losses: Vec<GadgetLoss> = (0..gid_of_index.len())
2220 .map(|index| {
2221 let gtype = gadgets.get(&gid_of_index[index]).unwrap().instance.gtype;
2222 GadgetLoss {
2223 loss_model: gadget_types.get(>ype).unwrap().loss_model.as_ref().unwrap(),
2224 observed: &loss_masks[index],
2225 output_links: &output_links_vec[index],
2226 }
2227 })
2228 .collect();
2229
2230 for site in build_cross_gadget_loss_sites(&gadget_losses) {
2231 let local_eid = local_eid_of_index[site.gadget_index];
2232 loss_sites.push(RawLossSite::from_compiled(site, local_eid));
2233 }
2234 loss_sites
2235 }
2236
2237 #[allow(clippy::too_many_arguments)]
2238 async fn causal_gap_problem(
2239 &self,
2240 hypergraph: DecodingHypergraph,
2241 syndrome: BitVector,
2242 decoder_seed: Option<u64>,
2243 baseline: &ParityFactor,
2244 errors: &ProjectedErrors,
2245 committing_cids: &HashSet<u64>,
2246 logical_targets: &[CorrectionBasis],
2247 window_gids: &HashSet<u64>,
2248 relative_program: &RelativeProgram,
2249 mapping: &RelativeMapping,
2250 error_models: &HashMap<u64, ErrorModel>,
2251 ) -> Result<CausalGapProblem, Status> {
2252 let mut detector_keys = vec![(0, 0); usize::try_from(hypergraph.vertex_num).unwrap()];
2253 for gadget in &relative_program.local_gadgets {
2254 if let Some(check_model) = &gadget.check_model {
2255 let local_cid = usize::try_from(check_model.cid).unwrap();
2256 let start = mapping.start_indices[local_cid];
2257 for check in 0..check_model.count_checks {
2258 detector_keys[start + check] = (mapping.global_cid_of[local_cid], u64::try_from(check).unwrap());
2259 }
2260 }
2261 }
2262 let vertices = detector_keys
2263 .iter()
2264 .copied()
2265 .enumerate()
2266 .map(|(index, key)| (key, u64::try_from(index).unwrap()))
2267 .collect();
2268 let selected: HashSet<_> = baseline.subgraph.iter().copied().collect();
2269 let mut edges = Vec::with_capacity(errors.len());
2270 let mut committed = HashMap::<u64, Vec<RecordedGapEdge>>::new();
2271 {
2272 let error_types = self.error_model_types.read().await;
2273 let check_models = self.check_models.read().await;
2274 for (index, hyperedge) in hypergraph.hyperedges.iter().enumerate() {
2275 let reference = &errors[index];
2276 let model = &error_models[&mapping.global_eid_of[reference.eid]];
2277 let error = &error_types[&model.instance.etype].errors[reference.error_index];
2278 let gid = check_models[&model.instance.cid].instance.gid;
2279 let edge = RecordedGapEdge {
2280 gid,
2281 checks: hyperedge
2282 .vertices
2283 .iter()
2284 .map(|&vertex| detector_keys[usize::try_from(vertex).unwrap()])
2285 .collect(),
2286 probability: hyperedge.probability,
2287 residual: error.residual.clone(),
2288 readouts: error.readout_flips.clone(),
2289 selected: selected.contains(&u64::try_from(index).unwrap()),
2290 };
2291 if committing_cids.contains(&model.instance.cid) {
2292 committed.entry(gid).or_default().push(edge.clone());
2293 }
2294 edges.push(edge);
2295 }
2296 }
2297 let mut snapshot = CausalGapSnapshot {
2298 hypergraph,
2299 syndrome,
2300 baseline: baseline.clone(),
2301 edges,
2302 };
2303 let mut state = self.forced_gap_state.as_ref().unwrap().write().await;
2304 snapshot.restore_history(
2305 mapping
2306 .global_gid_of
2307 .iter()
2308 .filter_map(|gid| state.history.get(gid))
2309 .flatten()
2310 .cloned(),
2311 &vertices,
2312 );
2313 let logical_cache = state
2314 .symbolic
2315 .logical_flip_cache(window_gids.iter().copied(), logical_targets);
2316 let logical_flips: Vec<_> = snapshot
2317 .edges
2318 .iter()
2319 .map(|edge| logical_cache.propagated_logical_flips(edge.gid, &edge.residual, &edge.readouts))
2320 .collect();
2321 let mut by_gadget = HashMap::new();
2322 let mut problems = Vec::with_capacity(logical_targets.len());
2323 for target in logical_targets {
2324 let (CorrectionBasis::Readout { gid, .. } | CorrectionBasis::Residual { gid, .. }) = *target;
2325 let problem = by_gadget.entry(gid).or_insert_with(|| {
2326 let ancestors = state.symbolic.causal_gadgets(gid, window_gids);
2327 let retained: Vec<_> = snapshot.edges.iter().map(|edge| ancestors.contains(&edge.gid)).collect();
2328 CommitRegionDecoder::new(&snapshot.hypergraph, &logical_flips, &retained, logical_targets.len(), false)
2329 .problem(
2330 self.gap_decoder().clone(),
2331 &snapshot.hypergraph,
2332 snapshot.syndrome.clone(),
2333 decoder_seed,
2334 &snapshot.baseline,
2335 vec![],
2336 false,
2337 )
2338 .map(Arc::new)
2339 });
2340 problems.push(problem.clone()?);
2341 }
2342 state.history.extend(committed);
2343 Ok(CausalGapProblem { problems })
2344 }
2345
2346 #[allow(clippy::too_many_arguments)]
2347 async fn decode_parity_factor(
2348 &self,
2349 committing_cids: &HashSet<u64>,
2350 logical_targets: &[CorrectionBasis],
2351 window_gids: &HashSet<u64>,
2352 relative_program: &RelativeProgram,
2353 mapping: &RelativeMapping,
2354 span: &Span,
2355 error_models: &HashMap<u64, ErrorModel>,
2356 ) -> Result<
2357 (
2358 blackbox_decoder::ParityFactor,
2359 ProjectedErrors,
2360 Vec<f64>,
2361 Option<Result<CausalGapProblem, Status>>,
2362 ),
2363 Status,
2364 > {
2365 let decoder_seed = {
2366 let gadgets = self.gadgets.read().await;
2367 crate::coordinator::common_decoder_seed(
2368 window_gids
2369 .iter()
2370 .map(|gid| &gadgets[gid])
2371 .filter(|gadget| gadget.outcomes.borrow().is_some())
2372 .map(|gadget| gadget.decoder_seed),
2373 )?
2374 };
2375 let target_count = logical_targets.len();
2376 span.add_event(Event::new("calculate_syndrome"));
2378 let syndrome: BitVector = {
2379 let mut syndrome = bit_vector::from_sparse_indices(relative_program.count_checks as u64, &[]);
2380 let check_models = self.check_models.read().await;
2381 let mut start_index = 0;
2382 for &cid in &mapping.global_cid_of {
2383 let check_model = check_models.get(&cid).unwrap();
2384 let check_syndrome = check_model.syndrome.borrow().clone().unwrap();
2385 for i in 0..check_syndrome.size {
2386 if get_bit(&check_syndrome, i) {
2387 set_bit(&mut syndrome, start_index + i, true);
2388 }
2389 }
2390 start_index += check_syndrome.size;
2391 }
2392 assert!(start_index == syndrome.size);
2393 syndrome
2394 };
2395 span.add_event(Event::new("syndrome_calculated"));
2396 span.add_property(|| ("syndrome", format!("{syndrome:?}")));
2397
2398 let loss_sites = self.build_loss_sites(relative_program, mapping).await;
2403
2404 let deduplicate = self.config.merge_hyperedges && !self.loss_handler.hands_off_to_decoder();
2411
2412 let logical_flips_are_cacheable = if self.config.forced_gap && target_count != 0 {
2413 let gadgets = self.gadgets.read().await;
2414 window_gids.iter().all(|gid| gadgets[gid].instance.modifier.is_none())
2415 } else {
2416 true
2417 };
2418 let cache_key = if self.config.persistent_decoder && logical_flips_are_cacheable {
2419 let error_model_types = self.error_model_types.read().await;
2420 Some(DecoderCacheKey {
2424 relative_program: relative_program.clone(),
2425 error_model_fingerprints: build_modifier_fingerprints(mapping, error_models, &error_model_types),
2426 committing_local_cids: committing_local_cids_sorted(committing_cids, mapping),
2427 logical_flip_signature: logical_flip_signature(logical_targets, mapping),
2428 })
2429 } else {
2430 None
2431 };
2432
2433 if let Some(ref cache_key) = cache_key {
2434 let loaded = self.loaded_decoders.read().await.get(cache_key).cloned();
2435 if let Some(loaded) = loaded {
2436 let probability_reweights = self.projected_shot_probability_reweights(mapping, &loaded.projection).await;
2437 let projected = self
2438 .loss_handler
2439 .project_shot(&loaded.projection, &probability_reweights, &loss_sites);
2440 span.add_event(Event::new("decoding").with_property(|| ("type", "loaded")));
2442 let decode_syndrome = loaded.project_syndrome(syndrome.clone());
2443 let parity_factor = decode_projected(
2444 &self.decoder,
2445 &loaded,
2446 decode_syndrome.clone(),
2447 decoder_seed,
2448 projected.reweights.clone(),
2449 projected.loss,
2450 self.use_loaded_reweights,
2451 )
2452 .await?;
2453 if self.config.assert_parity_factor {
2454 assert_parity_factor(loaded.decoding_hypergraph.as_ref().unwrap(), &parity_factor, &decode_syndrome);
2455 }
2456 let weights = loaded.correction_weights(&parity_factor, &projected.reweights);
2457 let forced_gap_problem = if self.config.forced_gap {
2458 let mut graph = loaded.projection.base_hypergraph.clone();
2459 apply_reweights(&mut graph, probability_reweights.iter().copied());
2460 let (graph, _) = self
2461 .loss_handler
2462 .apply_sites(graph, &loss_sites, &loaded.projection.base_errors);
2463 let baseline =
2464 original_scoring_baseline(&loaded.projection.base_errors, &projected.errors, &parity_factor);
2465 let scoring_errors = ProjectedErrors::shared(Arc::clone(&loaded.projection.base_errors));
2466 Some(
2467 self.causal_gap_problem(
2468 graph,
2469 decode_syndrome,
2470 decoder_seed,
2471 &baseline,
2472 &scoring_errors,
2473 committing_cids,
2474 logical_targets,
2475 window_gids,
2476 relative_program,
2477 mapping,
2478 error_models,
2479 )
2480 .await,
2481 )
2482 } else {
2483 None
2484 };
2485 return Ok((parity_factor, projected.errors, weights, forced_gap_problem));
2486 }
2487 }
2488
2489 let committing_eids: HashSet<_> = {
2492 mapping
2493 .global_eid_of
2494 .iter()
2495 .enumerate()
2496 .filter_map(|(local_eid, eid)| {
2497 committing_cids.contains(&error_models[eid].instance.cid).then_some(local_eid)
2498 })
2499 .collect()
2500 };
2501 let (decoding_hypergraph, errors, logical_flips) = self
2502 .decoding_hypergraph(
2503 committing_cids,
2504 logical_targets,
2505 window_gids,
2506 relative_program,
2507 mapping,
2508 error_models,
2509 )
2510 .await;
2511
2512 let Some(cache_key) = cache_key else {
2513 let probability_reweights = self.shot_probability_reweights(mapping, &errors).await;
2514 let mut decoding_hypergraph = decoding_hypergraph;
2520 apply_reweights(&mut decoding_hypergraph, probability_reweights.iter().copied());
2521 let (mut decoding_hypergraph, loss) = self.loss_handler.apply_sites(decoding_hypergraph, &loss_sites, &errors);
2522 let scoring_input = self
2523 .config
2524 .forced_gap
2525 .then(|| (decoding_hypergraph.clone(), Arc::clone(&errors)));
2526 let mut errors = errors;
2527 let mut logical_flips = logical_flips;
2528 if deduplicate {
2529 let prepared = deduplicate_decoder_input(&decoding_hypergraph, &errors, &logical_flips, |error| {
2530 usize::from(committing_eids.contains(&error.eid))
2531 });
2532 decoding_hypergraph = prepared.hypergraph;
2533 errors = prepared.representatives;
2534 logical_flips = prepared.logical_flips.as_ref().clone();
2535 }
2536 let mut syndrome = syndrome;
2537 ignore_edge_isolated_history_vertices(&decoding_hypergraph, &mut syndrome);
2538 let hard_hypergraph = hard_decoding_hypergraph(decoding_hypergraph.clone(), &logical_flips);
2539 span.add_event(Event::new("decoding").with_property(|| ("type", "temporary")));
2540 let parity_factor = self
2541 .decoder
2542 .decode(blackbox_decoder::DecodingProblem {
2543 hypergraph: Some(hard_hypergraph),
2544 syndrome: Some(syndrome.clone()),
2545 loss,
2546 decoder_seed,
2547 })
2548 .await?;
2549 if self.config.assert_parity_factor {
2550 assert_parity_factor(&decoding_hypergraph, &parity_factor, &syndrome);
2551 }
2552 let weights = correction_weights(&decoding_hypergraph, &parity_factor);
2553 let errors: ProjectedErrors = errors.into();
2554 let forced_gap_problem = if let Some((graph, original_errors)) = scoring_input {
2555 let baseline = original_scoring_baseline(&original_errors, &errors, &parity_factor);
2556 let scoring_errors = ProjectedErrors::shared(original_errors);
2557 Some(
2558 self.causal_gap_problem(
2559 graph,
2560 syndrome,
2561 decoder_seed,
2562 &baseline,
2563 &scoring_errors,
2564 committing_cids,
2565 logical_targets,
2566 window_gids,
2567 relative_program,
2568 mapping,
2569 error_models,
2570 )
2571 .await,
2572 )
2573 } else {
2574 None
2575 };
2576 return Ok((parity_factor, errors, weights, forced_gap_problem));
2577 };
2578
2579 span.add_event(Event::new("decoding").with_property(|| ("type", "loading")));
2582 let retain_decoding_hypergraph =
2583 self.config.forced_gap || !self.use_loaded_reweights || self.config.assert_parity_factor;
2584 let (projection, prepared) = prepare_decoder(decoding_hypergraph, errors, logical_flips, deduplicate, |error| {
2585 usize::from(committing_eids.contains(&error.eid))
2586 });
2587 let loaded = load_projected_decoder(&self.decoder, projection, prepared, retain_decoding_hypergraph, true).await?;
2588 let probability_reweights = self.projected_shot_probability_reweights(mapping, &loaded.projection).await;
2589 let decode_syndrome = loaded.project_syndrome(syndrome);
2590 let projected = self
2591 .loss_handler
2592 .project_shot(&loaded.projection, &probability_reweights, &loss_sites);
2593 let mut loaded_decoders = self.loaded_decoders.write().await;
2594 loaded_decoders.insert(cache_key, loaded.clone());
2595 drop(loaded_decoders);
2596 let parity_factor = decode_projected(
2597 &self.decoder,
2598 &loaded,
2599 decode_syndrome.clone(),
2600 decoder_seed,
2601 projected.reweights.clone(),
2602 projected.loss,
2603 self.use_loaded_reweights,
2604 )
2605 .await?;
2606 if self.config.assert_parity_factor {
2607 assert_parity_factor(loaded.decoding_hypergraph.as_ref().unwrap(), &parity_factor, &decode_syndrome);
2608 }
2609 let weights = loaded.correction_weights(&parity_factor, &projected.reweights);
2610 let forced_gap_problem = if self.config.forced_gap {
2611 let mut graph = loaded.projection.base_hypergraph.clone();
2612 apply_reweights(&mut graph, probability_reweights.iter().copied());
2613 let (graph, _) = self
2614 .loss_handler
2615 .apply_sites(graph, &loss_sites, &loaded.projection.base_errors);
2616 let baseline = original_scoring_baseline(&loaded.projection.base_errors, &projected.errors, &parity_factor);
2617 let scoring_errors = ProjectedErrors::shared(Arc::clone(&loaded.projection.base_errors));
2618 Some(
2619 self.causal_gap_problem(
2620 graph,
2621 decode_syndrome,
2622 decoder_seed,
2623 &baseline,
2624 &scoring_errors,
2625 committing_cids,
2626 logical_targets,
2627 window_gids,
2628 relative_program,
2629 mapping,
2630 error_models,
2631 )
2632 .await,
2633 )
2634 } else {
2635 None
2636 };
2637 Ok((parity_factor, projected.errors, weights, forced_gap_problem))
2638 }
2639
2640 async fn bind_probability_modifiers(
2641 &self,
2642 gid: u64,
2643 modifiers: &[bin::ProbabilityModifier],
2644 ) -> Result<Vec<(u64, bin::ProbabilityModifier)>, Status> {
2645 if modifiers.is_empty() {
2646 return Ok(vec![]);
2647 }
2648 self.await_error_models(&HashSet::from([gid])).await?;
2649 let cid = self
2650 .gadgets
2651 .read()
2652 .await
2653 .get(&gid)
2654 .and_then(|gadget| gadget.binding_cid)
2655 .ok_or_else(|| Status::failed_precondition(format!("gid={gid} has no binding check model")))?;
2656 let (mut eids, terminal) = self
2657 .check_models
2658 .read()
2659 .await
2660 .get(&cid)
2661 .map(|check_model| {
2662 (
2663 check_model.attaching_eid_vec.clone(),
2664 check_model.instance.terminal_error_model.clone(),
2665 )
2666 })
2667 .ok_or_else(|| Status::failed_precondition(format!("cid={cid} is not loaded")))?;
2668 if eids.is_empty()
2669 && let Some(terminal) = &terminal
2670 {
2671 eids.push(terminal.eid);
2672 }
2673 if modifiers.len() > eids.len() {
2674 return Err(Status::invalid_argument(format!(
2675 "gid={gid} supplied {} probability modifiers for {} attached error models",
2676 modifiers.len(),
2677 eids.len()
2678 )));
2679 }
2680 let error_model_types = self.error_model_types.read().await;
2681 let error_models = self.error_models.read().await;
2682 let mut bound = Vec::with_capacity(modifiers.len());
2683 for (&eid, modifier) in eids.iter().zip(modifiers) {
2684 let error_model = error_models
2685 .get(&eid)
2686 .map(|model| &model.instance)
2687 .or_else(|| terminal.as_ref().filter(|model| model.eid == eid))
2688 .ok_or_else(|| Status::failed_precondition(format!("eid={eid} is not loaded")))?;
2689 let error_model_type = error_model_types
2690 .get(&error_model.etype)
2691 .ok_or_else(|| Status::failed_precondition(format!("etype={} is not loaded", error_model.etype)))?;
2692 validate_probability_modifier(modifier, error_model_type.errors.len()).map_err(Status::invalid_argument)?;
2693 bound.push((eid, modifier.clone()));
2694 }
2695 Ok(bound)
2696 }
2697
2698 async fn shot_probability_reweights(
2699 &self,
2700 mapping: &RelativeMapping,
2701 error_reference: &[ErrorIndex],
2702 ) -> Vec<(u64, f64)> {
2703 let gadgets = self.gadgets.read().await;
2704 probability_reweights(error_reference, Self::shot_probability_modifiers(mapping, &gadgets))
2705 }
2706
2707 async fn projected_shot_probability_reweights(
2708 &self,
2709 mapping: &RelativeMapping,
2710 projection: &crate::coordinator::reweight_handler::DecodeProjection,
2711 ) -> Vec<(u64, f64)> {
2712 let gadgets = self.gadgets.read().await;
2713 projection.probability_reweights(Self::shot_probability_modifiers(mapping, &gadgets))
2714 }
2715
2716 fn shot_probability_modifiers<'a>(
2717 mapping: &RelativeMapping,
2718 gadgets: &'a HashMap<u64, Gadget>,
2719 ) -> Vec<(usize, &'a bin::ProbabilityModifier)> {
2720 let mut modifiers: Vec<_> = mapping
2721 .global_gid_of
2722 .iter()
2723 .filter_map(|gid| gadgets.get(gid))
2724 .flat_map(|gadget| gadget.probability_modifiers.iter())
2725 .filter_map(|(eid, modifier)| mapping.local_eid_of.get(eid).map(|&local_eid| (local_eid, modifier)))
2726 .collect();
2727 modifiers.sort_unstable_by_key(|&(local_eid, _)| local_eid);
2728 modifiers
2729 }
2730
2731 async fn decoding_hypergraph(
2732 &self,
2733 committing_cids: &HashSet<u64>,
2734 logical_targets: &[CorrectionBasis],
2735 window_gids: &HashSet<u64>,
2736 relative_program: &RelativeProgram,
2737 mapping: &RelativeMapping,
2738 error_models: &HashMap<u64, ErrorModel>,
2739 ) -> (DecodingHypergraph, Arc<Vec<ErrorIndex>>, Vec<Vec<u64>>) {
2740 #[cfg(feature = "cli")]
2741 log::info!("constructing decoding hypergraph for cids={committing_cids:?}");
2742 let error_model_types = self.error_model_types.read().await;
2743 let logical_flip_cache = if self.config.forced_gap && !logical_targets.is_empty() {
2744 let state = self.forced_gap_state.as_ref().unwrap().read().await;
2745 Some(
2746 state
2747 .symbolic
2748 .logical_flip_cache(window_gids.iter().copied(), logical_targets),
2749 )
2750 } else {
2751 None
2752 };
2753
2754 let mut hyperedges: Vec<Hyperedge> = vec![];
2755 let mut error_reference: Vec<ErrorIndex> = vec![];
2756 let mut logical_flips = vec![];
2757
2758 for (local_gid, expanded_gadget) in relative_program.local_gadgets.iter().enumerate() {
2763 let has_local_check = expanded_gadget.check_model.is_some();
2764 let local_start_index = if let Some(ref cm) = expanded_gadget.check_model {
2765 mapping.start_indices[cm.cid as usize] as u64
2766 } else {
2767 0 };
2769 let is_in_commit_region = has_local_check
2770 && expanded_gadget
2771 .check_model
2772 .as_ref()
2773 .is_some_and(|cm| committing_cids.contains(&mapping.global_cid_of[cm.cid as usize]));
2774
2775 let local_eid_bias = mapping.local_eid_bias[local_gid];
2776 for (em_index, expanded_em) in expanded_gadget.error_models.iter().enumerate() {
2777 let local_eid = local_eid_bias + em_index;
2778 let eid = mapping.global_eid_of[local_eid];
2779 let error_model = error_models.get(&eid).unwrap();
2780 let error_model_type = error_model_types.get(&error_model.instance.etype).unwrap();
2781 let expanded_remotes = &expanded_em.remote_check_models;
2782 let mut errors = &error_model_type.errors;
2783 let modified_errors: Option<Vec<bin::error_model_type::Error>>;
2785 if let Some(modifier) = &error_model.instance.modifier
2786 && let Some(probability_modifier) = &modifier.probability_modifier
2787 {
2788 let mut new_errors = errors.clone();
2789 for (error_index, &probability) in probability_modifier.probabilities.iter().enumerate() {
2790 new_errors[error_index].probability = probability;
2791 }
2792 for (&error_index, &probability) in probability_modifier
2793 .sparse_indices
2794 .iter()
2795 .zip(probability_modifier.sparse_probabilities.iter())
2796 {
2797 new_errors[error_index as usize].probability = probability;
2798 }
2799 modified_errors = Some(new_errors);
2800 errors = modified_errors.as_ref().unwrap();
2801 }
2802 for (error_index, error) in errors.iter().enumerate() {
2803 if is_in_commit_region
2804 && error.checks.iter().any(|check| {
2805 check
2806 .remote_check_model
2807 .is_some_and(|remote| expanded_remotes[remote as usize].is_none())
2808 })
2809 {
2810 continue;
2811 }
2812 let mut vertices: Vec<u64> = vec![];
2818 for check in &error.checks {
2819 if let Some(ri) = check.remote_check_model {
2820 if let Some(remote_local_cid) = expanded_remotes[ri as usize] {
2821 let remote_start_index = mapping.start_indices[remote_local_cid as usize] as u64;
2822 vertices.push(
2823 remote_start_index
2824 + check.check_index
2825 + error_model.modified_remote_check_models[ri as usize]
2826 .as_ref()
2827 .unwrap()
2828 .check_bias,
2829 );
2830 }
2831 } else if has_local_check {
2832 vertices.push(local_start_index + check.check_index);
2834 }
2835 }
2838 let logical_readout_flips =
2839 if is_in_commit_region && let Some(logical_flip_cache) = logical_flip_cache.as_ref() {
2840 let gid = mapping.global_gid_of[local_gid];
2841 logical_flip_cache.propagated_logical_flips(gid, &error.residual, &error.readout_flips)
2842 } else {
2843 vec![]
2844 };
2845 if vertices.is_empty() && logical_readout_flips.is_empty() {
2846 continue;
2847 }
2848 error_reference.push(ErrorIndex {
2849 eid: local_eid,
2850 error_index,
2851 });
2852 hyperedges.push(Hyperedge {
2853 observable_flips: None,
2854 vertices,
2855 probability: error.probability,
2856 });
2857 logical_flips.push(logical_readout_flips);
2858 }
2859 }
2860 }
2861 let hypergraph = DecodingHypergraph {
2862 vertex_num: relative_program.count_checks as u64,
2863 hyperedges,
2864 };
2865 (hypergraph, Arc::new(error_reference), logical_flips)
2866 }
2867
2868 async fn available_remote_check_owners(&self, owners: &HashSet<u64>) -> HashMap<u64, HashSet<u64>> {
2869 let error_model_types = self.error_model_types.read().await;
2870 let gadgets = self.gadgets.read().await;
2871 let check_models = self.check_models.read().await;
2872 let error_models = self.error_models.read().await;
2873 let mut dependencies = HashMap::new();
2874 for &owner_gid in owners {
2875 let owner = &gadgets[&owner_gid];
2876 if owner.state.borrow().committed {
2877 continue;
2878 }
2879 let Some(cid) = owner.binding_cid else { continue };
2880 let mut targets = HashSet::new();
2881 for eid in &check_models[&cid].attaching_eid_vec {
2882 let error_model = &error_models[eid];
2883 let remotes = &error_model.modified_remote_check_models;
2884 let used_remotes: HashSet<_> = error_model_types[&error_model.instance.etype]
2885 .errors
2886 .iter()
2887 .flat_map(|error| &error.checks)
2888 .filter_map(|check| check.remote_check_model.map(|index| usize::try_from(index).unwrap()))
2889 .collect();
2890 let mut resolved = vec![None; remotes.len()];
2891 for remote_index in used_remotes {
2892 Self::resolve_remote_check_model_gid(&mut resolved, remote_index, remotes, owner_gid, &gadgets);
2893 let Some(remote) = &remotes[remote_index] else { continue };
2894 let target_cid = remote.absolute_cid.or_else(|| {
2895 resolved[remote_index]
2896 .and_then(|gid| gadgets.get(&gid))
2897 .and_then(|gadget| gadget.binding_cid)
2898 });
2899 if let Some(check_model) = target_cid.and_then(|cid| check_models.get(&cid))
2900 && check_model.syndrome.borrow().is_some()
2901 {
2902 targets.insert(check_model.instance.gid);
2903 }
2904 }
2905 }
2906 dependencies.insert(owner_gid, targets);
2907 }
2908 dependencies
2909 }
2910
2911 async fn await_error_models(&self, window: &HashSet<u64>) -> Result<HashSet<u64>, Status> {
2912 let token = self.cancellation.read().await.clone();
2913 let mut readiness = vec![];
2914 let mut terminal_gids = HashSet::new();
2915 {
2916 let gadgets = self.gadgets.read().await;
2917 let checks = self.check_models.read().await;
2918 let mut terminal_boundaries = None;
2919 for gid in window {
2920 let Some(gadget) = gadgets.get(gid) else { continue };
2921 let Some(check) = gadget.binding_cid.and_then(|cid| checks.get(&cid)) else {
2922 continue;
2923 };
2924 let expected = check.instance.error_model_count.unwrap_or(1);
2925 if gadget.state.borrow().committed || check.attaching_eid_vec.len() as u64 >= expected {
2926 continue;
2927 }
2928 if expected == 1
2929 && check.instance.terminal_error_model.is_some()
2930 && terminal_boundaries
2931 .get_or_insert_with(|| Self::terminal_boundary_gids(&gadgets, window))
2932 .contains(gid)
2933 {
2934 terminal_gids.insert(*gid);
2935 } else if let Err(handle) = check_or_receiver(&check.error_model_ready, token.clone()) {
2936 readiness.push(handle);
2937 }
2938 }
2939 }
2940 join_all(readiness).await;
2941 if token.is_cancelled() {
2942 return Err(Status::cancelled("error model readiness wait cancelled"));
2943 }
2944 Ok(terminal_gids)
2945 }
2946
2947 fn terminal_boundary_gids(gadgets: &HashMap<u64, Gadget>, window: &HashSet<u64>) -> HashSet<u64> {
2948 let mut terminal_gids = window.clone();
2949 let mut pending: Vec<_> = window
2950 .iter()
2951 .filter_map(|gid| gadgets.get(gid))
2952 .flat_map(|gadget| gadget.instance.connectors.iter().map(|connector| connector.gid))
2953 .collect();
2954 let mut visited = HashSet::new();
2955 while let Some(current) = pending.pop() {
2956 if window.contains(¤t) {
2957 terminal_gids.remove(¤t);
2958 continue;
2959 }
2960 if visited.insert(current) {
2961 pending.extend(gadgets[¤t].instance.connectors.iter().map(|connector| connector.gid));
2962 }
2963 }
2964 terminal_gids
2965 }
2966
2967 fn expand_remote_check_models_in_window(
2968 gid: u64,
2969 error_model: &ErrorModel,
2970 gadgets: &HashMap<u64, Gadget>,
2971 window: &HashSet<u64>,
2972 ) -> Vec<Option<u64>> {
2973 let remotes = &error_model.modified_remote_check_models;
2974 let mut resolved = vec![None; remotes.len()];
2975 for remote_index in 0..remotes.len() {
2976 Self::resolve_remote_check_model_gid(&mut resolved, remote_index, remotes, gid, gadgets);
2977 }
2978 remotes
2979 .iter()
2980 .zip(resolved)
2981 .map(|(remote, resolved_gid)| {
2982 let remote = remote.as_ref()?;
2983 remote.absolute_cid.or_else(|| {
2984 resolved_gid
2985 .filter(|gid| window.contains(gid))
2986 .and_then(|gid| gadgets.get(&gid))
2987 .and_then(|gadget| gadget.binding_cid)
2988 })
2989 })
2990 .collect()
2991 }
2992
2993 fn resolve_remote_check_model_gid(
2996 resolved: &mut Vec<Option<u64>>,
2997 ri: usize,
2998 remote_check_models: &Vec<Option<bin::error_model_type::RemoteCheckModel>>,
2999 owner_gid: u64,
3000 gadgets: &HashMap<u64, Gadget>,
3001 ) {
3002 if resolved[ri].is_some() || remote_check_models[ri].is_none() {
3003 return;
3004 }
3005 let rcm = remote_check_models[ri].as_ref().unwrap();
3006 if rcm.absolute_cid.is_some() {
3007 resolved[ri] = Some(u64::MAX);
3009 return;
3010 }
3011 let previous = if let Some(prev_ri) = rcm.previous_remote_check_model {
3012 Self::resolve_remote_check_model_gid(resolved, prev_ri as usize, remote_check_models, owner_gid, gadgets);
3013 match resolved[prev_ri as usize] {
3014 Some(gid) if gid != u64::MAX => gid,
3015 _ => {
3016 resolved[ri] = Some(u64::MAX);
3017 return;
3018 }
3019 }
3020 } else {
3021 owner_gid
3022 };
3023 let Some(gadget) = gadgets.get(&previous) else {
3024 resolved[ri] = Some(u64::MAX);
3025 return;
3026 };
3027 match rcm.port.unwrap() {
3028 bin::error_model_type::remote_check_model::Port::Output(port) => {
3029 if let Some(conn) = gadget.outputs[port as usize].borrow().as_ref() {
3030 resolved[ri] = Some(conn.gid);
3031 } else {
3032 resolved[ri] = Some(u64::MAX); }
3034 }
3035 bin::error_model_type::remote_check_model::Port::Input(port) => {
3036 let connector = &gadget.instance.connectors[port as usize];
3037 resolved[ri] = Some(connector.gid);
3038 }
3039 }
3040 }
3041
3042 fn find_blocking_port(
3049 ri: usize,
3050 resolved_gids: &[Option<u64>],
3051 remote_check_models: &[Option<bin::error_model_type::RemoteCheckModel>],
3052 owner_gid: u64,
3053 gadgets: &HashMap<u64, Gadget>,
3054 ) -> Option<(u64, u64)> {
3055 let rcm = remote_check_models[ri].as_ref()?;
3056 if rcm.absolute_cid.is_some() {
3057 return None;
3058 }
3059 let previous = if let Some(prev_ri) = rcm.previous_remote_check_model {
3060 if resolved_gids[prev_ri as usize] == Some(u64::MAX) {
3061 return Self::find_blocking_port(prev_ri as usize, resolved_gids, remote_check_models, owner_gid, gadgets);
3063 }
3064 match resolved_gids[prev_ri as usize] {
3065 Some(gid) if gid != u64::MAX => gid,
3066 _ => return None,
3067 }
3068 } else {
3069 owner_gid
3070 };
3071 let gadget = gadgets.get(&previous)?;
3072 match rcm.port {
3073 Some(bin::error_model_type::remote_check_model::Port::Output(port)) => {
3074 if gadget.outputs[port as usize].borrow().is_none() {
3075 Some((previous, port))
3076 } else {
3077 None }
3079 }
3080 _ => None,
3081 }
3082 }
3083
3084 async fn expand_remote_gadgets(
3088 check_model: &bin::CheckModel,
3089 modified_remote_gadgets: &Vec<Option<bin::check_model_type::RemoteGadget>>,
3090 gadgets: &RwLock<HashMap<u64, Gadget>>,
3091 token: CancellationToken,
3092 ) -> Vec<Option<u64>> {
3093 let mut expanded_remote_gid_vec: Vec<Option<u64>> = vec![None; modified_remote_gadgets.len()];
3095 for ri in 0..modified_remote_gadgets.len() {
3096 Self::expand_remote_gadget(
3097 &mut expanded_remote_gid_vec,
3098 ri,
3099 modified_remote_gadgets,
3100 check_model.gid,
3101 gadgets,
3102 token.clone(),
3103 )
3104 .await;
3105 }
3106 expanded_remote_gid_vec
3107 }
3108
3109 async fn expand_remote_gadget(
3110 expanded_remote_gid_vec: &mut Vec<Option<u64>>,
3111 ri: usize,
3112 remote_gadgets: &Vec<Option<bin::check_model_type::RemoteGadget>>,
3113 gid: u64,
3114 gadgets: &RwLock<HashMap<u64, Gadget>>,
3115 token: CancellationToken,
3116 ) {
3117 if expanded_remote_gid_vec[ri].is_some() || remote_gadgets[ri].is_none() {
3118 return; }
3120 let remote_gadget = remote_gadgets[ri].as_ref().unwrap();
3121 if let Some(absolute_gid) = remote_gadget.absolute_gid {
3123 expanded_remote_gid_vec[ri] = Some(absolute_gid);
3124 return;
3125 }
3126 let previous = if let Some(previous) = remote_gadget.previous_remote_gadget {
3129 Box::pin(Self::expand_remote_gadget(
3130 expanded_remote_gid_vec,
3131 previous as usize,
3132 remote_gadgets,
3133 gid,
3134 gadgets,
3135 token.clone(),
3136 ))
3137 .await;
3138 expanded_remote_gid_vec[previous as usize].unwrap()
3139 } else {
3140 gid
3141 };
3142 let gadgets = gadgets.read().await;
3143 let gadget = gadgets.get(&previous).unwrap();
3144 match remote_gadget.port.unwrap() {
3145 bin::check_model_type::remote_gadget::Port::Output(port) => {
3146 let next = get_or_receiver(&gadget.outputs[port as usize], token);
3147 drop(gadgets); let next = match next {
3149 Ok(next) => Some(next),
3150 Err(handle) => handle.await.unwrap_or(None),
3151 };
3152 if let Some(next) = next {
3153 expanded_remote_gid_vec[ri] = Some(next.gid);
3154 }
3155 }
3156 bin::check_model_type::remote_gadget::Port::Input(port) => {
3157 let connector = &gadget.instance.connectors[port as usize];
3158 expanded_remote_gid_vec[ri] = Some(connector.gid);
3159 }
3160 }
3161 }
3162}
3163
3164#[tonic::async_trait]
3165impl coordinator::coordinator_server::Coordinator for WindowCoordinator {
3166 #[cfg_attr(feature = "cli", fastrace::trace)]
3167 async fn load_library(&self, request: Request<bin::Library>) -> Result<Response<()>, Status> {
3168 let _task_guard = self
3169 .task_counter
3170 .try_guard()
3171 .ok_or_else(|| Status::unavailable("coordinator reset in progress"))?;
3172 let library = request.into_inner();
3173 self.loss_handler
3174 .validate_capability(&library.gadget_types, self.decoder.features())?;
3175 let mut port_types = self.port_types.write().await;
3176 for port_type in library.port_types.into_iter() {
3177 if port_types.contains_key(&port_type.ptype) {
3178 return Err(Status::already_exists(format!("ptype={}", port_type.ptype)));
3179 }
3180 port_types.insert(port_type.ptype, Arc::new(port_type));
3181 }
3182 drop(port_types);
3183 let mut gadget_types = self.gadget_types.write().await;
3184 for gadget_type in library.gadget_types.into_iter() {
3185 if gadget_types.contains_key(&gadget_type.gtype) {
3186 return Err(Status::already_exists(format!("gtype={}", gadget_type.gtype)));
3187 }
3188 gadget_types.insert(gadget_type.gtype, Arc::new(gadget_type));
3189 }
3190 drop(gadget_types);
3191 let mut check_model_types = self.check_model_types.write().await;
3192 for check_model_type in library.check_model_types.into_iter() {
3193 if check_model_types.contains_key(&check_model_type.ctype) {
3194 return Err(Status::already_exists(format!("ctype={}", check_model_type.ctype)));
3195 }
3196 check_model_types.insert(check_model_type.ctype, Arc::new(check_model_type));
3197 }
3198 drop(check_model_types);
3199 let mut error_model_types = self.error_model_types.write().await;
3200 for error_model_type in library.error_model_types.into_iter() {
3201 if error_model_types.contains_key(&error_model_type.etype) {
3202 return Err(Status::already_exists(format!("etype={}", error_model_type.etype)));
3203 }
3204 error_model_types.insert(error_model_type.etype, Arc::new(error_model_type));
3205 }
3206 drop(error_model_types);
3207 Ok(().into())
3208 }
3209
3210 async fn unload(&self, _unload: Request<coordinator::UnloadLibrary>) -> Result<Response<()>, Status> {
3211 unimplemented!()
3212 }
3213
3214 #[cfg_attr(feature = "cli", fastrace::trace)]
3215 async fn execute(&self, request: Request<bin::Instruction>) -> Result<Response<coordinator::ExecuteResponse>, Status> {
3216 let _task_guard = self
3217 .task_counter
3218 .try_guard()
3219 .ok_or_else(|| Status::unavailable("coordinator reset in progress"))?;
3220 let instruction = request.into_inner();
3221 let create = instruction
3222 .create
3223 .ok_or_else(|| Status::invalid_argument("unknown instruction"))?;
3224 let id = match create {
3225 bin::instruction::Create::Gadget(gadget) => {
3226 let port_types = self.port_types.read().await;
3227 let gadget_types = self.gadget_types.read().await;
3228 let mut gadgets = self.gadgets.write().await;
3229 let gid = if gadget.gid == 0 {
3230 let mut next_gid = self.next_gid.lock().await;
3232 while gadgets.contains_key(&*next_gid) {
3233 *next_gid += 1;
3234 }
3235 let gid = *next_gid;
3236 *next_gid += 1;
3237 gid
3238 } else {
3239 gadget.gid
3241 };
3242 let gadget_type = gadget_types
3243 .get(&gadget.gtype)
3244 .ok_or_else(|| Status::not_found(format!("gtype={}", gadget.gtype)))?;
3245 debug_assert!(gadget.connectors.len() == gadget_type.inputs.len());
3246 for (port, connector) in gadget.connectors.iter().enumerate() {
3247 debug_assert!(gadgets.contains_key(&connector.gid));
3248 debug_assert!({
3249 let peer_outputs = &gadgets[&connector.gid].outputs;
3250 (connector.port as usize) < peer_outputs.len()
3251 && peer_outputs[connector.port as usize].borrow().is_none()
3252 });
3253 gadgets.get_mut(&connector.gid).unwrap().outputs[connector.port as usize]
3254 .send_replace(Some(bin::gadget::Connector { gid, port: port as u64 }));
3255 }
3256 let is_free_hop = gadget_type.is_free_hop.unwrap_or(gadget_type.measurements.is_empty());
3257 let mut gadget = gadget;
3258 gadget.gid = gid;
3259 gadgets.insert(
3260 gid,
3261 Gadget {
3262 instance: gadget.clone(),
3263 outcomes: watch::channel(None).0,
3264 probability_modifiers: vec![],
3265 binding_cid: None,
3266 outputs: gadget_type.outputs.iter().map(|_| watch::channel(None).0).collect(),
3268 pauli_frame: watch::channel(None).0,
3269 correction_count: 0,
3270 correction_weight: 0.0,
3271 is_free_hop,
3272 state: watch::channel(GadgetState::default()).0,
3273 loss_mask: None,
3274 decoder_seed: None,
3275 },
3276 );
3277 {
3282 let has_pending = {
3283 let pending_by_port = self.pending_referring_by_port.lock().await;
3284 gadget
3285 .connectors
3286 .iter()
3287 .any(|c| pending_by_port.contains_key(&(c.gid, c.port)))
3288 };
3289 if has_pending {
3290 let mut check_models = self.check_models.write().await;
3291 let mut pending_by_gid = self.pending_referring_by_gid.lock().await;
3292 let mut pending_by_port = self.pending_referring_by_port.lock().await;
3293 for connector in gadget.connectors.iter() {
3294 if let Some(entries) = pending_by_port.remove(&(connector.gid, connector.port)) {
3295 for referral in entries {
3296 let mut resolved_gids = vec![None; referral.modified_remote.len()];
3297 for resolved_ri in 0..referral.modified_remote.len() {
3298 Self::resolve_remote_check_model_gid(
3299 &mut resolved_gids,
3300 resolved_ri,
3301 &referral.modified_remote,
3302 referral.owner_gid,
3303 &gadgets,
3304 );
3305 }
3306 let target_gid = resolved_gids[referral.ri].unwrap_or(u64::MAX);
3307 if target_gid == referral.owner_gid {
3308 continue;
3309 }
3310 if target_gid == u64::MAX {
3311 if let Some(blocker) = Self::find_blocking_port(
3312 referral.ri,
3313 &resolved_gids,
3314 &referral.modified_remote,
3315 referral.owner_gid,
3316 &gadgets,
3317 ) {
3318 pending_by_port.entry(blocker).or_default().push(referral);
3319 }
3320 } else if let Some(target_gadget) = gadgets.get(&target_gid) {
3321 if let Some(target_cid) = target_gadget.binding_cid {
3322 if let Some(target_cm) = check_models.get_mut(&target_cid)
3323 && !target_cm.referring_eids.contains(&referral.eid)
3324 {
3325 target_cm.referring_eids.push(referral.eid);
3326 }
3327 } else {
3328 pending_by_gid.entry(target_gid).or_default().push(referral.eid);
3329 }
3330 }
3331 }
3332 }
3333 }
3334 }
3335 }
3336 let mut tracker = self.pauli_frame_tracker.lock().await;
3337 tracker.add_gadget(gid, gadget_type, gadget.modifier.as_ref(), &port_types, &gadget.connectors);
3338 if let Some(state) = &self.forced_gap_state {
3339 state.write().await.symbolic.add_gadget(gid, &tracker.gadgets[&gid]);
3340 }
3341 self.record_event(trace::event::Event::ExecuteGadget(trace::ExecuteGadgetEvent {
3342 gadget: Some(gadget),
3343 num_outputs: gadget_type.outputs.len() as u32,
3344 }))
3345 .await;
3346 gid
3347 }
3348 bin::instruction::Create::CheckModel(check_model) => {
3349 let check_model_types = self.check_model_types.read().await;
3350 let check_model_type = check_model_types
3351 .get(&check_model.ctype)
3352 .ok_or_else(|| Status::not_found(format!("ctype={}", check_model.ctype)))?;
3353 let error_model_types = self.error_model_types.read().await;
3354 if let Some(terminal) = &check_model.terminal_error_model {
3355 if check_model.error_model_count.unwrap_or(1) != 1 {
3356 return Err(Status::invalid_argument(
3357 "terminal fallback requires exactly one full error model",
3358 ));
3359 }
3360 let model_type = error_model_types
3361 .get(&terminal.etype)
3362 .ok_or_else(|| Status::not_found(format!("terminal etype={}", terminal.etype)))?;
3363 if terminal.eid == 0
3364 || (model_type.ctype != WILDCARD && model_type.ctype != check_model.ctype)
3365 || model_type.remote_check_models.iter().any(|remote| {
3366 remote.absolute_cid != Some(crate::misc::index::FUTURE_CHECK_CID)
3367 || remote.previous_remote_check_model.is_some()
3368 || remote.port.is_some()
3369 || remote.check_bias != 0
3370 })
3371 || model_type.errors.iter().flat_map(|error| &error.checks).any(|check| {
3372 match check.remote_check_model {
3373 Some(index) => {
3374 index >= model_type.remote_check_models.len() as u64 || check.check_index != 0
3375 }
3376 None => check.check_index >= check_model_type.checks.len() as u64,
3377 }
3378 })
3379 {
3380 return Err(Status::invalid_argument(
3381 "terminal model requires a nonzero eid, compatible ctype, and valid local or reserved future checks",
3382 ));
3383 }
3384 if let Some(modifier) = &terminal.modifier {
3385 if !modifier.reroute_remote_check_models.is_empty() {
3386 return Err(Status::invalid_argument("terminal model cannot reroute checks"));
3387 }
3388 if let Some(probabilities) = &modifier.probability_modifier {
3389 validate_probability_modifier(probabilities, model_type.errors.len())
3390 .map_err(Status::invalid_argument)?;
3391 }
3392 }
3393 }
3394 drop(error_model_types);
3395 let mut gadgets = self.gadgets.write().await;
3396 let mut check_models = self.check_models.write().await;
3397 let modified_remote = Arc::new(
3398 apply_check_model_reroutes(&check_model_type.remote_gadgets, check_model.modifier.as_ref())
3399 .map_err(Status::invalid_argument)?,
3400 );
3401 let cid = if check_model.cid == 0 {
3402 let mut next_cid = self.next_cid.lock().await;
3404 while check_models.contains_key(&*next_cid) {
3405 *next_cid += 1;
3406 }
3407 let cid = *next_cid;
3408 *next_cid += 1;
3409 cid
3410 } else {
3411 check_model.cid
3413 };
3414 if cid == crate::misc::index::FUTURE_CHECK_CID {
3415 return Err(Status::invalid_argument("check model id is reserved for future checks"));
3416 }
3417 if let Some(terminal) = &check_model.terminal_error_model
3418 && terminal.cid != 0
3419 && terminal.cid != cid
3420 {
3421 return Err(Status::invalid_argument(
3422 "terminal model must belong to its containing check model",
3423 ));
3424 }
3425 let gadget = gadgets.get_mut(&check_model.gid).ok_or_else(|| {
3426 Status::invalid_argument(format!("cid={cid} binding to unknown gid={}", check_model.gid))
3427 })?;
3428 debug_assert!(check_model_type.gtype == WILDCARD || check_model_type.gtype == gadget.instance.gtype);
3429 debug_assert!(gadget.binding_cid.is_none());
3430 gadget.binding_cid.replace(cid);
3431 let mut deferred_referring_eids = {
3432 let mut pending_by_gid = self.pending_referring_by_gid.lock().await;
3433 pending_by_gid.remove(&check_model.gid).unwrap_or_default()
3434 };
3435 {
3436 let mut pending_absolute_referrals = self.pending_referring_by_cid.lock().await;
3437 deferred_referring_eids.extend(pending_absolute_referrals.remove(&cid).unwrap_or_default());
3438 }
3439 let mut seen_eids = HashSet::new();
3440 deferred_referring_eids.retain(|eid| seen_eids.insert(*eid));
3441 let mut check_model = check_model;
3442 check_model.cid = cid;
3443 if let Some(terminal) = check_model.terminal_error_model.as_mut() {
3444 terminal.cid = cid;
3445 }
3446 check_models.insert(
3447 cid,
3448 CheckModel {
3449 instance: check_model.clone(),
3450 attaching_eid_vec: vec![],
3451 error_model_ready: watch::channel((check_model.error_model_count == Some(0)).then_some(())).0,
3452 modified_remote_gadgets: modified_remote.clone(),
3453 expanded_remote_gadgets: None,
3454 syndrome: watch::channel(None).0,
3455 syndrome_count: 0,
3456 referring_eids: deferred_referring_eids,
3457 },
3458 );
3459 self.record_event(trace::event::Event::ExecuteCheckModel(trace::ExecuteCheckModelEvent {
3460 check_model: Some(check_model.clone()),
3461 }))
3462 .await;
3463 let gadgets = self.gadgets.clone();
3466 let check_models = self.check_models.clone();
3467 let check_model_types = self.check_model_types.clone();
3468 let num_checks = check_model_type.checks.len();
3469 let token = self.cancellation.read().await.clone();
3470 let _guard = self.task_counter.guard();
3471 let check_model_gid = check_model.gid;
3472 let trace_shot = self.trace_shot.clone();
3473 let has_trace = self.config.trace_filepath.is_some();
3474 tokio::spawn(async move {
3475 let _guard = _guard;
3476 let expanded_remote_gadgets =
3477 Self::expand_remote_gadgets(&check_model, &modified_remote, gadgets.as_ref(), token.clone()).await;
3478 let mut handles: Vec<JoinHandle<bool>> = vec![];
3480 {
3481 let gadgets = gadgets.read().await;
3482 for gid in [check_model.gid]
3483 .into_iter()
3484 .chain(expanded_remote_gadgets.iter().filter_map(|x| *x))
3485 {
3486 let gadget = gadgets.get(&gid).unwrap();
3487 match check_or_receiver(&gadget.outcomes, token.clone()) {
3488 Ok(_) => {}
3489 Err(handle) => {
3490 handles.push(handle);
3491 }
3492 }
3493 }
3494 }
3495 futures_util::future::join_all(handles).await;
3496 if token.is_cancelled() {
3501 return;
3502 }
3503 let mut syndrome = bit_vector::from_sparse_indices(num_checks as u64, &[]);
3505 let check_model_types = check_model_types.read().await;
3506 let check_model_type = check_model_types.get(&check_model.ctype).unwrap();
3507 let gadgets = gadgets.read().await;
3508 let gadget = gadgets.get(&check_model.gid).unwrap();
3509 let local_outcomes = gadget.outcomes.borrow().clone().unwrap();
3510 let mut syndrome_count = 0;
3511 for (check_index, check) in check_model_type.checks.iter().enumerate() {
3513 let mut is_defect = check.naturally_flipped;
3514 for measurement in &check.measurements {
3515 if let Some(ri) = measurement.remote_gadget {
3516 let remote_gid = expanded_remote_gadgets[ri as usize].unwrap();
3517 let remote_gadget = gadgets.get(&remote_gid).unwrap();
3518 is_defect ^= get_bit(
3519 remote_gadget.outcomes.borrow().as_ref().unwrap(),
3520 measurement.measurement_index
3521 + modified_remote[ri as usize].as_ref().unwrap().measurement_bias,
3522 );
3523 } else {
3524 is_defect ^= get_bit(&local_outcomes, measurement.measurement_index);
3525 }
3526 }
3527 set_bit(&mut syndrome, check_index as u64, is_defect);
3528 syndrome_count += u64::from(is_defect);
3529 }
3530 drop(gadgets);
3531 drop(check_model_types);
3532 let mut check_models = check_models.write().await;
3534 let check_model = check_models.get_mut(&cid).unwrap();
3535 check_model.expanded_remote_gadgets = Some(expanded_remote_gadgets);
3536 check_model.syndrome_count = syndrome_count;
3537 check_model.syndrome.send_replace(Some(syndrome));
3538 drop(check_models);
3539 if has_trace {
3541 trace_shot.lock().await.events.push(trace::Event {
3542 timestamp_ns: crate::misc::util::timestamp_ns(),
3543 event: Some(trace::event::Event::SyndromeReady(trace::SyndromeReadyEvent {
3544 cid,
3545 gid: check_model_gid,
3546 })),
3547 });
3548 }
3549 });
3550 cid
3551 }
3552 bin::instruction::Create::ErrorModel(error_model) => {
3553 let error_model_types = self.error_model_types.read().await;
3554
3555 let error_model_type = error_model_types
3557 .get(&error_model.etype)
3558 .ok_or_else(|| Status::not_found(format!("etype={}", error_model.etype)))?;
3559 if let Some(probability_modifier) = error_model
3560 .modifier
3561 .as_ref()
3562 .and_then(|modifier| modifier.probability_modifier.as_ref())
3563 {
3564 validate_probability_modifier(probability_modifier, error_model_type.errors.len())
3565 .map_err(Status::invalid_argument)?;
3566 }
3567 let modified_remote = Arc::new(
3568 apply_error_model_reroutes(&error_model_type.remote_check_models, error_model.modifier.as_ref())
3569 .map_err(Status::invalid_argument)?,
3570 );
3571
3572 let gadgets = self.gadgets.read().await;
3578 let mut check_models = self.check_models.write().await;
3579 let mut error_models = self.error_models.write().await;
3580
3581 let owner_gid = check_models.get(&error_model.cid).map(|c| c.instance.gid).unwrap_or(0);
3582
3583 let mut resolved_gids: Vec<Option<u64>> = vec![None; modified_remote.len()];
3585 for ri in 0..modified_remote.len() {
3586 Self::resolve_remote_check_model_gid(&mut resolved_gids, ri, &modified_remote, owner_gid, &gadgets);
3587 }
3588
3589 let eid = if error_model.eid == 0 {
3591 let mut next_eid = self.next_eid.lock().await;
3592 while error_models.contains_key(&*next_eid) {
3593 *next_eid += 1;
3594 }
3595 let eid = *next_eid;
3596 *next_eid += 1;
3597 eid
3598 } else {
3599 error_model.eid
3600 };
3601 let check_model = check_models.get_mut(&error_model.cid).ok_or_else(|| {
3602 Status::invalid_argument(format!("eid={eid} attaching to unknown cid={}", error_model.cid))
3603 })?;
3604 if error_models.contains_key(&eid) {
3605 return Err(Status::already_exists(format!("eid={eid}")));
3606 }
3607 if check_model.attaching_eid_vec.len() as u64 >= check_model.instance.error_model_count.unwrap_or(1) {
3608 return Err(Status::failed_precondition("all declared error models are already attached"));
3609 }
3610 debug_assert!(error_model_type.ctype == WILDCARD || error_model_type.ctype == check_model.instance.ctype);
3611 check_model.attaching_eid_vec.push(eid);
3612 if check_model.attaching_eid_vec.len() as u64 == check_model.instance.error_model_count.unwrap_or(1) {
3613 check_model.error_model_ready.send_replace(Some(()));
3614 }
3615
3616 {
3618 let mut pending_by_gid = self.pending_referring_by_gid.lock().await;
3619 let mut pending_absolute_referrals = self.pending_referring_by_cid.lock().await;
3620 let mut pending_by_port = self.pending_referring_by_port.lock().await;
3621 for (ri, target_gid) in resolved_gids.iter().enumerate() {
3622 let Some(remote) = modified_remote[ri].as_ref() else {
3623 continue;
3624 };
3625 if let Some(target_cid) = remote.absolute_cid {
3626 if target_cid == error_model.cid || target_cid == crate::misc::index::FUTURE_CHECK_CID {
3627 continue;
3628 }
3629 let referrals = if let Some(target_cm) = check_models.get_mut(&target_cid) {
3630 &mut target_cm.referring_eids
3631 } else {
3632 pending_absolute_referrals.entry(target_cid).or_default()
3633 };
3634 if !referrals.contains(&eid) {
3635 referrals.push(eid);
3636 }
3637 continue;
3638 }
3639 let Some(&target_gid) = target_gid.as_ref() else { continue };
3640 if target_gid == owner_gid {
3641 continue;
3642 }
3643 if target_gid == u64::MAX {
3644 if let Some(blocker) =
3646 Self::find_blocking_port(ri, &resolved_gids, &modified_remote, owner_gid, &gadgets)
3647 {
3648 pending_by_port.entry(blocker).or_default().push(PendingPortReferral {
3649 eid,
3650 ri,
3651 owner_gid,
3652 modified_remote: modified_remote.clone(),
3653 });
3654 }
3655 } else if let Some(target_gadget) = gadgets.get(&target_gid) {
3656 if let Some(target_cid) = target_gadget.binding_cid {
3657 if let Some(target_cm) = check_models.get_mut(&target_cid)
3659 && !target_cm.referring_eids.contains(&eid)
3660 {
3661 target_cm.referring_eids.push(eid);
3662 }
3663 } else {
3664 pending_by_gid.entry(target_gid).or_default().push(eid);
3666 }
3667 }
3668 }
3669 }
3670
3671 let mut error_model = error_model;
3672 error_model.eid = eid;
3673 error_models.insert(
3674 eid,
3675 ErrorModel {
3676 instance: error_model.clone(),
3677 modified_remote_check_models: modified_remote.clone(),
3678 },
3679 );
3680 self.record_event(trace::event::Event::ExecuteErrorModel(trace::ExecuteErrorModelEvent {
3681 error_model: Some(error_model.clone()),
3682 }))
3683 .await;
3684 eid
3685 }
3686 };
3687 Ok((coordinator::ExecuteResponse { id }).into())
3688 }
3689
3690 #[cfg_attr(feature = "cli", fastrace::trace)]
3691 async fn decode(&self, request: Request<coordinator::Outcomes>) -> Result<Response<coordinator::Readouts>, Status> {
3692 let outcomes = request.into_inner();
3693 let _task_guard = self
3694 .task_counter
3695 .try_guard()
3696 .ok_or_else(|| Status::unavailable("coordinator reset in progress"))?;
3697 let gid = outcomes.gid;
3698 let probability_modifiers = self.bind_probability_modifiers(gid, &outcomes.modifiers).await?;
3699
3700 let is_free_hop;
3702 {
3703 let gadget_types = self.gadget_types.read().await;
3704 let mut gadgets = self.gadgets.write().await;
3705 let gadget = gadgets
3706 .get_mut(&gid)
3707 .ok_or_else(|| Status::not_found(format!("gid={}", gid)))?;
3708 is_free_hop = gadget.is_free_hop;
3709 let mut outcome_data = outcomes
3710 .outcomes
3711 .ok_or_else(|| Status::invalid_argument("missing outcomes"))?;
3712 let gadget_type = gadget_types
3713 .get(&gadget.instance.gtype)
3714 .ok_or_else(|| Status::failed_precondition(format!("gtype={} is not loaded", gadget.instance.gtype)))?;
3715 validate_outcomes(
3716 &outcome_data,
3717 outcomes.loss_mask.as_ref(),
3718 u64::try_from(gadget_type.measurements.len()).unwrap(),
3719 )
3720 .map_err(Status::invalid_argument)?;
3721 if let Some(seed) = self.loss_imputation_seed {
3726 apply_loss_random_imputation(&mut outcome_data, outcomes.loss_mask.as_ref(), seed, gid);
3727 }
3728 if self.loss_handler.tracks_losses()
3732 && has_loss_model(&gadget_types, gadget.instance.gtype)
3733 && let Some(loss_mask) = outcomes.loss_mask.as_ref()
3734 && (0..loss_mask.size).any(|index| get_bit(loss_mask, index))
3735 {
3736 gadget.loss_mask = Some(loss_mask.clone());
3737 }
3738 gadget.decoder_seed = outcomes.decoder_seed;
3739 gadget.outcomes.send_replace(Some(outcome_data));
3740 gadget.probability_modifiers = probability_modifiers;
3741 let mut readouts = Vec::with_capacity(gadget_type.readouts.len());
3742 let data: BitVector = gadget.outcomes.borrow().as_ref().unwrap().clone();
3743 for readout in gadget_type.readouts.iter() {
3744 let mut value = false;
3745 for &mi in readout.measurement_indices.iter() {
3746 value ^= get_bit(&data, mi);
3747 }
3748 readouts.push(value);
3749 }
3750 self.pauli_frame_tracker.lock().await.load_raw(gid, &readouts, &data);
3751 }
3752
3753 if is_free_hop && self.config.buffer_radius > 0 {
3754 self.record_event(trace::event::Event::Decode(trace::DecodeEvent {
3758 gid,
3759 is_leader: false,
3760 ..Default::default()
3761 }))
3762 .await;
3763 return self.wait_for_pauli_frame(gid).await;
3764 }
3765
3766 let mut explored = self
3770 .explore_mandatory_zone(gid)
3771 .await
3772 .ok_or_else(|| Status::cancelled("decode cancelled"))?;
3773
3774 self.await_mandatory_zone_syndrome(&explored)
3777 .await
3778 .ok_or_else(|| Status::cancelled("decode cancelled"))?;
3779
3780 self.await_error_models(&explored.gadgets).await?;
3781 self.wait_for_history_commitment(gid).await?;
3782
3783 self.explore_lookahead_zone(&mut explored)
3785 .await
3786 .ok_or_else(|| Status::cancelled("decode cancelled"))?;
3787
3788 loop {
3791 let token = self.cancellation.read().await.clone();
3792 if token.is_cancelled() {
3793 return Err(Status::cancelled("decode cancelled"));
3794 }
3795
3796 self.await_error_models(&explored.gadgets).await?;
3797 let remote_check_owners = if self.config.buffer_radius == 0 {
3798 HashMap::new()
3799 } else {
3800 self.available_remote_check_owners(&explored.gadgets).await
3801 };
3802 let blocking_gids: Vec<u64>;
3803 {
3804 let mut gadgets = self.gadgets.write().await;
3805
3806 let my_state = gadgets.get(&gid).map(|g| g.state.borrow().clone()).unwrap_or_default();
3808 if my_state.committed {
3809 drop(gadgets);
3811 self.record_event(trace::event::Event::Decode(trace::DecodeEvent {
3812 gid,
3813 is_leader: false,
3814 ..Default::default()
3815 }))
3816 .await;
3817 return self.wait_for_pauli_frame(gid).await;
3818 }
3819 if let Some(other) = my_state.reserved_by
3820 && other != gid
3821 {
3822 let gadget = gadgets.get(&gid).ok_or_else(|| Status::not_found(format!("gid={gid}")))?;
3825 let mut rx = gadget.state.subscribe();
3826 let token_c = token.clone();
3827 drop(gadgets);
3828 let resolved = tokio::select! {
3829 result = rx.wait_for(|state| state.reserved_by.is_none()) => {
3830 result.ok().map(|state| state.clone())
3831 }
3832 () = token_c.cancelled() => None
3833 };
3834 if resolved.is_some_and(|state| state.committed) {
3835 self.record_event(trace::event::Event::Decode(trace::DecodeEvent {
3836 gid,
3837 is_leader: false,
3838 leader_gid: other,
3839 ..Default::default()
3840 }))
3841 .await;
3842 return self.wait_for_pauli_frame(gid).await;
3843 }
3844 continue;
3845 }
3846
3847 let mut blocked: Vec<u64> = Vec::new();
3849 for &wgid in &explored.gadgets {
3850 if let Some(g) = gadgets.get(&wgid)
3851 && g.state.borrow().reserved_by.is_some()
3852 {
3853 blocked.push(wgid);
3854 }
3855 }
3856
3857 if blocked.is_empty() {
3858 self.select_commit_region(&mut explored, &gadgets);
3860
3861 self.shrink_window(&mut explored, &gadgets);
3863
3864 let remote_context: HashSet<_> = explored
3865 .decoder_window
3866 .iter()
3867 .filter(|gid| !gadgets[*gid].state.borrow().committed)
3868 .filter_map(|gid| remote_check_owners.get(gid))
3869 .flatten()
3870 .copied()
3871 .collect();
3872 explored.decoder_window.extend(remote_context);
3873 blocked.extend(
3874 explored
3875 .decoder_window
3876 .iter()
3877 .copied()
3878 .filter(|gid| gadgets[gid].state.borrow().reserved_by.is_some()),
3879 );
3880 }
3881
3882 if blocked.is_empty() {
3883 let mandatory_zone_gids: Vec<u64> = explored
3885 .gadgets
3886 .iter()
3887 .filter(|g| explored.phase.get(*g) == Some(&ExplorePhase::MandatoryZone))
3888 .copied()
3889 .collect();
3890 let lookahead_zone_gids: Vec<u64> = explored
3891 .gadgets
3892 .iter()
3893 .filter(|g| explored.phase.get(*g) == Some(&ExplorePhase::LookaheadZone))
3894 .copied()
3895 .collect();
3896 self.record_event(trace::event::Event::WindowExplore(trace::WindowExploreEvent {
3897 center_gid: gid,
3898 mandatory_zone_gids,
3899 lookahead_zone_gids,
3900 commit_region_gids: explored.commit_region.iter().copied().collect(),
3901 decoder_window_gids: explored.decoder_window.iter().copied().collect(),
3902 }))
3903 .await;
3904
3905 for &wgid in &explored.decoder_window {
3907 if let Some(g) = gadgets.get_mut(&wgid) {
3908 g.state.send_modify(|state| state.reserved_by = Some(gid));
3909 }
3910 }
3911 break;
3912 }
3913
3914 blocking_gids = blocked;
3915 }
3916
3917 let gadgets = self.gadgets.read().await;
3919 let mut watchers: Vec<JoinHandle<()>> = Vec::new();
3920 for &bgid in &blocking_gids {
3921 if let Some(g) = gadgets.get(&bgid) {
3922 let mut rx = g.state.subscribe();
3923 let token = token.clone();
3924 watchers.push(tokio::spawn(async move {
3925 tokio::select! {
3926 result = rx.wait_for(|state| state.reserved_by.is_none()) => {
3927 result.map(|_| ()).unwrap_or(())
3928 }
3929 _ = token.cancelled() => {}
3930 }
3931 }));
3932 }
3933 }
3934 drop(gadgets);
3935 futures_util::future::join_all(watchers).await;
3936 }
3937
3938 if let Err(error) = self
3941 .decode_and_commit(
3942 gid,
3943 &explored.commit_region,
3944 &explored.committing_cids,
3945 &explored.decoder_window,
3946 )
3947 .await
3948 {
3949 self.cancel_pending().await;
3950 return Err(error);
3951 }
3952
3953 self.wait_for_pauli_frame(gid).await
3956 }
3957
3958 #[cfg_attr(feature = "cli", fastrace::trace)]
3959 async fn reset(&self, request: Request<coordinator::ResetRequest>) -> Result<Response<()>, Status> {
3960 let flags = request.into_inner();
3961 let _pause = self
3962 .task_counter
3963 .try_pause()
3964 .ok_or_else(|| Status::unavailable("coordinator reset already in progress"))?;
3965 {
3968 let token = self.cancellation.read().await;
3969 token.cancel();
3970 }
3971 self.task_counter.wait_for_zero().await;
3972 {
3973 let mut token = self.cancellation.write().await;
3974 *token = CancellationToken::new();
3975 }
3976 if flags.reset_library {
3977 self.port_types.write().await.clear();
3978 self.gadget_types.write().await.clear();
3979 self.check_model_types.write().await.clear();
3980 self.error_model_types.write().await.clear();
3981 }
3982 self.gadgets.write().await.clear();
3983 self.check_models.write().await.clear();
3984 self.error_models.write().await.clear();
3985 self.pending_referring_by_gid.lock().await.clear();
3986 self.pending_referring_by_cid.lock().await.clear();
3987 self.pending_referring_by_port.lock().await.clear();
3988 *self.next_gid.lock().await = 1;
3989 *self.next_cid.lock().await = 1;
3990 *self.next_eid.lock().await = 1;
3991 self.pauli_frame_tracker.lock().await.reset();
3992 if let Some(state) = &self.forced_gap_state {
3993 state.write().await.reset();
3994 }
3995 if flags.reset_library || flags.reset_decoder_service {
3996 self.loaded_decoders.write().await.clear();
3997 }
3998 self.decoder
4000 .reset(blackbox_decoder::ResetRequest {
4001 reset_hypergraphs: flags.reset_decoder_service,
4002 ..Default::default()
4003 })
4004 .await
4005 .map_err(|e| Status::internal(format!("reset decoder service error: {}", e)))?;
4006 if let Some(decoder) = &self.gap_decoder {
4007 decoder
4008 .reset(blackbox_decoder::ResetRequest {
4009 reset_hypergraphs: flags.reset_decoder_service,
4010 ..Default::default()
4011 })
4012 .await
4013 .map_err(|error| Status::internal(format!("reset gap decoder service error: {error}")))?;
4014 }
4015 {
4017 let shot = std::mem::take(&mut *self.trace_shot.lock().await);
4018 let mut trace = self.trace.lock().await;
4019 trace.shots.push(shot);
4020 if let Some(filepath) = &self.config.trace_filepath {
4021 let buf = trace.encode_to_vec();
4022 std::fs::write(filepath, buf).map_err(|e| Status::internal(format!("failed to write trace: {e}")))?;
4023 }
4024 }
4025 Ok(().into())
4026 }
4027}
4028
4029#[cfg(test)]
4030#[path = "../../tests/unit/window_coordinator_test.rs"]
4031mod tests;
4032
4033#[cfg(test)]
4034mod incoming_tests {
4035 use super::*;
4047
4048 #[tokio::test]
4049 async fn radius_zero_does_not_include_adjacent_free_hops() {
4050 let coordinator = WindowCoordinator::new(
4051 serde_json::json!({"buffer_radius": 0, "lookahead_radius": 0}),
4052 DynDecoder::Mock(Arc::new(crate::decoder::MockDecoder::new())),
4053 );
4054 for gid in [1, 2] {
4055 coordinator.gadgets.write().await.insert(
4056 gid,
4057 Gadget {
4058 instance: bin::Gadget {
4059 gid,
4060 connectors: if gid == 2 {
4061 vec![bin::gadget::Connector { gid: 1, port: 0 }]
4062 } else {
4063 vec![]
4064 },
4065 ..Default::default()
4066 },
4067 outcomes: watch::channel(None).0,
4068 probability_modifiers: vec![],
4069 loss_mask: None,
4070 decoder_seed: None,
4071 binding_cid: None,
4072 outputs: if gid == 1 {
4073 vec![watch::channel(Some(bin::gadget::Connector { gid: 2, port: 0 })).0]
4074 } else {
4075 vec![]
4076 },
4077 pauli_frame: watch::channel(None).0,
4078 is_free_hop: true,
4079 state: watch::channel(GadgetState {
4080 committed: false,
4081 reserved_by: None,
4082 })
4083 .0,
4084 correction_count: 0,
4085 correction_weight: 0.0,
4086 },
4087 );
4088 }
4089 for gid in [1, 2] {
4090 let explored = coordinator.explore_mandatory_zone(gid).await.unwrap();
4091 assert_eq!(explored.gadgets, HashSet::from([gid]));
4092 assert!(explored.frontier.is_empty());
4093 }
4094 }
4095
4096 #[tokio::test]
4097 async fn missing_window_state_is_internal_not_cancelled() {
4098 for (entity, id) in [
4099 ("gadget", 11),
4100 ("check model", 22),
4101 ("check model type", 33),
4102 ("expanded remote gadgets for check model", 22),
4103 ("error model", 44),
4104 ("referring error model", 55),
4105 ("cancellation", 0),
4106 ] {
4107 let coordinator = WindowCoordinator::new(
4108 serde_json::json!({}),
4109 DynDecoder::Mock(Arc::new(crate::decoder::MockDecoder::new())),
4110 );
4111 coordinator.gadgets.write().await.insert(
4112 11,
4113 Gadget {
4114 instance: bin::Gadget {
4115 gid: 11,
4116 ..Default::default()
4117 },
4118 outcomes: watch::channel(Some(BitVector::default())).0,
4119 probability_modifiers: vec![],
4120 loss_mask: None,
4121 decoder_seed: None,
4122 binding_cid: Some(22),
4123 outputs: vec![],
4124 pauli_frame: watch::channel(None).0,
4125 is_free_hop: false,
4126 state: watch::channel(GadgetState {
4127 committed: false,
4128 reserved_by: Some(11),
4129 })
4130 .0,
4131 correction_count: 0,
4132 correction_weight: 0.0,
4133 },
4134 );
4135 coordinator.check_models.write().await.insert(
4136 22,
4137 CheckModel {
4138 instance: bin::CheckModel {
4139 cid: 22,
4140 ctype: 33,
4141 gid: 11,
4142 ..Default::default()
4143 },
4144 attaching_eid_vec: vec![44],
4145 error_model_ready: watch::channel(Some(())).0,
4146 modified_remote_gadgets: Arc::new(vec![]),
4147 expanded_remote_gadgets: Some(vec![]),
4148 syndrome: watch::channel(Some(BitVector::default())).0,
4149 referring_eids: vec![],
4150 syndrome_count: 0,
4151 },
4152 );
4153 coordinator
4154 .check_model_types
4155 .write()
4156 .await
4157 .insert(33, Arc::new(bin::CheckModelType::default()));
4158 match entity {
4159 "gadget" => coordinator.gadgets.write().await.clear(),
4160 "check model" => coordinator.check_models.write().await.clear(),
4161 "check model type" => coordinator.check_model_types.write().await.clear(),
4162 "expanded remote gadgets for check model" => {
4163 coordinator
4164 .check_models
4165 .write()
4166 .await
4167 .get_mut(&22)
4168 .unwrap()
4169 .expanded_remote_gadgets = None;
4170 }
4171 "referring error model" => {
4172 let mut models = coordinator.check_models.write().await;
4173 let model = models.get_mut(&22).unwrap();
4174 model.attaching_eid_vec.clear();
4175 model.referring_eids.push(55);
4176 }
4177 "cancellation" => coordinator.cancel_pending().await,
4178 _ => {}
4179 }
4180 let error = tokio::time::timeout(
4181 std::time::Duration::from_secs(5),
4182 coordinator.decode_and_commit(11, &HashSet::from([11]), &HashSet::from([22]), &HashSet::from([11])),
4183 )
4184 .await
4185 .expect("ready window must not block")
4186 .unwrap_err();
4187 if entity == "cancellation" {
4188 assert_eq!(error.code(), tonic::Code::Cancelled);
4189 } else {
4190 assert_eq!(error.code(), tonic::Code::Internal, "{entity}: {error}");
4191 let expected_entity = if entity == "referring error model" {
4192 "error model"
4193 } else {
4194 entity
4195 };
4196 assert_eq!(
4197 error.message(),
4198 format!("window centered at gid=11: missing {expected_entity} for id={id}")
4199 );
4200 assert!(!coordinator.cancellation.read().await.is_cancelled());
4201 }
4202 }
4203 }
4204}