1use crate::bin;
25use crate::coordinator;
26use crate::coordinator::forced_gap_handler::{ForcedGapGraph, ForcedGapProblem};
27use crate::coordinator::loss_handler::{RawLossSite, apply_loss_random_imputation, has_loss_model};
28use crate::coordinator::reweight_handler::{
29 ProjectedErrors, apply_reweights, correction_weights, decode_projected, deduplicate_decoder_input,
30 hard_decoding_hypergraph, load_projected_decoder, prepare_decoder, probability_reweights,
31};
32use crate::coordinator::{
33 DecoderCacheKey, DecoderReweighting, FingerprintSource, LoadedDecoder, LossHandler, LossStrategy,
34 build_modifier_fingerprints,
35};
36use crate::decoder::DynDecoder;
37use crate::decoder::blackbox_decoder::{self, DecodingHypergraph, Hyperedge};
38use crate::decoder::blackbox_util::assert_parity_factor;
39use crate::decoder::decoder_features::DecoderFeatures;
40use crate::jit::loss_compiler::{GadgetLoss, build_cross_gadget_loss_sites, build_cross_gadget_output_links};
41use crate::misc::bit_vector::{self, get_bit, set_bit};
42use crate::misc::index::{ErrorIndex, WILDCARD};
43use crate::misc::pauli_frame_symbolic_propagator::{CorrectionBasis, PauliFrameSymbolicPropagator};
44use crate::misc::pauli_frame_tracker::PauliFrameTracker;
45use crate::misc::relative_program::{self, RelativeMapping, RelativeProgram};
46use crate::misc::sync::{TaskCounter, check_or_receiver, get_or_receiver, get_value};
47use crate::misc::union_find::{UnionFindGeneric, UnionNodeTrait};
48use crate::misc::validation::{
49 apply_check_model_reroutes, apply_error_model_reroutes, validate_outcomes, validate_probability_modifier,
50};
51use crate::util::BitVector;
52use binar::{BitVec, BitwiseMut};
53use hashbrown::{HashMap, HashSet};
54
55use serde::{Deserialize, Serialize};
56use std::sync::Arc;
57#[cfg(feature = "cli")]
58use structdoc::StructDoc;
59use tokio::sync::{Mutex, RwLock, oneshot, watch};
60use tokio_util::sync::CancellationToken;
61use tonic::{Request, Response, Status};
62
63#[derive(Clone)]
65pub struct MonolithicDecoderCacheEntry {
66 pub(crate) decoder: LoadedDecoder,
67 pub(crate) scoring: Option<Arc<ForcedGapGraph>>,
68}
69
70#[derive(Debug, Clone, Serialize, Deserialize)]
71#[cfg_attr(feature = "cli", derive(StructDoc))]
72#[serde(deny_unknown_fields)]
73pub struct MonolithicCoordinatorConfig {
74 #[serde(default)]
77 pub assert_parity_factor: bool,
78 #[serde(default = "default_true")]
83 pub merge_hyperedges: bool,
84 #[serde(default = "default_true")]
87 pub async_expand: bool,
88 #[serde(default = "default_true")]
93 pub persistent_decoder: bool,
94 #[serde(default)]
99 pub decoder_reweighting: DecoderReweighting,
100 #[serde(default)]
104 pub forced_gap: bool,
105 #[serde(default = "default_true")]
109 pub loss_random_imputation: bool,
110 #[serde(default)]
113 pub loss_random_imputation_seed: Option<u64>,
114 #[serde(default)]
125 pub loss_strategy: LossStrategy,
126 #[serde(default)]
132 #[cfg_attr(feature = "cli", structdoc(leaf = "JSON object"))]
133 pub loss_config: serde_json::Value,
134}
135
136fn default_true() -> bool {
137 true
138}
139
140pub struct MonolithicCoordinator {
143 pub config: MonolithicCoordinatorConfig,
144 pub port_types: RwLock<HashMap<u64, Arc<bin::PortType>>>,
146 pub gadget_types: RwLock<HashMap<u64, Arc<bin::GadgetType>>>,
147 pub check_model_types: RwLock<HashMap<u64, Arc<bin::CheckModelType>>>,
148 pub error_model_types: RwLock<HashMap<u64, Arc<bin::ErrorModelType>>>,
149 pub gadgets: Arc<RwLock<HashMap<u64, Gadget>>>,
151 pub check_models: Arc<RwLock<HashMap<u64, CheckModel>>>,
152 pub error_models: Arc<RwLock<HashMap<u64, ErrorModel>>>,
153 pub next_gid: Mutex<u64>,
155 pub next_cid: Mutex<u64>,
156 pub next_eid: Mutex<u64>,
157 pub pending_subgraphs: Mutex<UnionFindGeneric<MonolithicUnionNode>>,
159 pub gid_to_union_index: Mutex<HashMap<u64, usize>>,
161 pub loaded_decoders: RwLock<HashMap<DecoderCacheKey, MonolithicDecoderCacheEntry>>,
172 pub decoder: DynDecoder,
174 gap_decoder: Option<DynDecoder>,
175 gap_use_loaded_reweights: bool,
176 pub pauli_frame_tracker: Mutex<PauliFrameTracker>,
178 symbolic_propagator: Option<Mutex<PauliFrameSymbolicPropagator>>,
179 pub cancellation: RwLock<CancellationToken>,
181 pub task_counter: Arc<TaskCounter>,
183 loss_imputation_seed: Option<u64>,
185 pub loss_handler: LossHandler,
188 pub use_loaded_reweights: bool,
190}
191
192impl FingerprintSource for ErrorModel {
195 fn instance(&self) -> &bin::ErrorModel {
196 &self.instance
197 }
198 fn modified_remote_check_models(&self) -> &Arc<Vec<Option<bin::error_model_type::RemoteCheckModel>>> {
199 &self.modified_remote_check_models
200 }
201}
202
203pub struct Gadget {
204 pub instance: bin::Gadget,
205 pub outcomes: Option<BitVector>,
206 pub probability_modifiers: Vec<(u64, bin::ProbabilityModifier)>,
207 pub loss_mask: Option<BitVector>,
208 pub decoder_seed: Option<u64>,
210 pub binding_cid: watch::Sender<Option<u64>>,
212 pub outputs: Vec<watch::Sender<Option<bin::gadget::Connector>>>,
214 pub tx: oneshot::Sender<Result<coordinator::Readouts, Status>>,
218 pub rx: Option<oneshot::Receiver<Result<coordinator::Readouts, Status>>>,
220}
221
222pub struct CheckModel {
223 pub instance: bin::CheckModel,
224 pub attaching_eid_vec: Vec<u64>,
226 pub error_model_ready: watch::Sender<Option<()>>,
227 pub modified_remote_gadgets: Arc<Vec<Option<bin::check_model_type::RemoteGadget>>>,
229 pub expanded_remote_gadgets: watch::Sender<Option<Vec<Option<u64>>>>,
231}
232
233pub struct ErrorModel {
234 pub instance: bin::ErrorModel,
235 pub modified_remote_check_models: Arc<Vec<Option<bin::error_model_type::RemoteCheckModel>>>,
237 pub expanded_remote_check_models: watch::Sender<Option<Vec<Option<u64>>>>,
239}
240
241impl MonolithicCoordinator {
242 pub fn new(config: serde_json::Value, decoder: DynDecoder) -> Self {
243 Self::with_gap_decoder(config, decoder, None)
244 }
245
246 #[must_use]
252 pub fn with_gap_decoder(config: serde_json::Value, decoder: DynDecoder, gap_decoder: Option<DynDecoder>) -> Self {
253 let config: MonolithicCoordinatorConfig = serde_json::from_value(config).unwrap();
254 let use_loaded_reweights = config
255 .decoder_reweighting
256 .use_loaded(config.persistent_decoder, decoder.features())
257 .unwrap_or_else(|error| panic!("invalid decoder reweighting configuration: {error}"));
258 let gap_use_loaded_reweights = config
259 .decoder_reweighting
260 .use_loaded(config.persistent_decoder, gap_decoder.as_ref().unwrap_or(&decoder).features())
261 .unwrap_or_else(|error| panic!("invalid gap decoder reweighting configuration: {error}"));
262 let loss_imputation_seed = if config.loss_random_imputation {
263 use rand::Rng;
264 Some(config.loss_random_imputation_seed.unwrap_or_else(|| rand::rng().next_u64()))
265 } else {
266 None
267 };
268 let loss_handler = LossHandler::new(config.loss_strategy, config.loss_config.clone())
269 .unwrap_or_else(|error| panic!("invalid loss configuration: {error}"));
270 let symbolic_propagator = (config.forced_gap || decoder.features().contains(DecoderFeatures::OBSERVABLES))
271 .then(|| Mutex::new(PauliFrameSymbolicPropagator::new()));
272 assert!(
273 !config.forced_gap || !loss_handler.hands_off_to_decoder(),
274 "forced_gap does not support loss_strategy \"handoff\"; use \"reweight\" or \"ignore\""
275 );
276 Self {
277 config,
278 port_types: Default::default(),
279 gadget_types: Default::default(),
280 check_model_types: Default::default(),
281 error_model_types: Default::default(),
282 gadgets: Default::default(),
283 check_models: Default::default(),
284 error_models: Default::default(),
285 next_gid: Mutex::new(1),
286 next_cid: Mutex::new(1),
287 next_eid: Mutex::new(1),
288 pending_subgraphs: Mutex::new(UnionFindGeneric::new(0)),
289 gid_to_union_index: Mutex::new(HashMap::new()),
290 loaded_decoders: Default::default(),
291 decoder,
292 gap_decoder,
293 gap_use_loaded_reweights,
294 pauli_frame_tracker: Default::default(),
295 symbolic_propagator,
296 cancellation: RwLock::default(),
297 task_counter: TaskCounter::new(),
298 loss_imputation_seed,
299 loss_handler,
300 use_loaded_reweights,
301 }
302 }
303
304 fn gap_decoder(&self) -> &DynDecoder {
305 self.gap_decoder.as_ref().unwrap_or(&self.decoder)
306 }
307
308 pub async fn cancel_pending(&self) {
315 let token = self.cancellation.read().await;
316 token.cancel();
317 }
318
319 async fn await_error_models(&self, gid: u64) -> Result<(), Status> {
320 let token = self.cancellation.read().await.clone();
321 let readiness = {
322 let gadgets = self.gadgets.read().await;
323 let check_models = self.check_models.read().await;
324 let gadget = gadgets.get(&gid).ok_or_else(|| Status::not_found(format!("gid={gid}")))?;
325 let Some(cid) = *gadget.binding_cid.borrow() else {
326 return Ok(());
327 };
328 let check_model = check_models
329 .get(&cid)
330 .ok_or_else(|| Status::failed_precondition(format!("cid={cid} is not loaded")))?;
331 check_or_receiver(&check_model.error_model_ready, token.clone())
332 };
333 if let Err(receiver) = readiness {
334 receiver.await.map_err(|error| Status::internal(error.to_string()))?;
335 }
336 if token.is_cancelled() {
337 return Err(Status::cancelled("error model readiness wait cancelled"));
338 }
339 Ok(())
340 }
341
342 async fn get_subgraph(&self, gid: u64) -> HashSet<u64> {
345 let gadgets = self.gadgets.read().await;
346 let mut subgraph: HashSet<u64> = HashSet::new();
347 subgraph.insert(gid);
348 let mut boundary_gadgets: Vec<u64> = vec![gid];
349 while !boundary_gadgets.is_empty() {
350 let mut new_boundary_gadgets = vec![];
351 for boundary_gid in boundary_gadgets.into_iter() {
352 let gadget = gadgets.get(&boundary_gid).unwrap();
353 for next in gadget
354 .outputs
355 .iter()
356 .map(|x| x.borrow().unwrap())
357 .chain(gadget.instance.connectors.iter().copied())
358 {
359 if !subgraph.contains(&next.gid) {
360 subgraph.insert(next.gid);
361 new_boundary_gadgets.push(next.gid);
362 }
363 }
364 }
365 boundary_gadgets = new_boundary_gadgets;
366 }
367 subgraph
368 }
369
370 async fn take_subgraph(&self, gid: u64) -> (HashMap<u64, Gadget>, HashMap<u64, CheckModel>, HashMap<u64, ErrorModel>) {
372 let subgraph = self.get_subgraph(gid).await;
373
374 if self.config.async_expand {
376 let token = self.cancellation.read().await.clone();
377 let mut handles = vec![];
378 let gadgets = self.gadgets.read().await;
379 let check_models = self.check_models.read().await;
380 let error_models = self.error_models.read().await;
381 for &gid in subgraph.iter() {
382 let gadget = &gadgets[&gid];
383 if let Some(&cid) = gadget.binding_cid.borrow().as_ref() {
384 let check_model = &check_models[&cid];
385 if let Err(receiver) = check_or_receiver(&check_model.expanded_remote_gadgets, token.clone()) {
386 handles.push(receiver);
387 }
388 for &eid in check_model.attaching_eid_vec.iter() {
389 let error_model = &error_models[&eid];
390 match check_or_receiver(&error_model.expanded_remote_check_models, token.clone()) {
391 Ok(..) => {}
392 Err(receiver) => handles.push(receiver),
393 }
394 }
395 }
396 }
397 drop(gadgets);
398 drop(check_models);
399 drop(error_models);
400 futures_util::future::join_all(handles).await;
401 }
402
403 let gadgets: HashMap<u64, Gadget> = {
404 let mut gadgets = self.gadgets.write().await;
405 subgraph.iter().map(|gid| (*gid, gadgets.remove(gid).unwrap())).collect()
406 };
407
408 let check_models: HashMap<u64, CheckModel> = {
409 let mut check_models = self.check_models.write().await;
410 subgraph
411 .iter()
412 .filter_map(|gid| {
413 let gadget = &gadgets[gid];
414 if let Some(&cid) = gadget.binding_cid.borrow().as_ref() {
415 Some((cid, check_models.remove(&cid).unwrap()))
416 } else {
417 None
418 }
419 })
420 .collect()
421 };
422
423 let error_models: HashMap<u64, ErrorModel> = {
424 let mut error_models = self.error_models.write().await;
425 check_models
426 .iter()
427 .flat_map(|(_, check_model)| {
428 check_model
429 .attaching_eid_vec
430 .iter()
431 .map(|eid| {
432 let error_model = error_models.remove(eid).unwrap();
433 (*eid, error_model)
434 })
435 .collect::<Vec<_>>()
436 .into_iter()
437 })
438 .collect()
439 };
440
441 (gadgets, check_models, error_models)
442 }
443
444 async fn batch_expand(
445 &self,
446 gadgets: HashMap<u64, Gadget>,
447 mut check_models: HashMap<u64, CheckModel>,
448 mut error_models: HashMap<u64, ErrorModel>,
449 ) -> (HashMap<u64, Gadget>, HashMap<u64, CheckModel>, HashMap<u64, ErrorModel>) {
450 let token = self.cancellation.read().await.clone();
451 let gadgets_locked = RwLock::new(gadgets);
452 for check_model in check_models.values_mut() {
453 let expanded_remote_gadgets = Self::expand_remote_gadgets(
454 &check_model.instance,
455 &check_model.modified_remote_gadgets,
456 &gadgets_locked,
457 token.clone(),
458 )
459 .await;
460 check_model
461 .expanded_remote_gadgets
462 .send_replace(Some(expanded_remote_gadgets));
463 }
464
465 let check_models_locked = RwLock::new(check_models);
466 for error_model in error_models.values_mut() {
467 let expanded_remote_check_models = Self::expand_remote_check_models(
468 &error_model.instance,
469 &error_model.modified_remote_check_models,
470 &gadgets_locked,
471 &check_models_locked,
472 token.clone(),
473 )
474 .await;
475 error_model
476 .expanded_remote_check_models
477 .send_replace(Some(expanded_remote_check_models));
478 }
479
480 (gadgets_locked.into_inner(), check_models_locked.into_inner(), error_models)
481 }
482
483 async fn decode_subgraph(&self, gid: u64) {
484 let (mut gadgets, mut check_models, mut error_models) = self.take_subgraph(gid).await;
486
487 if !self.config.async_expand {
489 (gadgets, check_models, error_models) = self.batch_expand(gadgets, check_models, error_models).await;
490 }
491
492 let mut expanded_gadgets: Vec<relative_program::ExpandedGadget> = vec![];
493 let mut gid_vec: Vec<_> = gadgets.keys().cloned().collect();
494 gid_vec.sort();
495 let token = self.cancellation.read().await.clone();
496 for &gid in gid_vec.iter() {
497 let gadget = gadgets.get(&gid).unwrap();
498 let inputs: Vec<_> = gadget.instance.connectors.iter().cloned().map(Some).collect();
499 let outputs: Vec<_> = gadget.outputs.iter().map(|v| v.borrow().unwrap()).map(Some).collect();
500 let gtype = gadget.instance.gtype;
501 let cid = gadget.binding_cid.borrow().as_ref().cloned();
502 let (check_model, error_models) = if let Some(cid) = cid {
503 let check_model = check_models.get(&cid).unwrap();
504 let remote_gadgets = get_value(&check_model.expanded_remote_gadgets, token.clone()).await;
505 let Some(remote_gadgets) = remote_gadgets else { return };
506 let expanded_check_model = relative_program::ExpandedCheckModel {
507 cid,
508 ctype: check_model.instance.ctype,
509 remote_gadgets,
510 count_checks: self
511 .check_model_types
512 .read()
513 .await
514 .get(&check_model.instance.ctype)
515 .unwrap()
516 .checks
517 .len(),
518 };
519 let mut expanded_error_models = vec![];
520 for &eid in check_model.attaching_eid_vec.iter() {
521 let error_model = error_models.get(&eid).unwrap();
522 let remote_check_models = get_value(&error_model.expanded_remote_check_models, token.clone()).await;
523 let Some(remote_check_models) = remote_check_models else {
524 return;
525 };
526 expanded_error_models.push(relative_program::ExpandedErrorModel {
527 eid,
528 etype: error_model.instance.etype,
529 remote_check_models,
530 });
531 }
532 (Some(expanded_check_model), expanded_error_models)
533 } else {
534 (None, vec![])
535 };
536 expanded_gadgets.push(relative_program::ExpandedGadget {
537 gid,
538 gtype,
539 inputs,
540 outputs,
541 check_model,
542 error_models,
543 });
544 }
545 let (relative_program, mapping) = RelativeProgram::new(&expanded_gadgets);
546
547 let (syndrome, syndrome_counts) = self.get_syndrome(&relative_program, &mapping, &gadgets, &check_models).await;
548 let decoded = self
549 .decode_parity_factor(syndrome, &relative_program, &mapping, &gadgets, &check_models, &error_models)
550 .await;
551 let (parity_factor, errors, correction_weights, forced_gap_problem) = match decoded {
552 Ok(decoded) => decoded,
553 Err(error) => {
554 for gadget in gadgets.into_values() {
555 let _ = gadget.tx.send(Err(error.clone()));
556 }
557 return;
558 }
559 };
560 let mut correction_statistics = HashMap::new();
561 for (&error_index, &weight) in parity_factor.subgraph.iter().zip(&correction_weights) {
562 let eid = mapping.global_eid_of[errors[usize::try_from(error_index).unwrap()].eid];
563 let gid = check_models[&error_models[&eid].instance.cid].instance.gid;
564 let (count, total_weight) = correction_statistics.entry(gid).or_insert((0_u64, 0.0));
565 *count += 1;
566 *total_weight += weight;
567 }
568
569 let probabilities = if let Some(problem) = forced_gap_problem {
570 match problem.probabilities().await {
571 Ok(probabilities) => probabilities,
572 Err(error) => {
573 for gadget in gadgets.into_values() {
574 let _ = gadget.tx.send(Err(error.clone()));
575 }
576 return;
577 }
578 }
579 } else {
580 vec![]
581 };
582
583 let updates = self
584 .update_pauli_frame(&parity_factor, &errors, &relative_program, &mapping, &error_models)
585 .await;
586
587 let mut probability_offset = 0;
588 for (gid, readouts) in updates {
589 let probability_end = probability_offset + usize::try_from(readouts.size).unwrap();
590 let readout_probabilities = if probabilities.is_empty() {
591 vec![]
592 } else {
593 probabilities[probability_offset..probability_end].to_vec()
594 };
595 probability_offset = probability_end;
596 let gadget = gadgets.remove(&gid).unwrap();
597 let (correction_count, correction_weight) = correction_statistics.remove(&gid).unwrap_or_default();
598 let _ = gadget.tx.send(Ok(coordinator::Readouts {
599 gid,
600 readouts: Some(readouts),
601 probabilities: readout_probabilities,
602 syndrome_count: syndrome_counts.get(&gid).copied().unwrap_or(0),
603 correction_count,
604 correction_weight,
605 frame_probabilities: vec![],
606 }));
607 }
608 }
609
610 async fn update_pauli_frame(
611 &self,
612 parity_factor: &blackbox_decoder::ParityFactor,
613 errors: &ProjectedErrors,
614 relative_program: &RelativeProgram,
615 mapping: &RelativeMapping,
616 error_models: &HashMap<u64, ErrorModel>,
617 ) -> Vec<(u64, BitVector)> {
618 let error_model_types = self.error_model_types.read().await;
619 let mut tracker = self.pauli_frame_tracker.lock().await;
620
621 let mut residual_vec: Vec<BitVec> = Vec::with_capacity(relative_program.local_gadgets.len());
623 let mut readout_flips_vec: Vec<BitVec> = Vec::with_capacity(relative_program.local_gadgets.len());
624 for &gid in mapping.global_gid_of.iter() {
625 let Some(gadget) = tracker.gadgets.get(&gid) else {
626 return vec![];
628 };
629 residual_vec.push(BitVec::zeros(gadget.num_output_observables()));
630 readout_flips_vec.push(BitVec::zeros(gadget.num_readouts()));
631 }
632
633 for &ei in parity_factor.subgraph.iter() {
635 let local_error = &errors[ei as usize];
636 let local_eid = local_error.eid;
637 let eid = mapping.global_eid_of[local_eid];
638 let error_index = local_error.error_index;
639 let error_model = error_models.get(&eid).unwrap();
640 let error_model_type = error_model_types.get(&error_model.instance.etype).unwrap();
641 let error = &error_model_type.errors[error_index];
642 let local_gid = mapping.local_gid_of_local_eid[local_eid];
644 let residual = &mut residual_vec[local_gid];
645 let readout_flips = &mut readout_flips_vec[local_gid];
646 for &ri in error.residual.iter() {
647 residual.negate_index(ri as usize);
648 }
649 for &ri in error.readout_flips.iter() {
650 readout_flips.negate_index(ri as usize);
651 }
652 }
653
654 let mut updates = vec![];
657 for ((&gid, residual), readout_flips) in mapping.global_gid_of.iter().zip(residual_vec).zip(readout_flips_vec) {
658 let mut single_update = tracker.load_correction(gid, residual, readout_flips);
659 debug_assert_eq!(single_update.keys().cloned().collect::<Vec<_>>(), vec![gid]);
660 updates.push((gid, single_update.remove(&gid).unwrap()));
661 }
662 updates
663 }
664
665 async fn decode_parity_factor(
666 &self,
667 syndrome: BitVector,
668 relative_program: &RelativeProgram,
669 mapping: &RelativeMapping,
670 gadgets: &HashMap<u64, Gadget>,
671 check_models: &HashMap<u64, CheckModel>,
672 error_models: &HashMap<u64, ErrorModel>,
673 ) -> Result<
674 (
675 blackbox_decoder::ParityFactor,
676 ProjectedErrors,
677 Vec<f64>,
678 Option<ForcedGapProblem>,
679 ),
680 Status,
681 > {
682 let decoder_seed = crate::coordinator::common_decoder_seed(gadgets.values().map(|gadget| gadget.decoder_seed))?;
683 let logical_targets = if let Some(symbolic) = &self.symbolic_propagator {
684 let symbolic = symbolic.lock().await;
685 let targets = symbolic.readout_targets(mapping.global_gid_of.iter().copied());
686 if self.decoder.features().contains(DecoderFeatures::OBSERVABLES)
687 && !targets.is_empty()
688 && !symbolic.remote_dependencies_are_closed(&mapping.global_gid_of)
689 {
690 return Err(Status::failed_precondition(
691 "observable effects require remote conditional corrections to remain within one monolithic decode subgraph",
692 ));
693 }
694 targets
695 } else {
696 vec![]
697 };
698 let target_count = logical_targets.len();
699 let has_forced_gap_targets = self.config.forced_gap && target_count != 0;
700
701 let loss_sites = self.build_loss_sites(mapping, gadgets, check_models).await;
706
707 let deduplicate = self.config.merge_hyperedges && !self.loss_handler.hands_off_to_decoder();
714
715 let logical_flips_are_cacheable = target_count == 0
716 || mapping
717 .global_gid_of
718 .iter()
719 .all(|gid| gadgets[gid].instance.modifier.is_none());
720 let cache_key = if self.config.persistent_decoder && logical_flips_are_cacheable {
721 let error_model_types = self.error_model_types.read().await;
722 Some(DecoderCacheKey {
726 relative_program: relative_program.clone(),
727 error_model_fingerprints: build_modifier_fingerprints(mapping, error_models, &error_model_types),
728 committing_local_cids: Vec::new(),
729 logical_flip_signature: vec![],
732 })
733 } else {
734 None
735 };
736
737 if let Some(ref cache_key) = cache_key {
738 let loaded = self.loaded_decoders.read().await.get(cache_key).cloned();
739 if let Some(loaded) = loaded {
740 let probability_reweights = loaded
741 .decoder
742 .projection
743 .probability_reweights(Self::shot_probability_modifiers(mapping, gadgets));
744 let projected =
745 self.loss_handler
746 .project_shot(&loaded.decoder.projection, &probability_reweights, &loss_sites);
747 let parity_factor = decode_projected(
748 &self.decoder,
749 &loaded.decoder,
750 syndrome.clone(),
751 decoder_seed,
752 projected.reweights.clone(),
753 projected.loss,
754 self.use_loaded_reweights,
755 )
756 .await?;
757 if self.config.assert_parity_factor {
758 assert_parity_factor(
759 loaded.decoder.decoding_hypergraph.as_ref().unwrap(),
760 &parity_factor,
761 &syndrome,
762 );
763 }
764 let weights = loaded.decoder.correction_weights(&parity_factor, &projected.reweights);
765 let forced_gap_problem = loaded.scoring.as_ref().map(|graph| {
766 graph.problem(
767 self.gap_decoder().clone(),
768 syndrome,
769 decoder_seed,
770 parity_factor.clone(),
771 projected.reweights,
772 self.gap_use_loaded_reweights,
773 )
774 });
775 return Ok((parity_factor, projected.errors, weights, forced_gap_problem));
776 }
777 }
778
779 let (decoding_hypergraph, errors, logical_flips) = self
782 .decoding_hypergraph(&logical_targets, relative_program, mapping, check_models, error_models)
783 .await;
784
785 let Some(cache_key) = cache_key else {
786 let probability_reweights = Self::shot_probability_reweights(mapping, gadgets, &errors);
787 let mut decoding_hypergraph = decoding_hypergraph;
790 apply_reweights(&mut decoding_hypergraph, probability_reweights.iter().copied());
791 let (mut decoding_hypergraph, loss) = self.loss_handler.apply_sites(decoding_hypergraph, &loss_sites, &errors);
792 let mut errors = errors;
793 let mut logical_flips = logical_flips;
794 if deduplicate {
795 let prepared = deduplicate_decoder_input(&decoding_hypergraph, &errors, &logical_flips, |_| 0);
796 decoding_hypergraph = prepared.hypergraph;
797 errors = prepared.representatives;
798 logical_flips = prepared.logical_flips.as_ref().clone();
799 }
800 let hard_hypergraph = hard_decoding_hypergraph(decoding_hypergraph.clone(), &logical_flips);
801 let parity_factor = self
802 .decoder
803 .decode(blackbox_decoder::DecodingProblem {
804 hypergraph: Some(hard_hypergraph),
805 syndrome: Some(syndrome.clone()),
806 loss,
807 decoder_seed,
808 })
809 .await?;
810 if self.config.assert_parity_factor {
811 assert_parity_factor(&decoding_hypergraph, &parity_factor, &syndrome);
812 }
813 let errors = errors.into();
814 let weights = correction_weights(&decoding_hypergraph, &parity_factor);
815 let forced_gap_problem = has_forced_gap_targets.then(|| {
816 Arc::new(ForcedGapGraph::new(
817 Arc::new(decoding_hypergraph),
818 Arc::new(logical_flips),
819 target_count,
820 false,
821 ))
822 .problem(
823 self.gap_decoder().clone(),
824 syndrome,
825 decoder_seed,
826 parity_factor.clone(),
827 vec![],
828 false,
829 )
830 });
831 return Ok((parity_factor, errors, weights, forced_gap_problem));
832 };
833
834 let retain_decoding_hypergraph =
837 has_forced_gap_targets || !self.use_loaded_reweights || self.config.assert_parity_factor;
838 let (projection, prepared) = prepare_decoder(decoding_hypergraph, errors, logical_flips, deduplicate, |_| 0);
839 let decoder = load_projected_decoder(&self.decoder, projection, prepared, retain_decoding_hypergraph, false).await?;
840 let scoring = has_forced_gap_targets.then(|| {
841 Arc::new(ForcedGapGraph::new(
842 Arc::clone(decoder.decoding_hypergraph.as_ref().unwrap()),
843 Arc::clone(&decoder.logical_flips),
844 target_count,
845 true,
846 ))
847 });
848 let loaded = MonolithicDecoderCacheEntry { decoder, scoring };
849 let probability_reweights = loaded
850 .decoder
851 .projection
852 .probability_reweights(Self::shot_probability_modifiers(mapping, gadgets));
853 let projected = self
854 .loss_handler
855 .project_shot(&loaded.decoder.projection, &probability_reweights, &loss_sites);
856 let mut loaded_decoders = self.loaded_decoders.write().await;
857 loaded_decoders.insert(cache_key, loaded.clone());
858 drop(loaded_decoders);
859 let parity_factor = decode_projected(
860 &self.decoder,
861 &loaded.decoder,
862 syndrome.clone(),
863 decoder_seed,
864 projected.reweights.clone(),
865 projected.loss,
866 self.use_loaded_reweights,
867 )
868 .await?;
869 if self.config.assert_parity_factor {
870 assert_parity_factor(
871 loaded.decoder.decoding_hypergraph.as_ref().unwrap(),
872 &parity_factor,
873 &syndrome,
874 );
875 }
876 let weights = loaded.decoder.correction_weights(&parity_factor, &projected.reweights);
877 let forced_gap_problem = loaded.scoring.as_ref().map(|graph| {
878 graph.problem(
879 self.gap_decoder().clone(),
880 syndrome,
881 decoder_seed,
882 parity_factor.clone(),
883 projected.reweights,
884 self.gap_use_loaded_reweights,
885 )
886 });
887 Ok((parity_factor, projected.errors, weights, forced_gap_problem))
888 }
889
890 async fn bind_probability_modifiers(
891 &self,
892 gid: u64,
893 modifiers: &[bin::ProbabilityModifier],
894 ) -> Result<Vec<(u64, bin::ProbabilityModifier)>, Status> {
895 if modifiers.is_empty() {
896 return Ok(vec![]);
897 }
898 let cid = self
899 .gadgets
900 .read()
901 .await
902 .get(&gid)
903 .and_then(|gadget| *gadget.binding_cid.borrow())
904 .ok_or_else(|| Status::failed_precondition(format!("gid={gid} has no binding check model")))?;
905 let eids = self
906 .check_models
907 .read()
908 .await
909 .get(&cid)
910 .map(|check_model| check_model.attaching_eid_vec.clone())
911 .ok_or_else(|| Status::failed_precondition(format!("cid={cid} is not loaded")))?;
912 if modifiers.len() > eids.len() {
913 return Err(Status::invalid_argument(format!(
914 "gid={gid} supplied {} probability modifiers for {} attached error models",
915 modifiers.len(),
916 eids.len()
917 )));
918 }
919 let error_model_types = self.error_model_types.read().await;
920 let error_models = self.error_models.read().await;
921 let mut bound = Vec::with_capacity(modifiers.len());
922 for (&eid, modifier) in eids.iter().zip(modifiers) {
923 let error_model = error_models
924 .get(&eid)
925 .ok_or_else(|| Status::failed_precondition(format!("eid={eid} is not loaded")))?;
926 let error_model_type = error_model_types
927 .get(&error_model.instance.etype)
928 .ok_or_else(|| Status::failed_precondition(format!("etype={} is not loaded", error_model.instance.etype)))?;
929 validate_probability_modifier(modifier, error_model_type.errors.len()).map_err(Status::invalid_argument)?;
930 bound.push((eid, modifier.clone()));
931 }
932 Ok(bound)
933 }
934
935 fn shot_probability_reweights(
936 mapping: &RelativeMapping,
937 gadgets: &HashMap<u64, Gadget>,
938 error_reference: &[ErrorIndex],
939 ) -> Vec<(u64, f64)> {
940 probability_reweights(error_reference, Self::shot_probability_modifiers(mapping, gadgets))
941 }
942
943 fn shot_probability_modifiers<'a>(
944 mapping: &RelativeMapping,
945 gadgets: &'a HashMap<u64, Gadget>,
946 ) -> Vec<(usize, &'a bin::ProbabilityModifier)> {
947 let mut modifiers: Vec<_> = mapping
948 .global_gid_of
949 .iter()
950 .filter_map(|gid| gadgets.get(gid))
951 .flat_map(|gadget| gadget.probability_modifiers.iter())
952 .filter_map(|(eid, modifier)| mapping.local_eid_of.get(eid).map(|&local_eid| (local_eid, modifier)))
953 .collect();
954 modifiers.sort_unstable_by_key(|&(local_eid, _)| local_eid);
955 modifiers
956 }
957
958 async fn get_syndrome(
959 &self,
960 relative_program: &RelativeProgram,
961 mapping: &RelativeMapping,
962 gadgets: &HashMap<u64, Gadget>,
963 check_models: &HashMap<u64, CheckModel>,
964 ) -> (BitVector, HashMap<u64, u64>) {
965 let mut syndrome: BitVector = bit_vector::from_sparse_indices(relative_program.count_checks as u64, &[]);
966 let mut syndrome_counts = HashMap::new();
967 let check_model_types = self.check_model_types.read().await;
968 for (&cid, &start_index) in mapping.global_cid_of.iter().zip(mapping.start_indices.iter()) {
969 let check_model = check_models.get(&cid).unwrap();
970 let check_model_type = check_model_types.get(&check_model.instance.ctype).unwrap();
971 let gid = check_model.instance.gid;
972 let gadget = gadgets.get(&gid).unwrap();
973 let expanded_remote_ref = check_model.expanded_remote_gadgets.borrow();
974 let expanded_remotes = expanded_remote_ref.as_ref().unwrap();
975 let local_outcomes = gadget.outcomes.as_ref().unwrap();
976 for (check_index, check) in check_model_type.checks.iter().enumerate() {
978 let mut is_defect = check.naturally_flipped;
979 for measurement in &check.measurements {
980 if let Some(ri) = measurement.remote_gadget {
981 let remote_gid = expanded_remotes[ri as usize].unwrap();
982 let remote_gadget = gadgets.get(&remote_gid).unwrap();
983 is_defect ^= get_bit(
984 remote_gadget.outcomes.as_ref().unwrap(),
985 measurement.measurement_index
986 + check_model.modified_remote_gadgets[ri as usize]
987 .as_ref()
988 .unwrap()
989 .measurement_bias,
990 );
991 } else {
992 is_defect ^= get_bit(local_outcomes, measurement.measurement_index);
993 }
994 }
995 set_bit(&mut syndrome, (start_index + check_index) as u64, is_defect);
996 *syndrome_counts.entry(gid).or_insert(0) += u64::from(is_defect);
997 }
998 }
999 (syndrome, syndrome_counts)
1000 }
1001
1002 async fn build_loss_sites(
1009 &self,
1010 mapping: &RelativeMapping,
1011 gadgets: &HashMap<u64, Gadget>,
1012 check_models: &HashMap<u64, CheckModel>,
1013 ) -> Vec<RawLossSite> {
1014 let mut loss_sites = Vec::new();
1015 if !self.loss_handler.tracks_losses() || !gadgets.values().any(|g| g.loss_mask.is_some()) {
1016 return loss_sites;
1017 }
1018
1019 let gadget_types = self.gadget_types.read().await;
1024 let mut gid_of_index: Vec<u64> = Vec::new();
1025 let mut index_of_gid: HashMap<u64, usize> = HashMap::new();
1026 for &gid in &mapping.global_gid_of {
1027 let Some(gadget) = gadgets.get(&gid) else { continue };
1028 if !has_loss_model(&gadget_types, gadget.instance.gtype) {
1029 continue;
1030 }
1031 index_of_gid.insert(gid, gid_of_index.len());
1032 gid_of_index.push(gid);
1033 }
1034 if gid_of_index.is_empty() {
1035 return loss_sites;
1036 }
1037
1038 let local_eid_of_index: Vec<Option<usize>> = gid_of_index
1041 .iter()
1042 .map(|&gid| {
1043 let cid = (*gadgets.get(&gid)?.binding_cid.borrow())?;
1044 let eid = *check_models.get(&cid)?.attaching_eid_vec.first()?;
1045 mapping.local_eid_of.get(&eid).copied()
1046 })
1047 .collect();
1048
1049 let loss_masks: Vec<BitVector> = gid_of_index
1050 .iter()
1051 .map(|&gid| gadgets.get(&gid).and_then(|g| g.loss_mask.clone()).unwrap_or_default())
1052 .collect();
1053
1054 let port_types = self.port_types.read().await;
1057 let gadget_instances: Vec<&bin::Gadget> =
1058 gid_of_index.iter().map(|&gid| &gadgets.get(&gid).unwrap().instance).collect();
1059 let output_links_vec = build_cross_gadget_output_links(&gadget_instances, &index_of_gid, &gadget_types, &port_types);
1060
1061 let gadget_losses: Vec<GadgetLoss> = (0..gid_of_index.len())
1062 .map(|index| {
1063 let gtype = gadgets.get(&gid_of_index[index]).unwrap().instance.gtype;
1064 GadgetLoss {
1065 loss_model: gadget_types.get(>ype).unwrap().loss_model.as_ref().unwrap(),
1066 observed: &loss_masks[index],
1067 output_links: &output_links_vec[index],
1068 }
1069 })
1070 .collect();
1071
1072 for site in build_cross_gadget_loss_sites(&gadget_losses) {
1077 let local_eid = local_eid_of_index[site.gadget_index];
1078 loss_sites.push(RawLossSite::from_compiled(site, local_eid));
1079 }
1080 loss_sites
1081 }
1082
1083 async fn decoding_hypergraph(
1084 &self,
1085 logical_targets: &[CorrectionBasis],
1086 relative_program: &RelativeProgram,
1087 mapping: &RelativeMapping,
1088 check_models: &HashMap<u64, CheckModel>,
1089 error_models: &HashMap<u64, ErrorModel>,
1090 ) -> (DecodingHypergraph, Arc<Vec<ErrorIndex>>, Vec<Vec<u64>>) {
1091 let error_model_types = self.error_model_types.read().await;
1094 let logical_flip_cache = if logical_targets.is_empty() {
1095 None
1096 } else {
1097 let symbolic = self.symbolic_propagator.as_ref().unwrap().lock().await;
1098 assert!(
1099 symbolic.remote_dependencies_are_closed(&mapping.global_gid_of),
1100 "forced_gap requires remote conditional corrections to remain within one monolithic decode subgraph"
1101 );
1102 Some(symbolic.logical_flip_cache(mapping.global_gid_of.iter().copied(), logical_targets))
1103 };
1104
1105 let mut hyperedges: Vec<Hyperedge> = vec![];
1106 let mut error_reference: Vec<ErrorIndex> = vec![];
1107 let mut logical_flips = vec![];
1108 for (local_cid, &cid) in mapping.global_cid_of.iter().enumerate() {
1109 let check_model = check_models.get(&cid).unwrap();
1110 for &eid in &check_model.attaching_eid_vec {
1111 let local_eid = mapping.local_eid_of[&eid];
1112 let error_model = error_models.get(&eid).unwrap();
1113 let error_model_type = error_model_types.get(&error_model.instance.etype).unwrap();
1114 let expanded_remote_ref = error_model.expanded_remote_check_models.borrow();
1115 let expanded_remotes = expanded_remote_ref.as_ref().unwrap();
1116 let mut errors = &error_model_type.errors;
1117 let modified_errors: Option<Vec<bin::error_model_type::Error>>;
1119 if let Some(modifier) = &error_model.instance.modifier
1120 && let Some(probability_modifier) = &modifier.probability_modifier
1121 {
1122 let mut new_errors = errors.clone();
1123 for (error_index, &probability) in probability_modifier.probabilities.iter().enumerate() {
1124 new_errors[error_index].probability = probability;
1125 }
1126 for (&error_index, &probability) in probability_modifier
1127 .sparse_indices
1128 .iter()
1129 .zip(probability_modifier.sparse_probabilities.iter())
1130 {
1131 new_errors[error_index as usize].probability = probability;
1132 }
1133 modified_errors = Some(new_errors);
1134 errors = modified_errors.as_ref().unwrap();
1135 }
1136 let local_start_index = mapping.start_indices[local_cid] as u64;
1137 for (error_index, error) in errors.iter().enumerate() {
1138 let probability = error.probability;
1147 let mut vertices: Vec<u64> = vec![];
1148 for check in &error.checks {
1149 if let Some(ri) = check.remote_check_model {
1150 let remote_cid = expanded_remotes[ri as usize].unwrap();
1151 let remote_local_cid = mapping.local_cid_of[&remote_cid];
1152 let remote_start_index = mapping.start_indices[remote_local_cid] as u64;
1153 vertices.push(
1154 remote_start_index
1155 + check.check_index
1156 + error_model.modified_remote_check_models[ri as usize]
1157 .as_ref()
1158 .unwrap()
1159 .check_bias,
1160 );
1161 } else {
1162 vertices.push(local_start_index + check.check_index);
1163 }
1164 }
1165 let logical_readout_flips = if let Some(logical_flip_cache) = logical_flip_cache.as_ref() {
1166 let owner_local_gid = mapping.local_gid_of_local_eid[local_eid];
1167 let gid = mapping.global_gid_of[owner_local_gid];
1168 logical_flip_cache.propagated_logical_flips(gid, &error.residual, &error.readout_flips)
1169 } else {
1170 vec![]
1171 };
1172 if vertices.is_empty() && logical_readout_flips.is_empty() {
1173 continue; }
1175 error_reference.push(ErrorIndex {
1176 eid: local_eid,
1177 error_index,
1178 });
1179 hyperedges.push(Hyperedge {
1180 observable_flips: self.decoder.features().contains(DecoderFeatures::OBSERVABLES).then(|| {
1181 blackbox_decoder::ObservableFlips {
1182 indices: logical_readout_flips.clone(),
1183 }
1184 }),
1185 vertices,
1186 probability,
1187 });
1188 logical_flips.push(logical_readout_flips);
1189 }
1190 }
1191 }
1192 let hypergraph = DecodingHypergraph {
1193 vertex_num: relative_program.count_checks as u64,
1194 hyperedges,
1195 };
1196 (hypergraph, Arc::new(error_reference), logical_flips)
1197 }
1198
1199 async fn expand_remote_gadgets(
1203 check_model: &bin::CheckModel,
1204 modified_remote_gadgets: &Vec<Option<bin::check_model_type::RemoteGadget>>,
1205 gadgets: &RwLock<HashMap<u64, Gadget>>,
1206 token: CancellationToken,
1207 ) -> Vec<Option<u64>> {
1208 let mut expanded_remote_gid_vec: Vec<Option<u64>> = vec![None; modified_remote_gadgets.len()];
1210 for ri in 0..modified_remote_gadgets.len() {
1211 Self::expand_remote_gadget(
1212 &mut expanded_remote_gid_vec,
1213 ri,
1214 modified_remote_gadgets,
1215 check_model.gid,
1216 gadgets,
1217 token.clone(),
1218 )
1219 .await;
1220 }
1221 expanded_remote_gid_vec
1222 }
1223
1224 async fn expand_remote_gadget(
1225 expanded_remote_gid_vec: &mut Vec<Option<u64>>,
1226 ri: usize,
1227 remote_gadgets: &Vec<Option<bin::check_model_type::RemoteGadget>>,
1228 gid: u64,
1229 gadgets: &RwLock<HashMap<u64, Gadget>>,
1230 token: CancellationToken,
1231 ) {
1232 if expanded_remote_gid_vec[ri].is_some() || remote_gadgets[ri].is_none() {
1233 return; }
1235 let remote_gadget = remote_gadgets[ri].as_ref().unwrap();
1236 if let Some(absolute_gid) = remote_gadget.absolute_gid {
1238 expanded_remote_gid_vec[ri] = Some(absolute_gid);
1239 return;
1240 }
1241 let previous = if let Some(previous) = remote_gadget.previous_remote_gadget {
1244 Box::pin(Self::expand_remote_gadget(
1245 expanded_remote_gid_vec,
1246 previous as usize,
1247 remote_gadgets,
1248 gid,
1249 gadgets,
1250 token.clone(),
1251 ))
1252 .await;
1253 expanded_remote_gid_vec[previous as usize].unwrap()
1254 } else {
1255 gid
1256 };
1257 let gadgets = gadgets.read().await;
1258 let gadget = gadgets.get(&previous).unwrap();
1259 match remote_gadget.port.unwrap() {
1260 bin::check_model_type::remote_gadget::Port::Output(port) => {
1261 let next = get_or_receiver(&gadget.outputs[port as usize], token);
1262 drop(gadgets); let next = match next {
1264 Ok(next) => Some(next),
1265 Err(handle) => handle.await.unwrap_or(None),
1266 };
1267 if let Some(next) = next {
1268 expanded_remote_gid_vec[ri] = Some(next.gid);
1269 }
1270 }
1271 bin::check_model_type::remote_gadget::Port::Input(port) => {
1272 let connector = &gadget.instance.connectors[port as usize];
1273 expanded_remote_gid_vec[ri] = Some(connector.gid);
1274 }
1275 }
1276 }
1277
1278 async fn expand_remote_check_models(
1282 error_model: &bin::ErrorModel,
1283 modified_remote_check_models: &Vec<Option<bin::error_model_type::RemoteCheckModel>>,
1284 gadgets: &RwLock<HashMap<u64, Gadget>>,
1285 check_models: &RwLock<HashMap<u64, CheckModel>>,
1286 token: CancellationToken,
1287 ) -> Vec<Option<u64>> {
1288 let gid = check_models.read().await.get(&error_model.cid).unwrap().instance.gid;
1290 let mut expanded_remote_gid_vec: Vec<Option<u64>> = vec![None; modified_remote_check_models.len()];
1291 for ri in 0..modified_remote_check_models.len() {
1292 Self::expand_remote_check_model(
1293 &mut expanded_remote_gid_vec,
1294 ri,
1295 modified_remote_check_models,
1296 gid,
1297 gadgets,
1298 token.clone(),
1299 )
1300 .await;
1301 }
1302 let mut expanded_remote_cid_vec = Vec::with_capacity(modified_remote_check_models.len());
1303 let mut gadgets_read = gadgets.read().await;
1304 for (ri, gid) in expanded_remote_gid_vec.into_iter().enumerate() {
1305 if let Some(gid) = gid {
1306 if gid == u64::MAX - 1 {
1308 let absolute_cid = modified_remote_check_models[ri]
1309 .as_ref()
1310 .unwrap()
1311 .absolute_cid
1312 .expect("absolute_cid should be present when sentinel is used");
1313 expanded_remote_cid_vec.push(Some(absolute_cid));
1314 continue;
1315 }
1316 let gadget = gadgets_read.get(&gid).unwrap();
1317 let cid = if let Some(&cid) = gadget.binding_cid.borrow().as_ref() {
1318 cid
1319 } else {
1320 let mut rx = gadget.binding_cid.subscribe();
1321 drop(gadgets_read);
1323 let cid = tokio::select! {
1324 result = rx.wait_for(|v| v.is_some()) => {
1325 match result {
1326 Ok(v) => v.unwrap(),
1327 Err(_) => return expanded_remote_cid_vec,
1328 }
1329 }
1330 _ = token.cancelled() => { return expanded_remote_cid_vec; }
1331 };
1332 gadgets_read = gadgets.read().await;
1333 cid
1334 };
1335 expanded_remote_cid_vec.push(Some(cid));
1336 } else {
1337 expanded_remote_cid_vec.push(None);
1338 }
1339 }
1340 expanded_remote_cid_vec
1341 }
1342
1343 async fn expand_remote_check_model(
1344 expanded_remotes: &mut Vec<Option<u64>>,
1345 ri: usize,
1346 remote_check_models: &Vec<Option<bin::error_model_type::RemoteCheckModel>>,
1347 gid: u64,
1348 gadgets: &RwLock<HashMap<u64, Gadget>>,
1349 token: CancellationToken,
1350 ) {
1351 if expanded_remotes[ri].is_some() || remote_check_models[ri].is_none() {
1352 return; }
1354 let remote_check_model = remote_check_models[ri].as_ref().unwrap();
1355 if remote_check_model.absolute_cid.is_some() {
1359 expanded_remotes[ri] = Some(u64::MAX - 1); return;
1363 }
1364 let previous = if let Some(previous) = remote_check_model.previous_remote_check_model {
1367 Box::pin(Self::expand_remote_check_model(
1368 expanded_remotes,
1369 previous as usize,
1370 remote_check_models,
1371 gid,
1372 gadgets,
1373 token.clone(),
1374 ))
1375 .await;
1376 expanded_remotes[previous as usize].unwrap()
1377 } else {
1378 gid
1379 };
1380 let gadgets = gadgets.read().await;
1381 let gadget = gadgets.get(&previous).unwrap();
1382 match remote_check_model.port.unwrap() {
1383 bin::error_model_type::remote_check_model::Port::Output(port) => {
1384 let next = get_or_receiver(&gadget.outputs[port as usize], token);
1385 drop(gadgets); let next = match next {
1387 Ok(gid) => Some(gid),
1388 Err(handle) => handle.await.unwrap_or(None),
1389 };
1390 if let Some(next) = next {
1391 expanded_remotes[ri] = Some(next.gid);
1392 }
1393 }
1394 bin::error_model_type::remote_check_model::Port::Input(port) => {
1395 let connector = &gadget.instance.connectors[port as usize];
1396 expanded_remotes[ri] = Some(connector.gid);
1397 }
1398 }
1399 }
1400}
1401
1402#[tonic::async_trait]
1403impl coordinator::coordinator_server::Coordinator for MonolithicCoordinator {
1404 async fn load_library(&self, request: Request<bin::Library>) -> Result<Response<()>, Status> {
1405 let _task_guard = self
1406 .task_counter
1407 .try_guard()
1408 .ok_or_else(|| Status::unavailable("coordinator reset in progress"))?;
1409 let library = request.into_inner();
1410 self.loss_handler
1411 .validate_capability(&library.gadget_types, self.decoder.features())?;
1412 let mut port_types = self.port_types.write().await;
1413 for port_type in library.port_types.into_iter() {
1414 if port_types.contains_key(&port_type.ptype) {
1415 return Err(Status::already_exists(format!("ptype={}", port_type.ptype)));
1416 }
1417 port_types.insert(port_type.ptype, Arc::new(port_type));
1418 }
1419 drop(port_types);
1420 let mut gadget_types = self.gadget_types.write().await;
1421 for gadget_type in library.gadget_types.into_iter() {
1422 if gadget_types.contains_key(&gadget_type.gtype) {
1423 return Err(Status::already_exists(format!("gtype={}", gadget_type.gtype)));
1424 }
1425 gadget_types.insert(gadget_type.gtype, Arc::new(gadget_type));
1426 }
1427 drop(gadget_types);
1428 let mut check_model_types = self.check_model_types.write().await;
1429 for check_model_type in library.check_model_types.into_iter() {
1430 if check_model_types.contains_key(&check_model_type.ctype) {
1431 return Err(Status::already_exists(format!("ctype={}", check_model_type.ctype)));
1432 }
1433 check_model_types.insert(check_model_type.ctype, Arc::new(check_model_type));
1434 }
1435 drop(check_model_types);
1436 let mut error_model_types = self.error_model_types.write().await;
1437 for error_model_type in library.error_model_types.into_iter() {
1438 if error_model_types.contains_key(&error_model_type.etype) {
1439 return Err(Status::already_exists(format!("etype={}", error_model_type.etype)));
1440 }
1441 error_model_types.insert(error_model_type.etype, Arc::new(error_model_type));
1442 }
1443 drop(error_model_types);
1444 Ok(().into())
1445 }
1446
1447 async fn unload(&self, _unload: Request<coordinator::UnloadLibrary>) -> Result<Response<()>, Status> {
1448 unimplemented!()
1449 }
1450
1451 async fn execute(&self, request: Request<bin::Instruction>) -> Result<Response<coordinator::ExecuteResponse>, Status> {
1452 let _task_guard = self
1453 .task_counter
1454 .try_guard()
1455 .ok_or_else(|| Status::unavailable("coordinator reset in progress"))?;
1456 let instruction = request.into_inner();
1457 let create = instruction
1458 .create
1459 .ok_or_else(|| Status::invalid_argument("unknown instruction"))?;
1460 let id = match create {
1461 bin::instruction::Create::Gadget(gadget) => {
1462 let port_types = self.port_types.read().await;
1463 let gadget_types = self.gadget_types.read().await;
1464 let mut gadgets = self.gadgets.write().await;
1465 let gid = if gadget.gid == 0 {
1466 let mut next_gid = self.next_gid.lock().await;
1468 while gadgets.contains_key(&*next_gid) {
1469 *next_gid += 1;
1470 }
1471 let gid = *next_gid;
1472 *next_gid += 1;
1473 gid
1474 } else {
1475 gadget.gid
1477 };
1478 let gadget_type = gadget_types
1479 .get(&gadget.gtype)
1480 .ok_or_else(|| Status::not_found(format!("gtype={}", gadget.gtype)))?;
1481 debug_assert!(gadget.connectors.len() == gadget_type.inputs.len());
1482 let mut pending_subgraphs = self.pending_subgraphs.lock().await;
1484 let mut gid_to_union_index = self.gid_to_union_index.lock().await;
1485 let union_index = pending_subgraphs.payload.len();
1486 pending_subgraphs.insert(MonolithicUnionNode::default());
1487 gid_to_union_index.insert(gid, union_index);
1488 for (port, connector) in gadget.connectors.iter().enumerate() {
1490 debug_assert!(gadgets.contains_key(&connector.gid));
1491 debug_assert!({
1492 let peer_outputs = &gadgets[&connector.gid].outputs;
1493 (connector.port as usize) < peer_outputs.len()
1494 && peer_outputs[connector.port as usize].borrow().is_none()
1495 });
1496 let peer_union_index = gid_to_union_index[&connector.gid];
1497 pending_subgraphs.union(union_index, peer_union_index);
1498 gadgets.get_mut(&connector.gid).unwrap().outputs[connector.port as usize]
1499 .send_replace(Some(bin::gadget::Connector { gid, port: port as u64 }));
1500 }
1501 let node = pending_subgraphs.get_mut(union_index);
1502 node.num_unconnected_outputs += gadget_type.outputs.len();
1503 node.num_unconnected_outputs -= gadget.connectors.len();
1504 node.num_unloaded_gadgets += 1;
1505 let mut tracker = self.pauli_frame_tracker.lock().await;
1506 tracker.add_gadget(gid, gadget_type, gadget.modifier.as_ref(), &port_types, &gadget.connectors);
1507 if let Some(symbolic) = &self.symbolic_propagator {
1508 symbolic.lock().await.add_gadget(gid, &tracker.gadgets[&gid]);
1509 }
1510 let (tx, rx) = oneshot::channel();
1511 let mut gadget = gadget;
1512 gadget.gid = gid;
1513 gadgets.insert(
1514 gid,
1515 Gadget {
1516 instance: gadget,
1517 outcomes: None,
1518 probability_modifiers: vec![],
1519 loss_mask: None,
1520 decoder_seed: None,
1521 binding_cid: watch::channel(None).0,
1522 outputs: gadget_type.outputs.iter().map(|_| watch::channel(None).0).collect(),
1524 tx,
1525 rx: Some(rx),
1526 },
1527 );
1528 gid
1529 }
1530 bin::instruction::Create::CheckModel(check_model) => {
1531 let check_model_types = self.check_model_types.read().await;
1532 let mut gadgets = self.gadgets.write().await;
1533 let mut check_models = self.check_models.write().await;
1534 let check_model_type = check_model_types
1535 .get(&check_model.ctype)
1536 .ok_or_else(|| Status::not_found(format!("ctype={}", check_model.ctype)))?;
1537 let modified_remote = Arc::new(
1538 apply_check_model_reroutes(&check_model_type.remote_gadgets, check_model.modifier.as_ref())
1539 .map_err(Status::invalid_argument)?,
1540 );
1541 let cid = if check_model.cid == 0 {
1542 let mut next_cid = self.next_cid.lock().await;
1544 while check_models.contains_key(&*next_cid) {
1545 *next_cid += 1;
1546 }
1547 let cid = *next_cid;
1548 *next_cid += 1;
1549 cid
1550 } else {
1551 check_model.cid
1553 };
1554 let gadget = gadgets.get_mut(&check_model.gid).ok_or_else(|| {
1555 Status::invalid_argument(format!("cid={cid} binding to unknown gid={}", check_model.gid))
1556 })?;
1557 debug_assert!(check_model_type.gtype == WILDCARD || check_model_type.gtype == gadget.instance.gtype);
1558 debug_assert!(gadget.binding_cid.borrow().is_none());
1559 gadget.binding_cid.send_replace(Some(cid));
1560 let mut check_model = check_model;
1561 check_model.cid = cid;
1562 check_models.insert(
1563 cid,
1564 CheckModel {
1565 instance: check_model.clone(),
1566 attaching_eid_vec: vec![],
1567 error_model_ready: watch::channel((check_model.error_model_count == Some(0)).then_some(())).0,
1568 modified_remote_gadgets: modified_remote.clone(),
1569 expanded_remote_gadgets: watch::channel(None).0,
1570 },
1571 );
1572 let gadgets = self.gadgets.clone();
1575 let check_models = self.check_models.clone();
1576 if self.config.async_expand {
1577 let token = self.cancellation.read().await.clone();
1578 let _guard = self.task_counter.guard();
1579 tokio::spawn(async move {
1580 let _guard = _guard;
1581 let expanded_remote_gadgets =
1582 Self::expand_remote_gadgets(&check_model, &modified_remote, gadgets.as_ref(), token).await;
1583 let mut check_models = check_models.write().await;
1584 if let Some(cm) = check_models.get_mut(&cid) {
1585 cm.expanded_remote_gadgets.send_replace(Some(expanded_remote_gadgets));
1586 }
1587 });
1588 }
1589 cid
1590 }
1591 bin::instruction::Create::ErrorModel(error_model) => {
1592 let error_model_types = self.error_model_types.read().await;
1593 let mut check_models = self.check_models.write().await;
1594 let mut error_models = self.error_models.write().await;
1595 let error_model_type = error_model_types
1596 .get(&error_model.etype)
1597 .ok_or_else(|| Status::not_found(format!("etype={}", error_model.etype)))?;
1598 if let Some(probability_modifier) = error_model
1599 .modifier
1600 .as_ref()
1601 .and_then(|modifier| modifier.probability_modifier.as_ref())
1602 {
1603 validate_probability_modifier(probability_modifier, error_model_type.errors.len())
1604 .map_err(Status::invalid_argument)?;
1605 }
1606 let modified_remote = Arc::new(
1607 apply_error_model_reroutes(&error_model_type.remote_check_models, error_model.modifier.as_ref())
1608 .map_err(Status::invalid_argument)?,
1609 );
1610 let eid = if error_model.eid == 0 {
1611 let mut next_eid = self.next_eid.lock().await;
1613 while error_models.contains_key(&*next_eid) {
1614 *next_eid += 1;
1615 }
1616 let eid = *next_eid;
1617 *next_eid += 1;
1618 eid
1619 } else {
1620 error_model.eid
1622 };
1623 let check_model = check_models.get_mut(&error_model.cid).ok_or_else(|| {
1624 Status::invalid_argument(format!("eid={eid} attaching to unknown cid={}", error_model.cid))
1625 })?;
1626 if error_models.contains_key(&eid) {
1627 return Err(Status::already_exists(format!("eid={eid}")));
1628 }
1629 if check_model.attaching_eid_vec.len() as u64 >= check_model.instance.error_model_count.unwrap_or(1) {
1630 return Err(Status::failed_precondition("all declared error models are already attached"));
1631 }
1632 debug_assert!(error_model_type.ctype == WILDCARD || error_model_type.ctype == check_model.instance.ctype);
1633 check_model.attaching_eid_vec.push(eid);
1634 if check_model.attaching_eid_vec.len() as u64 == check_model.instance.error_model_count.unwrap_or(1) {
1635 check_model.error_model_ready.send_replace(Some(()));
1636 }
1637 let mut error_model = error_model;
1638 error_model.eid = eid;
1639 error_models.insert(
1640 eid,
1641 ErrorModel {
1642 instance: error_model.clone(),
1643 modified_remote_check_models: modified_remote.clone(),
1644 expanded_remote_check_models: watch::channel(None).0,
1645 },
1646 );
1647 let gadgets = self.gadgets.clone();
1650 let check_models = self.check_models.clone();
1651 let error_models = self.error_models.clone();
1652 if self.config.async_expand {
1653 let token = self.cancellation.read().await.clone();
1654 let _guard = self.task_counter.guard();
1655 tokio::spawn(async move {
1656 let _guard = _guard;
1657 let expanded_remote_check_models = Self::expand_remote_check_models(
1658 &error_model,
1659 &modified_remote,
1660 gadgets.as_ref(),
1661 check_models.as_ref(),
1662 token,
1663 )
1664 .await;
1665 let mut error_models = error_models.write().await;
1666 if let Some(em) = error_models.get_mut(&eid) {
1667 em.expanded_remote_check_models
1668 .send_replace(Some(expanded_remote_check_models));
1669 }
1670 });
1671 }
1672 eid
1673 }
1674 };
1675 Ok((coordinator::ExecuteResponse { id }).into())
1676 }
1677
1678 async fn decode(&self, request: Request<coordinator::Outcomes>) -> Result<Response<coordinator::Readouts>, Status> {
1679 let outcomes = request.into_inner();
1680 let _task_guard = self
1681 .task_counter
1682 .try_guard()
1683 .ok_or_else(|| Status::unavailable("coordinator reset in progress"))?;
1684 let gid = outcomes.gid;
1685 let token = self.cancellation.read().await.clone();
1686 self.await_error_models(gid).await?;
1687 let probability_modifiers = self.bind_probability_modifiers(gid, &outcomes.modifiers).await?;
1688 let gadget_types = self.gadget_types.read().await;
1689 let mut gadgets = self.gadgets.write().await;
1690 let gadget = gadgets
1691 .get_mut(&gid)
1692 .ok_or_else(|| Status::not_found(format!("gid={}", gid)))?;
1693 if gadget.outcomes.is_some() {
1694 return Err(Status::already_exists(format!("gid={} outcomes loaded", gid)));
1695 }
1696 let mut outcome_data = outcomes
1698 .outcomes
1699 .ok_or_else(|| Status::invalid_argument("missing outcomes"))?;
1700 let gadget_type = gadget_types
1701 .get(&gadget.instance.gtype)
1702 .ok_or_else(|| Status::failed_precondition(format!("gtype={} is not loaded", gadget.instance.gtype)))?;
1703 validate_outcomes(
1704 &outcome_data,
1705 outcomes.loss_mask.as_ref(),
1706 u64::try_from(gadget_type.measurements.len()).unwrap(),
1707 )
1708 .map_err(Status::invalid_argument)?;
1709 if let Some(seed) = self.loss_imputation_seed {
1714 apply_loss_random_imputation(&mut outcome_data, outcomes.loss_mask.as_ref(), seed, gid);
1715 }
1716 if self.loss_handler.tracks_losses()
1720 && has_loss_model(&gadget_types, gadget.instance.gtype)
1721 && let Some(loss_mask) = outcomes.loss_mask.as_ref()
1722 && (0..loss_mask.size).any(|index| get_bit(loss_mask, index))
1723 {
1724 gadget.loss_mask = Some(loss_mask.clone());
1725 }
1726 gadget.outcomes.replace(outcome_data);
1727 gadget.probability_modifiers = probability_modifiers;
1728 gadget.decoder_seed = outcomes.decoder_seed;
1729 let mut pending_subgraphs = self.pending_subgraphs.lock().await;
1730 let gid_to_union_index = self.gid_to_union_index.lock().await;
1731 let union_index = gid_to_union_index[&gid];
1732 let node = pending_subgraphs.get_mut(union_index);
1733 node.num_unloaded_gadgets -= 1;
1734 let is_final_gadget = node.num_unloaded_gadgets == 0 && node.num_unconnected_outputs == 0;
1736 let rx = gadget.rx.take().unwrap();
1737 let mut readouts = Vec::with_capacity(gadget_type.readouts.len());
1739 let data: &BitVector = gadget.outcomes.as_ref().unwrap();
1740 for readout in gadget_type.readouts.iter() {
1741 let mut value = false;
1742 for &mi in readout.measurement_indices.iter() {
1743 value ^= get_bit(data, mi);
1744 }
1745 readouts.push(value);
1746 }
1747 self.pauli_frame_tracker.lock().await.load_raw(gid, &readouts, data);
1748 drop(gid_to_union_index);
1749 drop(pending_subgraphs);
1750 drop(gadgets);
1751 drop(gadget_types);
1752 if is_final_gadget {
1753 self.decode_subgraph(gid).await;
1756 }
1757 let result = tokio::select! {
1758 result = rx => result.map_err(|_| Status::internal(format!("gid={gid} receive error")))??,
1759 () = token.cancelled() => return Err(Status::cancelled("decode cancelled")),
1760 };
1761 return Ok(result.into());
1762 }
1763
1764 async fn reset(&self, request: Request<coordinator::ResetRequest>) -> Result<Response<()>, Status> {
1765 let flags = request.into_inner();
1766 let _pause = self
1767 .task_counter
1768 .try_pause()
1769 .ok_or_else(|| Status::unavailable("coordinator reset already in progress"))?;
1770 {
1773 let token = self.cancellation.read().await;
1774 token.cancel();
1775 }
1776 self.task_counter.wait_for_zero().await;
1777 {
1778 let mut token = self.cancellation.write().await;
1779 *token = CancellationToken::new();
1780 }
1781 if flags.reset_library {
1782 self.port_types.write().await.clear();
1783 self.gadget_types.write().await.clear();
1784 self.check_model_types.write().await.clear();
1785 self.error_model_types.write().await.clear();
1786 }
1787 self.gadgets.write().await.clear();
1788 self.check_models.write().await.clear();
1789 self.error_models.write().await.clear();
1790 *self.next_gid.lock().await = 1;
1791 *self.next_cid.lock().await = 1;
1792 *self.next_eid.lock().await = 1;
1793 let mut pending_subgraphs = self.pending_subgraphs.lock().await;
1794 pending_subgraphs.remove_all();
1795 self.gid_to_union_index.lock().await.clear();
1796 self.pauli_frame_tracker.lock().await.reset();
1797 if let Some(symbolic) = &self.symbolic_propagator {
1798 symbolic.lock().await.reset();
1799 }
1800 if flags.reset_library || flags.reset_decoder_service {
1801 self.loaded_decoders.write().await.clear();
1802 }
1803 self.decoder
1805 .reset(blackbox_decoder::ResetRequest {
1806 reset_hypergraphs: flags.reset_decoder_service,
1807 ..Default::default()
1808 })
1809 .await
1810 .map_err(|e| Status::internal(format!("reset decoder service error: {}", e)))?;
1811 if let Some(decoder) = &self.gap_decoder {
1812 decoder
1813 .reset(blackbox_decoder::ResetRequest {
1814 reset_hypergraphs: flags.reset_decoder_service,
1815 ..Default::default()
1816 })
1817 .await
1818 .map_err(|error| Status::internal(format!("reset gap decoder service error: {error}")))?;
1819 }
1820 Ok(().into())
1821 }
1822}
1823
1824#[derive(Debug, Serialize, Deserialize, Clone)]
1826pub struct MonolithicUnionNode {
1827 pub set_size: usize,
1828 pub num_unloaded_gadgets: usize,
1829 pub num_unconnected_outputs: usize,
1830}
1831
1832impl UnionNodeTrait for MonolithicUnionNode {
1834 #[inline]
1835 fn union(left: &Self, right: &Self) -> (bool, Self) {
1836 let result = Self {
1837 set_size: left.set_size + right.set_size,
1838 num_unloaded_gadgets: left.num_unloaded_gadgets + right.num_unloaded_gadgets,
1839 num_unconnected_outputs: left.num_unconnected_outputs + right.num_unconnected_outputs,
1840 };
1841 (left.set_size >= right.set_size, result)
1843 }
1844 #[inline]
1845 fn clear(&mut self) {
1846 self.set_size = 1;
1847 }
1848 #[inline]
1849 fn default() -> Self {
1850 Self {
1851 set_size: 1,
1852 num_unloaded_gadgets: 0,
1853 num_unconnected_outputs: 0,
1854 }
1855 }
1856}
1857
1858#[cfg(test)]
1859#[path = "../../tests/unit/monolithic_coordinator_test.rs"]
1860mod tests;