use crate::bin;
use crate::coordinator;
use crate::coordinator::{DecoderCacheKey, DecoderReweighting, FingerprintSource, LoadedDecoder, LossHandler, LossStrategy};
use crate::decoder::DynDecoder;
use crate::misc::index::CollectedIds;
use crate::misc::pauli_frame_tracker::PauliFrameTracker;
use crate::misc::sync::TaskCounter;
use crate::util::BitVector;
use hashbrown::{HashMap, HashSet};
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use std::sync::atomic::AtomicU64;
#[cfg(feature = "cli")]
use structdoc::StructDoc;
use tokio::sync::{Mutex, RwLock, Semaphore, watch};
use tokio_util::sync::CancellationToken;
mod collection;
mod decoding;
mod error_metadata;
mod exploration;
mod scoring;
mod service;
use decoding::PreparedWindow;
use error_metadata::CachedErrorRemotes;
pub use exploration::{ExplorePhase, ExploredWindow};
pub use scoring::CommitRegionDecoder;
use scoring::ForcedGapState;
pub mod trace {
include!("../proto/deq.coordinator.window_coordinator.rs");
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
#[cfg_attr(feature = "cli", derive(StructDoc))]
pub struct WindowCoordinatorConfig {
#[serde(default)]
pub assert_parity_factor: bool,
#[serde(default = "default_true")]
pub merge_hyperedges: bool,
#[serde(default = "default_true")]
pub persistent_decoder: bool,
#[serde(default)]
pub decoder_reweighting: DecoderReweighting,
#[serde(default)]
pub forced_gap: bool,
#[serde(default)]
pub frame_probabilities: bool,
#[serde(default)]
pub forced_gap_strategy: ForcedGapStrategy,
#[serde(default)]
pub window_parallelism: WindowParallelism,
#[serde(default = "default_buffer_radius")]
pub buffer_radius: usize,
#[serde(default, alias = "center_radius")]
pub lookahead_radius: Option<usize>,
#[serde(default)]
pub trace_filepath: Option<String>,
#[serde(default = "default_true")]
pub loss_random_imputation: bool,
#[serde(default)]
pub loss_random_imputation_seed: Option<u64>,
#[serde(default)]
pub loss_strategy: LossStrategy,
#[serde(default)]
#[cfg_attr(feature = "cli", structdoc(leaf = "JSON object"))]
pub loss_config: serde_json::Value,
}
impl WindowCoordinatorConfig {
pub fn lookahead_radius(&self) -> usize {
if self.buffer_radius == 0 {
return 0;
}
self.lookahead_radius.unwrap_or(self.buffer_radius)
}
pub fn effective_window_radius(&self) -> usize {
self.buffer_radius + self.lookahead_radius()
}
}
fn default_true() -> bool {
true
}
fn default_buffer_radius() -> usize {
1
}
pub struct WindowCoordinator {
pub config: WindowCoordinatorConfig,
pub port_types: RwLock<HashMap<u64, Arc<bin::PortType>>>,
pub gadget_types: RwLock<HashMap<u64, Arc<bin::GadgetType>>>,
pub check_model_types: Arc<RwLock<HashMap<u64, Arc<bin::CheckModelType>>>>,
pub error_model_types: RwLock<HashMap<u64, Arc<bin::ErrorModelType>>>,
referenced_error_remotes: RwLock<HashMap<u64, CachedErrorRemotes>>,
pub gadgets: Arc<RwLock<HashMap<u64, Gadget>>>,
pub check_models: Arc<RwLock<HashMap<u64, CheckModel>>>,
pub error_models: Arc<RwLock<HashMap<u64, ErrorModel>>>,
completed_readouts: Mutex<HashSet<u64>>,
collection_requests: AtomicU64,
collection_completed: Mutex<u64>,
pinned_gadgets: Arc<std::sync::Mutex<HashMap<u64, usize>>>,
collected_gids: Mutex<CollectedIds>,
collected_cids: Mutex<CollectedIds>,
collected_eids: Mutex<CollectedIds>,
pending_type_unloads: Mutex<coordinator::UnloadLibrary>,
pending_referring_by_gid: Mutex<HashMap<u64, Vec<u64>>>,
pending_referring_by_cid: Mutex<HashMap<u64, Vec<u64>>>,
pending_referring_by_port: Mutex<HashMap<(u64, u64), Vec<PendingPortReferral>>>,
pub next_gid: Mutex<u64>,
pub next_cid: Mutex<u64>,
pub next_eid: Mutex<u64>,
pub loaded_decoders: RwLock<HashMap<DecoderCacheKey, LoadedDecoder>>,
prepared_windows: std::sync::Mutex<lru::LruCache<DecoderCacheKey, Arc<PreparedWindow>>>,
persistent_prepared_windows: std::sync::Mutex<HashMap<DecoderCacheKey, Arc<PreparedWindow>>>,
prepared_window_preparation: Mutex<()>,
pub decoder: DynDecoder,
pub decoding_slots: Arc<Semaphore>,
gap_decoder: Option<DynDecoder>,
pub pauli_frame_tracker: Mutex<PauliFrameTracker>,
forced_gap_state: Option<RwLock<ForcedGapState>>,
pub cancellation: RwLock<CancellationToken>,
pub task_counter: Arc<TaskCounter>,
loss_imputation_seed: Option<u64>,
pub loss_handler: LossHandler,
pub use_loaded_reweights: bool,
pub trace_shot: Arc<Mutex<trace::Shot>>,
pub trace: Mutex<trace::WindowCoordinatorTrace>,
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct GadgetState {
pub committed: bool,
pub reserved_by: Option<u64>,
}
impl GadgetState {
fn release_buffer(&self, leader_gid: u64) -> Option<Self> {
(self.reserved_by == Some(leader_gid)).then_some(Self {
committed: self.committed,
reserved_by: None,
})
}
}
pub struct Gadget {
pub instance: bin::Gadget,
pub outcomes: watch::Sender<Option<BitVector>>,
pub probability_modifiers: Vec<(u64, bin::ProbabilityModifier)>,
pub loss_mask: Option<BitVector>,
pub decoder_seed: Option<u64>,
pub binding_cid: Option<u64>,
pub outputs: Vec<watch::Sender<Option<bin::gadget::Connector>>>,
pub pauli_frame: watch::Sender<Option<BitVector>>,
pub correction_count: u64,
pub correction_weight: f64,
pub is_free_hop: bool,
pub state: watch::Sender<GadgetState>,
}
pub struct CheckModel {
pub instance: bin::CheckModel,
pub attaching_eid_vec: Vec<u64>,
pub error_model_ready: watch::Sender<Option<()>>,
pub modified_remote_gadgets: Arc<Vec<Option<bin::check_model_type::RemoteGadget>>>,
pub expanded_remote_gadgets: Option<Vec<Option<u64>>>,
pub syndrome: watch::Sender<Option<BitVector>>,
pub syndrome_count: u64,
pub referring_eids: Vec<u64>,
}
#[derive(Clone)]
pub struct ErrorModel {
pub instance: bin::ErrorModel,
pub modified_remote_check_models: Arc<Vec<Option<bin::error_model_type::RemoteCheckModel>>>,
}
impl FingerprintSource for ErrorModel {
fn instance(&self) -> &bin::ErrorModel {
&self.instance
}
fn modified_remote_check_models(&self) -> &Arc<Vec<Option<bin::error_model_type::RemoteCheckModel>>> {
&self.modified_remote_check_models
}
}
struct PendingPortReferral {
eid: u64,
ri: usize,
owner_gid: u64,
modified_remote: Arc<Vec<Option<bin::error_model_type::RemoteCheckModel>>>,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "cli", derive(StructDoc))]
#[serde(rename_all = "snake_case")]
pub enum WindowParallelism {
#[default]
Sliding,
FullyParallel,
Serial,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "cli", derive(StructDoc))]
#[serde(rename_all = "snake_case")]
pub enum ForcedGapStrategy {
Eager,
#[default]
Lazy,
}
impl WindowCoordinator {
pub fn new(config: serde_json::Value, decoder: DynDecoder) -> Self {
Self::with_gap_decoder(config, decoder, None)
}
#[must_use]
pub fn with_gap_decoder(config: serde_json::Value, decoder: DynDecoder, gap_decoder: Option<DynDecoder>) -> Self {
let config: WindowCoordinatorConfig = serde_json::from_value(config).unwrap();
assert!(
!config.frame_probabilities || config.forced_gap,
"frame_probabilities requires forced_gap"
);
let use_loaded_reweights = config
.decoder_reweighting
.use_loaded(config.persistent_decoder, decoder.features())
.unwrap_or_else(|error| panic!("invalid decoder reweighting configuration: {error}"));
config
.decoder_reweighting
.use_loaded(config.persistent_decoder, gap_decoder.as_ref().unwrap_or(&decoder).features())
.unwrap_or_else(|error| panic!("invalid gap decoder reweighting configuration: {error}"));
let loss_imputation_seed = if config.loss_random_imputation {
use rand::Rng;
Some(config.loss_random_imputation_seed.unwrap_or_else(|| rand::rng().next_u64()))
} else {
None
};
let loss_handler = LossHandler::new(config.loss_strategy, config.loss_config.clone())
.unwrap_or_else(|error| panic!("invalid loss configuration: {error}"));
let forced_gap_state = config.forced_gap.then(|| RwLock::new(ForcedGapState::new()));
let decoding_slots = Arc::new(Semaphore::new(
decoder
.thread_pool()
.or_else(|| gap_decoder.as_ref().and_then(DynDecoder::thread_pool))
.map_or(Semaphore::MAX_PERMITS, |pool| {
pool.current_num_threads().saturating_mul(2).min(Semaphore::MAX_PERMITS)
}),
));
assert!(
!config.forced_gap || !loss_handler.hands_off_to_decoder(),
"forced_gap does not support loss_strategy \"handoff\"; use \"reweight\" or \"ignore\""
);
Self {
config,
port_types: Default::default(),
gadget_types: Default::default(),
check_model_types: Default::default(),
error_model_types: Default::default(),
referenced_error_remotes: RwLock::default(),
gadgets: Default::default(),
check_models: Default::default(),
error_models: Default::default(),
completed_readouts: Mutex::default(),
collection_requests: AtomicU64::new(0),
collection_completed: Mutex::new(0),
pinned_gadgets: Arc::default(),
collected_gids: Mutex::default(),
collected_cids: Mutex::default(),
collected_eids: Mutex::default(),
pending_type_unloads: Mutex::default(),
pending_referring_by_gid: Default::default(),
pending_referring_by_cid: Mutex::default(),
pending_referring_by_port: Default::default(),
next_gid: Mutex::new(1),
next_cid: Mutex::new(1),
next_eid: Mutex::new(1),
loaded_decoders: Default::default(),
prepared_windows: std::sync::Mutex::new(lru::LruCache::new(std::num::NonZeroUsize::new(128).unwrap())),
persistent_prepared_windows: std::sync::Mutex::default(),
prepared_window_preparation: Mutex::new(()),
decoder,
decoding_slots,
gap_decoder,
pauli_frame_tracker: Default::default(),
forced_gap_state,
cancellation: RwLock::default(),
task_counter: TaskCounter::new(),
loss_imputation_seed,
loss_handler,
use_loaded_reweights,
trace_shot: Arc::new(Mutex::new(trace::Shot::default())),
trace: Mutex::new(trace::WindowCoordinatorTrace::default()),
}
}
fn gap_decoder(&self) -> &DynDecoder {
self.gap_decoder.as_ref().unwrap_or(&self.decoder)
}
pub async fn cancel_pending(&self) {
let token = self.cancellation.read().await;
token.cancel();
}
async fn record_event(&self, event: trace::event::Event) {
if self.config.trace_filepath.is_some() {
self.trace_shot.lock().await.events.push(trace::Event {
timestamp_ns: crate::misc::util::timestamp_ns(),
event: Some(event),
});
}
}
}
#[cfg(test)]
#[path = "../../tests/unit/window_coordinator_test.rs"]
mod tests;