Skip to main content

tycho_simulation/evm/
stream.rs

1//! Builder for configuring a multi-protocol stream.
2//!
3//! Provides a builder for creating a multi-protocol stream that produces
4//! protocol state update messages. It runs one synchronization worker per protocol
5//! and a supervisor that aggregates updates, ensuring gap‑free streaming
6//! and robust state tracking.
7//!
8//! ## Context
9//!
10//! This stream wraps a `TychoStream` from `tycho-client`. It decodes `FeedMessage`s
11//! into protocol state updates. Internally, each protocol runs in its own
12//! synchronization worker, and a supervisor aggregates their messages per block.
13//!
14//! ### Protocol Synchronization Worker
15//! A synchronization worker runs the snapshot + delta protocol from `tycho-indexer`.
16//! - It first downloads components and their snapshots.
17//! - It then streams deltas.
18//! - It reacts to new or paused components by pulling snapshots or removing them from the active
19//!   set.
20//!
21//! Each worker emits snapshots and deltas to the supervisor.
22//!
23//! ### Stream Supervisor
24//! The supervisor aggregates worker messages by block and assigns sync status.
25//! - It ensures workers produce gap-free messages.
26//! - It flags late workers as `Delayed`, and marks them `Stale` if they exceed `max_missed_blocks`.
27//! - It marks workers with terminal errors as `Ended`.
28//!
29//! Aggregating by block adds small latency, since the supervisor waits briefly for
30//! all workers to emit. This latency only applies to workers in `Ready` or `Delayed`.
31//!
32//! The stream ends only when **all** workers are `Stale` or `Ended`.
33//!
34//! ## Configuration
35//!
36//! The builder lets you customize:
37//!
38//! ### Protocols
39//! Select which protocols to synchronize.
40//!
41//! ### Tokens & Minimum Token Quality
42//! Provide token metadata up front so the decoder can initialize protocol states from startup
43//! snapshots. `set_tokens` does not act as an ongoing filter — components arriving after startup
44//! include their own token metadata. To restrict processing to specific tokens, apply that filter
45//! in your consumer when reading `new_components`. New tokens arriving via stream deltas are added
46//! automatically when their quality exceeds `min_token_quality`.
47//!
48//! ### StreamEndPolicy
49//! Control when the stream ends based on worker states. By default, it ends when all
50//! workers are `Stale` or `Ended`.
51//!
52//! ## Stream
53//! The stream emits one protocol state update every `block_time`. Each update
54//! reports protocol synchronization states and any changes.
55//!
56//! The `new_components` field lists newly deployed components and their tokens.
57//!
58//! The stream aims to run indefinitely. Internal retry and reconnect logic handle
59//! most errors, so users should rarely need to restart it manually.
60//!
61//! ## Example
62//! ```no_run
63//! use tycho_common::models::Chain;
64//! use tycho_simulation::evm::stream::ProtocolStreamBuilder;
65//! use tycho_simulation::utils::load_all_tokens;
66//! use futures::StreamExt;
67//! use tycho_client::feed::component_tracker::ComponentFilter;
68//! use tycho_simulation::evm::protocol::uniswap_v2::state::UniswapV2State;
69//!
70//! #[tokio::main]
71//! async fn main() {
72//!     let all_tokens = load_all_tokens(
73//!         "tycho-beta.propellerheads.xyz",
74//!         false,
75//!         Some("sampletoken"),
76//!         true,
77//!         Chain::Ethereum,
78//!         None,
79//!         None,
80//!     )
81//!     .await
82//!     .expect("Failed loading tokens");
83//!
84//!     let protocol_stream =
85//!         ProtocolStreamBuilder::new("tycho-beta.propellerheads.xyz", Chain::Ethereum)
86//!             .auth_key(Some("sampletoken".to_string()))
87//!             .skip_state_decode_failures(true)
88//!             .exchange::<UniswapV2State>(
89//!                 "uniswap_v2", ComponentFilter::with_tvl_range(5.0, 10.0), None
90//!             )
91//!             .set_tokens(all_tokens)
92//!             .await
93//!             .build()
94//!             .await
95//!             .expect("Failed building protocol stream");
96//!     tokio::pin!(protocol_stream);
97//!
98//!     // Loop through block updates
99//!     while let Some(msg) = protocol_stream.next().await {
100//!         dbg!(msg).expect("failed decoding");
101//!     }
102//! }
103//! ```
104use std::{
105    collections::{HashMap, HashSet},
106    sync::Arc,
107    time,
108};
109
110use futures::{future::Either, Stream, StreamExt};
111use tokio_stream::wrappers::ReceiverStream;
112use tracing::{debug, error, warn};
113use tycho_client::{
114    feed::{
115        component_tracker::ComponentFilter, synchronizer::ComponentWithState, BlockHeader,
116        BlockSynchronizerError, FeedMessage, SynchronizerState,
117    },
118    stream::{RetryConfiguration, StreamError, TychoStreamBuilder},
119};
120use tycho_common::{
121    models::{token::Token, Chain},
122    simulation::protocol_sim::ProtocolSim,
123    traits::TxDeltaIndexer,
124    Bytes,
125};
126
127use crate::{
128    evm::{
129        decoder::{StreamDecodeError, TychoStreamDecoder},
130        override_stream::{self, StateOverrideProvider},
131        pending::PendingBlockProcessor,
132        protocol::{
133            filters::uniswap_v4_non_angstrom_hook_pool_filter,
134            native_wrapper::state::NativeWrapperState,
135            uniswap_v4::hooks::hook_handler_creator::initialize_hook_handlers,
136        },
137    },
138    protocol::{
139        errors::InvalidSnapshotError,
140        models::{DecoderContext, TryFromWithBlock, Update},
141    },
142    utils::default_blocklist,
143};
144
145const EXCHANGES_REQUIRING_FILTER: [&str; 5] =
146    ["vm:balancer_v2", "fluid_v1", "erc4626", "ekubo_v3", "vm:curve"];
147
148/// The client-side filter exchange `name` always gets, in addition to any filter the caller
149/// provides.
150///
151/// `uniswap_v4_hooks`: without `ANGSTROM_API_KEY`, Angstrom swaps cannot be encoded (they carry
152/// per-block attestations from the Angstrom API), so Angstrom pools are excluded up front rather
153/// than failing every route that selects them at encoding time. A caller's own hook filter does
154/// not replace this one: the encoder still has no key.
155fn mandatory_filter_fn(name: &str) -> Option<fn(&ComponentWithState) -> bool> {
156    if name == "uniswap_v4_hooks" && std::env::var("ANGSTROM_API_KEY").is_err() {
157        warn!(
158            "ANGSTROM_API_KEY is not set: excluding Angstrom pools from '{name}'. \
159             Set the key to include them."
160        );
161        return Some(uniswap_v4_non_angstrom_hook_pool_filter);
162    }
163    None
164}
165
166#[derive(Default, Debug, Clone, Copy)]
167pub enum StreamEndPolicy {
168    /// End stream if all states are Stale or Ended (default)
169    #[default]
170    AllEndedOrStale,
171    /// End stream if any protocol ended
172    AnyEnded,
173    /// End stream if any protocol ended or is stale
174    AnyEndedOrStale,
175    /// End stream if any protocol is stale
176    AnyStale,
177}
178
179impl StreamEndPolicy {
180    fn should_end<'a>(&self, states: impl IntoIterator<Item = &'a SynchronizerState>) -> bool {
181        let mut it = states.into_iter();
182        match self {
183            StreamEndPolicy::AllEndedOrStale => false,
184            StreamEndPolicy::AnyEnded => it.any(|s| matches!(s, SynchronizerState::Ended(_))),
185            StreamEndPolicy::AnyStale => it.any(|s| matches!(s, SynchronizerState::Stale(_))),
186            StreamEndPolicy::AnyEndedOrStale => {
187                it.any(|s| matches!(s, SynchronizerState::Stale(_) | SynchronizerState::Ended(_)))
188            }
189        }
190    }
191}
192
193/// Handle returned by [`ProtocolStreamBuilder::with_step_controller`] that gives external
194/// control over when each buffered block is released for decoding.
195///
196/// Intended for complex test scenarios where the caller needs to observe what the next
197/// block contains before allowing the decoder pipeline to process it.
198///
199/// ## Drop behaviour
200///
201/// Dropping this controller ungates the stream: the gating task detects the closed trigger
202/// channel, forwards the currently-buffered block (if any), then continues passing subsequent
203/// blocks through without waiting for triggers — exactly as if step-control had never been
204/// enabled. The stream runs to its natural end.
205pub struct BlockStepController {
206    /// Sends a trigger signal to release the next buffered block.
207    trigger_tx: tokio::sync::mpsc::UnboundedSender<()>,
208    /// Watch channel containing the next buffered raw message, or `None` if no block is pending.
209    peek_rx: tokio::sync::watch::Receiver<Option<FeedMessage<BlockHeader>>>,
210}
211
212impl BlockStepController {
213    /// Releases the next buffered block for decoding and emission.
214    ///
215    /// Returns an error if the stream has already ended and the sender is disconnected.
216    pub fn trigger_next_block(&self) -> Result<(), tokio::sync::mpsc::error::SendError<()>> {
217        // Send a unit value on the trigger channel to unblock the gating task.
218        self.trigger_tx.send(())
219    }
220
221    /// Returns the currently buffered block immediately, or `None` if no block is buffered yet.
222    pub fn try_peek_next_block(&self) -> Option<FeedMessage<BlockHeader>> {
223        self.peek_rx.borrow().clone()
224    }
225
226    /// Waits until a block is buffered and returns it without consuming it.
227    ///
228    /// Returns `None` only if the stream has ended and no further blocks will arrive.
229    /// If a block is already buffered when this is called, it returns immediately.
230    pub async fn peek_next_block(&self) -> Option<FeedMessage<BlockHeader>> {
231        // Clone so we don't hold a mutable borrow on self; wait_for checks the current
232        // value first, so this returns immediately if a block is already present.
233        let mut rx = self.peek_rx.clone();
234        let guard = rx
235            .wait_for(|v| v.is_some())
236            .await
237            .ok()?;
238        guard.clone()
239    }
240}
241
242/// Builds and configures the multi protocol stream described in the [module-level docs](self).
243///
244/// See the module documentation for details on protocols, configuration options, and
245/// stream behavior.
246pub struct ProtocolStreamBuilder {
247    decoder: TychoStreamDecoder<BlockHeader>,
248    stream_builder: TychoStreamBuilder,
249    stream_end_policy: StreamEndPolicy,
250    chain: Chain,
251    pending_indexers: HashMap<String, Box<dyn TxDeltaIndexer>>,
252    /// Watch sender used to publish the currently-buffered raw block so the controller can peek
253    /// at it before triggering. `Some` iff step-control mode is active.
254    step_peek_tx: Option<tokio::sync::watch::Sender<Option<FeedMessage<BlockHeader>>>>,
255    /// Receiver half of the trigger channel. Held here until `build()` / `build_with_pending()`
256    /// transfers ownership to the gating task. `Some` iff step-control mode is active.
257    step_trigger_rx: Option<tokio::sync::mpsc::UnboundedReceiver<()>>,
258    /// State-override providers explicitly registered by the consumer, keyed by `protocol_system`.
259    /// These take precedence over the built-in default registry and are installed onto the decoder
260    /// at build time.
261    override_providers: HashMap<String, Arc<dyn StateOverrideProvider>>,
262    /// Names of all exchanges registered on the builder, used to decide which built-in override
263    /// providers to auto-register at build time.
264    registered_exchanges: HashSet<String>,
265}
266
267impl ProtocolStreamBuilder {
268    /// Creates a new builder for a multi-protocol stream.
269    ///
270    /// The shipped pool blocklist is applied by default, excluding components known to break
271    /// simulation. Use [`blocklist_components`](Self::blocklist_components) to exclude additional
272    /// components.
273    ///
274    /// See the [module-level docs](self) for full details on stream behavior and configuration.
275    pub fn new(tycho_url: &str, chain: Chain) -> Self {
276        Self {
277            decoder: TychoStreamDecoder::new(chain),
278            stream_builder: TychoStreamBuilder::new(tycho_url, chain)
279                .blocklisted_ids(default_blocklist()),
280            stream_end_policy: StreamEndPolicy::default(),
281            chain,
282            pending_indexers: HashMap::new(),
283            step_peek_tx: None,
284            step_trigger_rx: None,
285            override_providers: HashMap::new(),
286            registered_exchanges: HashSet::new(),
287        }
288    }
289
290    /// Adds a specific exchange to the stream.
291    ///
292    /// This configures the builder to include a new protocol synchronizer for `name`,
293    /// filtering its components according to `filter` and optionally `filter_fn`.
294    ///
295    /// The type parameter `T` specifies the decoder type for this exchange. All
296    /// component states for this exchange will be decoded into instances of `T`.
297    ///
298    /// # Parameters
299    ///
300    /// - `name`: The protocol or exchange name (e.g., `"uniswap_v4"`, `"vm:balancer_v2"`).
301    /// - `filter`: Defines the set of components to include in the stream.
302    /// - `filter_fn`: Optional custom filter function for client-side filtering of components not
303    ///   expressible in `filter`.
304    ///
305    /// # Notes
306    ///
307    /// For certain protocols (e.g., `"uniswap_v4"`, `"vm:balancer_v2"`, `"vm:curve"`), omitting
308    /// `filter_fn` may cause decoding errors or incorrect results. In these cases, a proper
309    /// filter function is required to ensure correct decoding and quoting logic.
310    pub fn exchange<T>(
311        mut self,
312        name: &str,
313        filter: ComponentFilter,
314        filter_fn: Option<fn(&ComponentWithState) -> bool>,
315    ) -> Self
316    where
317        T: ProtocolSim
318            + TryFromWithBlock<ComponentWithState, BlockHeader, Error = InvalidSnapshotError>
319            + Send
320            + 'static,
321    {
322        self.stream_builder = self
323            .stream_builder
324            .exchange(name, filter);
325        self.registered_exchanges
326            .insert(name.to_string());
327        self.decoder.register_decoder::<T>(name);
328        if let Some(predicate) = filter_fn {
329            self.decoder
330                .register_filter(name, predicate);
331        }
332        if let Some(predicate) = mandatory_filter_fn(name) {
333            self.decoder
334                .register_filter(name, predicate);
335        }
336
337        if EXCHANGES_REQUIRING_FILTER.contains(&name) && filter_fn.is_none() {
338            warn!(
339                "Warning: For exchange type '{}', it is necessary to set a filter function because not all pools are supported. See all filters at src/evm/protocol/filters.rs",
340                name
341            );
342        }
343
344        self
345    }
346
347    /// Adds a specific exchange to the stream with decoder context.
348    ///
349    /// This configures the builder to include a new protocol synchronizer for `name`,
350    /// filtering its components according to `filter` and optionally `filter_fn`. It also registers
351    /// the DecoderContext (this is useful to test protocols that are not live yet)
352    ///
353    /// The type parameter `T` specifies the decoder type for this exchange. All
354    /// component states for this exchange will be decoded into instances of `T`.
355    ///
356    /// # Parameters
357    ///
358    /// - `name`: The protocol or exchange name (e.g., `"uniswap_v4"`, `"vm:balancer_v2"`).
359    /// - `filter`: Defines the set of components to include in the stream.
360    /// - `filter_fn`: Optional custom filter function for client-side filtering of components not
361    ///   expressible in `filter`.
362    /// - `decoder_context`: The decoder context for this exchange
363    ///
364    /// # Notes
365    ///
366    /// For certain protocols (e.g., `"uniswap_v4"`, `"vm:balancer_v2"`, `"vm:curve"`), omitting
367    /// `filter_fn` may cause decoding errors or incorrect results. In these cases, a proper
368    /// filter function is required to ensure correct decoding and quoting logic.
369    pub fn exchange_with_decoder_context<T>(
370        mut self,
371        name: &str,
372        filter: ComponentFilter,
373        filter_fn: Option<fn(&ComponentWithState) -> bool>,
374        decoder_context: DecoderContext,
375    ) -> Self
376    where
377        T: ProtocolSim
378            + TryFromWithBlock<ComponentWithState, BlockHeader, Error = InvalidSnapshotError>
379            + Send
380            + 'static,
381    {
382        self.stream_builder = self
383            .stream_builder
384            .exchange(name, filter);
385        self.registered_exchanges
386            .insert(name.to_string());
387        self.decoder
388            .register_decoder_with_context::<T>(name, decoder_context);
389        if let Some(predicate) = filter_fn {
390            self.decoder
391                .register_filter(name, predicate);
392        }
393        if let Some(predicate) = mandatory_filter_fn(name) {
394            self.decoder
395                .register_filter(name, predicate);
396        }
397
398        if EXCHANGES_REQUIRING_FILTER.contains(&name) && filter_fn.is_none() {
399            warn!(
400                "Warning: For exchange type '{}', it is necessary to set a filter function because not all pools are supported. See all filters at src/evm/protocol/filters.rs",
401                name
402            );
403        }
404
405        self
406    }
407
408    /// Sets the block time interval for the stream.
409    ///
410    /// This controls how often the stream produces updates.
411    pub fn block_time(mut self, block_time: u64) -> Self {
412        self.stream_builder = self
413            .stream_builder
414            .block_time(block_time);
415        self
416    }
417
418    /// Sets the network operation timeout (deprecated).
419    ///
420    /// Use [`latency_buffer()`](Self::latency_buffer) instead for controlling latency.
421    /// This method is retained for backwards compatibility.
422    #[deprecated = "Use latency_buffer instead"]
423    pub fn timeout(mut self, timeout: u64) -> Self {
424        self.stream_builder = self.stream_builder.timeout(timeout);
425        self
426    }
427
428    /// Sets the latency buffer to aggregate same-block messages.
429    ///
430    /// This allows the supervisor to wait a short interval for all synchronizers to emit
431    /// before aggregating.
432    pub fn latency_buffer(mut self, timeout: u64) -> Self {
433        self.stream_builder = self.stream_builder.timeout(timeout);
434        self
435    }
436
437    /// Sets the maximum number of blocks a synchronizer may miss before being marked as `Stale`.
438    pub fn max_missed_blocks(mut self, n: u64) -> Self {
439        self.stream_builder = self.stream_builder.max_missed_blocks(n);
440        self
441    }
442
443    /// Sets how long a synchronizer may take to process the initial message.
444    ///
445    /// Useful for data-intensive protocols where startup decoding takes longer.
446    pub fn startup_timeout(mut self, timeout: time::Duration) -> Self {
447        self.stream_builder = self
448            .stream_builder
449            .startup_timeout(timeout);
450        self
451    }
452
453    /// Configures the stream to exclude state updates.
454    ///
455    /// This reduces bandwidth and decoding workload if protocol state is not of
456    /// interest (e.g. only process new tokens).
457    pub fn no_state(mut self, no_state: bool) -> Self {
458        self.stream_builder = self.stream_builder.no_state(no_state);
459        self
460    }
461
462    /// Sets the API key for authenticating with the Tycho server.
463    pub fn auth_key(mut self, auth_key: Option<String>) -> Self {
464        self.stream_builder = self.stream_builder.auth_key(auth_key);
465        self
466    }
467
468    /// Adds client-metadata entries forwarded to the server in the `X-Tycho-Client-Metadata`
469    /// header.
470    ///
471    /// See [`TychoStreamBuilder::add_client_metadata`]. Values are self-reported and may surface in
472    /// the server's metrics and logs — do not include secrets or personally identifiable
473    /// information.
474    pub fn add_client_metadata<I, K, V>(mut self, metadata: I) -> Self
475    where
476        I: IntoIterator<Item = (K, V)>,
477        K: Into<String>,
478        V: Into<String>,
479    {
480        self.stream_builder = self
481            .stream_builder
482            .add_client_metadata(metadata);
483        self
484    }
485
486    /// Disables TLS/ SSL for the connection, using http and ws protocols.
487    ///
488    /// This is not recommended for production use.
489    pub fn no_tls(mut self, no_tls: bool) -> Self {
490        self.stream_builder = self.stream_builder.no_tls(no_tls);
491        self
492    }
493
494    /// Disable compression for the connection.
495    pub fn disable_compression(mut self) -> Self {
496        self.stream_builder = self
497            .stream_builder
498            .disable_compression();
499        self
500    }
501
502    /// Enables partial block updates (flashblocks).
503    pub fn enable_partial_blocks(mut self) -> Self {
504        self.stream_builder = self
505            .stream_builder
506            .enable_partial_blocks();
507        self
508    }
509
510    /// Sets the number of deltas buffered for each underlying Tycho WebSocket subscription.
511    ///
512    /// See [`TychoStreamBuilder::subscription_buffer_size`].
513    pub fn subscription_buffer_size(mut self, subscription_buffer_size: usize) -> Self {
514        self.stream_builder = self
515            .stream_builder
516            .subscription_buffer_size(subscription_buffer_size);
517        self
518    }
519
520    /// Exclude additional component IDs from all registered exchanges.
521    ///
522    /// These IDs are added to the shipped blocklist that is already applied by default (see
523    /// [`new`](Self::new)).
524    pub fn blocklist_components(mut self, ids: HashSet<String>) -> Self {
525        if !ids.is_empty() {
526            tracing::info!("Blocklisting {} components", ids.len());
527            self.stream_builder = self.stream_builder.blocklisted_ids(ids);
528        }
529        self
530    }
531
532    /// Sets the stream end policy.
533    ///
534    /// Controls when the stream should stop based on synchronizer states.
535    ///
536    /// ## Note
537    /// The stream always ends latest if all protocols are stale or ended independent of
538    /// this configuration. This allows you to end the stream earlier than that.
539    ///
540    /// See [self::StreamEndPolicy] for possible configuration options.
541    pub fn stream_end_policy(mut self, stream_end_policy: StreamEndPolicy) -> Self {
542        self.stream_end_policy = stream_end_policy;
543        self
544    }
545
546    /// Provides token metadata used to decode startup snapshots and initialize protocol states.
547    ///
548    /// This is not a stream filter — components arriving after startup include their own token
549    /// metadata. To restrict to specific tokens, filter in your consumer logic. New tokens
550    /// arriving via stream deltas are added automatically if they meet the quality threshold.
551    pub async fn set_tokens(self, tokens: HashMap<Bytes, Token>) -> Self {
552        self.decoder.set_tokens(tokens).await;
553        self
554    }
555
556    /// Skips decoding errors for component state updates.
557    ///
558    /// Allows the stream to continue processing even if some states fail to decode,
559    /// logging a warning instead of panicking.
560    pub fn skip_state_decode_failures(mut self, skip: bool) -> Self {
561        self.decoder
562            .skip_state_decode_failures(skip);
563        self
564    }
565
566    /// Sets the minimum token quality for tokens added via the stream.
567    ///
568    /// Tokens arriving in stream deltas below this threshold are ignored. Defaults to 100.
569    /// Set this to the same value used in [`load_all_tokens()`](crate::utils::load_all_tokens) to
570    /// apply consistent filtering.
571    pub fn min_token_quality(mut self, quality: u32) -> Self {
572        self.decoder.min_token_quality(quality);
573        self
574    }
575
576    /// Configures the retry policy for websocket reconnects.
577    pub fn websocket_retry_config(mut self, config: &RetryConfiguration) -> Self {
578        self.stream_builder = self
579            .stream_builder
580            .websockets_retry_config(config);
581        self
582    }
583
584    /// Configures the retry policy for state synchronization.
585    pub fn state_synchronizer_retry_config(mut self, config: &RetryConfiguration) -> Self {
586        self.stream_builder = self
587            .stream_builder
588            .state_synchronizer_retry_config(config);
589        self
590    }
591
592    /// Sets the maximum random delay before a restarted synchronizer requests a snapshot.
593    ///
594    /// Defaults to 60 seconds. A `jitter` below 5 seconds is raised to 5 seconds.
595    pub fn snapshot_jitter(mut self, jitter: time::Duration) -> Self {
596        self.stream_builder = self
597            .stream_builder
598            .snapshot_jitter(jitter);
599        self
600    }
601
602    pub fn get_decoder(&self) -> &TychoStreamDecoder<BlockHeader> {
603        &self.decoder
604    }
605
606    /// Registers a [`TxDeltaIndexer`] for ephemeral pending-block simulation.
607    ///
608    /// The indexer is associated with `extractor` (the protocol synchronizer name, e.g.
609    /// `"uniswap_v3"`). Use [`build_with_pending`](Self::build_with_pending) to obtain both
610    /// the confirmed stream and the pending processor.
611    ///
612    /// The exchange must decode into a state whose `delta_transition` can rebuild it from the
613    /// `state_deltas` the indexer produces, because that is all
614    /// [`apply_deltas_ephemeral`](crate::evm::decoder::TychoStreamDecoder::apply_deltas_ephemeral)
615    /// applies. Native and hybrid states qualify; the generic VM adapter does not, because it
616    /// re-reads pool state from the VM database — an indexer registered for one still gets its
617    /// balance and block-environment attributes applied, but every storage-derived value stays at
618    /// the confirmed block, with no error.
619    #[expect(
620        clippy::unnecessary_wraps,
621        reason = "public API; changing the signature is a breaking change"
622    )]
623    pub fn with_pending_indexer(
624        mut self,
625        extractor: &str,
626        indexer: Box<dyn TxDeltaIndexer>,
627    ) -> Result<Self, StreamError> {
628        self.pending_indexers
629            .insert(extractor.to_string(), indexer);
630        Ok(self)
631    }
632
633    /// Enables controlled-step mode for testing.
634    ///
635    /// Returns a [`BlockStepController`] that lets the caller decide when each buffered block
636    /// is released for decoding. Call this before [`build`](Self::build) or
637    /// [`build_with_pending`](Self::build_with_pending) — both detect and wire up the gating
638    /// automatically.
639    ///
640    /// In production code, do not call this method; the stream runs at full speed.
641    pub fn with_step_controller(mut self) -> (Self, BlockStepController) {
642        let (trigger_tx, trigger_rx) = tokio::sync::mpsc::unbounded_channel::<()>();
643        let (peek_tx, peek_rx) =
644            tokio::sync::watch::channel::<Option<FeedMessage<BlockHeader>>>(None);
645
646        self.step_peek_tx = Some(peek_tx);
647        self.step_trigger_rx = Some(trigger_rx);
648
649        let controller = BlockStepController { trigger_tx, peek_rx };
650        (self, controller)
651    }
652
653    /// Spawns a background task that gates `FeedMessage` delivery.
654    ///
655    /// The task buffers each incoming message, publishes it to `peek_tx` so the
656    /// [`BlockStepController`] can inspect it, waits for a trigger, then forwards the message to
657    /// `output_tx` for the decode pipeline. If `advance_tx` is `Some`, a clone of the message is
658    /// also forwarded there (used by the pending-processor path) before the decode step.
659    /// When the input channel closes or a terminal error is received according to
660    /// `stream_end_policy`, the task exits and all output channels are dropped.
661    fn run_gating_task(
662        raw_rx: tokio::sync::mpsc::Receiver<
663            Result<FeedMessage<BlockHeader>, BlockSynchronizerError>,
664        >,
665        mut trigger_rx: tokio::sync::mpsc::UnboundedReceiver<()>,
666        peek_tx: tokio::sync::watch::Sender<Option<FeedMessage<BlockHeader>>>,
667        output_tx: tokio::sync::mpsc::Sender<FeedMessage<BlockHeader>>,
668        stream_end_policy: StreamEndPolicy,
669    ) {
670        tokio::spawn(async move {
671            let mut raw_stream = ReceiverStream::new(raw_rx);
672            loop {
673                let msg = match raw_stream.next().await {
674                    Some(Ok(msg)) => msg,
675                    Some(Err(e)) => {
676                        error!("Block stream ended with terminal error: {e}");
677                        break;
678                    }
679                    None => break,
680                };
681
682                if stream_end_policy.should_end(msg.sync_states.values()) {
683                    error!(
684                        "Block stream ended due to {:?}: {:?}",
685                        stream_end_policy, msg.sync_states
686                    );
687                    break;
688                }
689
690                // Publish the buffered message so the caller can peek before triggering.
691                #[expect(
692                    clippy::let_underscore_must_use,
693                    reason = "the peek slot is optional; a send error only means the controller dropped its receiver"
694                )]
695                let _ = peek_tx.send(Some(msg.clone()));
696
697                // Block until the controller fires trigger_next_block(), or until it is dropped.
698                if trigger_rx.recv().await.is_none() {
699                    // Controller dropped — forward the buffered message and drain the rest
700                    // without gating, so the stream continues to its natural end.
701                    #[expect(
702                        clippy::let_underscore_must_use,
703                        reason = "the peek slot is optional; a send error only means the controller dropped its receiver"
704                    )]
705                    let _ = peek_tx.send(None);
706                    if output_tx.send(msg).await.is_err() {
707                        break;
708                    }
709                    while let Some(item) = raw_stream.next().await {
710                        let Ok(msg) = item else { break };
711                        if stream_end_policy.should_end(msg.sync_states.values()) {
712                            break;
713                        }
714                        if output_tx.send(msg).await.is_err() {
715                            break;
716                        }
717                    }
718                    break;
719                }
720
721                // Clear the peek slot before decoding so callers see None between blocks.
722                #[expect(
723                    clippy::let_underscore_must_use,
724                    reason = "the peek slot is optional; a send error only means the controller dropped its receiver"
725                )]
726                let _ = peek_tx.send(None);
727
728                if output_tx.send(msg).await.is_err() {
729                    break;
730                }
731            }
732        });
733    }
734
735    /// Registers `provider` as the live override source for `protocol_system`.
736    ///
737    /// Explicit registrations take precedence over the built-in default registry, so this is how
738    /// you swap a venue (e.g. `vm:bopamm`) onto a different provider. Registering the same provider
739    /// for several protocols is cheap — it is shared via `Arc`, not duplicated.
740    pub fn with_override_provider(
741        mut self,
742        protocol_system: impl Into<String>,
743        provider: Arc<dyn StateOverrideProvider>,
744    ) -> Self {
745        self.override_providers
746            .insert(protocol_system.into(), provider);
747        self
748    }
749
750    /// Installs override providers onto the decoder before the stream is built.
751    ///
752    /// Explicit consumer registrations win; the built-in default registry (see
753    /// [`default_override_providers`](crate::evm::override_stream::default_override_providers))
754    /// then fills every remaining protocol it can serve.
755    fn install_override_providers(&mut self) {
756        let explicit = std::mem::take(&mut self.override_providers);
757        // Protocols eligible for a built-in default provider: registered exchanges not explicitly
758        // overridden by the consumer.
759        let uncovered = self
760            .registered_exchanges
761            .clone()
762            .into_iter()
763            .filter(|exchange| !explicit.contains_key(exchange));
764        let defaults = override_stream::default_override_providers(uncovered);
765        for (protocol_system, provider) in defaults.into_iter().chain(explicit) {
766            self.decoder
767                .set_override_provider(protocol_system, provider);
768        }
769    }
770
771    /// Builds the confirmed protocol stream and a [`PendingBlockProcessor`] that stays
772    /// in sync with it automatically.
773    ///
774    /// The stream pipeline forwards every confirmed [`FeedMessage`] to the processor via an
775    /// internal unbounded channel — it never blocks waiting for the consumer. The consumer
776    /// owns the returned `PendingBlockProcessor` exclusively and may wrap it in whatever
777    /// synchronisation primitive suits their use case (e.g. `Mutex` for shared access,
778    /// nothing for single-threaded use).
779    ///
780    /// Call [`generate_pending_update`](PendingBlockProcessor::generate_pending_update) to
781    /// simulate a candidate bundle; it drains the channel automatically before computing.
782    pub async fn build_with_pending(
783        mut self,
784    ) -> Result<
785        (impl Stream<Item = Result<Update, StreamDecodeError>>, PendingBlockProcessor),
786        StreamError,
787    > {
788        initialize_hook_handlers().map_err(|e| {
789            StreamError::SetUpError(format!("Error initializing hook handlers: {e:?}"))
790        })?;
791        self.install_override_providers();
792        let (_, rx) = self.stream_builder.build().await?;
793        let decoder = Arc::new(self.decoder);
794
795        let (advance_tx, advance_rx) =
796            tokio::sync::mpsc::unbounded_channel::<FeedMessage<BlockHeader>>();
797        let pending = PendingBlockProcessor::new(
798            self.pending_indexers,
799            decoder.clone(),
800            self.chain,
801            advance_rx,
802        );
803
804        let chain = self.chain;
805        let stream_end_policy = self.stream_end_policy;
806
807        let decode_stream: Box<dyn Stream<Item = FeedMessage<BlockHeader>> + Send + Unpin> =
808            if let (Some(peek_tx), Some(trigger_rx)) = (self.step_peek_tx, self.step_trigger_rx) {
809                let (gated_tx, gated_rx) =
810                    tokio::sync::mpsc::channel::<FeedMessage<BlockHeader>>(1);
811                Self::run_gating_task(rx, trigger_rx, peek_tx, gated_tx, stream_end_policy);
812                Box::new(ReceiverStream::new(gated_rx))
813            } else {
814                let normal = ReceiverStream::new(rx)
815                    .take_while(move |msg| match msg {
816                        Ok(msg) => {
817                            let states = msg.sync_states.values();
818                            if stream_end_policy.should_end(states) {
819                                error!(
820                                    "Block stream ended due to {:?}: {:?}",
821                                    stream_end_policy, msg.sync_states
822                                );
823                                futures::future::ready(false)
824                            } else {
825                                futures::future::ready(true)
826                            }
827                        }
828                        Err(e) => {
829                            error!("Block stream ended with terminal error: {e}");
830                            futures::future::ready(false)
831                        }
832                    })
833                    .map(|msg| msg.expect("Safe since stream ends if we receive an error"));
834                Box::new(Box::pin(normal))
835            };
836
837        let stream = Box::pin(decode_stream.then({
838            let decoder = decoder;
839            move |msg| {
840                let decoder = decoder.clone();
841                let advance_tx = advance_tx.clone();
842                async move {
843                    if let Err(e) = advance_tx.send(msg.clone()) {
844                        warn!(error = %e, "Pending block processor is gone; block not forwarded");
845                    }
846                    decoder.decode(&msg).await.map_err(|e| {
847                        debug!(msg=?msg, "Decode error: {}", e);
848                        e
849                    })
850                }
851            }
852        }));
853        let stream = inject_native_wrapper(stream, chain);
854        Ok((stream, pending))
855    }
856
857    /// Builds and returns the configured protocol stream.
858    ///
859    /// See the module-level docs for details on stream behavior and emitted messages.
860    /// This method applies all builder settings and starts the stream.
861    pub async fn build(
862        mut self,
863    ) -> Result<impl Stream<Item = Result<Update, StreamDecodeError>>, StreamError> {
864        initialize_hook_handlers().map_err(|e| {
865            StreamError::SetUpError(format!("Error initializing hook handlers: {e:?}"))
866        })?;
867        self.install_override_providers();
868        let (_, rx) = self.stream_builder.build().await?;
869        let decoder = Arc::new(self.decoder);
870        let chain = self.chain;
871        let stream_end_policy = self.stream_end_policy;
872
873        let decode_stream: Box<dyn Stream<Item = FeedMessage<BlockHeader>> + Send + Unpin> =
874            if let (Some(peek_tx), Some(trigger_rx)) = (self.step_peek_tx, self.step_trigger_rx) {
875                let (gated_tx, gated_rx) =
876                    tokio::sync::mpsc::channel::<FeedMessage<BlockHeader>>(1);
877                Self::run_gating_task(rx, trigger_rx, peek_tx, gated_tx, stream_end_policy);
878                Box::new(ReceiverStream::new(gated_rx))
879            } else {
880                let normal = ReceiverStream::new(rx)
881                    .take_while(move |msg| match msg {
882                        Ok(msg) => {
883                            let states = msg.sync_states.values();
884                            if stream_end_policy.should_end(states) {
885                                error!(
886                                    "Block stream ended due to {:?}: {:?}",
887                                    stream_end_policy, msg.sync_states
888                                );
889                                futures::future::ready(false)
890                            } else {
891                                futures::future::ready(true)
892                            }
893                        }
894                        Err(e) => {
895                            error!("Block stream ended with terminal error: {e}");
896                            futures::future::ready(false)
897                        }
898                    })
899                    .map(|msg| msg.expect("Safe since stream ends if we receive an error"));
900                Box::new(Box::pin(normal))
901            };
902
903        let stream = Box::pin(decode_stream.then({
904            let decoder = decoder;
905            move |msg| {
906                let decoder = decoder.clone();
907                async move {
908                    decoder.decode(&msg).await.map_err(|e| {
909                        debug!(msg=?msg, "Decode error: {}", e);
910                        e
911                    })
912                }
913            }
914        }));
915        let stream = inject_native_wrapper(stream, chain);
916        Ok(stream)
917    }
918}
919
920/// Wraps a decoded protocol stream to inject a `NativeWrapperState` component
921/// on the first successful update.
922///
923/// Skips injection when the chain's native asset has no real wrapper contract.
924fn inject_native_wrapper(
925    inner: impl Stream<Item = Result<Update, StreamDecodeError>> + Unpin + Send + 'static,
926    chain: Chain,
927) -> impl Stream<Item = Result<Update, StreamDecodeError>> + Send {
928    let Some(native_wrapper) = NativeWrapperState::new(chain) else {
929        return Either::Left(inner);
930    };
931
932    let mut pending_wrapper = Some(native_wrapper);
933    Either::Right(inner.map(move |result| {
934        result.map(|mut update| {
935            if let Some(native_wrapper) = pending_wrapper.take() {
936                let component = native_wrapper.component();
937                let id = component.id.to_string();
938                update
939                    .new_pairs
940                    .insert(id.clone(), component);
941                update
942                    .states
943                    .insert(id, Box::new(native_wrapper));
944                debug!("Injected native_wrapper component for {chain}");
945            }
946            update
947        })
948    }))
949}
950
951#[cfg(test)]
952mod tests {
953    use std::collections::HashMap;
954
955    use futures::{stream, StreamExt};
956    use tycho_common::models::Chain;
957
958    use super::*;
959    use crate::{
960        evm::protocol::native_wrapper::state::NATIVE_WRAPPER_ID, protocol::models::Update,
961    };
962
963    fn empty_update(block: u64) -> Update {
964        Update::new(block, HashMap::new(), HashMap::new())
965    }
966
967    #[tokio::test]
968    async fn test_inject_native_wrapper_first_message_only() {
969        let updates = vec![Ok(empty_update(1)), Ok(empty_update(2)), Ok(empty_update(3))];
970        let input = stream::iter(updates);
971
972        let results: Vec<_> = inject_native_wrapper(input, Chain::Ethereum)
973            .collect()
974            .await;
975
976        assert_eq!(results.len(), 3);
977
978        let expected_id = NativeWrapperState::new(Chain::Ethereum)
979            .expect("Ethereum should have a wrapper")
980            .component()
981            .id
982            .to_string();
983
984        let first = results[0]
985            .as_ref()
986            .expect("first update ok");
987        assert!(
988            first
989                .new_pairs
990                .contains_key(&expected_id),
991            "first message should have native_wrapper component"
992        );
993        assert!(
994            first.states.contains_key(&expected_id),
995            "first message should have native_wrapper state"
996        );
997
998        let second = results[1]
999            .as_ref()
1000            .expect("second update ok");
1001        assert!(
1002            !second
1003                .new_pairs
1004                .contains_key(&expected_id),
1005            "second message should NOT have native_wrapper component"
1006        );
1007        assert!(
1008            !second.states.contains_key(&expected_id),
1009            "second message should NOT have native_wrapper state"
1010        );
1011    }
1012
1013    #[tokio::test]
1014    async fn test_inject_native_wrapper_after_initial_decode_error() {
1015        let updates = vec![
1016            Err(StreamDecodeError::Fatal("decode failed".to_string())),
1017            Ok(empty_update(1)),
1018            Ok(empty_update(2)),
1019        ];
1020        let results: Vec<_> = inject_native_wrapper(stream::iter(updates), Chain::Ethereum)
1021            .collect()
1022            .await;
1023
1024        assert!(results[0].is_err(), "the decode error must pass through unchanged");
1025        let expected_id = NativeWrapperState::new(Chain::Ethereum)
1026            .expect("Ethereum should have a wrapper")
1027            .component()
1028            .id
1029            .to_string();
1030        let first_success = results[1]
1031            .as_ref()
1032            .expect("second stream item should decode");
1033        assert!(first_success
1034            .new_pairs
1035            .contains_key(&expected_id));
1036        assert!(first_success
1037            .states
1038            .contains_key(&expected_id));
1039
1040        let second_success = results[2]
1041            .as_ref()
1042            .expect("third stream item should decode");
1043        assert!(!second_success
1044            .new_pairs
1045            .contains_key(&expected_id));
1046        assert!(!second_success
1047            .states
1048            .contains_key(&expected_id));
1049    }
1050
1051    #[tokio::test]
1052    async fn test_does_not_inject_native_wrapper_for_arc_shared_balance() {
1053        let results: Vec<_> =
1054            inject_native_wrapper(stream::iter([Ok(empty_update(1))]), Chain::Arc)
1055                .collect()
1056                .await;
1057
1058        let update = results[0]
1059            .as_ref()
1060            .expect("update should decode");
1061        assert!(!update
1062            .new_pairs
1063            .contains_key(NATIVE_WRAPPER_ID));
1064        assert!(!update
1065            .states
1066            .contains_key(NATIVE_WRAPPER_ID));
1067    }
1068
1069    /// Verifies that `with_step_controller` returns both a modified builder and a controller.
1070    ///
1071    /// This test only checks that the builder method is callable and that the returned controller
1072    /// compiles — it does not start any network connection.
1073    #[tokio::test]
1074    async fn test_with_step_controller_returns_controller() {
1075        let builder = ProtocolStreamBuilder::new("tycho-beta.propellerheads.xyz", Chain::Ethereum);
1076        let (_builder, controller) = builder.with_step_controller();
1077        // The controller was successfully returned — verifying the public API is callable.
1078        drop(controller);
1079    }
1080
1081    #[tokio::test]
1082    async fn test_subscription_buffer_size_forwards_zero_rejection() {
1083        let error = ProtocolStreamBuilder::new("not a valid endpoint", Chain::Ethereum)
1084            .subscription_buffer_size(0)
1085            .build()
1086            .await;
1087
1088        let Err(error) = error else {
1089            panic!("a zero subscription buffer size must be rejected during setup");
1090        };
1091
1092        assert!(
1093            error
1094                .to_string()
1095                .contains("subscription buffer size must be greater than zero"),
1096            "the underlying Tycho stream builder should return the configuration error: {error}"
1097        );
1098    }
1099
1100    #[tokio::test]
1101    async fn test_subscription_buffer_size_forwards_upper_bound_rejection() {
1102        let error = ProtocolStreamBuilder::new("not a valid endpoint", Chain::Ethereum)
1103            .subscription_buffer_size(usize::MAX)
1104            .build()
1105            .await;
1106
1107        let Err(error) = error else {
1108            panic!("an oversized subscription buffer size must be rejected during setup");
1109        };
1110
1111        assert!(
1112            error
1113                .to_string()
1114                .contains("subscription buffer size must not exceed"),
1115            "the underlying Tycho stream builder should return the configuration error: {error}"
1116        );
1117    }
1118
1119    /// Connects to a live Tycho instance, verifies that the stream blocks until
1120    /// `trigger_next_block` is called, and that `peek_next_block` exposes the buffered message.
1121    #[ignore = "requires live Tycho connection (TYCHO_AUTH_TOKEN env var)"]
1122    #[tokio::test]
1123    async fn test_step_controller_trigger_releases_block() {
1124        use std::{env, time::Duration};
1125
1126        use crate::evm::protocol::uniswap_v2::state::UniswapV2State;
1127
1128        let auth = env::var("TYCHO_AUTH_TOKEN").expect("TYCHO_AUTH_TOKEN must be set");
1129
1130        // Track a single well-known pool to minimise startup latency.
1131        let usdc_weth_v2 = "0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc".to_string();
1132        let (builder, controller) =
1133            ProtocolStreamBuilder::new("tycho-beta.propellerheads.xyz", Chain::Ethereum)
1134                .auth_key(Some(auth))
1135                .exchange::<UniswapV2State>(
1136                    "uniswap_v2",
1137                    ComponentFilter::Ids(vec![usdc_weth_v2]),
1138                    None,
1139                )
1140                .with_step_controller();
1141
1142        let (stream, _pending) = builder
1143            .build_with_pending()
1144            .await
1145            .expect("build_with_pending failed");
1146        tokio::pin!(stream);
1147
1148        // Wait up to 60 s for the first block to arrive in the gating buffer.
1149        let peeked = tokio::time::timeout(Duration::from_secs(60), controller.peek_next_block())
1150            .await
1151            .expect("timed out waiting for first block to buffer")
1152            .expect("stream ended before a block arrived");
1153
1154        assert!(!peeked.sync_states.is_empty(), "peeked block should carry sync states");
1155
1156        // Stream must be empty before we trigger — the gating task should be holding the block.
1157        let pre_trigger = tokio::time::timeout(Duration::from_millis(200), stream.next()).await;
1158        assert!(
1159            pre_trigger.is_err(),
1160            "stream should be blocked before trigger_next_block, got an item"
1161        );
1162
1163        // Release the block.
1164        controller
1165            .trigger_next_block()
1166            .expect("trigger_next_block failed");
1167
1168        // Stream should now yield the decoded update within one block time.
1169        let update = tokio::time::timeout(Duration::from_secs(30), stream.next())
1170            .await
1171            .expect("timed out waiting for update after trigger")
1172            .expect("stream ended unexpectedly");
1173
1174        assert!(update.is_ok(), "decoded update should be Ok, got: {:?}", update);
1175    }
1176}