Skip to main content

tycho_simulation/evm/
stream.rs

1//! Builder for configuring a multi-protocol stream.
2//!
3//! Provides a builder for creating a multi-protocol stream that produces
4//! protocol state update messages. It runs one synchronization worker per protocol
5//! and a supervisor that aggregates updates, ensuring gap‑free streaming
6//! and robust state tracking.
7//!
8//! ## Context
9//!
10//! This stream wraps a `TychoStream` from `tycho-client`. It decodes `FeedMessage`s
11//! into protocol state updates. Internally, each protocol runs in its own
12//! synchronization worker, and a supervisor aggregates their messages per block.
13//!
14//! ### Protocol Synchronization Worker
15//! A synchronization worker runs the snapshot + delta protocol from `tycho-indexer`.
16//! - It first downloads components and their snapshots.
17//! - It then streams deltas.
18//! - It reacts to new or paused components by pulling snapshots or removing them from the active
19//!   set.
20//!
21//! Each worker emits snapshots and deltas to the supervisor.
22//!
23//! ### Stream Supervisor
24//! The supervisor aggregates worker messages by block and assigns sync status.
25//! - It ensures workers produce gap-free messages.
26//! - It flags late workers as `Delayed`, and marks them `Stale` if they exceed `max_missed_blocks`.
27//! - It marks workers with terminal errors as `Ended`.
28//!
29//! Aggregating by block adds small latency, since the supervisor waits briefly for
30//! all workers to emit. This latency only applies to workers in `Ready` or `Delayed`.
31//!
32//! The stream ends only when **all** workers are `Stale` or `Ended`.
33//!
34//! ## Configuration
35//!
36//! The builder lets you customize:
37//!
38//! ### Protocols
39//! Select which protocols to synchronize.
40//!
41//! ### Tokens & Minimum Token Quality
42//! Provide token metadata up front so the decoder can initialize protocol states from startup
43//! snapshots. `set_tokens` does not act as an ongoing filter — components arriving after startup
44//! include their own token metadata. To restrict processing to specific tokens, apply that filter
45//! in your consumer when reading `new_components`. New tokens arriving via stream deltas are added
46//! automatically when their quality exceeds `min_token_quality`.
47//!
48//! ### StreamEndPolicy
49//! Control when the stream ends based on worker states. By default, it ends when all
50//! workers are `Stale` or `Ended`.
51//!
52//! ## Stream
53//! The stream emits one protocol state update every `block_time`. Each update
54//! reports protocol synchronization states and any changes.
55//!
56//! The `new_components` field lists newly deployed components and their tokens.
57//!
58//! The stream aims to run indefinitely. Internal retry and reconnect logic handle
59//! most errors, so users should rarely need to restart it manually.
60//!
61//! ## Example
62//! ```no_run
63//! use tycho_common::models::Chain;
64//! use tycho_simulation::evm::stream::ProtocolStreamBuilder;
65//! use tycho_simulation::utils::load_all_tokens;
66//! use futures::StreamExt;
67//! use tycho_client::feed::component_tracker::ComponentFilter;
68//! use tycho_simulation::evm::protocol::uniswap_v2::state::UniswapV2State;
69//!
70//! #[tokio::main]
71//! async fn main() {
72//!     let all_tokens = load_all_tokens(
73//!         "tycho-beta.propellerheads.xyz",
74//!         false,
75//!         Some("sampletoken"),
76//!         true,
77//!         Chain::Ethereum,
78//!         None,
79//!         None,
80//!     )
81//!     .await
82//!     .expect("Failed loading tokens");
83//!
84//!     let protocol_stream =
85//!         ProtocolStreamBuilder::new("tycho-beta.propellerheads.xyz", Chain::Ethereum)
86//!             .auth_key(Some("sampletoken".to_string()))
87//!             .skip_state_decode_failures(true)
88//!             .exchange::<UniswapV2State>(
89//!                 "uniswap_v2", ComponentFilter::with_tvl_range(5.0, 10.0), None
90//!             )
91//!             .set_tokens(all_tokens)
92//!             .await
93//!             .build()
94//!             .await
95//!             .expect("Failed building protocol stream");
96//!     tokio::pin!(protocol_stream);
97//!
98//!     // Loop through block updates
99//!     while let Some(msg) = protocol_stream.next().await {
100//!         dbg!(msg).expect("failed decoding");
101//!     }
102//! }
103//! ```
104use std::{
105    collections::{HashMap, HashSet},
106    sync::Arc,
107    time,
108};
109
110use futures::{future::Either, Stream, StreamExt};
111use tokio_stream::wrappers::ReceiverStream;
112use tracing::{debug, error, warn};
113use tycho_client::{
114    feed::{
115        component_tracker::ComponentFilter, synchronizer::ComponentWithState, BlockHeader,
116        BlockSynchronizerError, FeedMessage, SynchronizerState,
117    },
118    stream::{RetryConfiguration, StreamError, TychoStreamBuilder},
119};
120use tycho_common::{
121    models::{token::Token, Chain},
122    simulation::protocol_sim::ProtocolSim,
123    traits::TxDeltaIndexer,
124    Bytes,
125};
126
127use crate::{
128    evm::{
129        decoder::{StreamDecodeError, TychoStreamDecoder},
130        override_stream::{self, StateOverrideProvider},
131        pending::PendingBlockProcessor,
132        protocol::{
133            filters::uniswap_v4_non_angstrom_hook_pool_filter,
134            native_wrapper::state::NativeWrapperState,
135            uniswap_v4::hooks::hook_handler_creator::initialize_hook_handlers,
136        },
137    },
138    protocol::{
139        errors::InvalidSnapshotError,
140        models::{DecoderContext, TryFromWithBlock, Update},
141    },
142    utils::default_blocklist,
143};
144
145const EXCHANGES_REQUIRING_FILTER: [&str; 5] =
146    ["vm:balancer_v2", "fluid_v1", "erc4626", "ekubo_v3", "vm:curve"];
147
148/// The client-side filter exchange `name` always gets, in addition to any filter the caller
149/// provides.
150///
151/// `uniswap_v4_hooks`: without `ANGSTROM_API_KEY`, Angstrom swaps cannot be encoded (they carry
152/// per-block attestations from the Angstrom API), so Angstrom pools are excluded up front rather
153/// than failing every route that selects them at encoding time. A caller's own hook filter does
154/// not replace this one: the encoder still has no key.
155fn mandatory_filter_fn(name: &str) -> Option<fn(&ComponentWithState) -> bool> {
156    if name == "uniswap_v4_hooks" && std::env::var("ANGSTROM_API_KEY").is_err() {
157        warn!(
158            "ANGSTROM_API_KEY is not set: excluding Angstrom pools from '{name}'. \
159             Set the key to include them."
160        );
161        return Some(uniswap_v4_non_angstrom_hook_pool_filter);
162    }
163    None
164}
165
166#[derive(Default, Debug, Clone, Copy)]
167pub enum StreamEndPolicy {
168    /// End stream if all states are Stale or Ended (default)
169    #[default]
170    AllEndedOrStale,
171    /// End stream if any protocol ended
172    AnyEnded,
173    /// End stream if any protocol ended or is stale
174    AnyEndedOrStale,
175    /// End stream if any protocol is stale
176    AnyStale,
177}
178
179impl StreamEndPolicy {
180    fn should_end<'a>(&self, states: impl IntoIterator<Item = &'a SynchronizerState>) -> bool {
181        let mut it = states.into_iter();
182        match self {
183            StreamEndPolicy::AllEndedOrStale => false,
184            StreamEndPolicy::AnyEnded => it.any(|s| matches!(s, SynchronizerState::Ended(_))),
185            StreamEndPolicy::AnyStale => it.any(|s| matches!(s, SynchronizerState::Stale(_))),
186            StreamEndPolicy::AnyEndedOrStale => {
187                it.any(|s| matches!(s, SynchronizerState::Stale(_) | SynchronizerState::Ended(_)))
188            }
189        }
190    }
191}
192
193/// Handle returned by [`ProtocolStreamBuilder::with_step_controller`] that gives external
194/// control over when each buffered block is released for decoding.
195///
196/// Intended for complex test scenarios where the caller needs to observe what the next
197/// block contains before allowing the decoder pipeline to process it.
198///
199/// ## Drop behaviour
200///
201/// Dropping this controller ungates the stream: the gating task detects the closed trigger
202/// channel, forwards the currently-buffered block (if any), then continues passing subsequent
203/// blocks through without waiting for triggers — exactly as if step-control had never been
204/// enabled. The stream runs to its natural end.
205pub struct BlockStepController {
206    /// Sends a trigger signal to release the next buffered block.
207    trigger_tx: tokio::sync::mpsc::UnboundedSender<()>,
208    /// Watch channel containing the next buffered raw message, or `None` if no block is pending.
209    peek_rx: tokio::sync::watch::Receiver<Option<FeedMessage<BlockHeader>>>,
210}
211
212impl BlockStepController {
213    /// Releases the next buffered block for decoding and emission.
214    ///
215    /// Returns an error if the stream has already ended and the sender is disconnected.
216    pub fn trigger_next_block(&self) -> Result<(), tokio::sync::mpsc::error::SendError<()>> {
217        // Send a unit value on the trigger channel to unblock the gating task.
218        self.trigger_tx.send(())
219    }
220
221    /// Returns the currently buffered block immediately, or `None` if no block is buffered yet.
222    pub fn try_peek_next_block(&self) -> Option<FeedMessage<BlockHeader>> {
223        self.peek_rx.borrow().clone()
224    }
225
226    /// Waits until a block is buffered and returns it without consuming it.
227    ///
228    /// Returns `None` only if the stream has ended and no further blocks will arrive.
229    /// If a block is already buffered when this is called, it returns immediately.
230    pub async fn peek_next_block(&self) -> Option<FeedMessage<BlockHeader>> {
231        // Clone so we don't hold a mutable borrow on self; wait_for checks the current
232        // value first, so this returns immediately if a block is already present.
233        let mut rx = self.peek_rx.clone();
234        let guard = rx
235            .wait_for(|v| v.is_some())
236            .await
237            .ok()?;
238        guard.clone()
239    }
240}
241
242/// Builds and configures the multi protocol stream described in the [module-level docs](self).
243///
244/// See the module documentation for details on protocols, configuration options, and
245/// stream behavior.
246pub struct ProtocolStreamBuilder {
247    decoder: TychoStreamDecoder<BlockHeader>,
248    stream_builder: TychoStreamBuilder,
249    stream_end_policy: StreamEndPolicy,
250    chain: Chain,
251    pending_indexers: HashMap<String, Box<dyn TxDeltaIndexer>>,
252    /// Watch sender used to publish the currently-buffered raw block so the controller can peek
253    /// at it before triggering. `Some` iff step-control mode is active.
254    step_peek_tx: Option<tokio::sync::watch::Sender<Option<FeedMessage<BlockHeader>>>>,
255    /// Receiver half of the trigger channel. Held here until `build()` / `build_with_pending()`
256    /// transfers ownership to the gating task. `Some` iff step-control mode is active.
257    step_trigger_rx: Option<tokio::sync::mpsc::UnboundedReceiver<()>>,
258    /// State-override providers explicitly registered by the consumer, keyed by `protocol_system`.
259    /// These take precedence over the built-in default registry and are installed onto the decoder
260    /// at build time.
261    override_providers: HashMap<String, Arc<dyn StateOverrideProvider>>,
262    /// Names of all exchanges registered on the builder, used to decide which built-in override
263    /// providers to auto-register at build time.
264    registered_exchanges: HashSet<String>,
265}
266
267impl ProtocolStreamBuilder {
268    /// Creates a new builder for a multi-protocol stream.
269    ///
270    /// The shipped pool blocklist is applied by default, excluding components known to break
271    /// simulation. Use [`blocklist_components`](Self::blocklist_components) to exclude additional
272    /// components.
273    ///
274    /// See the [module-level docs](self) for full details on stream behavior and configuration.
275    pub fn new(tycho_url: &str, chain: Chain) -> Self {
276        Self {
277            decoder: TychoStreamDecoder::new(chain),
278            stream_builder: TychoStreamBuilder::new(tycho_url, chain)
279                .blocklisted_ids(default_blocklist()),
280            stream_end_policy: StreamEndPolicy::default(),
281            chain,
282            pending_indexers: HashMap::new(),
283            step_peek_tx: None,
284            step_trigger_rx: None,
285            override_providers: HashMap::new(),
286            registered_exchanges: HashSet::new(),
287        }
288    }
289
290    /// Adds a specific exchange to the stream.
291    ///
292    /// This configures the builder to include a new protocol synchronizer for `name`,
293    /// filtering its components according to `filter` and optionally `filter_fn`.
294    ///
295    /// The type parameter `T` specifies the decoder type for this exchange. All
296    /// component states for this exchange will be decoded into instances of `T`.
297    ///
298    /// # Parameters
299    ///
300    /// - `name`: The protocol or exchange name (e.g., `"uniswap_v4"`, `"vm:balancer_v2"`).
301    /// - `filter`: Defines the set of components to include in the stream.
302    /// - `filter_fn`: Optional custom filter function for client-side filtering of components not
303    ///   expressible in `filter`.
304    ///
305    /// # Notes
306    ///
307    /// For certain protocols (e.g., `"uniswap_v4"`, `"vm:balancer_v2"`, `"vm:curve"`), omitting
308    /// `filter_fn` may cause decoding errors or incorrect results. In these cases, a proper
309    /// filter function is required to ensure correct decoding and quoting logic.
310    pub fn exchange<T>(
311        mut self,
312        name: &str,
313        filter: ComponentFilter,
314        filter_fn: Option<fn(&ComponentWithState) -> bool>,
315    ) -> Self
316    where
317        T: ProtocolSim
318            + TryFromWithBlock<ComponentWithState, BlockHeader, Error = InvalidSnapshotError>
319            + Send
320            + 'static,
321    {
322        self.stream_builder = self
323            .stream_builder
324            .exchange(name, filter);
325        self.registered_exchanges
326            .insert(name.to_string());
327        self.decoder.register_decoder::<T>(name);
328        if let Some(predicate) = filter_fn {
329            self.decoder
330                .register_filter(name, predicate);
331        }
332        if let Some(predicate) = mandatory_filter_fn(name) {
333            self.decoder
334                .register_filter(name, predicate);
335        }
336
337        if EXCHANGES_REQUIRING_FILTER.contains(&name) && filter_fn.is_none() {
338            warn!(
339                "Warning: For exchange type '{}', it is necessary to set a filter function because not all pools are supported. See all filters at src/evm/protocol/filters.rs",
340                name
341            );
342        }
343
344        self
345    }
346
347    /// Adds a specific exchange to the stream with decoder context.
348    ///
349    /// This configures the builder to include a new protocol synchronizer for `name`,
350    /// filtering its components according to `filter` and optionally `filter_fn`. It also registers
351    /// the DecoderContext (this is useful to test protocols that are not live yet)
352    ///
353    /// The type parameter `T` specifies the decoder type for this exchange. All
354    /// component states for this exchange will be decoded into instances of `T`.
355    ///
356    /// # Parameters
357    ///
358    /// - `name`: The protocol or exchange name (e.g., `"uniswap_v4"`, `"vm:balancer_v2"`).
359    /// - `filter`: Defines the set of components to include in the stream.
360    /// - `filter_fn`: Optional custom filter function for client-side filtering of components not
361    ///   expressible in `filter`.
362    /// - `decoder_context`: The decoder context for this exchange
363    ///
364    /// # Notes
365    ///
366    /// For certain protocols (e.g., `"uniswap_v4"`, `"vm:balancer_v2"`, `"vm:curve"`), omitting
367    /// `filter_fn` may cause decoding errors or incorrect results. In these cases, a proper
368    /// filter function is required to ensure correct decoding and quoting logic.
369    pub fn exchange_with_decoder_context<T>(
370        mut self,
371        name: &str,
372        filter: ComponentFilter,
373        filter_fn: Option<fn(&ComponentWithState) -> bool>,
374        decoder_context: DecoderContext,
375    ) -> Self
376    where
377        T: ProtocolSim
378            + TryFromWithBlock<ComponentWithState, BlockHeader, Error = InvalidSnapshotError>
379            + Send
380            + 'static,
381    {
382        self.stream_builder = self
383            .stream_builder
384            .exchange(name, filter);
385        self.registered_exchanges
386            .insert(name.to_string());
387        self.decoder
388            .register_decoder_with_context::<T>(name, decoder_context);
389        if let Some(predicate) = filter_fn {
390            self.decoder
391                .register_filter(name, predicate);
392        }
393        if let Some(predicate) = mandatory_filter_fn(name) {
394            self.decoder
395                .register_filter(name, predicate);
396        }
397
398        if EXCHANGES_REQUIRING_FILTER.contains(&name) && filter_fn.is_none() {
399            warn!(
400                "Warning: For exchange type '{}', it is necessary to set a filter function because not all pools are supported. See all filters at src/evm/protocol/filters.rs",
401                name
402            );
403        }
404
405        self
406    }
407
408    /// Sets the block time interval for the stream.
409    ///
410    /// This controls how often the stream produces updates.
411    pub fn block_time(mut self, block_time: u64) -> Self {
412        self.stream_builder = self
413            .stream_builder
414            .block_time(block_time);
415        self
416    }
417
418    /// Sets the network operation timeout (deprecated).
419    ///
420    /// Use [`latency_buffer()`](Self::latency_buffer) instead for controlling latency.
421    /// This method is retained for backwards compatibility.
422    #[deprecated = "Use latency_buffer instead"]
423    pub fn timeout(mut self, timeout: u64) -> Self {
424        self.stream_builder = self.stream_builder.timeout(timeout);
425        self
426    }
427
428    /// Sets the latency buffer to aggregate same-block messages.
429    ///
430    /// This allows the supervisor to wait a short interval for all synchronizers to emit
431    /// before aggregating.
432    pub fn latency_buffer(mut self, timeout: u64) -> Self {
433        self.stream_builder = self.stream_builder.timeout(timeout);
434        self
435    }
436
437    /// Sets the maximum number of blocks a synchronizer may miss before being marked as `Stale`.
438    pub fn max_missed_blocks(mut self, n: u64) -> Self {
439        self.stream_builder = self.stream_builder.max_missed_blocks(n);
440        self
441    }
442
443    /// Sets how long a synchronizer may take to process the initial message.
444    ///
445    /// Useful for data-intensive protocols where startup decoding takes longer.
446    pub fn startup_timeout(mut self, timeout: time::Duration) -> Self {
447        self.stream_builder = self
448            .stream_builder
449            .startup_timeout(timeout);
450        self
451    }
452
453    /// Configures the stream to exclude state updates.
454    ///
455    /// This reduces bandwidth and decoding workload if protocol state is not of
456    /// interest (e.g. only process new tokens).
457    pub fn no_state(mut self, no_state: bool) -> Self {
458        self.stream_builder = self.stream_builder.no_state(no_state);
459        self
460    }
461
462    /// Sets the API key for authenticating with the Tycho server.
463    pub fn auth_key(mut self, auth_key: Option<String>) -> Self {
464        self.stream_builder = self.stream_builder.auth_key(auth_key);
465        self
466    }
467
468    /// Adds client-metadata entries forwarded to the server in the `X-Tycho-Client-Metadata`
469    /// header.
470    ///
471    /// See [`TychoStreamBuilder::add_client_metadata`]. Values are self-reported and may surface in
472    /// the server's metrics and logs — do not include secrets or personally identifiable
473    /// information.
474    pub fn add_client_metadata<I, K, V>(mut self, metadata: I) -> Self
475    where
476        I: IntoIterator<Item = (K, V)>,
477        K: Into<String>,
478        V: Into<String>,
479    {
480        self.stream_builder = self
481            .stream_builder
482            .add_client_metadata(metadata);
483        self
484    }
485
486    /// Disables TLS/ SSL for the connection, using http and ws protocols.
487    ///
488    /// This is not recommended for production use.
489    pub fn no_tls(mut self, no_tls: bool) -> Self {
490        self.stream_builder = self.stream_builder.no_tls(no_tls);
491        self
492    }
493
494    /// Disable compression for the connection.
495    pub fn disable_compression(mut self) -> Self {
496        self.stream_builder = self
497            .stream_builder
498            .disable_compression();
499        self
500    }
501
502    /// Enables partial block updates (flashblocks).
503    pub fn enable_partial_blocks(mut self) -> Self {
504        self.stream_builder = self
505            .stream_builder
506            .enable_partial_blocks();
507        self
508    }
509
510    /// Sets the number of deltas buffered for each underlying Tycho WebSocket subscription.
511    ///
512    /// See [`TychoStreamBuilder::subscription_buffer_size`].
513    pub fn subscription_buffer_size(mut self, subscription_buffer_size: usize) -> Self {
514        self.stream_builder = self
515            .stream_builder
516            .subscription_buffer_size(subscription_buffer_size);
517        self
518    }
519
520    /// Exclude additional component IDs from all registered exchanges.
521    ///
522    /// These IDs are added to the shipped blocklist that is already applied by default (see
523    /// [`new`](Self::new)).
524    pub fn blocklist_components(mut self, ids: HashSet<String>) -> Self {
525        if !ids.is_empty() {
526            tracing::info!("Blocklisting {} components", ids.len());
527            self.stream_builder = self.stream_builder.blocklisted_ids(ids);
528        }
529        self
530    }
531
532    /// Sets the stream end policy.
533    ///
534    /// Controls when the stream should stop based on synchronizer states.
535    ///
536    /// ## Note
537    /// The stream always ends latest if all protocols are stale or ended independent of
538    /// this configuration. This allows you to end the stream earlier than that.
539    ///
540    /// See [self::StreamEndPolicy] for possible configuration options.
541    pub fn stream_end_policy(mut self, stream_end_policy: StreamEndPolicy) -> Self {
542        self.stream_end_policy = stream_end_policy;
543        self
544    }
545
546    /// Provides token metadata used to decode startup snapshots and initialize protocol states.
547    ///
548    /// This is not a stream filter — components arriving after startup include their own token
549    /// metadata. To restrict to specific tokens, filter in your consumer logic. New tokens
550    /// arriving via stream deltas are added automatically if they meet the quality threshold.
551    pub async fn set_tokens(self, tokens: HashMap<Bytes, Token>) -> Self {
552        self.decoder.set_tokens(tokens).await;
553        self
554    }
555
556    /// Skips decoding errors for component state updates.
557    ///
558    /// Allows the stream to continue processing even if some states fail to decode,
559    /// logging a warning instead of panicking.
560    pub fn skip_state_decode_failures(mut self, skip: bool) -> Self {
561        self.decoder
562            .skip_state_decode_failures(skip);
563        self
564    }
565
566    /// Sets the minimum token quality for tokens added via the stream.
567    ///
568    /// Tokens arriving in stream deltas below this threshold are ignored. Defaults to 100.
569    /// Set this to the same value used in [`load_all_tokens()`](crate::utils::load_all_tokens) to
570    /// apply consistent filtering.
571    pub fn min_token_quality(mut self, quality: u32) -> Self {
572        self.decoder.min_token_quality(quality);
573        self
574    }
575
576    /// Configures the retry policy for websocket reconnects.
577    pub fn websocket_retry_config(mut self, config: &RetryConfiguration) -> Self {
578        self.stream_builder = self
579            .stream_builder
580            .websockets_retry_config(config);
581        self
582    }
583
584    /// Configures the retry policy for state synchronization.
585    pub fn state_synchronizer_retry_config(mut self, config: &RetryConfiguration) -> Self {
586        self.stream_builder = self
587            .stream_builder
588            .state_synchronizer_retry_config(config);
589        self
590    }
591
592    /// Sets the maximum random delay before a restarted synchronizer requests a snapshot.
593    ///
594    /// Defaults to 60 seconds. A `jitter` below 5 seconds is raised to 5 seconds.
595    pub fn snapshot_jitter(mut self, jitter: time::Duration) -> Self {
596        self.stream_builder = self
597            .stream_builder
598            .snapshot_jitter(jitter);
599        self
600    }
601
602    pub fn get_decoder(&self) -> &TychoStreamDecoder<BlockHeader> {
603        &self.decoder
604    }
605
606    /// Registers a [`TxDeltaIndexer`] for ephemeral pending-block simulation.
607    ///
608    /// `protocol_system` is the protocol system of the indexer's components, which is also the
609    /// exchange name (e.g. `"uniswap_v3"`). The indexer's account deltas reach only pools of that
610    /// protocol system, so the indexer must emit the writes to every account those pools read. Use
611    /// [`build_with_pending`](Self::build_with_pending) to obtain both the confirmed stream and
612    /// the pending processor.
613    ///
614    /// The exchange must decode into a state whose `delta_transition` can rebuild it from the
615    /// `state_deltas` the indexer produces, because
616    /// [`apply_deltas_ephemeral`](crate::evm::decoder::TychoStreamDecoder::apply_deltas_ephemeral)
617    /// writes nothing to the VM database. Native and hybrid states qualify; the generic VM adapter
618    /// does not, because it re-reads pool state from the VM database — an indexer registered for
619    /// one still gets its balance and block-environment attributes applied, but every
620    /// storage-derived value stays at the confirmed block, with no error.
621    #[expect(
622        clippy::unnecessary_wraps,
623        reason = "public API; changing the signature is a breaking change"
624    )]
625    pub fn with_pending_indexer(
626        mut self,
627        protocol_system: &str,
628        indexer: Box<dyn TxDeltaIndexer>,
629    ) -> Result<Self, StreamError> {
630        self.pending_indexers
631            .insert(protocol_system.to_string(), indexer);
632        Ok(self)
633    }
634
635    /// Enables controlled-step mode for testing.
636    ///
637    /// Returns a [`BlockStepController`] that lets the caller decide when each buffered block
638    /// is released for decoding. Call this before [`build`](Self::build) or
639    /// [`build_with_pending`](Self::build_with_pending) — both detect and wire up the gating
640    /// automatically.
641    ///
642    /// In production code, do not call this method; the stream runs at full speed.
643    pub fn with_step_controller(mut self) -> (Self, BlockStepController) {
644        let (trigger_tx, trigger_rx) = tokio::sync::mpsc::unbounded_channel::<()>();
645        let (peek_tx, peek_rx) =
646            tokio::sync::watch::channel::<Option<FeedMessage<BlockHeader>>>(None);
647
648        self.step_peek_tx = Some(peek_tx);
649        self.step_trigger_rx = Some(trigger_rx);
650
651        let controller = BlockStepController { trigger_tx, peek_rx };
652        (self, controller)
653    }
654
655    /// Spawns a background task that gates `FeedMessage` delivery.
656    ///
657    /// The task buffers each incoming message, publishes it to `peek_tx` so the
658    /// [`BlockStepController`] can inspect it, waits for a trigger, then forwards the message to
659    /// `output_tx` for the decode pipeline. If `advance_tx` is `Some`, a clone of the message is
660    /// also forwarded there (used by the pending-processor path) before the decode step.
661    /// When the input channel closes or a terminal error is received according to
662    /// `stream_end_policy`, the task exits and all output channels are dropped.
663    fn run_gating_task(
664        raw_rx: tokio::sync::mpsc::Receiver<
665            Result<FeedMessage<BlockHeader>, BlockSynchronizerError>,
666        >,
667        mut trigger_rx: tokio::sync::mpsc::UnboundedReceiver<()>,
668        peek_tx: tokio::sync::watch::Sender<Option<FeedMessage<BlockHeader>>>,
669        output_tx: tokio::sync::mpsc::Sender<FeedMessage<BlockHeader>>,
670        stream_end_policy: StreamEndPolicy,
671    ) {
672        tokio::spawn(async move {
673            let mut raw_stream = ReceiverStream::new(raw_rx);
674            loop {
675                let msg = match raw_stream.next().await {
676                    Some(Ok(msg)) => msg,
677                    Some(Err(e)) => {
678                        error!("Block stream ended with terminal error: {e}");
679                        break;
680                    }
681                    None => break,
682                };
683
684                if stream_end_policy.should_end(msg.sync_states.values()) {
685                    error!(
686                        "Block stream ended due to {:?}: {:?}",
687                        stream_end_policy, msg.sync_states
688                    );
689                    break;
690                }
691
692                // Publish the buffered message so the caller can peek before triggering.
693                #[expect(
694                    clippy::let_underscore_must_use,
695                    reason = "the peek slot is optional; a send error only means the controller dropped its receiver"
696                )]
697                let _ = peek_tx.send(Some(msg.clone()));
698
699                // Block until the controller fires trigger_next_block(), or until it is dropped.
700                if trigger_rx.recv().await.is_none() {
701                    // Controller dropped — forward the buffered message and drain the rest
702                    // without gating, so the stream continues to its natural end.
703                    #[expect(
704                        clippy::let_underscore_must_use,
705                        reason = "the peek slot is optional; a send error only means the controller dropped its receiver"
706                    )]
707                    let _ = peek_tx.send(None);
708                    if output_tx.send(msg).await.is_err() {
709                        break;
710                    }
711                    while let Some(item) = raw_stream.next().await {
712                        let Ok(msg) = item else { break };
713                        if stream_end_policy.should_end(msg.sync_states.values()) {
714                            break;
715                        }
716                        if output_tx.send(msg).await.is_err() {
717                            break;
718                        }
719                    }
720                    break;
721                }
722
723                // Clear the peek slot before decoding so callers see None between blocks.
724                #[expect(
725                    clippy::let_underscore_must_use,
726                    reason = "the peek slot is optional; a send error only means the controller dropped its receiver"
727                )]
728                let _ = peek_tx.send(None);
729
730                if output_tx.send(msg).await.is_err() {
731                    break;
732                }
733            }
734        });
735    }
736
737    /// Registers `provider` as the live override source for `protocol_system`.
738    ///
739    /// Explicit registrations take precedence over the built-in default registry, so this is how
740    /// you swap a venue (e.g. `vm:bopamm`) onto a different provider. Registering the same provider
741    /// for several protocols is cheap — it is shared via `Arc`, not duplicated.
742    pub fn with_override_provider(
743        mut self,
744        protocol_system: impl Into<String>,
745        provider: Arc<dyn StateOverrideProvider>,
746    ) -> Self {
747        self.override_providers
748            .insert(protocol_system.into(), provider);
749        self
750    }
751
752    /// Installs override providers onto the decoder before the stream is built.
753    ///
754    /// Explicit consumer registrations win; the built-in default registry (see
755    /// [`default_override_providers`](crate::evm::override_stream::default_override_providers))
756    /// then fills every remaining protocol it can serve.
757    fn install_override_providers(&mut self) {
758        let explicit = std::mem::take(&mut self.override_providers);
759        // Protocols eligible for a built-in default provider: registered exchanges not explicitly
760        // overridden by the consumer.
761        let uncovered = self
762            .registered_exchanges
763            .clone()
764            .into_iter()
765            .filter(|exchange| !explicit.contains_key(exchange));
766        let defaults = override_stream::default_override_providers(uncovered);
767        for (protocol_system, provider) in defaults.into_iter().chain(explicit) {
768            self.decoder
769                .set_override_provider(protocol_system, provider);
770        }
771    }
772
773    /// Builds the confirmed protocol stream and a [`PendingBlockProcessor`] that stays
774    /// in sync with it automatically.
775    ///
776    /// The stream pipeline forwards every confirmed [`FeedMessage`] to the processor via an
777    /// internal unbounded channel — it never blocks waiting for the consumer. The consumer
778    /// owns the returned `PendingBlockProcessor` exclusively and may wrap it in whatever
779    /// synchronisation primitive suits their use case (e.g. `Mutex` for shared access,
780    /// nothing for single-threaded use).
781    ///
782    /// Call [`generate_pending_update`](PendingBlockProcessor::generate_pending_update) to
783    /// simulate a candidate bundle; it drains the channel automatically before computing.
784    ///
785    /// Warns once for each pending indexer whose protocol system is not a registered exchange:
786    /// its deltas reach no pool.
787    pub async fn build_with_pending(
788        mut self,
789    ) -> Result<
790        (impl Stream<Item = Result<Update, StreamDecodeError>>, PendingBlockProcessor),
791        StreamError,
792    > {
793        for protocol_system in self.pending_indexers.keys() {
794            if !self
795                .registered_exchanges
796                .contains(protocol_system)
797            {
798                warn!(
799                    protocol_system,
800                    "Pending indexer reaches no pool: no exchange has this protocol system"
801                );
802            }
803        }
804        initialize_hook_handlers().map_err(|e| {
805            StreamError::SetUpError(format!("Error initializing hook handlers: {e:?}"))
806        })?;
807        self.install_override_providers();
808        let (_, rx) = self.stream_builder.build().await?;
809        let decoder = Arc::new(self.decoder);
810
811        let (advance_tx, advance_rx) =
812            tokio::sync::mpsc::unbounded_channel::<FeedMessage<BlockHeader>>();
813        let pending = PendingBlockProcessor::new(
814            self.pending_indexers,
815            decoder.clone(),
816            self.chain,
817            advance_rx,
818        );
819
820        let chain = self.chain;
821        let stream_end_policy = self.stream_end_policy;
822
823        let decode_stream: Box<dyn Stream<Item = FeedMessage<BlockHeader>> + Send + Unpin> =
824            if let (Some(peek_tx), Some(trigger_rx)) = (self.step_peek_tx, self.step_trigger_rx) {
825                let (gated_tx, gated_rx) =
826                    tokio::sync::mpsc::channel::<FeedMessage<BlockHeader>>(1);
827                Self::run_gating_task(rx, trigger_rx, peek_tx, gated_tx, stream_end_policy);
828                Box::new(ReceiverStream::new(gated_rx))
829            } else {
830                let normal = ReceiverStream::new(rx)
831                    .take_while(move |msg| match msg {
832                        Ok(msg) => {
833                            let states = msg.sync_states.values();
834                            if stream_end_policy.should_end(states) {
835                                error!(
836                                    "Block stream ended due to {:?}: {:?}",
837                                    stream_end_policy, msg.sync_states
838                                );
839                                futures::future::ready(false)
840                            } else {
841                                futures::future::ready(true)
842                            }
843                        }
844                        Err(e) => {
845                            error!("Block stream ended with terminal error: {e}");
846                            futures::future::ready(false)
847                        }
848                    })
849                    .map(|msg| msg.expect("Safe since stream ends if we receive an error"));
850                Box::new(Box::pin(normal))
851            };
852
853        let stream = Box::pin(decode_stream.then({
854            let decoder = decoder;
855            move |msg| {
856                let decoder = decoder.clone();
857                let advance_tx = advance_tx.clone();
858                async move {
859                    if let Err(e) = advance_tx.send(msg.clone()) {
860                        warn!(error = %e, "Pending block processor is gone; block not forwarded");
861                    }
862                    decoder.decode(&msg).await.map_err(|e| {
863                        debug!(msg=?msg, "Decode error: {}", e);
864                        e
865                    })
866                }
867            }
868        }));
869        let stream = inject_native_wrapper(stream, chain);
870        Ok((stream, pending))
871    }
872
873    /// Builds and returns the configured protocol stream.
874    ///
875    /// See the module-level docs for details on stream behavior and emitted messages.
876    /// This method applies all builder settings and starts the stream.
877    pub async fn build(
878        mut self,
879    ) -> Result<impl Stream<Item = Result<Update, StreamDecodeError>>, StreamError> {
880        initialize_hook_handlers().map_err(|e| {
881            StreamError::SetUpError(format!("Error initializing hook handlers: {e:?}"))
882        })?;
883        self.install_override_providers();
884        let (_, rx) = self.stream_builder.build().await?;
885        let decoder = Arc::new(self.decoder);
886        let chain = self.chain;
887        let stream_end_policy = self.stream_end_policy;
888
889        let decode_stream: Box<dyn Stream<Item = FeedMessage<BlockHeader>> + Send + Unpin> =
890            if let (Some(peek_tx), Some(trigger_rx)) = (self.step_peek_tx, self.step_trigger_rx) {
891                let (gated_tx, gated_rx) =
892                    tokio::sync::mpsc::channel::<FeedMessage<BlockHeader>>(1);
893                Self::run_gating_task(rx, trigger_rx, peek_tx, gated_tx, stream_end_policy);
894                Box::new(ReceiverStream::new(gated_rx))
895            } else {
896                let normal = ReceiverStream::new(rx)
897                    .take_while(move |msg| match msg {
898                        Ok(msg) => {
899                            let states = msg.sync_states.values();
900                            if stream_end_policy.should_end(states) {
901                                error!(
902                                    "Block stream ended due to {:?}: {:?}",
903                                    stream_end_policy, msg.sync_states
904                                );
905                                futures::future::ready(false)
906                            } else {
907                                futures::future::ready(true)
908                            }
909                        }
910                        Err(e) => {
911                            error!("Block stream ended with terminal error: {e}");
912                            futures::future::ready(false)
913                        }
914                    })
915                    .map(|msg| msg.expect("Safe since stream ends if we receive an error"));
916                Box::new(Box::pin(normal))
917            };
918
919        let stream = Box::pin(decode_stream.then({
920            let decoder = decoder;
921            move |msg| {
922                let decoder = decoder.clone();
923                async move {
924                    decoder.decode(&msg).await.map_err(|e| {
925                        debug!(msg=?msg, "Decode error: {}", e);
926                        e
927                    })
928                }
929            }
930        }));
931        let stream = inject_native_wrapper(stream, chain);
932        Ok(stream)
933    }
934}
935
936/// Wraps a decoded protocol stream to inject a `NativeWrapperState` component
937/// on the first successful update.
938///
939/// Skips injection when the chain's native asset has no real wrapper contract.
940fn inject_native_wrapper(
941    inner: impl Stream<Item = Result<Update, StreamDecodeError>> + Unpin + Send + 'static,
942    chain: Chain,
943) -> impl Stream<Item = Result<Update, StreamDecodeError>> + Send {
944    let Some(native_wrapper) = NativeWrapperState::new(chain) else {
945        return Either::Left(inner);
946    };
947
948    let mut pending_wrapper = Some(native_wrapper);
949    Either::Right(inner.map(move |result| {
950        result.map(|mut update| {
951            if let Some(native_wrapper) = pending_wrapper.take() {
952                let component = native_wrapper.component();
953                let id = component.id.to_string();
954                update
955                    .new_pairs
956                    .insert(id.clone(), component);
957                update
958                    .states
959                    .insert(id, Box::new(native_wrapper));
960                debug!("Injected native_wrapper component for {chain}");
961            }
962            update
963        })
964    }))
965}
966
967#[cfg(test)]
968mod tests {
969    use std::collections::HashMap;
970
971    use futures::{stream, StreamExt};
972    use tycho_common::models::Chain;
973
974    use super::*;
975    use crate::{
976        evm::protocol::native_wrapper::state::NATIVE_WRAPPER_ID, protocol::models::Update,
977    };
978
979    fn empty_update(block: u64) -> Update {
980        Update::new(block, HashMap::new(), HashMap::new())
981    }
982
983    #[tokio::test]
984    async fn test_inject_native_wrapper_first_message_only() {
985        let updates = vec![Ok(empty_update(1)), Ok(empty_update(2)), Ok(empty_update(3))];
986        let input = stream::iter(updates);
987
988        let results: Vec<_> = inject_native_wrapper(input, Chain::Ethereum)
989            .collect()
990            .await;
991
992        assert_eq!(results.len(), 3);
993
994        let expected_id = NativeWrapperState::new(Chain::Ethereum)
995            .expect("Ethereum should have a wrapper")
996            .component()
997            .id
998            .to_string();
999
1000        let first = results[0]
1001            .as_ref()
1002            .expect("first update ok");
1003        assert!(
1004            first
1005                .new_pairs
1006                .contains_key(&expected_id),
1007            "first message should have native_wrapper component"
1008        );
1009        assert!(
1010            first.states.contains_key(&expected_id),
1011            "first message should have native_wrapper state"
1012        );
1013
1014        let second = results[1]
1015            .as_ref()
1016            .expect("second update ok");
1017        assert!(
1018            !second
1019                .new_pairs
1020                .contains_key(&expected_id),
1021            "second message should NOT have native_wrapper component"
1022        );
1023        assert!(
1024            !second.states.contains_key(&expected_id),
1025            "second message should NOT have native_wrapper state"
1026        );
1027    }
1028
1029    #[tokio::test]
1030    async fn test_inject_native_wrapper_after_initial_decode_error() {
1031        let updates = vec![
1032            Err(StreamDecodeError::Fatal("decode failed".to_string())),
1033            Ok(empty_update(1)),
1034            Ok(empty_update(2)),
1035        ];
1036        let results: Vec<_> = inject_native_wrapper(stream::iter(updates), Chain::Ethereum)
1037            .collect()
1038            .await;
1039
1040        assert!(results[0].is_err(), "the decode error must pass through unchanged");
1041        let expected_id = NativeWrapperState::new(Chain::Ethereum)
1042            .expect("Ethereum should have a wrapper")
1043            .component()
1044            .id
1045            .to_string();
1046        let first_success = results[1]
1047            .as_ref()
1048            .expect("second stream item should decode");
1049        assert!(first_success
1050            .new_pairs
1051            .contains_key(&expected_id));
1052        assert!(first_success
1053            .states
1054            .contains_key(&expected_id));
1055
1056        let second_success = results[2]
1057            .as_ref()
1058            .expect("third stream item should decode");
1059        assert!(!second_success
1060            .new_pairs
1061            .contains_key(&expected_id));
1062        assert!(!second_success
1063            .states
1064            .contains_key(&expected_id));
1065    }
1066
1067    #[tokio::test]
1068    async fn test_does_not_inject_native_wrapper_for_arc_shared_balance() {
1069        let results: Vec<_> =
1070            inject_native_wrapper(stream::iter([Ok(empty_update(1))]), Chain::Arc)
1071                .collect()
1072                .await;
1073
1074        let update = results[0]
1075            .as_ref()
1076            .expect("update should decode");
1077        assert!(!update
1078            .new_pairs
1079            .contains_key(NATIVE_WRAPPER_ID));
1080        assert!(!update
1081            .states
1082            .contains_key(NATIVE_WRAPPER_ID));
1083    }
1084
1085    /// Verifies that `with_step_controller` returns both a modified builder and a controller.
1086    ///
1087    /// This test only checks that the builder method is callable and that the returned controller
1088    /// compiles — it does not start any network connection.
1089    #[tokio::test]
1090    async fn test_with_step_controller_returns_controller() {
1091        let builder = ProtocolStreamBuilder::new("tycho-beta.propellerheads.xyz", Chain::Ethereum);
1092        let (_builder, controller) = builder.with_step_controller();
1093        // The controller was successfully returned — verifying the public API is callable.
1094        drop(controller);
1095    }
1096
1097    #[tokio::test]
1098    async fn test_subscription_buffer_size_forwards_zero_rejection() {
1099        let error = ProtocolStreamBuilder::new("not a valid endpoint", Chain::Ethereum)
1100            .subscription_buffer_size(0)
1101            .build()
1102            .await;
1103
1104        let Err(error) = error else {
1105            panic!("a zero subscription buffer size must be rejected during setup");
1106        };
1107
1108        assert!(
1109            error
1110                .to_string()
1111                .contains("subscription buffer size must be greater than zero"),
1112            "the underlying Tycho stream builder should return the configuration error: {error}"
1113        );
1114    }
1115
1116    #[tokio::test]
1117    async fn test_subscription_buffer_size_forwards_upper_bound_rejection() {
1118        let error = ProtocolStreamBuilder::new("not a valid endpoint", Chain::Ethereum)
1119            .subscription_buffer_size(usize::MAX)
1120            .build()
1121            .await;
1122
1123        let Err(error) = error else {
1124            panic!("an oversized subscription buffer size must be rejected during setup");
1125        };
1126
1127        assert!(
1128            error
1129                .to_string()
1130                .contains("subscription buffer size must not exceed"),
1131            "the underlying Tycho stream builder should return the configuration error: {error}"
1132        );
1133    }
1134
1135    /// Connects to a live Tycho instance, verifies that the stream blocks until
1136    /// `trigger_next_block` is called, and that `peek_next_block` exposes the buffered message.
1137    #[ignore = "requires live Tycho connection (TYCHO_AUTH_TOKEN env var)"]
1138    #[tokio::test]
1139    async fn test_step_controller_trigger_releases_block() {
1140        use std::{env, time::Duration};
1141
1142        use crate::evm::protocol::uniswap_v2::state::UniswapV2State;
1143
1144        let auth = env::var("TYCHO_AUTH_TOKEN").expect("TYCHO_AUTH_TOKEN must be set");
1145
1146        // Track a single well-known pool to minimise startup latency.
1147        let usdc_weth_v2 = "0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc".to_string();
1148        let (builder, controller) =
1149            ProtocolStreamBuilder::new("tycho-beta.propellerheads.xyz", Chain::Ethereum)
1150                .auth_key(Some(auth))
1151                .exchange::<UniswapV2State>(
1152                    "uniswap_v2",
1153                    ComponentFilter::Ids(vec![usdc_weth_v2]),
1154                    None,
1155                )
1156                .with_step_controller();
1157
1158        let (stream, _pending) = builder
1159            .build_with_pending()
1160            .await
1161            .expect("build_with_pending failed");
1162        tokio::pin!(stream);
1163
1164        // Wait up to 60 s for the first block to arrive in the gating buffer.
1165        let peeked = tokio::time::timeout(Duration::from_secs(60), controller.peek_next_block())
1166            .await
1167            .expect("timed out waiting for first block to buffer")
1168            .expect("stream ended before a block arrived");
1169
1170        assert!(!peeked.sync_states.is_empty(), "peeked block should carry sync states");
1171
1172        // Stream must be empty before we trigger — the gating task should be holding the block.
1173        let pre_trigger = tokio::time::timeout(Duration::from_millis(200), stream.next()).await;
1174        assert!(
1175            pre_trigger.is_err(),
1176            "stream should be blocked before trigger_next_block, got an item"
1177        );
1178
1179        // Release the block.
1180        controller
1181            .trigger_next_block()
1182            .expect("trigger_next_block failed");
1183
1184        // Stream should now yield the decoded update within one block time.
1185        let update = tokio::time::timeout(Duration::from_secs(30), stream.next())
1186            .await
1187            .expect("timed out waiting for update after trigger")
1188            .expect("stream ended unexpectedly");
1189
1190        assert!(update.is_ok(), "decoded update should be Ok, got: {:?}", update);
1191    }
1192}