tycho_simulation/evm/stream.rs
1//! Builder for configuring a multi-protocol stream.
2//!
3//! Provides a builder for creating a multi-protocol stream that produces
4//! protocol state update messages. It runs one synchronization worker per protocol
5//! and a supervisor that aggregates updates, ensuring gap‑free streaming
6//! and robust state tracking.
7//!
8//! ## Context
9//!
10//! This stream wraps a `TychoStream` from `tycho-client`. It decodes `FeedMessage`s
11//! into protocol state updates. Internally, each protocol runs in its own
12//! synchronization worker, and a supervisor aggregates their messages per block.
13//!
14//! ### Protocol Synchronization Worker
15//! A synchronization worker runs the snapshot + delta protocol from `tycho-indexer`.
16//! - It first downloads components and their snapshots.
17//! - It then streams deltas.
18//! - It reacts to new or paused components by pulling snapshots or removing them from the active
19//! set.
20//!
21//! Each worker emits snapshots and deltas to the supervisor.
22//!
23//! ### Stream Supervisor
24//! The supervisor aggregates worker messages by block and assigns sync status.
25//! - It ensures workers produce gap-free messages.
26//! - It flags late workers as `Delayed`, and marks them `Stale` if they exceed `max_missed_blocks`.
27//! - It marks workers with terminal errors as `Ended`.
28//!
29//! Aggregating by block adds small latency, since the supervisor waits briefly for
30//! all workers to emit. This latency only applies to workers in `Ready` or `Delayed`.
31//!
32//! The stream ends only when **all** workers are `Stale` or `Ended`.
33//!
34//! ## Configuration
35//!
36//! The builder lets you customize:
37//!
38//! ### Protocols
39//! Select which protocols to synchronize.
40//!
41//! ### Tokens & Minimum Token Quality
42//! Provide token metadata up front so the decoder can initialize protocol states from startup
43//! snapshots. `set_tokens` does not act as an ongoing filter — components arriving after startup
44//! include their own token metadata. To restrict processing to specific tokens, apply that filter
45//! in your consumer when reading `new_components`. New tokens arriving via stream deltas are added
46//! automatically when their quality exceeds `min_token_quality`.
47//!
48//! ### StreamEndPolicy
49//! Control when the stream ends based on worker states. By default, it ends when all
50//! workers are `Stale` or `Ended`.
51//!
52//! ## Stream
53//! The stream emits one protocol state update every `block_time`. Each update
54//! reports protocol synchronization states and any changes.
55//!
56//! The `new_components` field lists newly deployed components and their tokens.
57//!
58//! The stream aims to run indefinitely. Internal retry and reconnect logic handle
59//! most errors, so users should rarely need to restart it manually.
60//!
61//! ## Example
62//! ```no_run
63//! use tycho_common::models::Chain;
64//! use tycho_simulation::evm::stream::ProtocolStreamBuilder;
65//! use tycho_simulation::utils::load_all_tokens;
66//! use futures::StreamExt;
67//! use tycho_client::feed::component_tracker::ComponentFilter;
68//! use tycho_simulation::evm::protocol::uniswap_v2::state::UniswapV2State;
69//!
70//! #[tokio::main]
71//! async fn main() {
72//! let all_tokens = load_all_tokens(
73//! "tycho-beta.propellerheads.xyz",
74//! false,
75//! Some("sampletoken"),
76//! true,
77//! Chain::Ethereum,
78//! None,
79//! None,
80//! )
81//! .await
82//! .expect("Failed loading tokens");
83//!
84//! let protocol_stream =
85//! ProtocolStreamBuilder::new("tycho-beta.propellerheads.xyz", Chain::Ethereum)
86//! .auth_key(Some("sampletoken".to_string()))
87//! .skip_state_decode_failures(true)
88//! .exchange::<UniswapV2State>(
89//! "uniswap_v2", ComponentFilter::with_tvl_range(5.0, 10.0), None
90//! )
91//! .set_tokens(all_tokens)
92//! .await
93//! .build()
94//! .await
95//! .expect("Failed building protocol stream");
96//! tokio::pin!(protocol_stream);
97//!
98//! // Loop through block updates
99//! while let Some(msg) = protocol_stream.next().await {
100//! dbg!(msg).expect("failed decoding");
101//! }
102//! }
103//! ```
104use std::{
105 collections::{HashMap, HashSet},
106 sync::Arc,
107 time,
108};
109
110use futures::{future::Either, Stream, StreamExt};
111use tokio_stream::wrappers::ReceiverStream;
112use tracing::{debug, error, warn};
113use tycho_client::{
114 feed::{
115 component_tracker::ComponentFilter, synchronizer::ComponentWithState, BlockHeader,
116 BlockSynchronizerError, FeedMessage, SynchronizerState,
117 },
118 stream::{RetryConfiguration, StreamError, TychoStreamBuilder},
119};
120use tycho_common::{
121 models::{token::Token, Chain},
122 simulation::protocol_sim::ProtocolSim,
123 traits::TxDeltaIndexer,
124 Bytes,
125};
126
127use crate::{
128 evm::{
129 decoder::{StreamDecodeError, TychoStreamDecoder},
130 override_stream::{self, StateOverrideProvider},
131 pending::PendingBlockProcessor,
132 protocol::{
133 filters::uniswap_v4_non_angstrom_hook_pool_filter,
134 native_wrapper::state::NativeWrapperState,
135 uniswap_v4::hooks::hook_handler_creator::initialize_hook_handlers,
136 },
137 },
138 protocol::{
139 errors::InvalidSnapshotError,
140 models::{DecoderContext, TryFromWithBlock, Update},
141 },
142 utils::default_blocklist,
143};
144
145const EXCHANGES_REQUIRING_FILTER: [&str; 5] =
146 ["vm:balancer_v2", "fluid_v1", "erc4626", "ekubo_v3", "vm:curve"];
147
148/// The client-side filter exchange `name` always gets, in addition to any filter the caller
149/// provides.
150///
151/// `uniswap_v4_hooks`: without `ANGSTROM_API_KEY`, Angstrom swaps cannot be encoded (they carry
152/// per-block attestations from the Angstrom API), so Angstrom pools are excluded up front rather
153/// than failing every route that selects them at encoding time. A caller's own hook filter does
154/// not replace this one: the encoder still has no key.
155fn mandatory_filter_fn(name: &str) -> Option<fn(&ComponentWithState) -> bool> {
156 if name == "uniswap_v4_hooks" && std::env::var("ANGSTROM_API_KEY").is_err() {
157 warn!(
158 "ANGSTROM_API_KEY is not set: excluding Angstrom pools from '{name}'. \
159 Set the key to include them."
160 );
161 return Some(uniswap_v4_non_angstrom_hook_pool_filter);
162 }
163 None
164}
165
166#[derive(Default, Debug, Clone, Copy)]
167pub enum StreamEndPolicy {
168 /// End stream if all states are Stale or Ended (default)
169 #[default]
170 AllEndedOrStale,
171 /// End stream if any protocol ended
172 AnyEnded,
173 /// End stream if any protocol ended or is stale
174 AnyEndedOrStale,
175 /// End stream if any protocol is stale
176 AnyStale,
177}
178
179impl StreamEndPolicy {
180 fn should_end<'a>(&self, states: impl IntoIterator<Item = &'a SynchronizerState>) -> bool {
181 let mut it = states.into_iter();
182 match self {
183 StreamEndPolicy::AllEndedOrStale => false,
184 StreamEndPolicy::AnyEnded => it.any(|s| matches!(s, SynchronizerState::Ended(_))),
185 StreamEndPolicy::AnyStale => it.any(|s| matches!(s, SynchronizerState::Stale(_))),
186 StreamEndPolicy::AnyEndedOrStale => {
187 it.any(|s| matches!(s, SynchronizerState::Stale(_) | SynchronizerState::Ended(_)))
188 }
189 }
190 }
191}
192
193/// Handle returned by [`ProtocolStreamBuilder::with_step_controller`] that gives external
194/// control over when each buffered block is released for decoding.
195///
196/// Intended for complex test scenarios where the caller needs to observe what the next
197/// block contains before allowing the decoder pipeline to process it.
198///
199/// ## Drop behaviour
200///
201/// Dropping this controller ungates the stream: the gating task detects the closed trigger
202/// channel, forwards the currently-buffered block (if any), then continues passing subsequent
203/// blocks through without waiting for triggers — exactly as if step-control had never been
204/// enabled. The stream runs to its natural end.
205pub struct BlockStepController {
206 /// Sends a trigger signal to release the next buffered block.
207 trigger_tx: tokio::sync::mpsc::UnboundedSender<()>,
208 /// Watch channel containing the next buffered raw message, or `None` if no block is pending.
209 peek_rx: tokio::sync::watch::Receiver<Option<FeedMessage<BlockHeader>>>,
210}
211
212impl BlockStepController {
213 /// Releases the next buffered block for decoding and emission.
214 ///
215 /// Returns an error if the stream has already ended and the sender is disconnected.
216 pub fn trigger_next_block(&self) -> Result<(), tokio::sync::mpsc::error::SendError<()>> {
217 // Send a unit value on the trigger channel to unblock the gating task.
218 self.trigger_tx.send(())
219 }
220
221 /// Returns the currently buffered block immediately, or `None` if no block is buffered yet.
222 pub fn try_peek_next_block(&self) -> Option<FeedMessage<BlockHeader>> {
223 self.peek_rx.borrow().clone()
224 }
225
226 /// Waits until a block is buffered and returns it without consuming it.
227 ///
228 /// Returns `None` only if the stream has ended and no further blocks will arrive.
229 /// If a block is already buffered when this is called, it returns immediately.
230 pub async fn peek_next_block(&self) -> Option<FeedMessage<BlockHeader>> {
231 // Clone so we don't hold a mutable borrow on self; wait_for checks the current
232 // value first, so this returns immediately if a block is already present.
233 let mut rx = self.peek_rx.clone();
234 let guard = rx
235 .wait_for(|v| v.is_some())
236 .await
237 .ok()?;
238 guard.clone()
239 }
240}
241
242/// Builds and configures the multi protocol stream described in the [module-level docs](self).
243///
244/// See the module documentation for details on protocols, configuration options, and
245/// stream behavior.
246pub struct ProtocolStreamBuilder {
247 decoder: TychoStreamDecoder<BlockHeader>,
248 stream_builder: TychoStreamBuilder,
249 stream_end_policy: StreamEndPolicy,
250 chain: Chain,
251 pending_indexers: HashMap<String, Box<dyn TxDeltaIndexer>>,
252 /// Watch sender used to publish the currently-buffered raw block so the controller can peek
253 /// at it before triggering. `Some` iff step-control mode is active.
254 step_peek_tx: Option<tokio::sync::watch::Sender<Option<FeedMessage<BlockHeader>>>>,
255 /// Receiver half of the trigger channel. Held here until `build()` / `build_with_pending()`
256 /// transfers ownership to the gating task. `Some` iff step-control mode is active.
257 step_trigger_rx: Option<tokio::sync::mpsc::UnboundedReceiver<()>>,
258 /// State-override providers explicitly registered by the consumer, keyed by `protocol_system`.
259 /// These take precedence over the built-in default registry and are installed onto the decoder
260 /// at build time.
261 override_providers: HashMap<String, Arc<dyn StateOverrideProvider>>,
262 /// Names of all exchanges registered on the builder, used to decide which built-in override
263 /// providers to auto-register at build time.
264 registered_exchanges: HashSet<String>,
265}
266
267impl ProtocolStreamBuilder {
268 /// Creates a new builder for a multi-protocol stream.
269 ///
270 /// The shipped pool blocklist is applied by default, excluding components known to break
271 /// simulation. Use [`blocklist_components`](Self::blocklist_components) to exclude additional
272 /// components.
273 ///
274 /// See the [module-level docs](self) for full details on stream behavior and configuration.
275 pub fn new(tycho_url: &str, chain: Chain) -> Self {
276 Self {
277 decoder: TychoStreamDecoder::new(chain),
278 stream_builder: TychoStreamBuilder::new(tycho_url, chain)
279 .blocklisted_ids(default_blocklist()),
280 stream_end_policy: StreamEndPolicy::default(),
281 chain,
282 pending_indexers: HashMap::new(),
283 step_peek_tx: None,
284 step_trigger_rx: None,
285 override_providers: HashMap::new(),
286 registered_exchanges: HashSet::new(),
287 }
288 }
289
290 /// Adds a specific exchange to the stream.
291 ///
292 /// This configures the builder to include a new protocol synchronizer for `name`,
293 /// filtering its components according to `filter` and optionally `filter_fn`.
294 ///
295 /// The type parameter `T` specifies the decoder type for this exchange. All
296 /// component states for this exchange will be decoded into instances of `T`.
297 ///
298 /// # Parameters
299 ///
300 /// - `name`: The protocol or exchange name (e.g., `"uniswap_v4"`, `"vm:balancer_v2"`).
301 /// - `filter`: Defines the set of components to include in the stream.
302 /// - `filter_fn`: Optional custom filter function for client-side filtering of components not
303 /// expressible in `filter`.
304 ///
305 /// # Notes
306 ///
307 /// For certain protocols (e.g., `"uniswap_v4"`, `"vm:balancer_v2"`, `"vm:curve"`), omitting
308 /// `filter_fn` may cause decoding errors or incorrect results. In these cases, a proper
309 /// filter function is required to ensure correct decoding and quoting logic.
310 pub fn exchange<T>(
311 mut self,
312 name: &str,
313 filter: ComponentFilter,
314 filter_fn: Option<fn(&ComponentWithState) -> bool>,
315 ) -> Self
316 where
317 T: ProtocolSim
318 + TryFromWithBlock<ComponentWithState, BlockHeader, Error = InvalidSnapshotError>
319 + Send
320 + 'static,
321 {
322 self.stream_builder = self
323 .stream_builder
324 .exchange(name, filter);
325 self.registered_exchanges
326 .insert(name.to_string());
327 self.decoder.register_decoder::<T>(name);
328 if let Some(predicate) = filter_fn {
329 self.decoder
330 .register_filter(name, predicate);
331 }
332 if let Some(predicate) = mandatory_filter_fn(name) {
333 self.decoder
334 .register_filter(name, predicate);
335 }
336
337 if EXCHANGES_REQUIRING_FILTER.contains(&name) && filter_fn.is_none() {
338 warn!(
339 "Warning: For exchange type '{}', it is necessary to set a filter function because not all pools are supported. See all filters at src/evm/protocol/filters.rs",
340 name
341 );
342 }
343
344 self
345 }
346
347 /// Adds a specific exchange to the stream with decoder context.
348 ///
349 /// This configures the builder to include a new protocol synchronizer for `name`,
350 /// filtering its components according to `filter` and optionally `filter_fn`. It also registers
351 /// the DecoderContext (this is useful to test protocols that are not live yet)
352 ///
353 /// The type parameter `T` specifies the decoder type for this exchange. All
354 /// component states for this exchange will be decoded into instances of `T`.
355 ///
356 /// # Parameters
357 ///
358 /// - `name`: The protocol or exchange name (e.g., `"uniswap_v4"`, `"vm:balancer_v2"`).
359 /// - `filter`: Defines the set of components to include in the stream.
360 /// - `filter_fn`: Optional custom filter function for client-side filtering of components not
361 /// expressible in `filter`.
362 /// - `decoder_context`: The decoder context for this exchange
363 ///
364 /// # Notes
365 ///
366 /// For certain protocols (e.g., `"uniswap_v4"`, `"vm:balancer_v2"`, `"vm:curve"`), omitting
367 /// `filter_fn` may cause decoding errors or incorrect results. In these cases, a proper
368 /// filter function is required to ensure correct decoding and quoting logic.
369 pub fn exchange_with_decoder_context<T>(
370 mut self,
371 name: &str,
372 filter: ComponentFilter,
373 filter_fn: Option<fn(&ComponentWithState) -> bool>,
374 decoder_context: DecoderContext,
375 ) -> Self
376 where
377 T: ProtocolSim
378 + TryFromWithBlock<ComponentWithState, BlockHeader, Error = InvalidSnapshotError>
379 + Send
380 + 'static,
381 {
382 self.stream_builder = self
383 .stream_builder
384 .exchange(name, filter);
385 self.registered_exchanges
386 .insert(name.to_string());
387 self.decoder
388 .register_decoder_with_context::<T>(name, decoder_context);
389 if let Some(predicate) = filter_fn {
390 self.decoder
391 .register_filter(name, predicate);
392 }
393 if let Some(predicate) = mandatory_filter_fn(name) {
394 self.decoder
395 .register_filter(name, predicate);
396 }
397
398 if EXCHANGES_REQUIRING_FILTER.contains(&name) && filter_fn.is_none() {
399 warn!(
400 "Warning: For exchange type '{}', it is necessary to set a filter function because not all pools are supported. See all filters at src/evm/protocol/filters.rs",
401 name
402 );
403 }
404
405 self
406 }
407
408 /// Sets the block time interval for the stream.
409 ///
410 /// This controls how often the stream produces updates.
411 pub fn block_time(mut self, block_time: u64) -> Self {
412 self.stream_builder = self
413 .stream_builder
414 .block_time(block_time);
415 self
416 }
417
418 /// Sets the network operation timeout (deprecated).
419 ///
420 /// Use [`latency_buffer()`](Self::latency_buffer) instead for controlling latency.
421 /// This method is retained for backwards compatibility.
422 #[deprecated = "Use latency_buffer instead"]
423 pub fn timeout(mut self, timeout: u64) -> Self {
424 self.stream_builder = self.stream_builder.timeout(timeout);
425 self
426 }
427
428 /// Sets the latency buffer to aggregate same-block messages.
429 ///
430 /// This allows the supervisor to wait a short interval for all synchronizers to emit
431 /// before aggregating.
432 pub fn latency_buffer(mut self, timeout: u64) -> Self {
433 self.stream_builder = self.stream_builder.timeout(timeout);
434 self
435 }
436
437 /// Sets the maximum number of blocks a synchronizer may miss before being marked as `Stale`.
438 pub fn max_missed_blocks(mut self, n: u64) -> Self {
439 self.stream_builder = self.stream_builder.max_missed_blocks(n);
440 self
441 }
442
443 /// Sets how long a synchronizer may take to process the initial message.
444 ///
445 /// Useful for data-intensive protocols where startup decoding takes longer.
446 pub fn startup_timeout(mut self, timeout: time::Duration) -> Self {
447 self.stream_builder = self
448 .stream_builder
449 .startup_timeout(timeout);
450 self
451 }
452
453 /// Configures the stream to exclude state updates.
454 ///
455 /// This reduces bandwidth and decoding workload if protocol state is not of
456 /// interest (e.g. only process new tokens).
457 pub fn no_state(mut self, no_state: bool) -> Self {
458 self.stream_builder = self.stream_builder.no_state(no_state);
459 self
460 }
461
462 /// Sets the API key for authenticating with the Tycho server.
463 pub fn auth_key(mut self, auth_key: Option<String>) -> Self {
464 self.stream_builder = self.stream_builder.auth_key(auth_key);
465 self
466 }
467
468 /// Adds client-metadata entries forwarded to the server in the `X-Tycho-Client-Metadata`
469 /// header.
470 ///
471 /// See [`TychoStreamBuilder::add_client_metadata`]. Values are self-reported and may surface in
472 /// the server's metrics and logs — do not include secrets or personally identifiable
473 /// information.
474 pub fn add_client_metadata<I, K, V>(mut self, metadata: I) -> Self
475 where
476 I: IntoIterator<Item = (K, V)>,
477 K: Into<String>,
478 V: Into<String>,
479 {
480 self.stream_builder = self
481 .stream_builder
482 .add_client_metadata(metadata);
483 self
484 }
485
486 /// Disables TLS/ SSL for the connection, using http and ws protocols.
487 ///
488 /// This is not recommended for production use.
489 pub fn no_tls(mut self, no_tls: bool) -> Self {
490 self.stream_builder = self.stream_builder.no_tls(no_tls);
491 self
492 }
493
494 /// Disable compression for the connection.
495 pub fn disable_compression(mut self) -> Self {
496 self.stream_builder = self
497 .stream_builder
498 .disable_compression();
499 self
500 }
501
502 /// Enables partial block updates (flashblocks).
503 pub fn enable_partial_blocks(mut self) -> Self {
504 self.stream_builder = self
505 .stream_builder
506 .enable_partial_blocks();
507 self
508 }
509
510 /// Sets the number of deltas buffered for each underlying Tycho WebSocket subscription.
511 ///
512 /// See [`TychoStreamBuilder::subscription_buffer_size`].
513 pub fn subscription_buffer_size(mut self, subscription_buffer_size: usize) -> Self {
514 self.stream_builder = self
515 .stream_builder
516 .subscription_buffer_size(subscription_buffer_size);
517 self
518 }
519
520 /// Exclude additional component IDs from all registered exchanges.
521 ///
522 /// These IDs are added to the shipped blocklist that is already applied by default (see
523 /// [`new`](Self::new)).
524 pub fn blocklist_components(mut self, ids: HashSet<String>) -> Self {
525 if !ids.is_empty() {
526 tracing::info!("Blocklisting {} components", ids.len());
527 self.stream_builder = self.stream_builder.blocklisted_ids(ids);
528 }
529 self
530 }
531
532 /// Sets the stream end policy.
533 ///
534 /// Controls when the stream should stop based on synchronizer states.
535 ///
536 /// ## Note
537 /// The stream always ends latest if all protocols are stale or ended independent of
538 /// this configuration. This allows you to end the stream earlier than that.
539 ///
540 /// See [self::StreamEndPolicy] for possible configuration options.
541 pub fn stream_end_policy(mut self, stream_end_policy: StreamEndPolicy) -> Self {
542 self.stream_end_policy = stream_end_policy;
543 self
544 }
545
546 /// Provides token metadata used to decode startup snapshots and initialize protocol states.
547 ///
548 /// This is not a stream filter — components arriving after startup include their own token
549 /// metadata. To restrict to specific tokens, filter in your consumer logic. New tokens
550 /// arriving via stream deltas are added automatically if they meet the quality threshold.
551 pub async fn set_tokens(self, tokens: HashMap<Bytes, Token>) -> Self {
552 self.decoder.set_tokens(tokens).await;
553 self
554 }
555
556 /// Skips decoding errors for component state updates.
557 ///
558 /// Allows the stream to continue processing even if some states fail to decode,
559 /// logging a warning instead of panicking.
560 pub fn skip_state_decode_failures(mut self, skip: bool) -> Self {
561 self.decoder
562 .skip_state_decode_failures(skip);
563 self
564 }
565
566 /// Sets the minimum token quality for tokens added via the stream.
567 ///
568 /// Tokens arriving in stream deltas below this threshold are ignored. Defaults to 100.
569 /// Set this to the same value used in [`load_all_tokens()`](crate::utils::load_all_tokens) to
570 /// apply consistent filtering.
571 pub fn min_token_quality(mut self, quality: u32) -> Self {
572 self.decoder.min_token_quality(quality);
573 self
574 }
575
576 /// Configures the retry policy for websocket reconnects.
577 pub fn websocket_retry_config(mut self, config: &RetryConfiguration) -> Self {
578 self.stream_builder = self
579 .stream_builder
580 .websockets_retry_config(config);
581 self
582 }
583
584 /// Configures the retry policy for state synchronization.
585 pub fn state_synchronizer_retry_config(mut self, config: &RetryConfiguration) -> Self {
586 self.stream_builder = self
587 .stream_builder
588 .state_synchronizer_retry_config(config);
589 self
590 }
591
592 /// Sets the maximum random delay before a restarted synchronizer requests a snapshot.
593 ///
594 /// Defaults to 60 seconds. A `jitter` below 5 seconds is raised to 5 seconds.
595 pub fn snapshot_jitter(mut self, jitter: time::Duration) -> Self {
596 self.stream_builder = self
597 .stream_builder
598 .snapshot_jitter(jitter);
599 self
600 }
601
602 pub fn get_decoder(&self) -> &TychoStreamDecoder<BlockHeader> {
603 &self.decoder
604 }
605
606 /// Registers a [`TxDeltaIndexer`] for ephemeral pending-block simulation.
607 ///
608 /// The indexer is associated with `extractor` (the protocol synchronizer name, e.g.
609 /// `"uniswap_v3"`). Use [`build_with_pending`](Self::build_with_pending) to obtain both
610 /// the confirmed stream and the pending processor.
611 ///
612 /// The exchange must decode into a state whose `delta_transition` can rebuild it from the
613 /// `state_deltas` the indexer produces, because that is all
614 /// [`apply_deltas_ephemeral`](crate::evm::decoder::TychoStreamDecoder::apply_deltas_ephemeral)
615 /// applies. Native and hybrid states qualify; the generic VM adapter does not, because it
616 /// re-reads pool state from the VM database — an indexer registered for one still gets its
617 /// balance and block-environment attributes applied, but every storage-derived value stays at
618 /// the confirmed block, with no error.
619 #[expect(
620 clippy::unnecessary_wraps,
621 reason = "public API; changing the signature is a breaking change"
622 )]
623 pub fn with_pending_indexer(
624 mut self,
625 extractor: &str,
626 indexer: Box<dyn TxDeltaIndexer>,
627 ) -> Result<Self, StreamError> {
628 self.pending_indexers
629 .insert(extractor.to_string(), indexer);
630 Ok(self)
631 }
632
633 /// Enables controlled-step mode for testing.
634 ///
635 /// Returns a [`BlockStepController`] that lets the caller decide when each buffered block
636 /// is released for decoding. Call this before [`build`](Self::build) or
637 /// [`build_with_pending`](Self::build_with_pending) — both detect and wire up the gating
638 /// automatically.
639 ///
640 /// In production code, do not call this method; the stream runs at full speed.
641 pub fn with_step_controller(mut self) -> (Self, BlockStepController) {
642 let (trigger_tx, trigger_rx) = tokio::sync::mpsc::unbounded_channel::<()>();
643 let (peek_tx, peek_rx) =
644 tokio::sync::watch::channel::<Option<FeedMessage<BlockHeader>>>(None);
645
646 self.step_peek_tx = Some(peek_tx);
647 self.step_trigger_rx = Some(trigger_rx);
648
649 let controller = BlockStepController { trigger_tx, peek_rx };
650 (self, controller)
651 }
652
653 /// Spawns a background task that gates `FeedMessage` delivery.
654 ///
655 /// The task buffers each incoming message, publishes it to `peek_tx` so the
656 /// [`BlockStepController`] can inspect it, waits for a trigger, then forwards the message to
657 /// `output_tx` for the decode pipeline. If `advance_tx` is `Some`, a clone of the message is
658 /// also forwarded there (used by the pending-processor path) before the decode step.
659 /// When the input channel closes or a terminal error is received according to
660 /// `stream_end_policy`, the task exits and all output channels are dropped.
661 fn run_gating_task(
662 raw_rx: tokio::sync::mpsc::Receiver<
663 Result<FeedMessage<BlockHeader>, BlockSynchronizerError>,
664 >,
665 mut trigger_rx: tokio::sync::mpsc::UnboundedReceiver<()>,
666 peek_tx: tokio::sync::watch::Sender<Option<FeedMessage<BlockHeader>>>,
667 output_tx: tokio::sync::mpsc::Sender<FeedMessage<BlockHeader>>,
668 stream_end_policy: StreamEndPolicy,
669 ) {
670 tokio::spawn(async move {
671 let mut raw_stream = ReceiverStream::new(raw_rx);
672 loop {
673 let msg = match raw_stream.next().await {
674 Some(Ok(msg)) => msg,
675 Some(Err(e)) => {
676 error!("Block stream ended with terminal error: {e}");
677 break;
678 }
679 None => break,
680 };
681
682 if stream_end_policy.should_end(msg.sync_states.values()) {
683 error!(
684 "Block stream ended due to {:?}: {:?}",
685 stream_end_policy, msg.sync_states
686 );
687 break;
688 }
689
690 // Publish the buffered message so the caller can peek before triggering.
691 #[expect(
692 clippy::let_underscore_must_use,
693 reason = "the peek slot is optional; a send error only means the controller dropped its receiver"
694 )]
695 let _ = peek_tx.send(Some(msg.clone()));
696
697 // Block until the controller fires trigger_next_block(), or until it is dropped.
698 if trigger_rx.recv().await.is_none() {
699 // Controller dropped — forward the buffered message and drain the rest
700 // without gating, so the stream continues to its natural end.
701 #[expect(
702 clippy::let_underscore_must_use,
703 reason = "the peek slot is optional; a send error only means the controller dropped its receiver"
704 )]
705 let _ = peek_tx.send(None);
706 if output_tx.send(msg).await.is_err() {
707 break;
708 }
709 while let Some(item) = raw_stream.next().await {
710 let Ok(msg) = item else { break };
711 if stream_end_policy.should_end(msg.sync_states.values()) {
712 break;
713 }
714 if output_tx.send(msg).await.is_err() {
715 break;
716 }
717 }
718 break;
719 }
720
721 // Clear the peek slot before decoding so callers see None between blocks.
722 #[expect(
723 clippy::let_underscore_must_use,
724 reason = "the peek slot is optional; a send error only means the controller dropped its receiver"
725 )]
726 let _ = peek_tx.send(None);
727
728 if output_tx.send(msg).await.is_err() {
729 break;
730 }
731 }
732 });
733 }
734
735 /// Registers `provider` as the live override source for `protocol_system`.
736 ///
737 /// Explicit registrations take precedence over the built-in default registry, so this is how
738 /// you swap a venue (e.g. `vm:bopamm`) onto a different provider. Registering the same provider
739 /// for several protocols is cheap — it is shared via `Arc`, not duplicated.
740 pub fn with_override_provider(
741 mut self,
742 protocol_system: impl Into<String>,
743 provider: Arc<dyn StateOverrideProvider>,
744 ) -> Self {
745 self.override_providers
746 .insert(protocol_system.into(), provider);
747 self
748 }
749
750 /// Installs override providers onto the decoder before the stream is built.
751 ///
752 /// Explicit consumer registrations win; the built-in default registry (see
753 /// [`default_override_providers`](crate::evm::override_stream::default_override_providers))
754 /// then fills every remaining protocol it can serve.
755 fn install_override_providers(&mut self) {
756 let explicit = std::mem::take(&mut self.override_providers);
757 // Protocols eligible for a built-in default provider: registered exchanges not explicitly
758 // overridden by the consumer.
759 let uncovered = self
760 .registered_exchanges
761 .clone()
762 .into_iter()
763 .filter(|exchange| !explicit.contains_key(exchange));
764 let defaults = override_stream::default_override_providers(uncovered);
765 for (protocol_system, provider) in defaults.into_iter().chain(explicit) {
766 self.decoder
767 .set_override_provider(protocol_system, provider);
768 }
769 }
770
771 /// Builds the confirmed protocol stream and a [`PendingBlockProcessor`] that stays
772 /// in sync with it automatically.
773 ///
774 /// The stream pipeline forwards every confirmed [`FeedMessage`] to the processor via an
775 /// internal unbounded channel — it never blocks waiting for the consumer. The consumer
776 /// owns the returned `PendingBlockProcessor` exclusively and may wrap it in whatever
777 /// synchronisation primitive suits their use case (e.g. `Mutex` for shared access,
778 /// nothing for single-threaded use).
779 ///
780 /// Call [`generate_pending_update`](PendingBlockProcessor::generate_pending_update) to
781 /// simulate a candidate bundle; it drains the channel automatically before computing.
782 pub async fn build_with_pending(
783 mut self,
784 ) -> Result<
785 (impl Stream<Item = Result<Update, StreamDecodeError>>, PendingBlockProcessor),
786 StreamError,
787 > {
788 initialize_hook_handlers().map_err(|e| {
789 StreamError::SetUpError(format!("Error initializing hook handlers: {e:?}"))
790 })?;
791 self.install_override_providers();
792 let (_, rx) = self.stream_builder.build().await?;
793 let decoder = Arc::new(self.decoder);
794
795 let (advance_tx, advance_rx) =
796 tokio::sync::mpsc::unbounded_channel::<FeedMessage<BlockHeader>>();
797 let pending = PendingBlockProcessor::new(
798 self.pending_indexers,
799 decoder.clone(),
800 self.chain,
801 advance_rx,
802 );
803
804 let chain = self.chain;
805 let stream_end_policy = self.stream_end_policy;
806
807 let decode_stream: Box<dyn Stream<Item = FeedMessage<BlockHeader>> + Send + Unpin> =
808 if let (Some(peek_tx), Some(trigger_rx)) = (self.step_peek_tx, self.step_trigger_rx) {
809 let (gated_tx, gated_rx) =
810 tokio::sync::mpsc::channel::<FeedMessage<BlockHeader>>(1);
811 Self::run_gating_task(rx, trigger_rx, peek_tx, gated_tx, stream_end_policy);
812 Box::new(ReceiverStream::new(gated_rx))
813 } else {
814 let normal = ReceiverStream::new(rx)
815 .take_while(move |msg| match msg {
816 Ok(msg) => {
817 let states = msg.sync_states.values();
818 if stream_end_policy.should_end(states) {
819 error!(
820 "Block stream ended due to {:?}: {:?}",
821 stream_end_policy, msg.sync_states
822 );
823 futures::future::ready(false)
824 } else {
825 futures::future::ready(true)
826 }
827 }
828 Err(e) => {
829 error!("Block stream ended with terminal error: {e}");
830 futures::future::ready(false)
831 }
832 })
833 .map(|msg| msg.expect("Safe since stream ends if we receive an error"));
834 Box::new(Box::pin(normal))
835 };
836
837 let stream = Box::pin(decode_stream.then({
838 let decoder = decoder;
839 move |msg| {
840 let decoder = decoder.clone();
841 let advance_tx = advance_tx.clone();
842 async move {
843 if let Err(e) = advance_tx.send(msg.clone()) {
844 warn!(error = %e, "Pending block processor is gone; block not forwarded");
845 }
846 decoder.decode(&msg).await.map_err(|e| {
847 debug!(msg=?msg, "Decode error: {}", e);
848 e
849 })
850 }
851 }
852 }));
853 let stream = inject_native_wrapper(stream, chain);
854 Ok((stream, pending))
855 }
856
857 /// Builds and returns the configured protocol stream.
858 ///
859 /// See the module-level docs for details on stream behavior and emitted messages.
860 /// This method applies all builder settings and starts the stream.
861 pub async fn build(
862 mut self,
863 ) -> Result<impl Stream<Item = Result<Update, StreamDecodeError>>, StreamError> {
864 initialize_hook_handlers().map_err(|e| {
865 StreamError::SetUpError(format!("Error initializing hook handlers: {e:?}"))
866 })?;
867 self.install_override_providers();
868 let (_, rx) = self.stream_builder.build().await?;
869 let decoder = Arc::new(self.decoder);
870 let chain = self.chain;
871 let stream_end_policy = self.stream_end_policy;
872
873 let decode_stream: Box<dyn Stream<Item = FeedMessage<BlockHeader>> + Send + Unpin> =
874 if let (Some(peek_tx), Some(trigger_rx)) = (self.step_peek_tx, self.step_trigger_rx) {
875 let (gated_tx, gated_rx) =
876 tokio::sync::mpsc::channel::<FeedMessage<BlockHeader>>(1);
877 Self::run_gating_task(rx, trigger_rx, peek_tx, gated_tx, stream_end_policy);
878 Box::new(ReceiverStream::new(gated_rx))
879 } else {
880 let normal = ReceiverStream::new(rx)
881 .take_while(move |msg| match msg {
882 Ok(msg) => {
883 let states = msg.sync_states.values();
884 if stream_end_policy.should_end(states) {
885 error!(
886 "Block stream ended due to {:?}: {:?}",
887 stream_end_policy, msg.sync_states
888 );
889 futures::future::ready(false)
890 } else {
891 futures::future::ready(true)
892 }
893 }
894 Err(e) => {
895 error!("Block stream ended with terminal error: {e}");
896 futures::future::ready(false)
897 }
898 })
899 .map(|msg| msg.expect("Safe since stream ends if we receive an error"));
900 Box::new(Box::pin(normal))
901 };
902
903 let stream = Box::pin(decode_stream.then({
904 let decoder = decoder;
905 move |msg| {
906 let decoder = decoder.clone();
907 async move {
908 decoder.decode(&msg).await.map_err(|e| {
909 debug!(msg=?msg, "Decode error: {}", e);
910 e
911 })
912 }
913 }
914 }));
915 let stream = inject_native_wrapper(stream, chain);
916 Ok(stream)
917 }
918}
919
920/// Wraps a decoded protocol stream to inject a `NativeWrapperState` component
921/// on the first successful update.
922///
923/// Skips injection when the chain's native asset has no real wrapper contract.
924fn inject_native_wrapper(
925 inner: impl Stream<Item = Result<Update, StreamDecodeError>> + Unpin + Send + 'static,
926 chain: Chain,
927) -> impl Stream<Item = Result<Update, StreamDecodeError>> + Send {
928 let Some(native_wrapper) = NativeWrapperState::new(chain) else {
929 return Either::Left(inner);
930 };
931
932 let mut pending_wrapper = Some(native_wrapper);
933 Either::Right(inner.map(move |result| {
934 result.map(|mut update| {
935 if let Some(native_wrapper) = pending_wrapper.take() {
936 let component = native_wrapper.component();
937 let id = component.id.to_string();
938 update
939 .new_pairs
940 .insert(id.clone(), component);
941 update
942 .states
943 .insert(id, Box::new(native_wrapper));
944 debug!("Injected native_wrapper component for {chain}");
945 }
946 update
947 })
948 }))
949}
950
951#[cfg(test)]
952mod tests {
953 use std::collections::HashMap;
954
955 use futures::{stream, StreamExt};
956 use tycho_common::models::Chain;
957
958 use super::*;
959 use crate::{
960 evm::protocol::native_wrapper::state::NATIVE_WRAPPER_ID, protocol::models::Update,
961 };
962
963 fn empty_update(block: u64) -> Update {
964 Update::new(block, HashMap::new(), HashMap::new())
965 }
966
967 #[tokio::test]
968 async fn test_inject_native_wrapper_first_message_only() {
969 let updates = vec![Ok(empty_update(1)), Ok(empty_update(2)), Ok(empty_update(3))];
970 let input = stream::iter(updates);
971
972 let results: Vec<_> = inject_native_wrapper(input, Chain::Ethereum)
973 .collect()
974 .await;
975
976 assert_eq!(results.len(), 3);
977
978 let expected_id = NativeWrapperState::new(Chain::Ethereum)
979 .expect("Ethereum should have a wrapper")
980 .component()
981 .id
982 .to_string();
983
984 let first = results[0]
985 .as_ref()
986 .expect("first update ok");
987 assert!(
988 first
989 .new_pairs
990 .contains_key(&expected_id),
991 "first message should have native_wrapper component"
992 );
993 assert!(
994 first.states.contains_key(&expected_id),
995 "first message should have native_wrapper state"
996 );
997
998 let second = results[1]
999 .as_ref()
1000 .expect("second update ok");
1001 assert!(
1002 !second
1003 .new_pairs
1004 .contains_key(&expected_id),
1005 "second message should NOT have native_wrapper component"
1006 );
1007 assert!(
1008 !second.states.contains_key(&expected_id),
1009 "second message should NOT have native_wrapper state"
1010 );
1011 }
1012
1013 #[tokio::test]
1014 async fn test_inject_native_wrapper_after_initial_decode_error() {
1015 let updates = vec![
1016 Err(StreamDecodeError::Fatal("decode failed".to_string())),
1017 Ok(empty_update(1)),
1018 Ok(empty_update(2)),
1019 ];
1020 let results: Vec<_> = inject_native_wrapper(stream::iter(updates), Chain::Ethereum)
1021 .collect()
1022 .await;
1023
1024 assert!(results[0].is_err(), "the decode error must pass through unchanged");
1025 let expected_id = NativeWrapperState::new(Chain::Ethereum)
1026 .expect("Ethereum should have a wrapper")
1027 .component()
1028 .id
1029 .to_string();
1030 let first_success = results[1]
1031 .as_ref()
1032 .expect("second stream item should decode");
1033 assert!(first_success
1034 .new_pairs
1035 .contains_key(&expected_id));
1036 assert!(first_success
1037 .states
1038 .contains_key(&expected_id));
1039
1040 let second_success = results[2]
1041 .as_ref()
1042 .expect("third stream item should decode");
1043 assert!(!second_success
1044 .new_pairs
1045 .contains_key(&expected_id));
1046 assert!(!second_success
1047 .states
1048 .contains_key(&expected_id));
1049 }
1050
1051 #[tokio::test]
1052 async fn test_does_not_inject_native_wrapper_for_arc_shared_balance() {
1053 let results: Vec<_> =
1054 inject_native_wrapper(stream::iter([Ok(empty_update(1))]), Chain::Arc)
1055 .collect()
1056 .await;
1057
1058 let update = results[0]
1059 .as_ref()
1060 .expect("update should decode");
1061 assert!(!update
1062 .new_pairs
1063 .contains_key(NATIVE_WRAPPER_ID));
1064 assert!(!update
1065 .states
1066 .contains_key(NATIVE_WRAPPER_ID));
1067 }
1068
1069 /// Verifies that `with_step_controller` returns both a modified builder and a controller.
1070 ///
1071 /// This test only checks that the builder method is callable and that the returned controller
1072 /// compiles — it does not start any network connection.
1073 #[tokio::test]
1074 async fn test_with_step_controller_returns_controller() {
1075 let builder = ProtocolStreamBuilder::new("tycho-beta.propellerheads.xyz", Chain::Ethereum);
1076 let (_builder, controller) = builder.with_step_controller();
1077 // The controller was successfully returned — verifying the public API is callable.
1078 drop(controller);
1079 }
1080
1081 #[tokio::test]
1082 async fn test_subscription_buffer_size_forwards_zero_rejection() {
1083 let error = ProtocolStreamBuilder::new("not a valid endpoint", Chain::Ethereum)
1084 .subscription_buffer_size(0)
1085 .build()
1086 .await;
1087
1088 let Err(error) = error else {
1089 panic!("a zero subscription buffer size must be rejected during setup");
1090 };
1091
1092 assert!(
1093 error
1094 .to_string()
1095 .contains("subscription buffer size must be greater than zero"),
1096 "the underlying Tycho stream builder should return the configuration error: {error}"
1097 );
1098 }
1099
1100 #[tokio::test]
1101 async fn test_subscription_buffer_size_forwards_upper_bound_rejection() {
1102 let error = ProtocolStreamBuilder::new("not a valid endpoint", Chain::Ethereum)
1103 .subscription_buffer_size(usize::MAX)
1104 .build()
1105 .await;
1106
1107 let Err(error) = error else {
1108 panic!("an oversized subscription buffer size must be rejected during setup");
1109 };
1110
1111 assert!(
1112 error
1113 .to_string()
1114 .contains("subscription buffer size must not exceed"),
1115 "the underlying Tycho stream builder should return the configuration error: {error}"
1116 );
1117 }
1118
1119 /// Connects to a live Tycho instance, verifies that the stream blocks until
1120 /// `trigger_next_block` is called, and that `peek_next_block` exposes the buffered message.
1121 #[ignore = "requires live Tycho connection (TYCHO_AUTH_TOKEN env var)"]
1122 #[tokio::test]
1123 async fn test_step_controller_trigger_releases_block() {
1124 use std::{env, time::Duration};
1125
1126 use crate::evm::protocol::uniswap_v2::state::UniswapV2State;
1127
1128 let auth = env::var("TYCHO_AUTH_TOKEN").expect("TYCHO_AUTH_TOKEN must be set");
1129
1130 // Track a single well-known pool to minimise startup latency.
1131 let usdc_weth_v2 = "0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc".to_string();
1132 let (builder, controller) =
1133 ProtocolStreamBuilder::new("tycho-beta.propellerheads.xyz", Chain::Ethereum)
1134 .auth_key(Some(auth))
1135 .exchange::<UniswapV2State>(
1136 "uniswap_v2",
1137 ComponentFilter::Ids(vec![usdc_weth_v2]),
1138 None,
1139 )
1140 .with_step_controller();
1141
1142 let (stream, _pending) = builder
1143 .build_with_pending()
1144 .await
1145 .expect("build_with_pending failed");
1146 tokio::pin!(stream);
1147
1148 // Wait up to 60 s for the first block to arrive in the gating buffer.
1149 let peeked = tokio::time::timeout(Duration::from_secs(60), controller.peek_next_block())
1150 .await
1151 .expect("timed out waiting for first block to buffer")
1152 .expect("stream ended before a block arrived");
1153
1154 assert!(!peeked.sync_states.is_empty(), "peeked block should carry sync states");
1155
1156 // Stream must be empty before we trigger — the gating task should be holding the block.
1157 let pre_trigger = tokio::time::timeout(Duration::from_millis(200), stream.next()).await;
1158 assert!(
1159 pre_trigger.is_err(),
1160 "stream should be blocked before trigger_next_block, got an item"
1161 );
1162
1163 // Release the block.
1164 controller
1165 .trigger_next_block()
1166 .expect("trigger_next_block failed");
1167
1168 // Stream should now yield the decoded update within one block time.
1169 let update = tokio::time::timeout(Duration::from_secs(30), stream.next())
1170 .await
1171 .expect("timed out waiting for update after trigger")
1172 .expect("stream ended unexpectedly");
1173
1174 assert!(update.is_ok(), "decoded update should be Ok, got: {:?}", update);
1175 }
1176}