tycho_simulation/evm/stream.rs
1//! Builder for configuring a multi-protocol stream.
2//!
3//! Provides a builder for creating a multi-protocol stream that produces
4//! protocol state update messages. It runs one synchronization worker per protocol
5//! and a supervisor that aggregates updates, ensuring gap‑free streaming
6//! and robust state tracking.
7//!
8//! ## Context
9//!
10//! This stream wraps a `TychoStream` from `tycho-client`. It decodes `FeedMessage`s
11//! into protocol state updates. Internally, each protocol runs in its own
12//! synchronization worker, and a supervisor aggregates their messages per block.
13//!
14//! ### Protocol Synchronization Worker
15//! A synchronization worker runs the snapshot + delta protocol from `tycho-indexer`.
16//! - It first downloads components and their snapshots.
17//! - It then streams deltas.
18//! - It reacts to new or paused components by pulling snapshots or removing them from the active
19//! set.
20//!
21//! Each worker emits snapshots and deltas to the supervisor.
22//!
23//! ### Stream Supervisor
24//! The supervisor aggregates worker messages by block and assigns sync status.
25//! - It ensures workers produce gap-free messages.
26//! - It flags late workers as `Delayed`, and marks them `Stale` if they exceed `max_missed_blocks`.
27//! - It marks workers with terminal errors as `Ended`.
28//!
29//! Aggregating by block adds small latency, since the supervisor waits briefly for
30//! all workers to emit. This latency only applies to workers in `Ready` or `Delayed`.
31//!
32//! The stream ends only when **all** workers are `Stale` or `Ended`.
33//!
34//! ## Configuration
35//!
36//! The builder lets you customize:
37//!
38//! ### Protocols
39//! Select which protocols to synchronize.
40//!
41//! ### Tokens & Minimum Token Quality
42//! Provide token metadata up front so the decoder can initialize protocol states from startup
43//! snapshots. `set_tokens` does not act as an ongoing filter — components arriving after startup
44//! include their own token metadata. To restrict processing to specific tokens, apply that filter
45//! in your consumer when reading `new_components`. New tokens arriving via stream deltas are added
46//! automatically when their quality exceeds `min_token_quality`.
47//!
48//! ### StreamEndPolicy
49//! Control when the stream ends based on worker states. By default, it ends when all
50//! workers are `Stale` or `Ended`.
51//!
52//! ## Stream
53//! The stream emits one protocol state update every `block_time`. Each update
54//! reports protocol synchronization states and any changes.
55//!
56//! The `new_components` field lists newly deployed components and their tokens.
57//!
58//! The stream aims to run indefinitely. Internal retry and reconnect logic handle
59//! most errors, so users should rarely need to restart it manually.
60//!
61//! ## Example
62//! ```no_run
63//! use tycho_common::models::Chain;
64//! use tycho_simulation::evm::stream::ProtocolStreamBuilder;
65//! use tycho_simulation::utils::load_all_tokens;
66//! use futures::StreamExt;
67//! use tycho_client::feed::component_tracker::ComponentFilter;
68//! use tycho_simulation::evm::protocol::uniswap_v2::state::UniswapV2State;
69//!
70//! #[tokio::main]
71//! async fn main() {
72//! let all_tokens = load_all_tokens(
73//! "tycho-beta.propellerheads.xyz",
74//! false,
75//! Some("sampletoken"),
76//! true,
77//! Chain::Ethereum,
78//! None,
79//! None,
80//! )
81//! .await
82//! .expect("Failed loading tokens");
83//!
84//! let protocol_stream =
85//! ProtocolStreamBuilder::new("tycho-beta.propellerheads.xyz", Chain::Ethereum)
86//! .auth_key(Some("sampletoken".to_string()))
87//! .skip_state_decode_failures(true)
88//! .exchange::<UniswapV2State>(
89//! "uniswap_v2", ComponentFilter::with_tvl_range(5.0, 10.0), None
90//! )
91//! .set_tokens(all_tokens)
92//! .await
93//! .build()
94//! .await
95//! .expect("Failed building protocol stream");
96//! tokio::pin!(protocol_stream);
97//!
98//! // Loop through block updates
99//! while let Some(msg) = protocol_stream.next().await {
100//! dbg!(msg).expect("failed decoding");
101//! }
102//! }
103//! ```
104use std::{
105 collections::{HashMap, HashSet},
106 sync::Arc,
107 time,
108};
109
110use futures::{future::Either, Stream, StreamExt};
111use tokio_stream::wrappers::ReceiverStream;
112use tracing::{debug, error, warn};
113use tycho_client::{
114 feed::{
115 component_tracker::ComponentFilter, synchronizer::ComponentWithState, BlockHeader,
116 BlockSynchronizerError, FeedMessage, SynchronizerState,
117 },
118 stream::{RetryConfiguration, StreamError, TychoStreamBuilder},
119};
120use tycho_common::{
121 models::{token::Token, Chain},
122 simulation::protocol_sim::ProtocolSim,
123 traits::TxDeltaIndexer,
124 Bytes,
125};
126
127use crate::{
128 evm::{
129 decoder::{StreamDecodeError, TychoStreamDecoder},
130 override_stream::{self, StateOverrideProvider},
131 pending::PendingBlockProcessor,
132 protocol::{
133 filters::uniswap_v4_non_angstrom_hook_pool_filter,
134 native_wrapper::state::NativeWrapperState,
135 uniswap_v4::hooks::hook_handler_creator::initialize_hook_handlers,
136 },
137 },
138 protocol::{
139 errors::InvalidSnapshotError,
140 models::{DecoderContext, TryFromWithBlock, Update},
141 },
142 utils::default_blocklist,
143};
144
145const EXCHANGES_REQUIRING_FILTER: [&str; 5] =
146 ["vm:balancer_v2", "fluid_v1", "erc4626", "ekubo_v3", "vm:curve"];
147
148/// The client-side filter exchange `name` always gets, in addition to any filter the caller
149/// provides.
150///
151/// `uniswap_v4_hooks`: without `ANGSTROM_API_KEY`, Angstrom swaps cannot be encoded (they carry
152/// per-block attestations from the Angstrom API), so Angstrom pools are excluded up front rather
153/// than failing every route that selects them at encoding time. A caller's own hook filter does
154/// not replace this one: the encoder still has no key.
155fn mandatory_filter_fn(name: &str) -> Option<fn(&ComponentWithState) -> bool> {
156 if name == "uniswap_v4_hooks" && std::env::var("ANGSTROM_API_KEY").is_err() {
157 warn!(
158 "ANGSTROM_API_KEY is not set: excluding Angstrom pools from '{name}'. \
159 Set the key to include them."
160 );
161 return Some(uniswap_v4_non_angstrom_hook_pool_filter);
162 }
163 None
164}
165
166#[derive(Default, Debug, Clone, Copy)]
167pub enum StreamEndPolicy {
168 /// End stream if all states are Stale or Ended (default)
169 #[default]
170 AllEndedOrStale,
171 /// End stream if any protocol ended
172 AnyEnded,
173 /// End stream if any protocol ended or is stale
174 AnyEndedOrStale,
175 /// End stream if any protocol is stale
176 AnyStale,
177}
178
179impl StreamEndPolicy {
180 fn should_end<'a>(&self, states: impl IntoIterator<Item = &'a SynchronizerState>) -> bool {
181 let mut it = states.into_iter();
182 match self {
183 StreamEndPolicy::AllEndedOrStale => false,
184 StreamEndPolicy::AnyEnded => it.any(|s| matches!(s, SynchronizerState::Ended(_))),
185 StreamEndPolicy::AnyStale => it.any(|s| matches!(s, SynchronizerState::Stale(_))),
186 StreamEndPolicy::AnyEndedOrStale => {
187 it.any(|s| matches!(s, SynchronizerState::Stale(_) | SynchronizerState::Ended(_)))
188 }
189 }
190 }
191}
192
193/// Handle returned by [`ProtocolStreamBuilder::with_step_controller`] that gives external
194/// control over when each buffered block is released for decoding.
195///
196/// Intended for complex test scenarios where the caller needs to observe what the next
197/// block contains before allowing the decoder pipeline to process it.
198///
199/// ## Drop behaviour
200///
201/// Dropping this controller ungates the stream: the gating task detects the closed trigger
202/// channel, forwards the currently-buffered block (if any), then continues passing subsequent
203/// blocks through without waiting for triggers — exactly as if step-control had never been
204/// enabled. The stream runs to its natural end.
205pub struct BlockStepController {
206 /// Sends a trigger signal to release the next buffered block.
207 trigger_tx: tokio::sync::mpsc::UnboundedSender<()>,
208 /// Watch channel containing the next buffered raw message, or `None` if no block is pending.
209 peek_rx: tokio::sync::watch::Receiver<Option<FeedMessage<BlockHeader>>>,
210}
211
212impl BlockStepController {
213 /// Releases the next buffered block for decoding and emission.
214 ///
215 /// Returns an error if the stream has already ended and the sender is disconnected.
216 pub fn trigger_next_block(&self) -> Result<(), tokio::sync::mpsc::error::SendError<()>> {
217 // Send a unit value on the trigger channel to unblock the gating task.
218 self.trigger_tx.send(())
219 }
220
221 /// Returns the currently buffered block immediately, or `None` if no block is buffered yet.
222 pub fn try_peek_next_block(&self) -> Option<FeedMessage<BlockHeader>> {
223 self.peek_rx.borrow().clone()
224 }
225
226 /// Waits until a block is buffered and returns it without consuming it.
227 ///
228 /// Returns `None` only if the stream has ended and no further blocks will arrive.
229 /// If a block is already buffered when this is called, it returns immediately.
230 pub async fn peek_next_block(&self) -> Option<FeedMessage<BlockHeader>> {
231 // Clone so we don't hold a mutable borrow on self; wait_for checks the current
232 // value first, so this returns immediately if a block is already present.
233 let mut rx = self.peek_rx.clone();
234 let guard = rx
235 .wait_for(|v| v.is_some())
236 .await
237 .ok()?;
238 guard.clone()
239 }
240}
241
242/// Builds and configures the multi protocol stream described in the [module-level docs](self).
243///
244/// See the module documentation for details on protocols, configuration options, and
245/// stream behavior.
246pub struct ProtocolStreamBuilder {
247 decoder: TychoStreamDecoder<BlockHeader>,
248 stream_builder: TychoStreamBuilder,
249 stream_end_policy: StreamEndPolicy,
250 chain: Chain,
251 pending_indexers: HashMap<String, Box<dyn TxDeltaIndexer>>,
252 /// Watch sender used to publish the currently-buffered raw block so the controller can peek
253 /// at it before triggering. `Some` iff step-control mode is active.
254 step_peek_tx: Option<tokio::sync::watch::Sender<Option<FeedMessage<BlockHeader>>>>,
255 /// Receiver half of the trigger channel. Held here until `build()` / `build_with_pending()`
256 /// transfers ownership to the gating task. `Some` iff step-control mode is active.
257 step_trigger_rx: Option<tokio::sync::mpsc::UnboundedReceiver<()>>,
258 /// State-override providers explicitly registered by the consumer, keyed by `protocol_system`.
259 /// These take precedence over the built-in default registry and are installed onto the decoder
260 /// at build time.
261 override_providers: HashMap<String, Arc<dyn StateOverrideProvider>>,
262 /// Names of all exchanges registered on the builder, used to decide which built-in override
263 /// providers to auto-register at build time.
264 registered_exchanges: HashSet<String>,
265}
266
267impl ProtocolStreamBuilder {
268 /// Creates a new builder for a multi-protocol stream.
269 ///
270 /// The shipped pool blocklist is applied by default, excluding components known to break
271 /// simulation. Use [`blocklist_components`](Self::blocklist_components) to exclude additional
272 /// components.
273 ///
274 /// See the [module-level docs](self) for full details on stream behavior and configuration.
275 pub fn new(tycho_url: &str, chain: Chain) -> Self {
276 Self {
277 decoder: TychoStreamDecoder::new(chain),
278 stream_builder: TychoStreamBuilder::new(tycho_url, chain)
279 .blocklisted_ids(default_blocklist()),
280 stream_end_policy: StreamEndPolicy::default(),
281 chain,
282 pending_indexers: HashMap::new(),
283 step_peek_tx: None,
284 step_trigger_rx: None,
285 override_providers: HashMap::new(),
286 registered_exchanges: HashSet::new(),
287 }
288 }
289
290 /// Adds a specific exchange to the stream.
291 ///
292 /// This configures the builder to include a new protocol synchronizer for `name`,
293 /// filtering its components according to `filter` and optionally `filter_fn`.
294 ///
295 /// The type parameter `T` specifies the decoder type for this exchange. All
296 /// component states for this exchange will be decoded into instances of `T`.
297 ///
298 /// # Parameters
299 ///
300 /// - `name`: The protocol or exchange name (e.g., `"uniswap_v4"`, `"vm:balancer_v2"`).
301 /// - `filter`: Defines the set of components to include in the stream.
302 /// - `filter_fn`: Optional custom filter function for client-side filtering of components not
303 /// expressible in `filter`.
304 ///
305 /// # Notes
306 ///
307 /// For certain protocols (e.g., `"uniswap_v4"`, `"vm:balancer_v2"`, `"vm:curve"`), omitting
308 /// `filter_fn` may cause decoding errors or incorrect results. In these cases, a proper
309 /// filter function is required to ensure correct decoding and quoting logic.
310 pub fn exchange<T>(
311 mut self,
312 name: &str,
313 filter: ComponentFilter,
314 filter_fn: Option<fn(&ComponentWithState) -> bool>,
315 ) -> Self
316 where
317 T: ProtocolSim
318 + TryFromWithBlock<ComponentWithState, BlockHeader, Error = InvalidSnapshotError>
319 + Send
320 + 'static,
321 {
322 self.stream_builder = self
323 .stream_builder
324 .exchange(name, filter);
325 self.registered_exchanges
326 .insert(name.to_string());
327 self.decoder.register_decoder::<T>(name);
328 if let Some(predicate) = filter_fn {
329 self.decoder
330 .register_filter(name, predicate);
331 }
332 if let Some(predicate) = mandatory_filter_fn(name) {
333 self.decoder
334 .register_filter(name, predicate);
335 }
336
337 if EXCHANGES_REQUIRING_FILTER.contains(&name) && filter_fn.is_none() {
338 warn!(
339 "Warning: For exchange type '{}', it is necessary to set a filter function because not all pools are supported. See all filters at src/evm/protocol/filters.rs",
340 name
341 );
342 }
343
344 self
345 }
346
347 /// Adds a specific exchange to the stream with decoder context.
348 ///
349 /// This configures the builder to include a new protocol synchronizer for `name`,
350 /// filtering its components according to `filter` and optionally `filter_fn`. It also registers
351 /// the DecoderContext (this is useful to test protocols that are not live yet)
352 ///
353 /// The type parameter `T` specifies the decoder type for this exchange. All
354 /// component states for this exchange will be decoded into instances of `T`.
355 ///
356 /// # Parameters
357 ///
358 /// - `name`: The protocol or exchange name (e.g., `"uniswap_v4"`, `"vm:balancer_v2"`).
359 /// - `filter`: Defines the set of components to include in the stream.
360 /// - `filter_fn`: Optional custom filter function for client-side filtering of components not
361 /// expressible in `filter`.
362 /// - `decoder_context`: The decoder context for this exchange
363 ///
364 /// # Notes
365 ///
366 /// For certain protocols (e.g., `"uniswap_v4"`, `"vm:balancer_v2"`, `"vm:curve"`), omitting
367 /// `filter_fn` may cause decoding errors or incorrect results. In these cases, a proper
368 /// filter function is required to ensure correct decoding and quoting logic.
369 pub fn exchange_with_decoder_context<T>(
370 mut self,
371 name: &str,
372 filter: ComponentFilter,
373 filter_fn: Option<fn(&ComponentWithState) -> bool>,
374 decoder_context: DecoderContext,
375 ) -> Self
376 where
377 T: ProtocolSim
378 + TryFromWithBlock<ComponentWithState, BlockHeader, Error = InvalidSnapshotError>
379 + Send
380 + 'static,
381 {
382 self.stream_builder = self
383 .stream_builder
384 .exchange(name, filter);
385 self.registered_exchanges
386 .insert(name.to_string());
387 self.decoder
388 .register_decoder_with_context::<T>(name, decoder_context);
389 if let Some(predicate) = filter_fn {
390 self.decoder
391 .register_filter(name, predicate);
392 }
393 if let Some(predicate) = mandatory_filter_fn(name) {
394 self.decoder
395 .register_filter(name, predicate);
396 }
397
398 if EXCHANGES_REQUIRING_FILTER.contains(&name) && filter_fn.is_none() {
399 warn!(
400 "Warning: For exchange type '{}', it is necessary to set a filter function because not all pools are supported. See all filters at src/evm/protocol/filters.rs",
401 name
402 );
403 }
404
405 self
406 }
407
408 /// Sets the block time interval for the stream.
409 ///
410 /// This controls how often the stream produces updates.
411 pub fn block_time(mut self, block_time: u64) -> Self {
412 self.stream_builder = self
413 .stream_builder
414 .block_time(block_time);
415 self
416 }
417
418 /// Sets the network operation timeout (deprecated).
419 ///
420 /// Use [`latency_buffer()`](Self::latency_buffer) instead for controlling latency.
421 /// This method is retained for backwards compatibility.
422 #[deprecated = "Use latency_buffer instead"]
423 pub fn timeout(mut self, timeout: u64) -> Self {
424 self.stream_builder = self.stream_builder.timeout(timeout);
425 self
426 }
427
428 /// Sets the latency buffer to aggregate same-block messages.
429 ///
430 /// This allows the supervisor to wait a short interval for all synchronizers to emit
431 /// before aggregating.
432 pub fn latency_buffer(mut self, timeout: u64) -> Self {
433 self.stream_builder = self.stream_builder.timeout(timeout);
434 self
435 }
436
437 /// Sets the maximum number of blocks a synchronizer may miss before being marked as `Stale`.
438 pub fn max_missed_blocks(mut self, n: u64) -> Self {
439 self.stream_builder = self.stream_builder.max_missed_blocks(n);
440 self
441 }
442
443 /// Sets how long a synchronizer may take to process the initial message.
444 ///
445 /// Useful for data-intensive protocols where startup decoding takes longer.
446 pub fn startup_timeout(mut self, timeout: time::Duration) -> Self {
447 self.stream_builder = self
448 .stream_builder
449 .startup_timeout(timeout);
450 self
451 }
452
453 /// Configures the stream to exclude state updates.
454 ///
455 /// This reduces bandwidth and decoding workload if protocol state is not of
456 /// interest (e.g. only process new tokens).
457 pub fn no_state(mut self, no_state: bool) -> Self {
458 self.stream_builder = self.stream_builder.no_state(no_state);
459 self
460 }
461
462 /// Sets the API key for authenticating with the Tycho server.
463 pub fn auth_key(mut self, auth_key: Option<String>) -> Self {
464 self.stream_builder = self.stream_builder.auth_key(auth_key);
465 self
466 }
467
468 /// Adds client-metadata entries forwarded to the server in the `X-Tycho-Client-Metadata`
469 /// header.
470 ///
471 /// See [`TychoStreamBuilder::add_client_metadata`]. Values are self-reported and may surface in
472 /// the server's metrics and logs — do not include secrets or personally identifiable
473 /// information.
474 pub fn add_client_metadata<I, K, V>(mut self, metadata: I) -> Self
475 where
476 I: IntoIterator<Item = (K, V)>,
477 K: Into<String>,
478 V: Into<String>,
479 {
480 self.stream_builder = self
481 .stream_builder
482 .add_client_metadata(metadata);
483 self
484 }
485
486 /// Disables TLS/ SSL for the connection, using http and ws protocols.
487 ///
488 /// This is not recommended for production use.
489 pub fn no_tls(mut self, no_tls: bool) -> Self {
490 self.stream_builder = self.stream_builder.no_tls(no_tls);
491 self
492 }
493
494 /// Disable compression for the connection.
495 pub fn disable_compression(mut self) -> Self {
496 self.stream_builder = self
497 .stream_builder
498 .disable_compression();
499 self
500 }
501
502 /// Enables partial block updates (flashblocks).
503 pub fn enable_partial_blocks(mut self) -> Self {
504 self.stream_builder = self
505 .stream_builder
506 .enable_partial_blocks();
507 self
508 }
509
510 /// Sets the number of deltas buffered for each underlying Tycho WebSocket subscription.
511 ///
512 /// See [`TychoStreamBuilder::subscription_buffer_size`].
513 pub fn subscription_buffer_size(mut self, subscription_buffer_size: usize) -> Self {
514 self.stream_builder = self
515 .stream_builder
516 .subscription_buffer_size(subscription_buffer_size);
517 self
518 }
519
520 /// Exclude additional component IDs from all registered exchanges.
521 ///
522 /// These IDs are added to the shipped blocklist that is already applied by default (see
523 /// [`new`](Self::new)).
524 pub fn blocklist_components(mut self, ids: HashSet<String>) -> Self {
525 if !ids.is_empty() {
526 tracing::info!("Blocklisting {} components", ids.len());
527 self.stream_builder = self.stream_builder.blocklisted_ids(ids);
528 }
529 self
530 }
531
532 /// Sets the stream end policy.
533 ///
534 /// Controls when the stream should stop based on synchronizer states.
535 ///
536 /// ## Note
537 /// The stream always ends latest if all protocols are stale or ended independent of
538 /// this configuration. This allows you to end the stream earlier than that.
539 ///
540 /// See [self::StreamEndPolicy] for possible configuration options.
541 pub fn stream_end_policy(mut self, stream_end_policy: StreamEndPolicy) -> Self {
542 self.stream_end_policy = stream_end_policy;
543 self
544 }
545
546 /// Provides token metadata used to decode startup snapshots and initialize protocol states.
547 ///
548 /// This is not a stream filter — components arriving after startup include their own token
549 /// metadata. To restrict to specific tokens, filter in your consumer logic. New tokens
550 /// arriving via stream deltas are added automatically if they meet the quality threshold.
551 pub async fn set_tokens(self, tokens: HashMap<Bytes, Token>) -> Self {
552 self.decoder.set_tokens(tokens).await;
553 self
554 }
555
556 /// Skips decoding errors for component state updates.
557 ///
558 /// Allows the stream to continue processing even if some states fail to decode,
559 /// logging a warning instead of panicking.
560 pub fn skip_state_decode_failures(mut self, skip: bool) -> Self {
561 self.decoder
562 .skip_state_decode_failures(skip);
563 self
564 }
565
566 /// Sets the minimum token quality for tokens added via the stream.
567 ///
568 /// Tokens arriving in stream deltas below this threshold are ignored. Defaults to 100.
569 /// Set this to the same value used in [`load_all_tokens()`](crate::utils::load_all_tokens) to
570 /// apply consistent filtering.
571 pub fn min_token_quality(mut self, quality: u32) -> Self {
572 self.decoder.min_token_quality(quality);
573 self
574 }
575
576 /// Configures the retry policy for websocket reconnects.
577 pub fn websocket_retry_config(mut self, config: &RetryConfiguration) -> Self {
578 self.stream_builder = self
579 .stream_builder
580 .websockets_retry_config(config);
581 self
582 }
583
584 /// Configures the retry policy for state synchronization.
585 pub fn state_synchronizer_retry_config(mut self, config: &RetryConfiguration) -> Self {
586 self.stream_builder = self
587 .stream_builder
588 .state_synchronizer_retry_config(config);
589 self
590 }
591
592 /// Sets the maximum random delay before a restarted synchronizer requests a snapshot.
593 ///
594 /// Defaults to 60 seconds. A `jitter` below 5 seconds is raised to 5 seconds.
595 pub fn snapshot_jitter(mut self, jitter: time::Duration) -> Self {
596 self.stream_builder = self
597 .stream_builder
598 .snapshot_jitter(jitter);
599 self
600 }
601
602 pub fn get_decoder(&self) -> &TychoStreamDecoder<BlockHeader> {
603 &self.decoder
604 }
605
606 /// Registers a [`TxDeltaIndexer`] for ephemeral pending-block simulation.
607 ///
608 /// `protocol_system` is the protocol system of the indexer's components, which is also the
609 /// exchange name (e.g. `"uniswap_v3"`). The indexer's account deltas reach only pools of that
610 /// protocol system, so the indexer must emit the writes to every account those pools read. Use
611 /// [`build_with_pending`](Self::build_with_pending) to obtain both the confirmed stream and
612 /// the pending processor.
613 ///
614 /// The exchange must decode into a state whose `delta_transition` can rebuild it from the
615 /// `state_deltas` the indexer produces, because
616 /// [`apply_deltas_ephemeral`](crate::evm::decoder::TychoStreamDecoder::apply_deltas_ephemeral)
617 /// writes nothing to the VM database. Native and hybrid states qualify; the generic VM adapter
618 /// does not, because it re-reads pool state from the VM database — an indexer registered for
619 /// one still gets its balance and block-environment attributes applied, but every
620 /// storage-derived value stays at the confirmed block, with no error.
621 #[expect(
622 clippy::unnecessary_wraps,
623 reason = "public API; changing the signature is a breaking change"
624 )]
625 pub fn with_pending_indexer(
626 mut self,
627 protocol_system: &str,
628 indexer: Box<dyn TxDeltaIndexer>,
629 ) -> Result<Self, StreamError> {
630 self.pending_indexers
631 .insert(protocol_system.to_string(), indexer);
632 Ok(self)
633 }
634
635 /// Enables controlled-step mode for testing.
636 ///
637 /// Returns a [`BlockStepController`] that lets the caller decide when each buffered block
638 /// is released for decoding. Call this before [`build`](Self::build) or
639 /// [`build_with_pending`](Self::build_with_pending) — both detect and wire up the gating
640 /// automatically.
641 ///
642 /// In production code, do not call this method; the stream runs at full speed.
643 pub fn with_step_controller(mut self) -> (Self, BlockStepController) {
644 let (trigger_tx, trigger_rx) = tokio::sync::mpsc::unbounded_channel::<()>();
645 let (peek_tx, peek_rx) =
646 tokio::sync::watch::channel::<Option<FeedMessage<BlockHeader>>>(None);
647
648 self.step_peek_tx = Some(peek_tx);
649 self.step_trigger_rx = Some(trigger_rx);
650
651 let controller = BlockStepController { trigger_tx, peek_rx };
652 (self, controller)
653 }
654
655 /// Spawns a background task that gates `FeedMessage` delivery.
656 ///
657 /// The task buffers each incoming message, publishes it to `peek_tx` so the
658 /// [`BlockStepController`] can inspect it, waits for a trigger, then forwards the message to
659 /// `output_tx` for the decode pipeline. If `advance_tx` is `Some`, a clone of the message is
660 /// also forwarded there (used by the pending-processor path) before the decode step.
661 /// When the input channel closes or a terminal error is received according to
662 /// `stream_end_policy`, the task exits and all output channels are dropped.
663 fn run_gating_task(
664 raw_rx: tokio::sync::mpsc::Receiver<
665 Result<FeedMessage<BlockHeader>, BlockSynchronizerError>,
666 >,
667 mut trigger_rx: tokio::sync::mpsc::UnboundedReceiver<()>,
668 peek_tx: tokio::sync::watch::Sender<Option<FeedMessage<BlockHeader>>>,
669 output_tx: tokio::sync::mpsc::Sender<FeedMessage<BlockHeader>>,
670 stream_end_policy: StreamEndPolicy,
671 ) {
672 tokio::spawn(async move {
673 let mut raw_stream = ReceiverStream::new(raw_rx);
674 loop {
675 let msg = match raw_stream.next().await {
676 Some(Ok(msg)) => msg,
677 Some(Err(e)) => {
678 error!("Block stream ended with terminal error: {e}");
679 break;
680 }
681 None => break,
682 };
683
684 if stream_end_policy.should_end(msg.sync_states.values()) {
685 error!(
686 "Block stream ended due to {:?}: {:?}",
687 stream_end_policy, msg.sync_states
688 );
689 break;
690 }
691
692 // Publish the buffered message so the caller can peek before triggering.
693 #[expect(
694 clippy::let_underscore_must_use,
695 reason = "the peek slot is optional; a send error only means the controller dropped its receiver"
696 )]
697 let _ = peek_tx.send(Some(msg.clone()));
698
699 // Block until the controller fires trigger_next_block(), or until it is dropped.
700 if trigger_rx.recv().await.is_none() {
701 // Controller dropped — forward the buffered message and drain the rest
702 // without gating, so the stream continues to its natural end.
703 #[expect(
704 clippy::let_underscore_must_use,
705 reason = "the peek slot is optional; a send error only means the controller dropped its receiver"
706 )]
707 let _ = peek_tx.send(None);
708 if output_tx.send(msg).await.is_err() {
709 break;
710 }
711 while let Some(item) = raw_stream.next().await {
712 let Ok(msg) = item else { break };
713 if stream_end_policy.should_end(msg.sync_states.values()) {
714 break;
715 }
716 if output_tx.send(msg).await.is_err() {
717 break;
718 }
719 }
720 break;
721 }
722
723 // Clear the peek slot before decoding so callers see None between blocks.
724 #[expect(
725 clippy::let_underscore_must_use,
726 reason = "the peek slot is optional; a send error only means the controller dropped its receiver"
727 )]
728 let _ = peek_tx.send(None);
729
730 if output_tx.send(msg).await.is_err() {
731 break;
732 }
733 }
734 });
735 }
736
737 /// Registers `provider` as the live override source for `protocol_system`.
738 ///
739 /// Explicit registrations take precedence over the built-in default registry, so this is how
740 /// you swap a venue (e.g. `vm:bopamm`) onto a different provider. Registering the same provider
741 /// for several protocols is cheap — it is shared via `Arc`, not duplicated.
742 pub fn with_override_provider(
743 mut self,
744 protocol_system: impl Into<String>,
745 provider: Arc<dyn StateOverrideProvider>,
746 ) -> Self {
747 self.override_providers
748 .insert(protocol_system.into(), provider);
749 self
750 }
751
752 /// Installs override providers onto the decoder before the stream is built.
753 ///
754 /// Explicit consumer registrations win; the built-in default registry (see
755 /// [`default_override_providers`](crate::evm::override_stream::default_override_providers))
756 /// then fills every remaining protocol it can serve.
757 fn install_override_providers(&mut self) {
758 let explicit = std::mem::take(&mut self.override_providers);
759 // Protocols eligible for a built-in default provider: registered exchanges not explicitly
760 // overridden by the consumer.
761 let uncovered = self
762 .registered_exchanges
763 .clone()
764 .into_iter()
765 .filter(|exchange| !explicit.contains_key(exchange));
766 let defaults = override_stream::default_override_providers(uncovered);
767 for (protocol_system, provider) in defaults.into_iter().chain(explicit) {
768 self.decoder
769 .set_override_provider(protocol_system, provider);
770 }
771 }
772
773 /// Builds the confirmed protocol stream and a [`PendingBlockProcessor`] that stays
774 /// in sync with it automatically.
775 ///
776 /// The stream pipeline forwards every confirmed [`FeedMessage`] to the processor via an
777 /// internal unbounded channel — it never blocks waiting for the consumer. The consumer
778 /// owns the returned `PendingBlockProcessor` exclusively and may wrap it in whatever
779 /// synchronisation primitive suits their use case (e.g. `Mutex` for shared access,
780 /// nothing for single-threaded use).
781 ///
782 /// Call [`generate_pending_update`](PendingBlockProcessor::generate_pending_update) to
783 /// simulate a candidate bundle; it drains the channel automatically before computing.
784 ///
785 /// Warns once for each pending indexer whose protocol system is not a registered exchange:
786 /// its deltas reach no pool.
787 pub async fn build_with_pending(
788 mut self,
789 ) -> Result<
790 (impl Stream<Item = Result<Update, StreamDecodeError>>, PendingBlockProcessor),
791 StreamError,
792 > {
793 for protocol_system in self.pending_indexers.keys() {
794 if !self
795 .registered_exchanges
796 .contains(protocol_system)
797 {
798 warn!(
799 protocol_system,
800 "Pending indexer reaches no pool: no exchange has this protocol system"
801 );
802 }
803 }
804 initialize_hook_handlers().map_err(|e| {
805 StreamError::SetUpError(format!("Error initializing hook handlers: {e:?}"))
806 })?;
807 self.install_override_providers();
808 let (_, rx) = self.stream_builder.build().await?;
809 let decoder = Arc::new(self.decoder);
810
811 let (advance_tx, advance_rx) =
812 tokio::sync::mpsc::unbounded_channel::<FeedMessage<BlockHeader>>();
813 let pending = PendingBlockProcessor::new(
814 self.pending_indexers,
815 decoder.clone(),
816 self.chain,
817 advance_rx,
818 );
819
820 let chain = self.chain;
821 let stream_end_policy = self.stream_end_policy;
822
823 let decode_stream: Box<dyn Stream<Item = FeedMessage<BlockHeader>> + Send + Unpin> =
824 if let (Some(peek_tx), Some(trigger_rx)) = (self.step_peek_tx, self.step_trigger_rx) {
825 let (gated_tx, gated_rx) =
826 tokio::sync::mpsc::channel::<FeedMessage<BlockHeader>>(1);
827 Self::run_gating_task(rx, trigger_rx, peek_tx, gated_tx, stream_end_policy);
828 Box::new(ReceiverStream::new(gated_rx))
829 } else {
830 let normal = ReceiverStream::new(rx)
831 .take_while(move |msg| match msg {
832 Ok(msg) => {
833 let states = msg.sync_states.values();
834 if stream_end_policy.should_end(states) {
835 error!(
836 "Block stream ended due to {:?}: {:?}",
837 stream_end_policy, msg.sync_states
838 );
839 futures::future::ready(false)
840 } else {
841 futures::future::ready(true)
842 }
843 }
844 Err(e) => {
845 error!("Block stream ended with terminal error: {e}");
846 futures::future::ready(false)
847 }
848 })
849 .map(|msg| msg.expect("Safe since stream ends if we receive an error"));
850 Box::new(Box::pin(normal))
851 };
852
853 let stream = Box::pin(decode_stream.then({
854 let decoder = decoder;
855 move |msg| {
856 let decoder = decoder.clone();
857 let advance_tx = advance_tx.clone();
858 async move {
859 if let Err(e) = advance_tx.send(msg.clone()) {
860 warn!(error = %e, "Pending block processor is gone; block not forwarded");
861 }
862 decoder.decode(&msg).await.map_err(|e| {
863 debug!(msg=?msg, "Decode error: {}", e);
864 e
865 })
866 }
867 }
868 }));
869 let stream = inject_native_wrapper(stream, chain);
870 Ok((stream, pending))
871 }
872
873 /// Builds and returns the configured protocol stream.
874 ///
875 /// See the module-level docs for details on stream behavior and emitted messages.
876 /// This method applies all builder settings and starts the stream.
877 pub async fn build(
878 mut self,
879 ) -> Result<impl Stream<Item = Result<Update, StreamDecodeError>>, StreamError> {
880 initialize_hook_handlers().map_err(|e| {
881 StreamError::SetUpError(format!("Error initializing hook handlers: {e:?}"))
882 })?;
883 self.install_override_providers();
884 let (_, rx) = self.stream_builder.build().await?;
885 let decoder = Arc::new(self.decoder);
886 let chain = self.chain;
887 let stream_end_policy = self.stream_end_policy;
888
889 let decode_stream: Box<dyn Stream<Item = FeedMessage<BlockHeader>> + Send + Unpin> =
890 if let (Some(peek_tx), Some(trigger_rx)) = (self.step_peek_tx, self.step_trigger_rx) {
891 let (gated_tx, gated_rx) =
892 tokio::sync::mpsc::channel::<FeedMessage<BlockHeader>>(1);
893 Self::run_gating_task(rx, trigger_rx, peek_tx, gated_tx, stream_end_policy);
894 Box::new(ReceiverStream::new(gated_rx))
895 } else {
896 let normal = ReceiverStream::new(rx)
897 .take_while(move |msg| match msg {
898 Ok(msg) => {
899 let states = msg.sync_states.values();
900 if stream_end_policy.should_end(states) {
901 error!(
902 "Block stream ended due to {:?}: {:?}",
903 stream_end_policy, msg.sync_states
904 );
905 futures::future::ready(false)
906 } else {
907 futures::future::ready(true)
908 }
909 }
910 Err(e) => {
911 error!("Block stream ended with terminal error: {e}");
912 futures::future::ready(false)
913 }
914 })
915 .map(|msg| msg.expect("Safe since stream ends if we receive an error"));
916 Box::new(Box::pin(normal))
917 };
918
919 let stream = Box::pin(decode_stream.then({
920 let decoder = decoder;
921 move |msg| {
922 let decoder = decoder.clone();
923 async move {
924 decoder.decode(&msg).await.map_err(|e| {
925 debug!(msg=?msg, "Decode error: {}", e);
926 e
927 })
928 }
929 }
930 }));
931 let stream = inject_native_wrapper(stream, chain);
932 Ok(stream)
933 }
934}
935
936/// Wraps a decoded protocol stream to inject a `NativeWrapperState` component
937/// on the first successful update.
938///
939/// Skips injection when the chain's native asset has no real wrapper contract.
940fn inject_native_wrapper(
941 inner: impl Stream<Item = Result<Update, StreamDecodeError>> + Unpin + Send + 'static,
942 chain: Chain,
943) -> impl Stream<Item = Result<Update, StreamDecodeError>> + Send {
944 let Some(native_wrapper) = NativeWrapperState::new(chain) else {
945 return Either::Left(inner);
946 };
947
948 let mut pending_wrapper = Some(native_wrapper);
949 Either::Right(inner.map(move |result| {
950 result.map(|mut update| {
951 if let Some(native_wrapper) = pending_wrapper.take() {
952 let component = native_wrapper.component();
953 let id = component.id.to_string();
954 update
955 .new_pairs
956 .insert(id.clone(), component);
957 update
958 .states
959 .insert(id, Box::new(native_wrapper));
960 debug!("Injected native_wrapper component for {chain}");
961 }
962 update
963 })
964 }))
965}
966
967#[cfg(test)]
968mod tests {
969 use std::collections::HashMap;
970
971 use futures::{stream, StreamExt};
972 use tycho_common::models::Chain;
973
974 use super::*;
975 use crate::{
976 evm::protocol::native_wrapper::state::NATIVE_WRAPPER_ID, protocol::models::Update,
977 };
978
979 fn empty_update(block: u64) -> Update {
980 Update::new(block, HashMap::new(), HashMap::new())
981 }
982
983 #[tokio::test]
984 async fn test_inject_native_wrapper_first_message_only() {
985 let updates = vec![Ok(empty_update(1)), Ok(empty_update(2)), Ok(empty_update(3))];
986 let input = stream::iter(updates);
987
988 let results: Vec<_> = inject_native_wrapper(input, Chain::Ethereum)
989 .collect()
990 .await;
991
992 assert_eq!(results.len(), 3);
993
994 let expected_id = NativeWrapperState::new(Chain::Ethereum)
995 .expect("Ethereum should have a wrapper")
996 .component()
997 .id
998 .to_string();
999
1000 let first = results[0]
1001 .as_ref()
1002 .expect("first update ok");
1003 assert!(
1004 first
1005 .new_pairs
1006 .contains_key(&expected_id),
1007 "first message should have native_wrapper component"
1008 );
1009 assert!(
1010 first.states.contains_key(&expected_id),
1011 "first message should have native_wrapper state"
1012 );
1013
1014 let second = results[1]
1015 .as_ref()
1016 .expect("second update ok");
1017 assert!(
1018 !second
1019 .new_pairs
1020 .contains_key(&expected_id),
1021 "second message should NOT have native_wrapper component"
1022 );
1023 assert!(
1024 !second.states.contains_key(&expected_id),
1025 "second message should NOT have native_wrapper state"
1026 );
1027 }
1028
1029 #[tokio::test]
1030 async fn test_inject_native_wrapper_after_initial_decode_error() {
1031 let updates = vec![
1032 Err(StreamDecodeError::Fatal("decode failed".to_string())),
1033 Ok(empty_update(1)),
1034 Ok(empty_update(2)),
1035 ];
1036 let results: Vec<_> = inject_native_wrapper(stream::iter(updates), Chain::Ethereum)
1037 .collect()
1038 .await;
1039
1040 assert!(results[0].is_err(), "the decode error must pass through unchanged");
1041 let expected_id = NativeWrapperState::new(Chain::Ethereum)
1042 .expect("Ethereum should have a wrapper")
1043 .component()
1044 .id
1045 .to_string();
1046 let first_success = results[1]
1047 .as_ref()
1048 .expect("second stream item should decode");
1049 assert!(first_success
1050 .new_pairs
1051 .contains_key(&expected_id));
1052 assert!(first_success
1053 .states
1054 .contains_key(&expected_id));
1055
1056 let second_success = results[2]
1057 .as_ref()
1058 .expect("third stream item should decode");
1059 assert!(!second_success
1060 .new_pairs
1061 .contains_key(&expected_id));
1062 assert!(!second_success
1063 .states
1064 .contains_key(&expected_id));
1065 }
1066
1067 #[tokio::test]
1068 async fn test_does_not_inject_native_wrapper_for_arc_shared_balance() {
1069 let results: Vec<_> =
1070 inject_native_wrapper(stream::iter([Ok(empty_update(1))]), Chain::Arc)
1071 .collect()
1072 .await;
1073
1074 let update = results[0]
1075 .as_ref()
1076 .expect("update should decode");
1077 assert!(!update
1078 .new_pairs
1079 .contains_key(NATIVE_WRAPPER_ID));
1080 assert!(!update
1081 .states
1082 .contains_key(NATIVE_WRAPPER_ID));
1083 }
1084
1085 /// Verifies that `with_step_controller` returns both a modified builder and a controller.
1086 ///
1087 /// This test only checks that the builder method is callable and that the returned controller
1088 /// compiles — it does not start any network connection.
1089 #[tokio::test]
1090 async fn test_with_step_controller_returns_controller() {
1091 let builder = ProtocolStreamBuilder::new("tycho-beta.propellerheads.xyz", Chain::Ethereum);
1092 let (_builder, controller) = builder.with_step_controller();
1093 // The controller was successfully returned — verifying the public API is callable.
1094 drop(controller);
1095 }
1096
1097 #[tokio::test]
1098 async fn test_subscription_buffer_size_forwards_zero_rejection() {
1099 let error = ProtocolStreamBuilder::new("not a valid endpoint", Chain::Ethereum)
1100 .subscription_buffer_size(0)
1101 .build()
1102 .await;
1103
1104 let Err(error) = error else {
1105 panic!("a zero subscription buffer size must be rejected during setup");
1106 };
1107
1108 assert!(
1109 error
1110 .to_string()
1111 .contains("subscription buffer size must be greater than zero"),
1112 "the underlying Tycho stream builder should return the configuration error: {error}"
1113 );
1114 }
1115
1116 #[tokio::test]
1117 async fn test_subscription_buffer_size_forwards_upper_bound_rejection() {
1118 let error = ProtocolStreamBuilder::new("not a valid endpoint", Chain::Ethereum)
1119 .subscription_buffer_size(usize::MAX)
1120 .build()
1121 .await;
1122
1123 let Err(error) = error else {
1124 panic!("an oversized subscription buffer size must be rejected during setup");
1125 };
1126
1127 assert!(
1128 error
1129 .to_string()
1130 .contains("subscription buffer size must not exceed"),
1131 "the underlying Tycho stream builder should return the configuration error: {error}"
1132 );
1133 }
1134
1135 /// Connects to a live Tycho instance, verifies that the stream blocks until
1136 /// `trigger_next_block` is called, and that `peek_next_block` exposes the buffered message.
1137 #[ignore = "requires live Tycho connection (TYCHO_AUTH_TOKEN env var)"]
1138 #[tokio::test]
1139 async fn test_step_controller_trigger_releases_block() {
1140 use std::{env, time::Duration};
1141
1142 use crate::evm::protocol::uniswap_v2::state::UniswapV2State;
1143
1144 let auth = env::var("TYCHO_AUTH_TOKEN").expect("TYCHO_AUTH_TOKEN must be set");
1145
1146 // Track a single well-known pool to minimise startup latency.
1147 let usdc_weth_v2 = "0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc".to_string();
1148 let (builder, controller) =
1149 ProtocolStreamBuilder::new("tycho-beta.propellerheads.xyz", Chain::Ethereum)
1150 .auth_key(Some(auth))
1151 .exchange::<UniswapV2State>(
1152 "uniswap_v2",
1153 ComponentFilter::Ids(vec![usdc_weth_v2]),
1154 None,
1155 )
1156 .with_step_controller();
1157
1158 let (stream, _pending) = builder
1159 .build_with_pending()
1160 .await
1161 .expect("build_with_pending failed");
1162 tokio::pin!(stream);
1163
1164 // Wait up to 60 s for the first block to arrive in the gating buffer.
1165 let peeked = tokio::time::timeout(Duration::from_secs(60), controller.peek_next_block())
1166 .await
1167 .expect("timed out waiting for first block to buffer")
1168 .expect("stream ended before a block arrived");
1169
1170 assert!(!peeked.sync_states.is_empty(), "peeked block should carry sync states");
1171
1172 // Stream must be empty before we trigger — the gating task should be holding the block.
1173 let pre_trigger = tokio::time::timeout(Duration::from_millis(200), stream.next()).await;
1174 assert!(
1175 pre_trigger.is_err(),
1176 "stream should be blocked before trigger_next_block, got an item"
1177 );
1178
1179 // Release the block.
1180 controller
1181 .trigger_next_block()
1182 .expect("trigger_next_block failed");
1183
1184 // Stream should now yield the decoded update within one block time.
1185 let update = tokio::time::timeout(Duration::from_secs(30), stream.next())
1186 .await
1187 .expect("timed out waiting for update after trigger")
1188 .expect("stream ended unexpectedly");
1189
1190 assert!(update.is_ok(), "decoded update should be Ok, got: {:?}", update);
1191 }
1192}