Skip to main content

near_indexer_client/
lib.rs

1//! Pull complete NEAR indexer messages over HTTP without running a node.
2//!
3//! [`Client::poll`] returns ordered rollback/application work. Process the whole
4//! [`Update`] and persist its checkpoint in the same transaction as consumer
5//! state. The client never advances a cursor or acknowledges queue delivery.
6
7use std::collections::BTreeSet;
8use std::time::Duration;
9
10pub use near_indexer_primitives::{CryptoHash, StreamerMessage};
11/// The `near-kit` version whose [`RpcClient`] [`Client::new`] takes and whose
12/// [`RpcError`] [`Error::Rpc`] carries, so callers need not pin it separately.
13pub use near_kit;
14use near_kit::rpc::{RpcClient, RpcError};
15use serde::de::Error as _;
16use serde::{Deserialize, Deserializer, Serialize};
17use serde_json::{Value, json};
18use thiserror::Error;
19
20/// The last block durably processed by a consumer, exclusive on the next poll.
21#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
22pub struct Checkpoint {
23    /// Hash anchors both branch identity and restart ancestry.
24    pub hash: CryptoHash,
25    /// Height validates the saved hash; skipped heights are allowed.
26    pub height: u64,
27}
28
29/// Minimal ancestry and finality context from the ordinary block RPC.
30#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)]
31pub struct Header {
32    /// This block's hash.
33    pub hash: CryptoHash,
34    /// Parent hash, which need not have height `height - 1`.
35    pub prev_hash: CryptoHash,
36    /// This block's height.
37    pub height: u64,
38    /// Finality marker recorded in this block, not a separately sampled head.
39    pub last_final_block: CryptoHash,
40}
41impl Header {
42    /// Construct the checkpoint to persist after processing this block.
43    pub fn checkpoint(&self) -> Checkpoint {
44        Checkpoint {
45            hash: self.hash,
46            height: self.height,
47        }
48    }
49}
50
51/// Official indexer payload with the endpoint's explicit execution coverage.
52#[derive(Clone, Debug, Serialize)]
53pub struct IndexerBlock {
54    /// Block, chunks, receipts, execution outcomes, and state changes.
55    #[serde(flatten)]
56    pub message: StreamerMessage,
57    /// Shards tracked by the serving node. This client requires full coverage.
58    pub tracked_shards: Vec<u64>,
59}
60
61// near-primitives 0.37.4 loses explicit null in Option<Option<TrieSplit>>.
62// Preserve it until the upstream double_option fix is available in a release.
63impl<'de> Deserialize<'de> for IndexerBlock {
64    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
65        #[derive(Deserialize)]
66        struct Wire {
67            #[serde(flatten)]
68            message: StreamerMessage,
69            tracked_shards: Vec<u64>,
70        }
71        let value = Value::deserialize(deserializer)?;
72        let explicit_null =
73            |header: &Value| header.get("proposed_split").is_some_and(Value::is_null);
74        let block_nulls: Vec<bool> = value["block"]["chunks"]
75            .as_array()
76            .into_iter()
77            .flatten()
78            .map(explicit_null)
79            .collect();
80        let shard_nulls: Vec<bool> = value["shards"]
81            .as_array()
82            .into_iter()
83            .flatten()
84            .map(|shard| explicit_null(&shard["chunk"]["header"]))
85            .collect();
86        let mut wire: Wire = serde_json::from_value(value).map_err(D::Error::custom)?;
87        for (header, was_null) in wire.message.block.chunks.iter_mut().zip(block_nulls) {
88            if was_null {
89                header.proposed_split = Some(None);
90            }
91        }
92        for (shard, was_null) in wire.message.shards.iter_mut().zip(shard_nulls) {
93            if was_null && let Some(chunk) = &mut shard.chunk {
94                chunk.header.proposed_split = Some(None);
95            }
96        }
97        Ok(Self {
98            message: wire.message,
99            tracked_shards: wire.tracked_shards,
100        })
101    }
102}
103
104/// Work to apply atomically with its checkpoint; dropping it acknowledges nothing.
105#[derive(Debug)]
106pub struct Update {
107    /// Finality requested when sampling the head; explicit targets are optimistic.
108    pub finality: Finality,
109    /// Sampled head and its recorded finality marker.
110    pub head: Header,
111    /// Undo this replaced ancestry in newest-first order. It can extend before
112    /// the initial starting block; consumers must tolerate absent undo entries.
113    pub rollback: Vec<Header>,
114    /// Process these complete messages in oldest-first order.
115    pub blocks: Vec<IndexerBlock>,
116    /// Persist only after successfully processing the entire update.
117    pub next_checkpoint: Checkpoint,
118}
119
120/// The headers of an [`Update`], before any block is fetched.
121///
122/// Fetch [`Self::apply`] with [`Client::fetch_block`] in chunks of any size and
123/// persist each chunk's last header as the checkpoint, so a long catch-up keeps
124/// bounded memory and a failed fetch repeats only that block.
125///
126/// ```no_run
127/// use near_indexer_client::{Checkpoint, Client, Error};
128///
129/// async fn catch_up(client: &Client, mut saved: Checkpoint) -> Result<Checkpoint, Error> {
130///     let head = client.head().await?;
131///     let plan = client.plan(Some(&saved), head).await?;
132///     // An optimistic consumer undoes `plan.rollback` first.
133///     for chunk in plan.apply.chunks(32) {
134///         for header in chunk {
135///             // Retry a failed fetch on its own, paced by a `Backoff`.
136///             let _block = client.fetch_block(header).await?;
137///         }
138///         if let Some(last) = chunk.last() {
139///             // Persist the chunk's effects and this checkpoint together.
140///             saved = last.checkpoint();
141///         }
142///     }
143///     Ok(saved)
144/// }
145/// ```
146#[derive(Clone, Debug, PartialEq, Eq)]
147pub struct Plan {
148    /// Sampled head; its checkpoint follows the last applied block.
149    pub head: Header,
150    /// Replaced ancestry to undo, newest first, as in [`Update::rollback`].
151    pub rollback: Vec<Header>,
152    /// Blocks to fetch and apply, oldest first, ending with [`Self::head`]
153    /// unless the head is the checkpoint itself.
154    pub apply: Vec<Header>,
155}
156
157/// Which head to follow. Optimistic consumers must implement rollback.
158#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
159#[serde(rename_all = "lowercase")]
160pub enum Finality {
161    /// Fully finalized head; any attempted rollback is an error.
162    Final,
163    /// Latest head, which can be replaced even at the same height.
164    Optimistic,
165}
166
167/// Every error leaves consumer-owned checkpoints untouched.
168#[derive(Debug, Error)]
169pub enum Error {
170    /// Transport, RPC, or typed payload decoding failure.
171    #[error(transparent)]
172    Rpc(#[from] Box<RpcError>),
173    /// Missing coverage, inconsistent hashes, or malformed ancestry.
174    #[error("invalid indexer data: {0}")]
175    InvalidData(&'static str),
176    /// Catch-up/reorg exceeds the configured number of ancestry steps.
177    #[error("ancestry limit exceeded ({0}); checkpoint not advanced")]
178    AncestryLimit(usize),
179    /// A finalized head would replace already processed state.
180    #[error("final head would roll back processed state")]
181    FinalityViolation,
182}
183
184impl From<RpcError> for Error {
185    fn from(error: RpcError) -> Self {
186        Self::Rpc(Box::new(error))
187    }
188}
189
190impl Error {
191    /// Classify the error, e.g. to choose a retry delay with [`Backoff`] or to
192    /// label it in metrics.
193    pub fn kind(&self) -> ErrorKind {
194        let rpc = match self {
195            Self::InvalidData(_) => return ErrorKind::InvalidData,
196            Self::AncestryLimit(_) => return ErrorKind::AncestryLimit,
197            Self::FinalityViolation => return ErrorKind::FinalityViolation,
198            Self::Rpc(rpc) => rpc.as_ref(),
199        };
200        let name = match rpc {
201            RpcError::Rpc { data, .. } => data
202                .as_ref()
203                .and_then(|data| data.get("name"))
204                .and_then(Value::as_str),
205            _ => None,
206        };
207        match name {
208            Some("BUSY") => ErrorKind::Busy,
209            Some("TIMEOUT_ERROR") => ErrorKind::Timeout,
210            Some("DATA_UNAVAILABLE") => ErrorKind::DataUnavailable,
211            Some("INCOMPLETE_DATA") => ErrorKind::IncompleteData,
212            Some("LIMIT_EXCEEDED") => ErrorKind::LimitExceeded,
213            Some("UNSUPPORTED") => ErrorKind::Unsupported,
214            Some(_) => ErrorKind::Rpc,
215            None if rpc.is_retryable() => ErrorKind::Unavailable,
216            None => ErrorKind::Rpc,
217        }
218    }
219}
220
221/// The class of an [`enum@Error`], from the endpoint's error name where it has one.
222#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
223#[non_exhaustive]
224pub enum ErrorKind {
225    /// The endpoint is at its concurrent indexer request limit.
226    Busy,
227    /// The endpoint timed out serving the request.
228    Timeout,
229    /// A transport failure or other RPC error near-kit deems retryable.
230    Unavailable,
231    /// The block or its execution data is not available, possibly not yet.
232    DataUnavailable,
233    /// The endpoint's data for the block is inconsistent or incomplete.
234    IncompleteData,
235    /// The block's response exceeds the endpoint's size cap.
236    LimitExceeded,
237    /// The endpoint is not configured to serve indexer requests.
238    Unsupported,
239    /// Any other RPC error.
240    Rpc,
241    /// See [`Error::InvalidData`].
242    InvalidData,
243    /// See [`Error::AncestryLimit`].
244    AncestryLimit,
245    /// See [`Error::FinalityViolation`].
246    FinalityViolation,
247}
248
249impl ErrorKind {
250    /// A stable snake_case name, suitable as a metric label.
251    pub fn as_str(self) -> &'static str {
252        match self {
253            Self::Busy => "busy",
254            Self::Timeout => "timeout",
255            Self::Unavailable => "unavailable",
256            Self::DataUnavailable => "data_unavailable",
257            Self::IncompleteData => "incomplete_data",
258            Self::LimitExceeded => "limit_exceeded",
259            Self::Unsupported => "unsupported",
260            Self::Rpc => "rpc",
261            Self::InvalidData => "invalid_data",
262            Self::AncestryLimit => "ancestry_limit",
263            Self::FinalityViolation => "finality_violation",
264        }
265    }
266}
267
268/// Whether a retry is expected to clear a failure on its own.
269#[derive(Clone, Copy, Debug, PartialEq, Eq)]
270pub enum Failure {
271    /// The endpoint is momentarily unable to answer.
272    Transient,
273    /// The same request will likely keep failing until the endpoint changes;
274    /// for a block error, every consumer of that block is affected.
275    Persistent,
276}
277
278/// Pacing for retrying one request at a time.
279///
280/// The client never retries or sleeps itself; the consumer retries the failed
281/// request (not the whole update) after the [`Backoff`] delay. Build the
282/// [`RpcClient`] with `RetryConfig::none()` so near-kit's retries do not stack
283/// underneath.
284///
285/// ```
286/// use near_indexer_client::{Error, Failure, RetryPolicy};
287/// use std::time::Duration;
288///
289/// let mut backoff = RetryPolicy::default().backoff();
290/// let error = Error::InvalidData("broken ancestry");
291/// let retry = backoff.next(&error, Duration::ZERO);
292/// assert_eq!(retry.failure, Failure::Persistent);
293/// assert_eq!(retry.delay, Duration::from_millis(500));
294/// ```
295#[derive(Clone, Copy, Debug, PartialEq, Eq)]
296pub struct RetryPolicy {
297    /// Delay after the first failure; it doubles with each further failure.
298    pub initial_delay: Duration,
299    /// Cap on the delay while failures are [`Failure::Transient`].
300    pub max_transient_delay: Duration,
301    /// Cap on the delay once failures are [`Failure::Persistent`].
302    pub max_persistent_delay: Duration,
303    /// How long [`ErrorKind::DataUnavailable`] counts as transient, measured
304    /// from the request's first failure. An endpoint may announce a head
305    /// before it has saved that block's execution data.
306    pub data_unavailable_grace: Duration,
307}
308
309impl Default for RetryPolicy {
310    /// 500 ms doubling to 10 s while transient and 60 s once persistent, with
311    /// a 30 s grace for unavailable data.
312    fn default() -> Self {
313        Self {
314            initial_delay: Duration::from_millis(500),
315            max_transient_delay: Duration::from_secs(10),
316            max_persistent_delay: Duration::from_secs(60),
317            data_unavailable_grace: Duration::from_secs(30),
318        }
319    }
320}
321
322impl RetryPolicy {
323    /// Classify an error of `kind` from a request failing for `failing_for`
324    /// since its first failure.
325    pub fn classify(&self, kind: ErrorKind, failing_for: Duration) -> Failure {
326        match kind {
327            ErrorKind::Busy | ErrorKind::Timeout | ErrorKind::Unavailable => Failure::Transient,
328            ErrorKind::DataUnavailable if failing_for < self.data_unavailable_grace => {
329                Failure::Transient
330            }
331            _ => Failure::Persistent,
332        }
333    }
334
335    /// Start pacing the retries of one failing request.
336    pub fn backoff(self) -> Backoff {
337        Backoff {
338            delay: self.initial_delay,
339            policy: self,
340        }
341    }
342}
343
344/// Retry delays for one failing request; start a new one per request.
345#[derive(Clone, Debug)]
346pub struct Backoff {
347    policy: RetryPolicy,
348    delay: Duration,
349}
350
351/// How to retry after one failed attempt.
352#[derive(Clone, Copy, Debug, PartialEq, Eq)]
353pub struct Retry {
354    /// The class of the error.
355    pub kind: ErrorKind,
356    /// Whether the error is expected to clear on its own.
357    pub failure: Failure,
358    /// How long to wait before the next attempt.
359    pub delay: Duration,
360}
361
362impl Backoff {
363    /// Classify `error`, which the request has been failing with for
364    /// `failing_for` since its first failure, and return the delay before the
365    /// next attempt.
366    pub fn next(&mut self, error: &Error, failing_for: Duration) -> Retry {
367        let kind = error.kind();
368        let failure = self.policy.classify(kind, failing_for);
369        Retry {
370            kind,
371            failure,
372            delay: self.next_delay(failure),
373        }
374    }
375
376    /// The delay before the next attempt after a failure the consumer
377    /// classified itself, such as one in its own processing.
378    pub fn next_delay(&mut self, failure: Failure) -> Duration {
379        let cap = match failure {
380            Failure::Transient => self.policy.max_transient_delay,
381            Failure::Persistent => self.policy.max_persistent_delay,
382        };
383        let delay = self.delay.min(cap);
384        self.delay = self.delay.saturating_mul(2).min(
385            self.policy
386                .max_persistent_delay
387                .max(self.policy.max_transient_delay),
388        );
389        delay
390    }
391}
392
393/// Sequential, pull-based indexing; consumer speed supplies backpressure.
394///
395/// The provided near-kit RPC client controls transport, timeouts, and retries.
396/// There is no background task, queue, database, or node runtime.
397pub struct Client {
398    rpc: RpcClient,
399    finality: Finality,
400    max_ancestry: usize,
401}
402impl Client {
403    /// Configure head finality and a positive bound on rollback + apply blocks.
404    ///
405    /// Zero rejects all polls. Exceeding the bound returns an error, never a
406    /// partial batch; choose a bound appropriate for the retained history.
407    pub fn new(rpc: RpcClient, finality: Finality, max_ancestry: usize) -> Self {
408        Self {
409            rpc,
410            finality,
411            max_ancestry,
412        }
413    }
414
415    /// Fetch one exact hash with full shard coverage. Missing data is an error.
416    pub async fn fetch(&self, hash: CryptoHash) -> Result<IndexerBlock, Error> {
417        let result: IndexerBlock = self
418            .rpc
419            .call("EXPERIMENTAL_indexer_block", json!({"block_hash":hash}))
420            .await?;
421        if result.message.block.header.hash != hash {
422            return Err(Error::InvalidData(
423                "requested hash differs from returned block",
424            ));
425        }
426        let expected: BTreeSet<u64> = result
427            .message
428            .block
429            .chunks
430            .iter()
431            .map(|c| c.shard_id.into())
432            .collect();
433        let tracked: BTreeSet<u64> = result.tracked_shards.iter().copied().collect();
434        let actual: BTreeSet<u64> = result
435            .message
436            .shards
437            .iter()
438            .map(|s| s.shard_id.into())
439            .collect();
440        if expected != tracked
441            || expected != actual
442            || result.message.block.chunks.len() != expected.len()
443            || result.message.shards.len() != expected.len()
444            || result.tracked_shards.len() != expected.len()
445        {
446            return Err(Error::InvalidData(
447                "node must supply every block shard exactly once",
448            ));
449        }
450        Ok(result)
451    }
452
453    /// Fetch the block `header` describes, rejecting a payload whose height,
454    /// parent or finality marker differs from it. Coverage rules match [`Self::fetch`].
455    pub async fn fetch_block(&self, header: &Header) -> Result<IndexerBlock, Error> {
456        let block = self.fetch(header.hash).await?;
457        let actual = &block.message.block.header;
458        if actual.height != header.height
459            || actual.prev_hash != header.prev_hash
460            || actual.last_final_block != header.last_final_block
461        {
462            return Err(Error::InvalidData(
463                "indexer payload differs from ancestry header",
464            ));
465        }
466        Ok(block)
467    }
468
469    /// Sample the head header at the configured finality.
470    pub async fn head(&self) -> Result<Header, Error> {
471        self.block_header(json!({"finality": self.finality})).await
472    }
473
474    /// Read the header of one exact block hash.
475    pub async fn header(&self, hash: CryptoHash) -> Result<Header, Error> {
476        let header = self.block_header(json!({"block_id": hash})).await?;
477        if header.hash != hash {
478            return Err(Error::InvalidData(
479                "requested hash differs from returned header",
480            ));
481        }
482        Ok(header)
483    }
484
485    /// Read the parent header of `child`, rejecting a parent that does not
486    /// match `child.prev_hash` or is not below it.
487    pub async fn parent(&self, child: &Header) -> Result<Header, Error> {
488        let parent = self
489            .block_header(json!({"block_id": child.prev_hash}))
490            .await?;
491        if parent.hash != child.prev_hash || parent.height >= child.height {
492            return Err(Error::InvalidData("broken ancestry or no common ancestor"));
493        }
494        Ok(parent)
495    }
496
497    /// Walk hash ancestry from `head` back to `checkpoint` without fetching any
498    /// block. Pass a head sampled by [`Self::head`]: [`Finality::Final`] rejects
499    /// any rollback. Ancestry and checkpoint rules match [`Self::poll`].
500    pub async fn plan(&self, checkpoint: Option<&Checkpoint>, head: Header) -> Result<Plan, Error> {
501        self.plan_with(checkpoint, head, self.finality).await
502    }
503
504    /// Sample a head, walk hash ancestry, then fetch the complete update.
505    ///
506    /// `None` starts with the current head only. A saved checkpoint resumes
507    /// exclusively after that block. An unchanged head returns an empty update.
508    /// Missing/pruned history, failed fetches, and excessive catch-up depth fail
509    /// closed. Retain the old checkpoint on failure and retry at your own pace.
510    /// For bounded memory during a long catch-up, use [`Self::plan`] instead.
511    pub async fn poll(&self, checkpoint: Option<&Checkpoint>) -> Result<Update, Error> {
512        let head = self.head().await?;
513        self.update(checkpoint, head, self.finality).await
514    }
515
516    /// Build an update to an exact target without resampling the moving head.
517    ///
518    /// Explicit targets carry [`Finality::Optimistic`]: a hash alone does not
519    /// establish finality. Ancestry, coverage, and checkpoint rules match [`Self::poll`].
520    pub async fn update_to(
521        &self,
522        checkpoint: Option<&Checkpoint>,
523        target: CryptoHash,
524    ) -> Result<Update, Error> {
525        let head = self.header(target).await?;
526        self.update(checkpoint, head, Finality::Optimistic).await
527    }
528
529    async fn update(
530        &self,
531        checkpoint: Option<&Checkpoint>,
532        head: Header,
533        finality: Finality,
534    ) -> Result<Update, Error> {
535        let plan = self.plan_with(checkpoint, head, finality).await?;
536        let mut blocks = Vec::with_capacity(plan.apply.len());
537        for header in &plan.apply {
538            blocks.push(self.fetch_block(header).await?);
539        }
540        Ok(Update {
541            finality,
542            head,
543            rollback: plan.rollback,
544            blocks,
545            next_checkpoint: head.checkpoint(),
546        })
547    }
548
549    async fn plan_with(
550        &self,
551        checkpoint: Option<&Checkpoint>,
552        head: Header,
553        finality: Finality,
554    ) -> Result<Plan, Error> {
555        if self.max_ancestry == 0 {
556            return Err(Error::AncestryLimit(0));
557        }
558        let (rollback, apply) = self.walk(checkpoint, head).await?;
559        if finality == Finality::Final && !rollback.is_empty() {
560            return Err(Error::FinalityViolation);
561        }
562        Ok(Plan {
563            head,
564            rollback,
565            apply,
566        })
567    }
568
569    async fn block_header(&self, params: Value) -> Result<Header, Error> {
570        #[derive(Deserialize)]
571        struct Block {
572            header: Header,
573        }
574        Ok(self.rpc.call::<_, Block>("block", params).await?.header)
575    }
576
577    async fn walk(
578        &self,
579        checkpoint: Option<&Checkpoint>,
580        head: Header,
581    ) -> Result<(Vec<Header>, Vec<Header>), Error> {
582        let Some(checkpoint) = checkpoint else {
583            return Ok((vec![], vec![head]));
584        };
585        let mut old = self
586            .block_header(json!({"block_id": checkpoint.hash}))
587            .await?;
588        if old.checkpoint() != *checkpoint {
589            return Err(Error::InvalidData("checkpoint does not match node history"));
590        }
591        let mut new = head;
592        let mut rollback = vec![];
593        let mut apply = vec![];
594        while old.hash != new.hash {
595            if rollback.len() + apply.len() >= self.max_ancestry {
596                return Err(Error::AncestryLimit(self.max_ancestry));
597            }
598            if old.height >= new.height {
599                let previous = self.parent(&old).await?;
600                rollback.push(old);
601                old = previous;
602            } else {
603                let previous = self.parent(&new).await?;
604                apply.push(new);
605                new = previous;
606            }
607        }
608        if old != new {
609            return Err(Error::InvalidData(
610                "same hash returned inconsistent headers",
611            ));
612        }
613        apply.reverse();
614        Ok((rollback, apply))
615    }
616}