faucet_stream/pagination/
mod.rs

1//! Pagination strategies for REST APIs.
2
3pub mod cursor;
4pub mod link_header;
5pub mod next_link_body;
6pub mod offset;
7pub mod page;
8
9use crate::error::FaucetError;
10use reqwest::header::HeaderMap;
11use serde_json::Value;
12use std::collections::HashMap;
13
14/// Supported pagination strategies.
15#[derive(Debug, Clone)]
16pub enum PaginationStyle {
17    None,
18    Cursor {
19        next_token_path: String,
20        param_name: String,
21    },
22    LinkHeader,
23    /// The full URL of the next page is embedded in the response body.
24    /// `next_link_path` is a JSONPath expression pointing to that URL field
25    /// (e.g. `"$.next_link"`).  Pagination stops when the field is absent,
26    /// null, or an empty string.
27    NextLinkInBody {
28        next_link_path: String,
29    },
30    PageNumber {
31        param_name: String,
32        start_page: usize,
33        page_size: Option<usize>,
34        page_size_param: Option<String>,
35    },
36    Offset {
37        offset_param: String,
38        limit_param: String,
39        limit: usize,
40        total_path: Option<String>,
41    },
42}
43
44/// Internal state tracked across pages.
45#[derive(Debug, Default)]
46pub struct PaginationState {
47    pub page: usize,
48    pub next_token: Option<String>,
49    pub offset: usize,
50    pub next_link: Option<String>,
51    /// The previous page's token/link, used for loop detection.
52    /// If `advance()` produces the same value twice in a row, pagination
53    /// is stuck and we stop rather than looping forever.
54    #[doc(hidden)]
55    pub previous_token: Option<String>,
56}
57
58impl PaginationStyle {
59    pub fn apply_params(&self, params: &mut HashMap<String, String>, state: &PaginationState) {
60        match self {
61            PaginationStyle::None => {}
62            PaginationStyle::Cursor { param_name, .. } => {
63                cursor::apply_params(params, param_name, &state.next_token);
64            }
65            PaginationStyle::LinkHeader => {}
66            PaginationStyle::NextLinkInBody { .. } => {}
67            PaginationStyle::PageNumber {
68                param_name,
69                start_page,
70                page_size,
71                page_size_param,
72            } => {
73                page::apply_params(
74                    params,
75                    param_name,
76                    *start_page,
77                    state.page,
78                    *page_size,
79                    page_size_param.as_deref(),
80                );
81            }
82            PaginationStyle::Offset {
83                offset_param,
84                limit_param,
85                limit,
86                ..
87            } => {
88                offset::apply_params(params, offset_param, limit_param, state.offset, *limit);
89            }
90        }
91    }
92
93    /// Advance pagination state based on the response body and headers.
94    /// Returns `true` if there is a next page to fetch.
95    ///
96    /// Includes **loop detection**: if a cursor or next-link value is identical
97    /// to the previous page's value, pagination stops with a warning instead of
98    /// looping forever.
99    pub fn advance(
100        &self,
101        body: &Value,
102        headers: &HeaderMap,
103        state: &mut PaginationState,
104        record_count: usize,
105    ) -> Result<bool, FaucetError> {
106        match self {
107            PaginationStyle::None => Ok(false),
108            PaginationStyle::Cursor {
109                next_token_path, ..
110            } => {
111                let has_next = cursor::advance(body, next_token_path, &mut state.next_token)?;
112                if has_next {
113                    if state.next_token == state.previous_token {
114                        tracing::warn!(
115                            "pagination loop detected: cursor {:?} repeated — stopping",
116                            state.next_token
117                        );
118                        return Ok(false);
119                    }
120                    state.previous_token = state.next_token.clone();
121                }
122                Ok(has_next)
123            }
124            PaginationStyle::LinkHeader => match link_header::extract_next_link(headers) {
125                Some(link) => {
126                    if Some(&link) == state.previous_token.as_ref() {
127                        tracing::warn!(
128                            "pagination loop detected: link {link:?} repeated — stopping"
129                        );
130                        state.next_link = None;
131                        return Ok(false);
132                    }
133                    state.previous_token = Some(link.clone());
134                    state.next_link = Some(link);
135                    Ok(true)
136                }
137                None => {
138                    state.next_link = None;
139                    Ok(false)
140                }
141            },
142            PaginationStyle::NextLinkInBody { next_link_path } => {
143                let has_next = next_link_body::advance(body, next_link_path, &mut state.next_link)?;
144                if has_next {
145                    if state.next_link == state.previous_token {
146                        tracing::warn!(
147                            "pagination loop detected: next_link {:?} repeated — stopping",
148                            state.next_link
149                        );
150                        return Ok(false);
151                    }
152                    state.previous_token = state.next_link.clone();
153                }
154                Ok(has_next)
155            }
156            PaginationStyle::PageNumber { .. } => {
157                state.page += 1;
158                Ok(record_count > 0)
159            }
160            PaginationStyle::Offset {
161                limit, total_path, ..
162            } => offset::advance(
163                body,
164                &mut state.offset,
165                record_count,
166                *limit,
167                total_path.as_deref(),
168            ),
169        }
170    }
171}
faucet_stream/pagination/mod.rs

faucet_stream/pagination/
mod.rs
