faucet_source_rest/pagination/
mod.rs

1//! Pagination strategies for REST APIs.
2
3pub mod cursor;
4pub mod link_header;
5pub mod next_link_body;
6pub mod offset;
7pub mod page;
8
9use faucet_core::FaucetError;
10use reqwest::header::HeaderMap;
11use schemars::JsonSchema;
12use serde::{Deserialize, Serialize};
13use serde_json::Value;
14use std::collections::HashMap;
15
16/// Supported pagination strategies.
17#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
18#[serde(tag = "type")]
19pub enum PaginationStyle {
20    None,
21    Cursor {
22        next_token_path: String,
23        param_name: String,
24    },
25    LinkHeader,
26    /// The full URL of the next page is embedded in the response body.
27    /// `next_link_path` is a JSONPath expression pointing to that URL field
28    /// (e.g. `"$.next_link"`).  Pagination stops when the field is absent,
29    /// null, or an empty string.
30    NextLinkInBody {
31        next_link_path: String,
32    },
33    PageNumber {
34        param_name: String,
35        start_page: usize,
36        page_size: Option<usize>,
37        page_size_param: Option<String>,
38    },
39    Offset {
40        offset_param: String,
41        limit_param: String,
42        limit: usize,
43        total_path: Option<String>,
44    },
45}
46
47/// Internal state tracked across pages.
48#[derive(Debug, Default)]
49pub struct PaginationState {
50    pub page: usize,
51    pub next_token: Option<String>,
52    pub offset: usize,
53    pub next_link: Option<String>,
54    /// The previous page's token/link, used for loop detection.
55    /// If `advance()` produces the same value twice in a row, pagination
56    /// is stuck and we stop rather than looping forever.
57    #[doc(hidden)]
58    pub previous_token: Option<String>,
59}
60
61impl PaginationStyle {
62    pub fn apply_params(&self, params: &mut HashMap<String, String>, state: &PaginationState) {
63        match self {
64            PaginationStyle::None => {}
65            PaginationStyle::Cursor { param_name, .. } => {
66                cursor::apply_params(params, param_name, &state.next_token);
67            }
68            PaginationStyle::LinkHeader => {}
69            PaginationStyle::NextLinkInBody { .. } => {}
70            PaginationStyle::PageNumber {
71                param_name,
72                start_page,
73                page_size,
74                page_size_param,
75            } => {
76                page::apply_params(
77                    params,
78                    param_name,
79                    *start_page,
80                    state.page,
81                    *page_size,
82                    page_size_param.as_deref(),
83                );
84            }
85            PaginationStyle::Offset {
86                offset_param,
87                limit_param,
88                limit,
89                ..
90            } => {
91                offset::apply_params(params, offset_param, limit_param, state.offset, *limit);
92            }
93        }
94    }
95
96    /// Advance pagination state based on the response body and headers.
97    /// Returns `true` if there is a next page to fetch.
98    ///
99    /// Includes **loop detection**: if a cursor or next-link value is identical
100    /// to the previous page's value, pagination stops with a warning instead of
101    /// looping forever.
102    pub fn advance(
103        &self,
104        body: &Value,
105        headers: &HeaderMap,
106        state: &mut PaginationState,
107        record_count: usize,
108    ) -> Result<bool, FaucetError> {
109        match self {
110            PaginationStyle::None => Ok(false),
111            PaginationStyle::Cursor {
112                next_token_path, ..
113            } => {
114                let has_next = cursor::advance(body, next_token_path, &mut state.next_token)?;
115                if has_next {
116                    if state.next_token == state.previous_token {
117                        tracing::warn!(
118                            "pagination loop detected: cursor {:?} repeated — stopping",
119                            state.next_token
120                        );
121                        return Ok(false);
122                    }
123                    state.previous_token = state.next_token.clone();
124                }
125                Ok(has_next)
126            }
127            PaginationStyle::LinkHeader => match link_header::extract_next_link(headers) {
128                Some(link) => {
129                    if Some(&link) == state.previous_token.as_ref() {
130                        tracing::warn!(
131                            "pagination loop detected: link {link:?} repeated — stopping"
132                        );
133                        state.next_link = None;
134                        return Ok(false);
135                    }
136                    state.previous_token = Some(link.clone());
137                    state.next_link = Some(link);
138                    Ok(true)
139                }
140                None => {
141                    state.next_link = None;
142                    Ok(false)
143                }
144            },
145            PaginationStyle::NextLinkInBody { next_link_path } => {
146                let has_next = next_link_body::advance(body, next_link_path, &mut state.next_link)?;
147                if has_next {
148                    if state.next_link == state.previous_token {
149                        tracing::warn!(
150                            "pagination loop detected: next_link {:?} repeated — stopping",
151                            state.next_link
152                        );
153                        return Ok(false);
154                    }
155                    state.previous_token = state.next_link.clone();
156                }
157                Ok(has_next)
158            }
159            PaginationStyle::PageNumber { .. } => {
160                state.page += 1;
161                Ok(record_count > 0)
162            }
163            PaginationStyle::Offset {
164                limit, total_path, ..
165            } => offset::advance(
166                body,
167                &mut state.offset,
168                record_count,
169                *limit,
170                total_path.as_deref(),
171            ),
172        }
173    }
174}
faucet_source_rest/pagination/mod.rs

faucet_source_rest/pagination/
mod.rs
