use futures::{Future, Stream, TryStreamExt};
#[derive(Debug, Clone, serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct PageInfo {
pub has_next_page: bool,
pub has_previous_page: bool,
pub start_cursor: Option<String>,
pub end_cursor: Option<String>,
}
#[derive(Debug, Clone)]
pub struct Page<T> {
pub nodes: Vec<T>,
pub page_info: PageInfo,
}
pub fn paginate<'a, T, F, Fut>(fetch: F) -> impl Stream<Item = crate::Result<T>> + 'a
where
T: 'a,
F: FnMut(Option<String>) -> Fut + 'a,
Fut: Future<Output = crate::Result<Page<T>>> + 'a,
{
futures::stream::try_unfold(
(fetch, None::<String>, false),
|(mut fetch, cursor, done)| async move {
if done {
return Ok(None);
}
let page = fetch(cursor.clone()).await?;
let next_cursor = page.page_info.end_cursor.clone();
let done = !page.page_info.has_next_page || next_cursor == cursor;
Ok(Some((page, (fetch, next_cursor, done))))
},
)
.map_ok(|page| futures::stream::iter(page.nodes.into_iter().map(Ok)))
.try_flatten()
}
pub async fn collect_all<'a, T, F, Fut>(fetch: F, limit: Option<usize>) -> crate::Result<Vec<T>>
where
T: 'a,
F: FnMut(Option<String>) -> Fut + 'a,
Fut: Future<Output = crate::Result<Page<T>>> + 'a,
{
let stream = paginate(fetch);
futures::pin_mut!(stream);
let mut items = Vec::new();
while let Some(item) = stream.try_next().await? {
items.push(item);
if limit.is_some_and(|limit| items.len() >= limit) {
break;
}
}
Ok(items)
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicUsize, Ordering};
fn page(nodes: Vec<u32>, end_cursor: Option<&str>, has_next: bool) -> Page<u32> {
Page {
nodes,
page_info: PageInfo {
has_next_page: has_next,
has_previous_page: false,
start_cursor: None,
end_cursor: end_cursor.map(str::to_owned),
},
}
}
#[tokio::test]
async fn paginate_walks_all_pages() {
let calls = AtomicUsize::new(0);
let items = collect_all(
|cursor| {
calls.fetch_add(1, Ordering::SeqCst);
async move {
Ok(match cursor.as_deref() {
None => page(vec![1, 2], Some("a"), true),
Some("a") => page(vec![3], Some("b"), false),
other => panic!("unexpected cursor {other:?}"),
})
}
},
None,
)
.await
.unwrap();
assert_eq!(items, vec![1, 2, 3]);
assert_eq!(calls.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn paginate_stops_on_repeated_cursor() {
let items = collect_all(
|_cursor| async move { Ok(page(vec![1], Some("stuck"), true)) },
None,
)
.await
.unwrap();
assert_eq!(items, vec![1, 1]);
}
#[tokio::test]
async fn collect_all_honors_limit() {
let items = collect_all(
|_cursor| async move { Ok(page(vec![1, 2, 3], Some("x"), false)) },
Some(2),
)
.await
.unwrap();
assert_eq!(items, vec![1, 2]);
}
}