use std::collections::{HashSet, VecDeque};
use std::future::Future;
use std::pin::Pin;
use futures_util::Stream;
use serde::Deserialize;
use crate::error::{ApiError, Error, MAX_SERVER_WAIT, Result};
#[derive(Debug, Clone, PartialEq, Deserialize)]
pub struct Page<T> {
#[serde(default = "Vec::new")]
pub items: Vec<T>,
#[serde(default)]
pub has_more: bool,
#[serde(default)]
pub next_cursor: Option<String>,
}
pub type ItemStream<'a, T> = Pin<Box<dyn Stream<Item = Result<T>> + Send + 'a>>;
struct State<F, T> {
fetch: F,
cursor: Option<String>,
buffer: VecDeque<T>,
seen: HashSet<String>,
yielded: usize,
max_items: Option<usize>,
done: bool,
}
pub(crate) fn paginate<'a, T, F, Fut>(
start: Option<String>,
max_items: Option<usize>,
fetch: F,
) -> ItemStream<'a, T>
where
T: Send + 'a,
F: FnMut(Option<String>) -> Fut + Send + 'a,
Fut: Future<Output = Result<Page<T>>> + Send + 'a,
{
let state = State {
fetch,
cursor: start,
buffer: VecDeque::new(),
seen: HashSet::new(),
yielded: 0,
max_items,
done: false,
};
Box::pin(futures_util::stream::unfold(state, |mut s| async move {
loop {
if s.max_items.is_some_and(|m| s.yielded >= m) {
return None;
}
if let Some(item) = s.buffer.pop_front() {
s.yielded += 1;
return Some((Ok(item), s));
}
if s.done {
return None;
}
match (s.fetch)(s.cursor.clone()).await {
Err(e) => {
s.done = true;
return Some((Err(e), s));
}
Ok(page) => {
for item in page.items {
s.buffer.push_back(item);
}
match page.next_cursor.filter(|c| page.has_more && !c.is_empty()) {
Some(next) if s.seen.insert(next.clone()) => s.cursor = Some(next),
_ => s.done = true,
}
}
}
}
}))
}
pub(crate) struct HistoryPage<T> {
pub(crate) rows: Vec<T>,
pub(crate) has_account: bool,
pub(crate) next_cursor: Option<String>,
}
struct HistoryState<F, W, T> {
fetch: F,
wait: W,
operation: &'static str,
max_busy_retries: u32,
cursor: Option<String>,
retries: u32,
buffer: VecDeque<T>,
yielded: usize,
max_items: Option<usize>,
done: bool,
pending: Option<Error>,
}
pub(crate) fn paginate_history<'a, T, F, Fut, W, WFut>(
operation: &'static str,
max_busy_retries: u32,
max_items: Option<usize>,
fetch: F,
wait: W,
) -> ItemStream<'a, T>
where
T: Send + 'a,
F: FnMut(Option<String>) -> Fut + Send + 'a,
Fut: Future<Output = Result<HistoryPage<T>>> + Send + 'a,
W: FnMut(u32, Error) -> WFut + Send + 'a,
WFut: Future<Output = ()> + Send + 'a,
{
let state = HistoryState {
fetch,
wait,
operation,
max_busy_retries,
cursor: None,
retries: 0,
buffer: VecDeque::new(),
yielded: 0,
max_items,
done: false,
pending: None,
};
Box::pin(futures_util::stream::unfold(state, |mut s| async move {
loop {
if s.max_items.is_some_and(|m| s.yielded >= m) {
return None;
}
if let Some(item) = s.buffer.pop_front() {
s.yielded += 1;
return Some((Ok(item), s));
}
if s.done {
return s.pending.take().map(|e| (Err(e), s));
}
let page = match (s.fetch)(s.cursor.clone()).await {
Ok(page) => page,
Err(e) if e.api().is_some_and(ApiError::is_futures_busy) => {
if s.retries >= s.max_busy_retries {
s.done = true;
let stalled = Error::PagingStalled {
operation: s.operation,
cursor: s.cursor.clone().unwrap_or_default(),
retries: s.retries,
source: e.api().cloned().map(|a| Box::new(a) as _),
};
return Some((Err(stalled), s));
}
if e.api()
.and_then(ApiError::busy_hint)
.is_some_and(|d| d > MAX_SERVER_WAIT)
{
s.done = true;
return Some((Err(e), s));
}
(s.wait)(s.retries, e).await;
s.retries += 1;
continue;
}
Err(e) => {
s.done = true;
return Some((Err(e), s));
}
};
if !page.has_account {
s.done = true;
continue;
}
let next = page.next_cursor;
if page.rows.is_empty() && next.is_some() && next == s.cursor {
let stalled = Error::PagingStalled {
operation: s.operation,
cursor: next.unwrap_or_default(),
retries: s.retries,
source: None,
};
if s.retries >= s.max_busy_retries {
s.done = true;
return Some((Err(stalled), s));
}
(s.wait)(s.retries, stalled).await;
s.retries += 1;
continue;
}
s.retries = 0;
s.buffer.extend(page.rows);
match next.filter(|n| !n.is_empty()) {
None => s.done = true,
Some(n) if s.cursor.as_ref() == Some(&n) => {
s.done = true;
s.pending = Some(Error::PagingCursorRepeated {
operation: s.operation,
cursor: n,
});
}
Some(n) => s.cursor = Some(n),
}
}
}))
}