use std::ops::Deref;
use futures_util::Stream;
use serde::de::DeserializeOwned;
use url::Url;
use crate::client::{AccountClient, Response};
use crate::error::Error;
use crate::hooks::OperationInfo;
use crate::operation::Operation;
use crate::route::Route;
#[derive(Debug, Clone)]
pub struct Page<T> {
value: T,
next: Option<Result<Url, String>>,
total_count: Option<u64>,
route: &'static Route,
info: OperationInfo,
origin: Url,
pinned: bool,
}
impl<T> Page<T> {
pub(crate) fn new(
value: T,
response: &Response,
route: &'static Route,
info: OperationInfo,
pinned: bool,
) -> Page<T> {
let next = response
.header("link")
.and_then(parse_next_link)
.map(|target| response.url.join(&target).map_err(|_| target));
let total_count = response
.header("x-total-count")
.and_then(|value| value.trim().parse().ok());
Page {
value,
next,
total_count,
route,
info,
origin: response.url.clone(),
pinned,
}
}
pub(crate) fn route(&self) -> &'static Route {
self.route
}
pub(crate) fn info(&self) -> &OperationInfo {
&self.info
}
pub(crate) fn origin(&self) -> &Url {
&self.origin
}
pub fn into_inner(self) -> T {
self.value
}
pub fn value(&self) -> &T {
&self.value
}
pub fn next_url(&self) -> Option<&Url> {
self.next.as_ref().and_then(|next| next.as_ref().ok())
}
pub fn has_next(&self) -> bool {
self.next.is_some()
}
pub fn is_pinned(&self) -> bool {
self.pinned
}
pub(crate) fn next_target(&self) -> Option<Result<Url, Error>> {
if self.pinned {
return None;
}
self.next.as_ref().map(|next| match next {
Ok(url) => Ok(url.clone()),
Err(_) => Err(Error::usage(
"pagination Link header names a target that is not a URL",
)),
})
}
pub fn total_count(&self) -> Option<u64> {
self.total_count
}
}
pub trait PageItems {
type Item;
fn into_items(self) -> Vec<Self::Item>;
}
impl<T> PageItems for Vec<T> {
type Item = T;
fn into_items(self) -> Vec<T> {
self
}
}
impl<T> Deref for Page<T> {
type Target = T;
fn deref(&self) -> &T {
&self.value
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct ListResult<T> {
pub items: Vec<T>,
pub meta: ListMeta,
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
#[non_exhaustive]
pub struct ListMeta {
pub total_count: u64,
pub truncated: bool,
pub next_url: Option<Url>,
}
impl AccountClient {
pub fn next_page<T: DeserializeOwned>(
&self,
page: &Page<T>,
) -> impl Future<Output = Result<Option<Page<T>>, Error>> + Send + '_ {
let operation = page.next_target().map(|next| self.follow_up(page, &next?));
async move {
match operation {
None => Ok(None),
Some(operation) => self.send_page(operation?).await.map(Some),
}
}
}
pub async fn collect_all<P: PageItems + DeserializeOwned>(
&self,
first: Page<P>,
max_items: Option<usize>,
) -> Result<ListResult<P::Item>, Error> {
let total_count = first.total_count().unwrap_or(0);
let max_pages = self.max_pages();
let mut page = first;
let mut pages = 1;
let mut items = Vec::new();
let route = page.route();
let info = page.info().clone();
let origin = page.origin().clone();
loop {
let next_target = page.next_target();
let more = page.has_next();
let named = page.next_url().cloned();
items.extend(page.into_inner().into_items());
if let Some(cap) = max_items
&& items.len() >= cap
{
let truncated = more || items.len() > cap;
items.truncate(cap);
return Ok(ListResult {
items,
meta: ListMeta {
total_count,
truncated,
next_url: named,
},
});
}
let Some(next) = next_target.transpose()? else {
return Ok(ListResult {
items,
meta: ListMeta {
total_count,
truncated: more,
next_url: named,
},
});
};
if pages >= max_pages {
return Ok(ListResult {
items,
meta: ListMeta {
total_count,
truncated: true,
next_url: Some(next),
},
});
}
let operation = self.follow_up_at(route, &info, &origin, &next)?;
page = self.send_page(operation).await?;
pages += 1;
}
}
}
enum Pending<T> {
Page(Page<T>),
Fetch(Operation),
Refused(Error),
}
impl AccountClient {
pub fn pages<T: DeserializeOwned + Send + 'static>(
&self,
first: Page<T>,
) -> impl Stream<Item = Result<Page<T>, Error>> + Send + '_ {
let max_pages = self.max_pages();
futures_util::stream::try_unfold(
(Some(Pending::Page(first)), 0usize),
move |(pending, yielded)| async move {
let page = match pending {
None => return Ok(None),
Some(Pending::Page(page)) => page,
Some(Pending::Fetch(operation)) => self.send_page(operation).await?,
Some(Pending::Refused(error)) => return Err(error),
};
let following = if yielded + 1 < max_pages {
self.pending_follow_up(&page)
} else {
None
};
Ok(Some((page, (following, yielded + 1))))
},
)
}
pub fn items<P>(&self, first: Page<P>) -> impl Stream<Item = Result<P::Item, Error>> + Send + '_
where
P: PageItems + DeserializeOwned + Send + 'static,
P::Item: Send + 'static,
{
use futures_util::TryStreamExt;
let max_pages = self.max_pages();
futures_util::stream::try_unfold(
(Some(Pending::Page(first)), 0usize),
move |(pending, yielded)| async move {
let page = match pending {
None => return Ok::<_, Error>(None),
Some(Pending::Page(page)) => page,
Some(Pending::Fetch(operation)) => self.send_page(operation).await?,
Some(Pending::Refused(error)) => return Err(error),
};
let capped = yielded + 1 >= max_pages;
let following = if capped {
None
} else {
self.pending_follow_up(&page)
};
let truncated = capped && page.has_next();
let mut items: Vec<Result<P::Item, Error>> =
page.into_inner().into_items().into_iter().map(Ok).collect();
if truncated {
items.push(Err(Error::usage(format!(
"collection truncated at the page cap (max_pages = {max_pages}) with a next page available"
))));
}
Ok(Some((
futures_util::stream::iter(items),
(following, yielded + 1),
)))
},
)
.try_flatten()
}
fn pending_follow_up<T>(&self, page: &Page<T>) -> Option<Pending<T>> {
let next = page.next_target()?;
Some(match next.and_then(|next| self.follow_up(page, &next)) {
Ok(operation) => Pending::Fetch(operation),
Err(error) => Pending::Refused(error),
})
}
}
pub fn parse_next_link(header: &str) -> Option<String> {
if header.is_empty() {
return None;
}
header
.split(',')
.map(str::trim)
.filter(|part| part.contains("rel=\"next\""))
.find_map(extract_angle_bracketed)
}
fn extract_angle_bracketed(part: &str) -> Option<String> {
let bytes = part.as_bytes();
let mut cursor = 0;
loop {
let start = cursor + bytes[cursor..].iter().position(|b| *b == b'<')?;
let end = start + 1 + bytes[start + 1..].iter().position(|b| *b == b'>')?;
if end > start + 1 {
return Some(part[start + 1..end].to_string());
}
cursor = start + 1;
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn finds_the_next_link_among_others() {
let header = r#"<https://3.basecampapi.com/999/projects.json?page=1>; rel="prev", <https://3.basecampapi.com/999/projects.json?page=3>; rel="next""#;
assert_eq!(
parse_next_link(header).as_deref(),
Some("https://3.basecampapi.com/999/projects.json?page=3")
);
assert_eq!(parse_next_link(r#"</x?page=2>; rel="last""#), None);
assert_eq!(parse_next_link(""), None);
}
#[test]
fn an_unextractable_next_part_does_not_hide_a_later_one() {
let header = r#"rel="next", </x?page=2>; rel="next""#;
assert_eq!(parse_next_link(header).as_deref(), Some("/x?page=2"));
assert_eq!(extract_angle_bracketed("<> <a>").as_deref(), Some("a"));
assert_eq!(extract_angle_bracketed("> <a>").as_deref(), Some("a"));
assert_eq!(extract_angle_bracketed("<<a>").as_deref(), Some("<a"));
assert_eq!(extract_angle_bracketed("<a"), None);
assert_eq!(extract_angle_bracketed("<é>").as_deref(), Some("é"));
}
}