use std::ops::Deref;
use crate::http::Method;
use serde_json::Value;
use url::Url;
use crate::client::{Client, Response};
use crate::error::Error;
use crate::observability::OperationInfo;
use crate::operation::Operation;
use crate::route::Route;
use crate::security::is_same_origin;
#[derive(Debug, Clone)]
pub struct Page<T> {
value: T,
next_url: Option<Url>,
next_cursor: Option<String>,
total_count: Option<u64>,
info: OperationInfo,
route: Option<&'static Route>,
}
impl<T> Page<T> {
pub(crate) fn new(
value: T,
response: &Response,
info: OperationInfo,
route: Option<&'static Route>,
) -> Page<T> {
let next_url = response
.headers
.get("link")
.and_then(|value| value.to_str().ok())
.and_then(next_link)
.and_then(|target| response.url.join(&target).ok());
let next_cursor = next_url.as_ref().and_then(|url| {
url.query_pairs()
.find(|(name, _)| name == "page")
.map(|(_, value)| value.into_owned())
});
let total_count = response
.headers
.get("x-total-count")
.and_then(|value| value.to_str().ok())
.and_then(|value| value.trim().parse().ok());
Page {
value,
next_url,
next_cursor,
total_count,
info,
route,
}
}
pub(crate) fn info(&self) -> &OperationInfo {
&self.info
}
pub(crate) fn route(&self) -> Option<&'static Route> {
self.route
}
pub fn into_inner(self) -> T {
self.value
}
pub fn value(&self) -> &T {
&self.value
}
pub fn next_page(&self) -> Option<&str> {
self.next_cursor.as_deref()
}
pub fn next_url(&self) -> Option<&Url> {
self.next_url.as_ref()
}
pub fn has_next(&self) -> bool {
self.next_url.is_some()
}
pub fn total_count(&self) -> Option<u64> {
self.total_count
}
pub fn map<U>(self, f: impl FnOnce(T) -> U) -> Page<U> {
Page {
value: f(self.value),
next_url: self.next_url,
next_cursor: self.next_cursor,
total_count: self.total_count,
info: self.info,
route: self.route,
}
}
}
impl<T> Deref for Page<T> {
type Target = T;
fn deref(&self) -> &T {
&self.value
}
}
impl Client {
pub async fn get_all(&self, path: &str) -> Result<Vec<Value>, Error> {
self.get_all_with_limit(path, 0).await
}
pub async fn get_all_with_limit(&self, path: &str, limit: usize) -> Result<Vec<Value>, Error> {
self.within_limit(Box::pin(async move {
let mut operation = self.raw(Method::GET, path)?;
let started_at = self.url_for(&operation)?;
let mut collected: Vec<Value> = Vec::new();
let mut pages = 0;
loop {
let response = self.execute(operation).await?;
collected.extend(response.json::<Vec<Value>>()?);
pages += 1;
if limit > 0 && collected.len() >= limit {
collected.truncate(limit);
break;
}
match next_page_url(&response, &started_at)? {
Some(next) if pages < self.max_pages() => {
operation = Operation::at(Method::GET, next);
}
Some(_) => return Err(Error::pagination_capped(self.max_pages())),
None => break,
}
}
Ok(collected)
}))
.await
}
pub async fn follow_pagination(
&self,
first: &Response,
first_page_count: usize,
limit: usize,
) -> Result<Vec<Value>, Error> {
self.within_limit(Box::pin(async move {
if limit > 0 && first_page_count >= limit {
return Ok(Vec::new());
}
let started_at = first.url.clone();
let mut next = next_page_url(first, &started_at)?;
let mut collected: Vec<Value> = Vec::new();
let mut count = first_page_count;
let mut pages = 1;
while let Some(url) = next {
if pages >= self.max_pages() {
return Err(Error::pagination_capped(self.max_pages()));
}
let response = self.execute(Operation::at(Method::GET, url)).await?;
let items: Vec<Value> = response.json()?;
count += items.len();
collected.extend(items);
pages += 1;
if limit > 0 && count >= limit {
collected.truncate(collected.len().saturating_sub(count - limit));
break;
}
next = next_page_url(&response, &started_at)?;
}
Ok(collected)
}))
.await
}
}
fn next_page_url(response: &Response, started_at: &Url) -> Result<Option<Url>, Error> {
match response.header("link").and_then(next_link) {
None => Ok(None),
Some(target) => {
let next = response.url.join(&target)?;
if is_same_origin(&next, started_at) {
Ok(Some(next))
} else {
Err(Error::usage(format!(
"pagination Link header points to a different origin: {next}"
)))
}
}
}
}
pub fn next_link(header: &str) -> Option<String> {
let mut remaining = header;
while let Some(start) = remaining.find('<') {
let after_start = &remaining[start + 1..];
let end = after_start.find('>')?;
let target = &after_start[..end];
let rest = &after_start[end + 1..];
let params_end = rest.find('<').unwrap_or(rest.len());
if link_is_next(&rest[..params_end]) {
return Some(target.to_string());
}
remaining = &rest[params_end..];
}
None
}
fn link_is_next(params: &str) -> bool {
params.split(';').any(|param| {
let mut parts = param.splitn(2, '=');
let name = parts.next().unwrap_or_default().trim();
let value = parts.next().unwrap_or_default().trim().trim_matches('"');
name.eq_ignore_ascii_case("rel")
&& value
.split_whitespace()
.any(|rel| rel.eq_ignore_ascii_case("next"))
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn finds_the_next_link_among_others() {
let header = r#"<https://app.hey.com/imbox.json?page=a,b>; rel="prev", <https://app.hey.com/imbox.json?page=c>; rel="next""#;
assert_eq!(
next_link(header).as_deref(),
Some("https://app.hey.com/imbox.json?page=c")
);
}
#[test]
fn matches_rel_sets_and_case() {
assert_eq!(
next_link(r#"</x?page=2>; REL="prev next""#).as_deref(),
Some("/x?page=2")
);
assert_eq!(next_link(r#"</x?page=2>; rel="last""#), None);
assert_eq!(next_link(""), None);
}
}