pub mod content;
pub mod costs;
pub mod metadata;
pub mod page;
pub mod search;
pub use content::{Body, MultiBody};
pub use costs::{Costs, Credits, Usd, CREDITS_PER_USD};
pub use metadata::Metadata;
pub use page::{FailedPage, Hint, Page, PageResult, Pages};
pub use search::{SearchEntry, SearchResults};
use crate::status::{ApiStatus, PageStatus};
use crate::thrift::ThriftReport;
use spider_route::RouteDecision;
use std::ops::Deref;
use std::time::Duration;
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct Attempt {
pub elapsed: Duration,
pub api: ApiStatus,
pub page: Option<PageStatus>,
pub cost: Credits,
pub charge_unknown: bool,
pub(crate) reserved: Credits,
}
impl Attempt {
pub(crate) fn new(
elapsed: Duration,
api: ApiStatus,
page: Option<PageStatus>,
cost: Credits,
) -> Attempt {
Attempt {
elapsed,
api,
page,
cost,
charge_unknown: false,
reserved: cost,
}
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct Outcome<T> {
pub overrun: Option<BudgetOverrun>,
pub value: T,
pub attempts: Vec<Attempt>,
pub cost: Credits,
pub thrift: ThriftReport,
pub route: Option<RouteDecision>,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct BudgetOverrun {
pub scope: BudgetScope,
pub cap: Credits,
pub spent: Credits,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BudgetScope {
Operation,
Run,
Page,
}
impl<T> Outcome<T> {
pub(crate) fn new(value: T, attempts: Vec<Attempt>) -> Outcome<T> {
let cost = attempts.iter().map(|a| a.cost).sum();
Outcome {
overrun: None,
value,
attempts,
cost,
thrift: ThriftReport::default(),
route: None,
}
}
pub(crate) fn with_cap(mut self, cap: Option<Credits>) -> Self {
self.overrun = cap.filter(|cap| self.cost > *cap).map(|cap| BudgetOverrun {
scope: BudgetScope::Operation,
cap,
spent: self.cost,
});
self
}
pub(crate) fn routed(mut self, decision: RouteDecision) -> Outcome<T> {
self.route = Some(decision);
self
}
pub(crate) fn reporting(mut self, thrift: ThriftReport) -> Outcome<T> {
self.thrift = thrift;
self
}
pub fn into_value(self) -> T {
self.value
}
pub fn attempt_count(&self) -> usize {
self.attempts.len()
}
pub fn elapsed(&self) -> Duration {
self.attempts.iter().map(|a| a.elapsed).sum()
}
pub fn escalated(&self) -> bool {
self.attempts.len() > 1
}
pub fn map<U>(self, f: impl FnOnce(T) -> U) -> Outcome<U> {
Outcome {
value: f(self.value),
overrun: self.overrun,
attempts: self.attempts,
cost: self.cost,
thrift: self.thrift,
route: self.route,
}
}
}
impl<T> Deref for Outcome<T> {
type Target = T;
fn deref(&self) -> &T {
&self.value
}
}
#[cfg(test)]
mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::indexing_slicing,
clippy::string_slice
)]
use super::page::PageParts;
use super::*;
#[test]
fn f2_mapping_an_outcome_preserves_its_overrun() {
let outcome = Outcome::new(
1,
vec![Attempt::new(
Duration::ZERO,
ApiStatus::new(200),
None,
Credits(2.0),
)],
)
.with_cap(Some(Credits(1.0)));
let expected = outcome.overrun;
let mapped = outcome.map(|value| value.to_string());
assert_eq!(mapped.overrun, expected);
assert_eq!(mapped.cost, Credits(2.0));
assert_eq!(mapped.value, "1");
}
fn attempt(code: u16, page: Option<u16>, cost: f64) -> Attempt {
Attempt::new(
Duration::from_millis(100),
ApiStatus::new(code),
page.map(PageStatus::new),
Credits(cost),
)
}
#[test]
fn an_outcome_adds_up_its_attempts() {
let page = PageResult::from_parts(PageParts::for_test("https://example.com/a", 200))
.into_ok()
.expect("a page");
let outcome = Outcome::new(
page,
vec![attempt(200, Some(403), 4.0), attempt(200, Some(200), 6.0)],
);
assert_eq!(outcome.cost, Credits(10.0));
assert_eq!(outcome.attempt_count(), 2);
assert!(outcome.escalated());
assert_eq!(outcome.elapsed(), Duration::from_millis(200));
}
#[test]
fn an_outcome_reads_through_to_its_value() {
let page = PageResult::from_parts(PageParts::for_test("https://example.com/a", 200))
.into_ok()
.expect("a page");
let outcome = Outcome::new(page, vec![attempt(200, Some(200), 4.0)]);
assert_eq!(outcome.text(), Some("hello"));
assert_eq!(outcome.status.code(), 200);
assert!(!outcome.escalated());
assert_eq!(outcome.into_value().url.as_str(), "https://example.com/a");
}
#[test]
fn an_attempt_keeps_the_two_planes_apart() {
let a = attempt(200, Some(403), 4.0);
assert_eq!(a.api.to_string(), "api status 200");
assert_eq!(
a.page.expect("a target status").to_string(),
"target status 403"
);
}
#[test]
fn an_attempt_that_never_reached_a_site_has_no_target_status() {
let a = attempt(401, None, 0.0);
assert!(a.page.is_none());
}
#[test]
fn mapping_keeps_the_trail() {
let outcome = Outcome::new(1u8, vec![attempt(200, Some(200), 2.0)]);
let mapped = outcome.map(|v| v as u32 + 1);
assert_eq!(mapped.value, 2);
assert_eq!(mapped.cost, Credits(2.0));
assert_eq!(mapped.attempt_count(), 1);
}
}