pub mod account;
pub mod clients;
pub mod local;
pub mod mcp;
pub mod optimize;
pub mod pages;
pub mod query;
pub mod router;
pub mod run;
use std::time::{Duration, Instant};
use serde_json::json;
use url::Url;
use spider_cloud_agent::response::{Outcome, PageResult, Pages};
use spider_cloud_agent::{Budget, RunBudget, Spider};
use crate::cli::{Format, Global};
use crate::emit::{Emitter, Item};
use crate::exit::{Code, Failure, Run};
use crate::records::{self, Report};
use crate::setup;
pub struct Caps {
credits: Option<RunBudget>,
wall: Option<u64>,
started: Instant,
}
impl Caps {
pub fn new(global: &Global, started: Instant, spider: &Spider) -> Caps {
Caps {
credits: spider.run_budget().cloned(),
wall: global.wall,
started,
}
}
pub fn spent(&self, _report: &Report) -> Option<&'static str> {
if let Some(cap) = &self.credits {
if cap.remaining() < spider_cloud_agent::policy::budget::ASSUMED_MINIMUM_COST {
return Some("budget");
}
}
if let Some(seconds) = self.wall {
if self.started.elapsed() >= Duration::from_secs(seconds) {
return Some("time");
}
}
None
}
pub fn budget(&self, global: &Global, _report: &Report) -> Budget {
let mut budget = setup::budget(global);
if let Some(seconds) = self.wall {
let left = Duration::from_secs(seconds).saturating_sub(self.started.elapsed());
budget = budget.with_wall(left);
}
budget
}
}
pub fn close_caps(report: &mut Report, stopped: Option<&'static str>, worst: &mut Option<Failure>) {
if stopped.is_some() {
report.stopped = stopped;
}
if let Some(reason @ ("budget" | "time")) = stopped {
worsen(
worst,
Failure::new(
Code::Budget,
format!(
"stopped on {reason}, after {} page(s) and {:.3} credits",
report.served + report.refused,
report.cost.get()
),
),
);
}
}
pub fn record(
emitter: &mut Emitter,
report: &mut Report,
worst: &mut Option<Failure>,
url: Option<&Url>,
failure: Failure,
) -> Run<bool> {
if failure.code == Code::Budget {
report.stopped = Some("budget");
}
emitter.write(error_item(url, &failure))?;
let fatal = is_fatal(failure.code);
worsen(worst, failure);
Ok(fatal)
}
pub fn absorb(emitter: &mut Emitter, report: &mut Report, outcome: Outcome<Pages>) -> Run<()> {
report.attempts += outcome.attempts.len();
report.cost += outcome.cost;
report.add_thrift(&outcome.thrift);
for result in outcome.value.0 {
match result {
PageResult::Ok(page) => {
report.served += 1;
emitter.write(records::page(&page))?;
}
PageResult::Failed(failed) => {
report.refused += 1;
emitter.write(records::failed(&failed))?;
}
}
}
Ok(())
}
pub fn failure(report: &mut Report, error: spider_cloud_agent::Error) -> Failure {
report.attempts += error.attempts().len();
report.cost += error.spent();
Failure::from(error.into_cause())
}
pub fn error_item(url: Option<&Url>, failure: &Failure) -> Item {
let value = json!({
"type": "error",
"url": url.map(Url::as_str),
"code": failure.code.label(),
"exit": failure.code.number(),
"message": failure.message,
});
match url {
Some(url) => Item::structured(value).with_url(url.clone()),
None => Item::structured(value),
}
}
pub const fn is_fatal(code: Code) -> bool {
matches!(code, Code::Auth | Code::Budget | Code::Output)
}
pub fn finish(
emitter: &mut Emitter,
report: &Report,
log: &crate::progress::Log,
worst: Option<Failure>,
) -> Run<Code> {
match emitter.format() {
Format::Text => log.say(report.line()),
Format::Json | Format::Ndjson => emitter.write_report(report.item())?,
}
emitter.finish()?;
match worst {
Some(failure) => {
if !failure.is_silent() {
log.failed(&failure.message);
}
Ok(failure.code)
}
None => Ok(Code::Ok),
}
}
pub fn worsen(worst: &mut Option<Failure>, failure: Failure) {
let rank = |code: Code| match code {
Code::Ok => 0,
Code::Failed => 1,
Code::Refused => 2,
Code::Transport => 3,
Code::Usage => 4,
Code::Output => 5,
Code::Budget => 6,
Code::Auth => 7,
};
match worst {
Some(held) if rank(held.code) >= rank(failure.code) => {}
_ => *worst = Some(failure),
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
#[test]
fn a_stop_from_inside_one_operation_survives_the_close() {
let mut report = Report {
stopped: Some("budget"),
..Report::default()
};
let mut worst = Some(Failure::new(Code::Budget, "the operation said so"));
close_caps(&mut report, None, &mut worst);
assert_eq!(report.stopped, Some("budget"));
assert_eq!(
worst.as_ref().map(|f| f.message.as_str()),
Some("the operation said so")
);
}
#[test]
fn a_cap_spent_between_two_pages_leaves_under_the_budget_code() {
let mut report = Report::default();
let mut worst = None;
close_caps(&mut report, Some("time"), &mut worst);
assert_eq!(report.stopped, Some("time"));
assert_eq!(worst.map(|f| f.code), Some(Code::Budget));
let mut worst = None;
close_caps(&mut report, Some("pages"), &mut worst);
assert!(
worst.is_none(),
"a page limit is the run doing what it was told"
);
}
}