use std::collections::VecDeque;
use std::time::{Duration, Instant};
use serde_json::Value;
use url::Url;
use spider_cloud_agent::{Budget, Need};
use crate::cli::{Format, Global, Goal, RunArgs};
use crate::commands::{close_caps, finish, record};
use crate::exit::{Code, Failure, Run};
use crate::progress::Log;
use crate::records::{self, Report};
use crate::setup::{self, pin};
struct Plan {
need: Need,
urls: Vec<Url>,
expand: usize,
max_pages: Option<usize>,
credits: Option<f64>,
}
pub async fn run(global: &Global, args: &RunArgs, log: Log) -> Run<Code> {
let plan = settle(global, args)?;
if plan.need == Need::screenshot() {
setup::refuse_bytes_on_a_terminal(global)?;
}
let spider = setup::client_with_credits(global, plan.credits, log)?;
let mut emitter = setup::emitter(global, Format::Ndjson)?;
let mut report = Report {
targets: plan.urls.len(),
..Report::default()
};
let started = Instant::now();
let mut worst = None;
let mut frontier: VecDeque<Url> = plan.urls.iter().cloned().collect();
let mut seen: Vec<String> = plan.urls.iter().map(|u| u.as_str().to_string()).collect();
let mut followed = 0usize;
let mut pages = 0usize;
let mut stopped = None;
while let Some(url) = frontier.pop_front() {
if let Some(cap) = plan.max_pages {
if pages >= cap {
stopped = Some("pages");
break;
}
}
if let Some(cap) = spider.run_budget() {
if cap.remaining() < spider_cloud_agent::policy::budget::ASSUMED_MINIMUM_COST {
stopped = Some("budget");
break;
}
}
if let Some(seconds) = global.wall {
if started.elapsed() >= Duration::from_secs(seconds) {
stopped = Some("time");
break;
}
}
let budget = remaining(global, started);
log.note(format!("working {url}"));
let mut call = pin!(
spider
.scrape(url.clone())
.need(plan.need.clone())
.budget(budget),
global
);
if plan.expand > 0 {
call.params_mut().return_page_links = Some(true);
}
pages += 1;
match call.send_all().await {
Ok(outcome) => {
report.attempts += outcome.attempts.len();
report.cost += outcome.cost;
report.add_thrift(&outcome.thrift);
if outcome.attempts.len() > 1 {
log.note(format!(
"{url}: took {} calls, the ladder answered the status",
outcome.attempts.len()
));
}
for result in outcome.value.0 {
match result {
spider_cloud_agent::response::PageResult::Ok(page) => {
report.served += 1;
if plan.expand > 0 && followed < plan.expand {
followed +=
queue(&mut frontier, &mut seen, &page, plan.expand - followed);
}
emitter.write(records::page(&page))?;
}
spider_cloud_agent::response::PageResult::Failed(failed) => {
report.refused += 1;
log.note(format!(
"{}: {} refused it, next move would be {}",
failed.url,
failed.status,
records::hint(failed.hint)
));
emitter.write(records::failed(&failed))?;
}
}
}
}
Err(error) => {
let failure = super::failure(&mut report, error);
if failure.code == Code::Budget {
stopped = Some("budget");
}
if record(&mut emitter, &mut report, &mut worst, Some(&url), failure)? {
break;
}
}
}
}
report.elapsed_ms = started.elapsed().as_millis() as u64;
close_caps(&mut report, stopped, &mut worst);
finish(&mut emitter, &report, &log, worst)
}
fn remaining(global: &Global, started: Instant) -> Budget {
let mut budget = setup::budget(global);
if let Some(seconds) = global.wall {
let left = Duration::from_secs(seconds).saturating_sub(started.elapsed());
budget = budget.with_wall(left);
}
budget
}
fn queue(
frontier: &mut VecDeque<Url>,
seen: &mut Vec<String>,
page: &spider_cloud_agent::response::Page,
room: usize,
) -> usize {
let mut added = 0usize;
let host = page.url.host_str().map(str::to_string);
for link in page.links.iter().flatten() {
if added >= room {
break;
}
if link.host_str().map(str::to_string) != host {
continue;
}
let key = link.as_str().to_string();
if seen.iter().any(|held| held == &key) {
continue;
}
seen.push(key);
frontier.push_back(link.clone());
added += 1;
}
added
}
fn settle(global: &Global, args: &RunArgs) -> Run<Plan> {
let plan: PlanFile = if let Some(spec) = &args.plan {
serde_json::from_str(&setup::source(spec)?)
.map_err(|e| Failure::usage(format!("the plan in {spec} is invalid: {e}")))?
} else {
PlanFile::default()
};
let planned_goal = plan.goal.as_deref().map(parse_goal).transpose()?;
let planned_budget = plan
.budget
.map(|n| crate::cli::credits(&n.to_string()))
.transpose()
.map_err(|e| Failure::usage(format!("plan budget: {e}")))?;
let planned_fields = plan
.selectors
.as_ref()
.map(|map| setup::fields_from_value(map, None))
.transpose()?;
let goal = args.goal.or(planned_goal).unwrap_or(Goal::Markdown);
let need = if let Some(spec) = &args.selectors {
setup::fields(spec, None)?
} else if let Some(fields) = planned_fields {
fields
} else {
setup::need(goal, None, None)?
};
let mut urls = args.targets.urls.clone();
urls.extend(plan.urls);
let targets = crate::cli::Targets {
urls,
urls_from: args.targets.urls_from.clone(),
};
Ok(Plan {
need,
urls: setup::addresses(&targets)?,
expand: args.expand.or(plan.expand).unwrap_or(0),
max_pages: args.max_pages.or(plan.max_pages),
credits: global.budget.or(planned_budget),
})
}
#[derive(Default, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct PlanFile {
#[serde(default, deserialize_with = "present")]
goal: Option<String>,
#[serde(default)]
urls: Vec<String>,
#[serde(default, deserialize_with = "present")]
expand: Option<usize>,
#[serde(default, deserialize_with = "present")]
max_pages: Option<usize>,
#[serde(default, deserialize_with = "present")]
budget: Option<f64>,
#[serde(default, deserialize_with = "present")]
selectors: Option<Value>,
}
fn present<'de, D, T>(deserializer: D) -> Result<Option<T>, D::Error>
where
D: serde::Deserializer<'de>,
T: serde::Deserialize<'de>,
{
T::deserialize(deserializer).map(Some)
}
fn parse_goal(named: &str) -> Run<Goal> {
use clap::ValueEnum;
Goal::from_str(named, false).map_err(|_| {
let goals: Vec<String> = Goal::value_variants()
.iter()
.filter_map(|goal| goal.to_possible_value().map(|v| v.get_name().to_owned()))
.collect();
Failure::usage(format!(
"{named} is not a goal. The goals are {}.",
goals.join(", ")
))
})
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
#[test]
fn typed_plan_values_are_checked_before_overrides() {
use clap::Parser;
let path = std::env::temp_dir().join(format!(
"spider-agent-plan-unit-{}.json",
std::process::id()
));
let parsed = crate::cli::Cli::try_parse_from([
"spider-agent",
"run",
"https://example.com/",
"--plan",
path.to_str().unwrap(),
"--goal",
"markdown",
"--expand",
"0",
"--budget",
"10",
])
.unwrap();
let Some(crate::cli::Command::Run(ref args)) = parsed.command else {
panic!("run arguments")
};
for body in [
r#"{"budget":"10"}"#,
r#"{"budget":-1}"#,
r#"{"budget":null}"#,
r#"{"expand":-1}"#,
r#"{"expand":1.5}"#,
r#"{"max_pages":"2"}"#,
r#"{"goal":"typo"}"#,
r#"{"urls":[1]}"#,
r#"{"selectors":{}}"#,
] {
std::fs::write(&path, body).unwrap();
let error = settle(&parsed.global, args).err().expect("invalid plan");
assert_eq!(error.code, Code::Usage, "{body}");
}
std::fs::write(
&path,
r#"{"goal":"html","expand":4,"budget":0,"urls":["https://example.com/b"]}"#,
)
.unwrap();
let plan = settle(&parsed.global, args).unwrap();
assert_eq!(plan.need, Need::Markdown);
assert_eq!(plan.expand, 0);
assert_eq!(plan.credits, Some(10.0));
assert_eq!(plan.urls.len(), 2);
std::fs::remove_file(path).unwrap();
}
#[test]
fn a_goal_that_is_not_one_names_the_goals() {
let failed = parse_goal("summarise").expect_err("a usage failure");
assert_eq!(failed.code, Code::Usage);
assert!(failed.message.contains("markdown"), "{failed}");
}
#[test]
fn every_goal_the_command_line_takes_is_readable_in_a_plan() {
for named in [
"text",
"markdown",
"html",
"links",
"metadata",
"fields",
"screenshot",
"raw",
] {
assert!(parse_goal(named).is_ok(), "{named}");
}
}
}