use std::io::{IsTerminal, Read};
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use serde_json::Value;
use url::Url;
use spider_cloud_agent::auth::router::{self, StoredRouter};
use spider_cloud_agent::params::{Country, ProxyPool, RequestMode};
use spider_cloud_agent::DeclaredNeed;
use spider_cloud_agent::{Budget, Credits, Error, Need, Spider};
use crate::cli::{Format, Global, Goal, Mode, Pool, Targets};
use crate::emit::{Emitter, FileRules};
use crate::exit::{Code, Failure, Run};
use crate::progress::Log;
pub fn source(spec: &str) -> Run<String> {
let text = if spec == "-" && crate::session::active() {
crate::session::input()
.ok_or_else(|| Failure::usage("this tool call carries no input to read"))?
} else if spec == "-" {
let mut text = String::new();
std::io::stdin()
.read_to_string(&mut text)
.map_err(|e| Failure::usage(format!("could not read stdin: {e}")))?;
text
} else {
std::fs::read_to_string(spec)
.map_err(|e| Failure::usage(format!("could not read {spec}: {e}")))?
};
Ok(without_bom(text))
}
fn without_bom(text: String) -> String {
match text.strip_prefix('\u{feff}') {
Some(rest) => rest.to_string(),
None => text,
}
}
pub fn addresses(targets: &Targets) -> Run<Vec<Url>> {
let mut raw: Vec<String> = targets.urls.clone();
if let Some(spec) = &targets.urls_from {
raw.extend(lines(&source(spec)?));
}
if raw.is_empty() && !crate::session::active() && !std::io::stdin().is_terminal() {
raw.extend(lines(&source("-")?));
}
if raw.is_empty() {
return Err(Failure::usage(
"no address. Name one, pass --urls-from, or pipe a list on stdin.",
));
}
parse_all(&raw)
}
fn lines(text: &str) -> Vec<String> {
text.lines()
.map(str::trim)
.filter(|line| !line.is_empty() && !line.starts_with('#'))
.map(str::to_string)
.collect()
}
pub fn parse_all(raw: &[String]) -> Run<Vec<Url>> {
let mut out = Vec::with_capacity(raw.len());
for one in raw {
out.push(Url::parse(one).map_err(|e| {
let hint = if one.chars().all(|c| c.is_ascii_alphanumeric() || c == '-') {
". If you meant a command, spider-agent --help lists them."
} else {
""
};
Failure::usage(format!("{one} is not an address: {e}{hint}"))
})?);
}
Ok(out)
}
pub fn need(goal: Goal, selectors: Option<&str>, at: Option<&str>) -> Run<Need> {
if let Some(spec) = selectors {
return fields(spec, at);
}
Ok(match goal {
Goal::Text => Need::Text,
Goal::Markdown => Need::Markdown,
Goal::Html => Need::Html,
Goal::Links => Need::Links,
Goal::Metadata => Need::Metadata,
Goal::Screenshot => Need::screenshot(),
Goal::Raw => Need::Raw,
Goal::Fields => {
return Err(Failure::usage(
"--goal fields needs --selectors, a JSON object of field name to selector.",
))
}
})
}
pub fn declared(goal: Goal) -> DeclaredNeed {
match goal {
Goal::Text => DeclaredNeed::Text,
Goal::Markdown => DeclaredNeed::Markdown,
Goal::Html => DeclaredNeed::Html,
Goal::Links => DeclaredNeed::Links,
Goal::Metadata => DeclaredNeed::Metadata,
Goal::Fields => DeclaredNeed::Fields,
Goal::Screenshot => DeclaredNeed::Screenshot,
Goal::Raw => DeclaredNeed::Raw,
}
}
pub fn fields(spec: &str, at: Option<&str>) -> Run<Need> {
let text = source(spec)?;
let value: Value = serde_json::from_str(&text)
.map_err(|e| Failure::usage(format!("the selector map in {spec} is not JSON: {e}")))?;
fields_from_value(&value, at)
}
pub fn fields_from_value(value: &Value, at: Option<&str>) -> Run<Need> {
let Value::Object(map) = value.clone() else {
return Err(Failure::usage(
"a selector map has to be a JSON object of field name to selector.",
));
};
if map.is_empty() {
return Err(Failure::usage(
"the selector map is empty, so the request would ask for nothing.",
));
}
let mut pairs: Vec<(String, String)> = Vec::new();
for (name, selector) in map {
match selector {
Value::String(one) => pairs.push((name, one)),
Value::Array(many) => {
for one in many {
match one {
Value::String(one) => pairs.push((name.clone(), one)),
other => {
return Err(Failure::usage(format!(
"the selectors for {name} have to be strings, and one is {other}."
)))
}
}
}
}
other => {
return Err(Failure::usage(format!(
"the selector for {name} has to be a string or a list of strings, and it is {other}."
)))
}
}
}
let mut spec = spider_cloud_agent::thrift::FieldSpec::new(Vec::<(String, String)>::new());
for (name, selector) in pairs {
spec.push(name, &selector);
}
if let Some(path) = at {
spec = spec.at(path);
}
Ok(Need::Fields(spec))
}
pub fn budget(global: &Global) -> Budget {
let mut budget = Budget::default();
if let Some(credits) = global.budget {
budget = budget.with_credits(Credits::new(credits));
}
if let Some(credits) = global.max_per_page {
budget = budget.with_per_page_credits(Credits::new(credits));
}
if let Some(attempts) = global.attempts {
budget = budget.with_attempts(attempts);
}
if let Some(seconds) = global.wall {
budget = budget.with_wall(Duration::from_secs(seconds));
}
budget
}
pub fn client(global: &Global) -> Run<Spider> {
build_client(global, global.budget, None, false)
}
pub fn scrape_client(global: &Global, log: Log) -> Run<Spider> {
let keyless = !keyless_refused() && !credentials_file_present();
let spider = build_client(global, global.budget, Some(log), keyless)?;
if !spider.raw().has_key() && !KEYLESS_NOTED.swap(true, Ordering::Relaxed) {
log.say("no key found, so this runs without one: scrape only, and rate limited. Run spider-agent login for anything more");
}
Ok(spider)
}
pub const NO_KEYLESS_ENV: &str = "SPIDER_AGENT_NO_KEYLESS";
fn keyless_refused() -> bool {
std::env::var_os("SPIDER_AGENT_NO_KEYLESS").is_some_and(|v| !v.is_empty())
}
fn credentials_file_present() -> bool {
let name = std::path::Path::new(spider_cloud_agent::auth::CREDENTIALS_PATH).file_name();
match (spider_cloud_agent::auth::store::spider_dir(), name) {
(Some(dir), Some(name)) => dir.join(name).exists(),
_ => false,
}
}
static KEYLESS_NOTED: AtomicBool = AtomicBool::new(false);
pub fn fetching_client(global: &Global, log: Log) -> Run<Spider> {
build_client(global, global.budget, Some(log), false)
}
pub fn client_with_credits(global: &Global, credits: Option<f64>, log: Log) -> Run<Spider> {
build_client(global, credits, Some(log), false)
}
pub const NO_ROUTER_ENV: &str = "SPIDER_AGENT_NO_ROUTER";
pub fn router_skipped(global: &Global) -> bool {
global.no_router || std::env::var_os("SPIDER_AGENT_NO_ROUTER").is_some_and(|v| !v.is_empty())
}
static ROUTER_NOTED: AtomicBool = AtomicBool::new(false);
#[cfg(feature = "optimize")]
static OPTIMIZER_NOTED: AtomicBool = AtomicBool::new(false);
fn build_client(
global: &Global,
credits: Option<f64>,
fetching: Option<Log>,
keyless: bool,
) -> Run<Spider> {
let mut builder = Spider::builder().budget(budget(global)).keyless(keyless);
if let Some(shared) = crate::session::http_client() {
builder = builder.http_client(shared);
}
if let Some(cap) = credits {
builder = builder.run_budget(spider_cloud_agent::RunBudget::new(Credits::new(cap)));
}
if fetching.is_some() && !router_skipped(global) {
if let Some(stored) = stored_router()? {
builder = builder.provider_router(stored);
}
}
#[cfg(feature = "optimize")]
let mut optimizer_note: Option<String> = None;
if fetching.is_some() {
let settings = crate::optimize::settings(global)?;
#[cfg(feature = "optimize")]
if let Some(ready) = crate::optimize::build(&settings)? {
builder = builder.optimizer(ready.optimizer);
optimizer_note = Some(ready.note);
}
#[cfg(not(feature = "optimize"))]
crate::optimize::build(&settings)?;
}
let spider = builder.build().map_err(Failure::from)?;
if let (Some(log), Some(stored)) = (fetching, spider.provider_router()) {
if !ROUTER_NOTED.swap(true, Ordering::Relaxed) {
log.say(router_note(stored));
}
}
#[cfg(feature = "optimize")]
if let (Some(log), Some(note)) = (fetching, optimizer_note) {
if !OPTIMIZER_NOTED.swap(true, Ordering::Relaxed) {
log.say(note);
}
}
Ok(spider)
}
pub fn stored_router() -> Run<Option<StoredRouter>> {
router::load().map_err(|error| {
let code = match &error {
Error::Config(_) => Code::Usage,
_ => Code::Output,
};
Failure::new(
code,
format!(
"the stored router cannot be used: {error}. Replace it with spider-agent router set, remove it with spider-agent router clear, or pass --no-router"
),
)
})
}
pub fn router_note(stored: &StoredRouter) -> String {
let router = &stored.router;
let parts: Vec<String> = [
("provider", &router.provider),
("mode", &router.mode),
("funding", &router.funding),
]
.into_iter()
.filter_map(|(name, value)| value.as_ref().map(|value| format!("{name} {value}")))
.collect();
if parts.is_empty() {
"using stored router".to_string()
} else {
format!("using stored router: {}", parts.join(", "))
}
}
pub fn mode(global: &Global) -> Option<RequestMode> {
global.mode.and_then(|mode| match mode {
Mode::Http => Some(RequestMode::Http),
Mode::Smart => Some(RequestMode::Smart),
Mode::Browser => Some(RequestMode::Browser),
Mode::Fallback | Mode::First | Mode::Off | Mode::Shadow | Mode::Apply => None,
})
}
pub fn pool(global: &Global) -> Option<ProxyPool> {
global.proxy.map(|pool| match pool {
Pool::Isp => ProxyPool::Isp,
Pool::Residential => ProxyPool::Residential,
})
}
pub fn country(global: &Global) -> Run<Option<Country>> {
match &global.country {
None => Ok(None),
Some(code) => match Country::new(code) {
Some(country) => Ok(Some(country)),
None => Err(Failure::usage(format!(
"{code} is not a two letter country code."
))),
},
}
}
macro_rules! pin {
($builder:expr, $global:expr) => {{
let mut builder = $builder;
if let Some(mode) = $crate::setup::mode($global) {
builder = builder.mode(mode);
}
if let Some(pool) = $crate::setup::pool($global) {
builder = builder.proxy(pool);
}
if let Some(country) = $crate::setup::country($global)? {
builder = builder.country(country);
}
if let Some(seconds) = $global.timeout {
builder = builder.timeout(std::time::Duration::from_secs(seconds));
}
if $global.session {
builder = builder.session(true);
}
if let Some(tokens) = $global.max_tokens {
builder = builder.max_tokens(tokens);
}
builder
}};
($builder:expr, $global:expr, links) => {{
let mut builder = $crate::setup::pin!($builder, $global);
if $global.with_links {
builder = builder.page_links(true);
}
builder
}};
}
pub(crate) use pin;
pub fn format(global: &Global, default: Format) -> Format {
if global.json {
return Format::Json;
}
if global.ndjson {
return Format::Ndjson;
}
global.format.unwrap_or(default)
}
pub fn emitter(global: &Global, default: Format) -> Run<Emitter> {
if let Some(sink) = crate::session::sink() {
return Ok(Emitter::to_stream(format(global, default), Box::new(sink)));
}
Emitter::open(
format(global, default),
global.output.as_deref(),
global.output_dir.as_deref(),
FileRules {
force: global.force,
mkdir: global.mkdir,
append: global.append,
},
)
}
pub fn refuse_bytes_on_a_terminal(global: &Global) -> Run<()> {
if global.output.is_none() && global.output_dir.is_none() && std::io::stdout().is_terminal() {
return Err(Failure::new(
Code::Usage,
"a picture needs somewhere to go. Pass --output or --output-dir, or redirect stdout.",
));
}
Ok(())
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
#[test]
fn comments_and_blank_lines_are_not_addresses() {
let read = lines("# a note\n\nhttps://example.com\n https://example.com/a \n");
assert_eq!(
read,
vec![
"https://example.com".to_string(),
"https://example.com/a".to_string()
]
);
}
#[test]
fn a_byte_order_mark_is_not_part_of_the_text() {
assert_eq!(
without_bom("\u{feff}https://example.com\n".to_string()),
"https://example.com\n"
);
assert_eq!(
without_bom("https://example.com\n".to_string()),
"https://example.com\n"
);
assert_eq!(without_bom("a\u{feff}b".to_string()), "a\u{feff}b");
}
#[test]
fn an_address_that_is_not_one_names_itself() {
let failed = parse_all(&["not a url".to_string()]).expect_err("a usage failure");
assert_eq!(failed.code, Code::Usage);
assert!(failed.message.contains("not a url"), "{failed}");
}
#[test]
fn a_goal_of_fields_without_selectors_is_a_usage_failure() {
let failed = need(Goal::Fields, None, None).expect_err("a usage failure");
assert_eq!(failed.code, Code::Usage);
}
#[test]
fn every_other_goal_needs_nothing_else() {
for goal in [
Goal::Text,
Goal::Markdown,
Goal::Html,
Goal::Links,
Goal::Metadata,
Goal::Screenshot,
Goal::Raw,
] {
assert!(need(goal, None, None).is_ok(), "{goal:?}");
}
}
}