use serde_json::{json, Map, Value};
use url::Url;
use spider_cloud_agent::response::{Body, FailedPage, Hint, Page, SearchEntry};
use spider_cloud_agent::thrift::ThriftReport;
use spider_cloud_agent::{Credits, RouteDecision};
use crate::emit::Item;
struct BodyShape {
kind: &'static str,
text: Option<String>,
bytes: Option<Vec<u8>>,
extension: &'static str,
fields: Option<Value>,
}
fn shape(body: &Body) -> BodyShape {
let plain = |kind: &'static str, text: &str, extension: &'static str| BodyShape {
kind,
text: Some(text.to_string()),
bytes: None,
extension,
fields: None,
};
match body {
Body::Text(text) => plain("text", text, "txt"),
Body::Markdown(text) => plain("markdown", text, "md"),
Body::Html(text) => plain("html", text, "html"),
Body::Xml(text) => plain("xml", text, "xml"),
Body::Bytes(bytes) => BodyShape {
kind: "bytes",
text: None,
bytes: Some(bytes.to_vec()),
extension: "bin",
fields: None,
},
Body::Screenshot(bytes) => BodyShape {
kind: "screenshot",
text: None,
bytes: Some(bytes.to_vec()),
extension: "png",
fields: None,
},
Body::Fields(fields) => {
let value = Value::Object(fields.clone().into_iter().collect::<Map<String, Value>>());
BodyShape {
kind: "fields",
text: serde_json::to_string(&value).ok(),
bytes: None,
extension: "json",
fields: Some(value),
}
}
Body::Multi(multi) => BodyShape {
kind: "multi",
text: multi.as_str().map(str::to_string),
bytes: None,
extension: "txt",
fields: None,
},
Body::Empty => BodyShape {
kind: "empty",
text: None,
bytes: None,
extension: "txt",
fields: None,
},
_ => BodyShape {
kind: "other",
text: None,
bytes: None,
extension: "bin",
fields: None,
},
}
}
pub fn page(page: &Page) -> Item {
let shape = shape(&page.body);
let mut value = Map::new();
value.insert("type".into(), json!("page"));
value.insert("url".into(), json!(page.url.as_str()));
value.insert("status".into(), json!(page.status.code()));
value.insert("content".into(), json!(shape.kind));
value.insert(
"body".into(),
match &shape.bytes {
Some(_) => Value::Null,
None => match &shape.fields {
Some(_) => Value::Null,
None => shape.text.clone().map(Value::String).unwrap_or(Value::Null),
},
},
);
if let Some(fields) = &shape.fields {
value.insert("fields".into(), fields.clone());
}
if let Some(metadata) = &page.metadata {
if let Ok(encoded) = serde_json::to_value(metadata) {
value.insert("metadata".into(), encoded);
}
}
if let Some(links) = &page.links {
value.insert(
"links".into(),
Value::Array(links.iter().map(|l| json!(l.as_str())).collect()),
);
}
value.insert("bytes".into(), json!(page.body.len()));
value.insert(
"duration_ms".into(),
json!(page.duration.as_millis() as u64),
);
value.insert("cost_credits".into(), json!(page.cost().get()));
let mut item = Item {
value: Value::Object(value),
text: shape.text,
bytes: shape.bytes,
url: Some(page.url.clone()),
extension: shape.extension,
};
if item.bytes.is_some() {
item.text = None;
}
item
}
pub fn failed(failed: &FailedPage) -> Item {
let value = json!({
"type": "failed",
"url": failed.url.as_str(),
"status": failed.status.code(),
"error": failed.error,
"hint": hint(failed.hint),
"billed": failed.was_billed(),
"cost_credits": failed.cost().get(),
});
Item::structured(value).with_url(failed.url.clone())
}
pub fn hint(hint: Hint) -> &'static str {
match hint {
Hint::TryBrowser => "try_browser",
Hint::TryResidentialProxy => "try_residential_proxy",
Hint::TryDifferentCountry => "try_different_country",
Hint::NeedsSession => "needs_session",
Hint::Permanent => "permanent",
Hint::SlowDown { .. } => "slow_down",
_ => "unknown",
}
}
pub fn link(found: &Url, from: &Url) -> Item {
let value = json!({
"type": "link",
"url": found.as_str(),
"from": from.as_str(),
});
Item::structured(value)
.with_text(found.as_str().to_string(), "txt")
.with_url(found.clone())
}
pub fn result(entry: &SearchEntry) -> Item {
let value = json!({
"type": "result",
"url": entry.url,
"title": entry.title,
"description": entry.description,
});
let item = Item::structured(value).with_text(entry.url.clone(), "txt");
match entry.url() {
Some(url) => item.with_url(url),
None => item,
}
}
pub fn row(row: &Value) -> Item {
let value = json!({ "type": "row", "data": row });
let text = serde_json::to_string(row).unwrap_or_default();
Item::structured(value).with_text(text, "json")
}
pub fn route(url: &Url, decision: &RouteDecision) -> Item {
let value = json!({
"type": "route",
"url": url.as_str(),
"mode": decision.mode().as_str(),
"proxy": decision.proxy().as_str(),
"wait_ms": decision.wait().millis(),
"country": decision.country.as_ref().map(|c| c.as_str()),
"start_rung": decision.start_rung,
"confidence": decision.confidence,
"source": source(decision),
});
let text = format!(
"{}\t{}\t{}",
url.as_str(),
decision.mode().as_str(),
decision.proxy().as_str()
);
Item::structured(value)
.with_text(text, "txt")
.with_url(url.clone())
}
fn source(decision: &RouteDecision) -> &'static str {
use spider_cloud_agent::RouteSource;
match decision.source {
RouteSource::Heuristic => "heuristic",
RouteSource::Model => "model",
RouteSource::Caller => "caller",
RouteSource::Memory => "memory",
RouteSource::Explore => "explore",
_ => "unknown",
}
}
pub struct Report {
pub targets: usize,
pub served: usize,
pub refused: usize,
pub attempts: usize,
pub cost: Credits,
pub elapsed_ms: u64,
pub thrift: ThriftReport,
pub stopped: Option<&'static str>,
}
impl Report {
pub fn item(&self) -> Item {
let value = json!({
"type": "report",
"targets": self.targets,
"served": self.served,
"refused": self.refused,
"attempts": self.attempts,
"cost_credits": self.cost.get(),
"elapsed_ms": self.elapsed_ms,
"wire_bytes": self.thrift.wire_bytes,
"returned_bytes": self.thrift.returned_bytes,
"approx_tokens_in": self.thrift.approx_tokens_in,
"approx_tokens_out": self.thrift.approx_tokens_out,
"stopped": self.stopped,
});
Item::structured(value)
}
pub fn line(&self) -> String {
let stopped = match self.stopped {
Some(reason) => format!(", stopped on {reason}"),
None => String::new(),
};
format!(
"{} served, {} refused, {} calls, {:.3} credits, {} ms{stopped}",
self.served, self.refused, self.attempts, self.cost.0, self.elapsed_ms
)
}
pub fn add_thrift(&mut self, thrift: &ThriftReport) {
self.thrift.wire_bytes += thrift.wire_bytes;
self.thrift.returned_bytes += thrift.returned_bytes;
self.thrift.approx_tokens_in += thrift.approx_tokens_in;
self.thrift.approx_tokens_out += thrift.approx_tokens_out;
}
}
impl Default for Report {
fn default() -> Report {
Report {
targets: 0,
served: 0,
refused: 0,
attempts: 0,
cost: Credits::ZERO,
elapsed_ms: 0,
thrift: ThriftReport::default(),
stopped: None,
}
}
}
#[cfg(test)]
mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::indexing_slicing
)]
use super::*;
#[test]
fn a_hint_has_a_stable_name() {
assert_eq!(hint(Hint::TryBrowser), "try_browser");
assert_eq!(hint(Hint::Permanent), "permanent");
assert_eq!(
hint(Hint::SlowDown {
after: std::time::Duration::from_secs(5)
}),
"slow_down"
);
}
#[test]
fn a_report_says_what_stopped_it() {
let report = Report {
served: 2,
refused: 1,
attempts: 4,
stopped: Some("budget"),
..Report::default()
};
let value = report.item().value;
assert_eq!(value["type"], json!("report"));
assert_eq!(value["stopped"], json!("budget"));
assert_eq!(value["served"], json!(2));
assert!(
report.line().contains("stopped on budget"),
"{}",
report.line()
);
}
#[test]
fn a_finished_run_reports_no_stop() {
let value = Report::default().item().value;
assert_eq!(value["stopped"], Value::Null);
}
}