use std::fs::File;
use std::io::Write;
use std::path::Path;
use std::sync::Arc;
use spider_route::{AttemptOutcome, FeatureVector, RouteDecision, RouteSource, StatusClass};
pub trait Recorder: Send + Sync {
fn observe(&self, features: &FeatureVector, chosen: &RouteDecision, outcome: &AttemptOutcome);
}
impl<T: Recorder + ?Sized> Recorder for Arc<T> {
fn observe(&self, features: &FeatureVector, chosen: &RouteDecision, outcome: &AttemptOutcome) {
(**self).observe(features, chosen, outcome);
}
}
impl<T: Recorder + ?Sized> Recorder for Box<T> {
fn observe(&self, features: &FeatureVector, chosen: &RouteDecision, outcome: &AttemptOutcome) {
(**self).observe(features, chosen, outcome);
}
}
pub const ROW_VERSION: u32 = 1;
pub struct JsonlRecorder<W> {
writer: W,
}
impl JsonlRecorder<File> {
pub fn create(path: impl AsRef<Path>) -> std::io::Result<JsonlRecorder<File>> {
let file = File::options().create(true).append(true).open(path)?;
Ok(JsonlRecorder::new(file))
}
}
impl<W> JsonlRecorder<W> {
pub fn new(writer: W) -> JsonlRecorder<W> {
JsonlRecorder { writer }
}
pub fn into_inner(self) -> W {
self.writer
}
}
impl<W> std::fmt::Debug for JsonlRecorder<W> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("JsonlRecorder")
}
}
impl<W> Recorder for JsonlRecorder<W>
where
W: Send + Sync,
for<'a> &'a W: Write,
{
fn observe(&self, features: &FeatureVector, chosen: &RouteDecision, outcome: &AttemptOutcome) {
let mut line = row(features, chosen, outcome);
line.push('\n');
let mut writer = &self.writer;
if let Err(error) = writer.write_all(line.as_bytes()) {
log::debug!("a routing row was not written: {error}");
}
}
}
fn row(features: &FeatureVector, chosen: &RouteDecision, outcome: &AttemptOutcome) -> String {
let slots = features.as_slice();
let mut out = String::with_capacity(slots.len() * 4 + 256);
out.push_str("{\"v\":");
out.push_str(&ROW_VERSION.to_string());
out.push_str(",\"mode\":\"");
out.push_str(chosen.mode().as_str());
out.push_str("\",\"proxy\":\"");
out.push_str(chosen.proxy().as_str());
out.push_str("\",\"wait_ms\":");
out.push_str(&chosen.wait().millis().to_string());
out.push_str(",\"start_rung\":");
out.push_str(&chosen.start_rung.to_string());
out.push_str(",\"source\":\"");
out.push_str(source_label(chosen.source));
out.push_str("\",\"confidence\":");
out.push_str(&number(chosen.confidence));
out.push_str(",\"success\":");
out.push_str(if outcome.success { "true" } else { "false" });
out.push_str(",\"status\":\"");
out.push_str(status_label(outcome.status));
out.push_str("\",\"bytes\":");
out.push_str(&outcome.bytes.to_string());
out.push_str(",\"millis\":");
out.push_str(&outcome.millis.to_string());
out.push_str(",\"features\":[");
for (at, value) in slots.iter().enumerate() {
if at > 0 {
out.push(',');
}
out.push_str(&number(*value));
}
out.push_str("]}");
out
}
fn number(value: f32) -> String {
if value.is_finite() {
format!("{value}")
} else {
"0".to_string()
}
}
fn source_label(source: RouteSource) -> &'static str {
match source {
RouteSource::Heuristic => "heuristic",
RouteSource::Model => "model",
RouteSource::Caller => "caller",
RouteSource::Memory => "memory",
RouteSource::Explore => "explore",
_ => "other",
}
}
fn status_label(status: StatusClass) -> &'static str {
match status {
StatusClass::Unknown => "unknown",
StatusClass::Ok => "ok",
StatusClass::Empty => "empty",
StatusClass::BadRequest => "bad_request",
StatusClass::NeedsLogin => "needs_login",
StatusClass::Blocked => "blocked",
StatusClass::NotFound => "not_found",
StatusClass::RateLimited => "rate_limited",
StatusClass::ServerError => "server_error",
_ => "other",
}
}
#[cfg(test)]
mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::indexing_slicing,
clippy::string_slice
)]
use super::*;
use spider_route::{featurize, Action, DeclaredNeed, ProxyPool, RequestMode, RouteInput};
use url::Url;
fn decision() -> RouteDecision {
RouteDecision::new(
Action::new(RequestMode::Browser).with_proxy(ProxyPool::Residential),
RouteSource::Memory,
0.8,
)
.starting_at(2)
}
struct Rows {
path: std::path::PathBuf,
}
impl Rows {
fn new(name: &str) -> Rows {
let path = std::env::temp_dir().join(format!(
"spider-cloud-agent-{name}-{}.jsonl",
std::process::id()
));
let _ = std::fs::remove_file(&path);
Rows { path }
}
fn recorder(&self) -> JsonlRecorder<File> {
JsonlRecorder::create(&self.path).expect("a file to write rows to")
}
fn read(&self) -> String {
std::fs::read_to_string(&self.path).unwrap_or_default()
}
}
impl Drop for Rows {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.path);
}
}
#[test]
fn a_row_is_one_line_of_json_with_every_feature_slot_in_it() {
let url = Url::parse("https://example.com/a/b").expect("a url");
let features = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown));
let line = row(&features, &decision(), &AttemptOutcome::ok(1_234, 567));
assert!(!line.contains('\n'));
assert!(line.starts_with("{\"v\":1,"));
assert!(line.contains("\"mode\":\"browser\""));
assert!(line.contains("\"proxy\":\"residential\""));
assert!(line.contains("\"source\":\"memory\""));
assert!(line.contains("\"start_rung\":2"));
assert!(line.contains("\"bytes\":1234"));
assert!(line.contains("\"millis\":567"));
let values = line
.split_once("\"features\":[")
.expect("a feature array")
.1
.trim_end_matches("]}")
.split(',')
.count();
assert_eq!(values, features.as_slice().len());
}
#[test]
fn a_recorded_row_holds_no_host() {
const DISTINCTIVE: &str = "zqxwvutsrqponmlk";
let url = Url::parse(&format!(
"https://shop.{DISTINCTIVE}.com/checkout/{DISTINCTIVE}-item?ref={DISTINCTIVE}"
))
.expect("a url");
let rows = Rows::new("no-host");
let features = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown));
rows.recorder().observe(
&features,
&decision(),
&AttemptOutcome::failed(StatusClass::Blocked, 900),
);
let out = rows.read();
assert!(
!out.is_empty(),
"nothing was written, so nothing was checked"
);
assert!(
!out.contains(DISTINCTIVE),
"the host reached the row: {out}"
);
assert!(!out.contains(".com"), "a suffix reached the row: {out}");
assert!(!out.contains("http"), "the address reached the row: {out}");
assert!(out.contains("\"status\":\"blocked\""), "{out}");
assert!(out.ends_with('\n'), "a row has to end its line");
}
#[test]
fn rows_append_rather_than_replace() {
let rows = Rows::new("append");
let recorder = rows.recorder();
let url = Url::parse("https://example.com/a").expect("a url");
let features = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown));
for _ in 0..3 {
recorder.observe(&features, &decision(), &AttemptOutcome::ok(10, 10));
}
assert_eq!(rows.read().lines().count(), 3);
}
#[test]
fn a_boxed_recorder_records() {
let rows = Rows::new("boxed");
let recorder: Box<dyn Recorder> = Box::new(rows.recorder());
let url = Url::parse("https://example.com/a").expect("a url");
let features = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown));
recorder.observe(&features, &decision(), &AttemptOutcome::ok(10, 10));
assert_eq!(rows.read().lines().count(), 1);
}
#[test]
fn an_unwritable_row_is_dropped_rather_than_raised() {
let path = std::env::temp_dir().join("spider-cloud-agent-unwritable.jsonl");
File::create(&path).expect("a file to reopen read only");
let recorder =
JsonlRecorder::new(File::open(&path).expect("a read only handle on that file"));
let url = Url::parse("https://example.com/a").expect("a url");
let features = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown));
recorder.observe(&features, &decision(), &AttemptOutcome::ok(10, 10));
let written = std::fs::metadata(&path)
.expect("the file still exists")
.len();
let _ = std::fs::remove_file(&path);
assert_eq!(written, 0, "the read only handle accepted a write");
}
}