1use crate::edit::EditSet;
16use crate::features::{value_bucket, EditFeatures, EDIT_FEATURE_VERSION};
17use crate::observe::Observation;
18use crate::params::Params;
19use crate::schema::{Key, SCHEMA_VERSION};
20use spider_route::{
21 DeclaredNeed, ExtClass, FeatureVector, RouteDecision, RouteSource, SiteMemory, StatusClass,
22 FEATURES_USED,
23};
24use std::fmt;
25
26pub const CMP_ROW_VERSION: u32 = 1;
28
29pub const BASE_FEATURE_VERSION: u16 = 1;
35
36const _: () = assert!(FEATURES_USED == 152);
37
38#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
40pub enum Arm {
41 Baseline,
43 Candidate,
45 Shadow,
47}
48
49#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
51pub enum MemoryState {
52 Cold,
54 Thin,
56 Warm,
58 Steady,
60}
61
62impl MemoryState {
63 pub const fn of(memory: Option<&SiteMemory>) -> MemoryState {
65 match memory {
66 None => MemoryState::Cold,
67 Some(memory) => match memory.observations {
68 0 => MemoryState::Cold,
69 1..=2 => MemoryState::Thin,
70 3..=9 => MemoryState::Warm,
71 _ => MemoryState::Steady,
72 },
73 }
74 }
75}
76
77#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
79pub struct IdentDescriptor {
80 pub class: u8,
82 pub share: u8,
84 pub count: u8,
86 pub third: bool,
88}
89
90#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
92pub struct EditDescriptor {
93 pub key: u8,
95 pub op: u8,
97 pub bucket: u8,
99 pub ident: Option<IdentDescriptor>,
101}
102
103impl EditDescriptor {
104 pub fn describe(edits: &EditSet, observation: &Observation) -> Option<EditDescriptor> {
111 let edit = edits
112 .get(Key::NetworkBlacklist)
113 .or_else(|| edits.edits().first())?;
114 let (bucket, ident) = match value_bucket(edit, observation) {
115 Some((bucket, ident)) => (bucket.min(254) as u8, ident),
116 None => (u8::MAX, None),
117 };
118 Some(EditDescriptor {
119 key: edit.key.index() as u8,
120 op: edit.op.code(),
121 bucket,
122 ident: ident.map(|ident| IdentDescriptor {
123 class: crate::features::ext_index(ident.class) as u8,
124 share: ident.share_bucket,
125 count: ident.count_bucket,
126 third: ident.third_party,
127 }),
128 })
129 }
130}
131
132pub fn pinned_mask(caller: &dyn Params) -> (u32, u64) {
137 let mut low = 0u32;
138 let mut high = 0u64;
139 for key in Key::ALL {
140 if !caller.is_set(*key) {
141 continue;
142 }
143 let at = key.index();
144 if at < 32 {
145 low |= 1 << at;
146 } else if at < 96 {
147 high |= 1 << (at - 32);
148 }
149 }
150 (low, high)
151}
152
153#[derive(Debug, Clone, Copy, PartialEq)]
155pub struct ComparisonRow<'a> {
156 pub pair: u64,
158 pub arm: Arm,
160 pub fallback: bool,
163 pub day: u32,
165 pub domain_key: u64,
168 pub need: DeclaredNeed,
170 pub ext: ExtClass,
172 pub tld: u8,
174 pub memory: MemoryState,
176 pub routed: &'a RouteDecision,
178 pub edit: Option<EditDescriptor>,
180 pub pinned: u32,
182 pub pinned_hi: u64,
184 pub success: bool,
186 pub status: StatusClass,
188 pub millis: u32,
190 pub bytes: u32,
192 pub credits: f64,
194 pub attempts: u8,
196 pub multiplier: f32,
198 pub fields_requested: u8,
200 pub fields_present: u8,
202 pub content_ok: Option<bool>,
204 pub fields_ok: Option<bool>,
206 pub shingle_jaccard: Option<f32>,
208 pub byte_ratio: Option<f32>,
210 pub base: &'a FeatureVector,
212 pub edit_feats: &'a EditFeatures,
214}
215
216struct Out(String);
218
219impl Out {
220 fn write_fmt(&mut self, args: fmt::Arguments<'_>) {
221 let _ = fmt::Write::write_fmt(&mut self.0, args);
223 }
224
225 fn raw(&mut self, text: &str) {
226 self.0.push_str(text);
227 }
228
229 fn float(&mut self, value: f32) {
230 if value.is_finite() {
231 write!(self, "{value}");
232 } else {
233 self.raw("0");
234 }
235 }
236
237 fn double(&mut self, value: f64) {
238 if value.is_finite() {
239 write!(self, "{value}");
240 } else {
241 self.raw("0");
242 }
243 }
244
245 fn flag(&mut self, value: bool) {
246 self.raw(if value { "true" } else { "false" });
247 }
248
249 fn label(&mut self, text: &str) {
250 self.raw("\"");
251 self.raw(text);
252 self.raw("\"");
253 }
254}
255
256pub fn comparison_row(r: &ComparisonRow<'_>) -> String {
258 let base = r.base.as_slice();
259 let base = base.get(..FEATURES_USED).unwrap_or(base);
260 let mut out = Out(String::with_capacity(4_096));
261
262 write!(
263 out,
264 "{{\"v\":{CMP_ROW_VERSION},\"schema_v\":{SCHEMA_VERSION},\"feat_v\":{BASE_FEATURE_VERSION},\"edit_feat_v\":{EDIT_FEATURE_VERSION},\"pair\":{},\"arm\":",
265 r.pair
266 );
267 out.label(arm_label(r.arm));
268 out.raw(",\"fallback\":");
269 out.flag(r.fallback);
270 write!(out, ",\"day\":{},\"dk\":{},\"need\":", r.day, r.domain_key);
271 out.label(need_label(r.need));
272 out.raw(",\"ext\":");
273 out.label(ext_label(r.ext));
274 write!(out, ",\"tld\":{},\"mem\":", r.tld);
275 out.label(memory_label(r.memory));
276
277 out.raw(",\"routed\":{\"mode\":");
278 out.label(r.routed.mode().as_str());
279 out.raw(",\"proxy\":");
280 out.label(r.routed.proxy().as_str());
281 write!(
282 out,
283 ",\"wait_ms\":{},\"start_rung\":{},\"source\":",
284 r.routed.wait().millis(),
285 r.routed.start_rung
286 );
287 out.label(source_label(r.routed.source));
288 out.raw(",\"confidence\":");
289 out.float(r.routed.confidence);
290 out.raw("}");
291
292 out.raw(",\"edit\":");
293 match r.edit {
294 None => out.raw("null"),
295 Some(edit) => {
296 write!(
297 out,
298 "{{\"key\":{},\"op\":{},\"bucket\":{},\"ident\":",
299 edit.key, edit.op, edit.bucket
300 );
301 match edit.ident {
302 None => out.raw("null"),
303 Some(ident) => {
304 write!(
305 out,
306 "{{\"class\":{},\"share\":{},\"count\":{},\"third\":",
307 ident.class, ident.share, ident.count
308 );
309 out.flag(ident.third);
310 out.raw("}");
311 }
312 }
313 out.raw("}");
314 }
315 }
316
317 write!(
318 out,
319 ",\"pinned\":{},\"pinned_hi\":{},\"success\":",
320 r.pinned, r.pinned_hi
321 );
322 out.flag(r.success);
323 out.raw(",\"status\":");
324 out.label(status_label(r.status));
325 write!(
326 out,
327 ",\"millis\":{},\"bytes\":{},\"credits\":",
328 r.millis, r.bytes
329 );
330 out.double(r.credits);
331 write!(out, ",\"attempts\":{},\"multiplier\":", r.attempts);
332 out.float(r.multiplier);
333 write!(
334 out,
335 ",\"fields_requested\":{},\"fields_present\":{},\"content_ok\":",
336 r.fields_requested, r.fields_present
337 );
338 optional_flag(&mut out, r.content_ok);
339 out.raw(",\"fields_ok\":");
340 optional_flag(&mut out, r.fields_ok);
341 out.raw(",\"shingle_jaccard\":");
342 optional_float(&mut out, r.shingle_jaccard);
343 out.raw(",\"byte_ratio\":");
344 optional_float(&mut out, r.byte_ratio);
345
346 out.raw(",\"base\":[");
347 array(&mut out, base);
348 out.raw("],\"edit_feats\":[");
349 array(&mut out, r.edit_feats.as_slice());
350 out.raw("]}");
351
352 out.0
353}
354
355fn optional_flag(out: &mut Out, value: Option<bool>) {
356 match value {
357 Some(value) => out.flag(value),
358 None => out.raw("null"),
359 }
360}
361
362fn optional_float(out: &mut Out, value: Option<f32>) {
363 match value {
364 Some(value) => out.float(value),
365 None => out.raw("null"),
366 }
367}
368
369fn array(out: &mut Out, values: &[f32]) {
370 for (at, value) in values.iter().enumerate() {
371 if at > 0 {
372 out.raw(",");
373 }
374 out.float(*value);
375 }
376}
377
378pub(crate) const fn arm_label(arm: Arm) -> &'static str {
380 match arm {
381 Arm::Baseline => "baseline",
382 Arm::Candidate => "candidate",
383 Arm::Shadow => "shadow",
384 }
385}
386
387pub(crate) const fn memory_label(memory: MemoryState) -> &'static str {
389 match memory {
390 MemoryState::Cold => "cold",
391 MemoryState::Thin => "thin",
392 MemoryState::Warm => "warm",
393 MemoryState::Steady => "steady",
394 }
395}
396
397#[cfg(test)]
399pub(crate) const ALL_NEEDS: [DeclaredNeed; 8] = [
400 DeclaredNeed::Text,
401 DeclaredNeed::Markdown,
402 DeclaredNeed::Html,
403 DeclaredNeed::Links,
404 DeclaredNeed::Metadata,
405 DeclaredNeed::Fields,
406 DeclaredNeed::Screenshot,
407 DeclaredNeed::Raw,
408];
409
410pub(crate) const fn need_label(need: DeclaredNeed) -> &'static str {
412 match need {
413 DeclaredNeed::Text => "text",
414 DeclaredNeed::Markdown => "markdown",
415 DeclaredNeed::Html => "html",
416 DeclaredNeed::Links => "links",
417 DeclaredNeed::Metadata => "metadata",
418 DeclaredNeed::Fields => "fields",
419 DeclaredNeed::Screenshot => "screenshot",
420 DeclaredNeed::Raw => "raw",
421 _ => "other",
422 }
423}
424
425#[cfg(test)]
427pub(crate) const ALL_STATUSES: [StatusClass; 9] = [
428 StatusClass::Unknown,
429 StatusClass::Ok,
430 StatusClass::Empty,
431 StatusClass::BadRequest,
432 StatusClass::NeedsLogin,
433 StatusClass::Blocked,
434 StatusClass::NotFound,
435 StatusClass::RateLimited,
436 StatusClass::ServerError,
437];
438
439pub(crate) const fn status_label(status: StatusClass) -> &'static str {
442 match status {
443 StatusClass::Unknown => "unknown",
444 StatusClass::Ok => "ok",
445 StatusClass::Empty => "empty",
446 StatusClass::BadRequest => "bad_request",
447 StatusClass::NeedsLogin => "needs_login",
448 StatusClass::Blocked => "blocked",
449 StatusClass::NotFound => "not_found",
450 StatusClass::RateLimited => "rate_limited",
451 StatusClass::ServerError => "server_error",
452 _ => "other",
453 }
454}
455
456pub(crate) const fn ext_label(ext: ExtClass) -> &'static str {
458 match ext {
459 ExtClass::None => "none",
460 ExtClass::Markup => "markup",
461 ExtClass::Xml => "xml",
462 ExtClass::Json => "json",
463 ExtClass::Feed => "feed",
464 ExtClass::Text => "text",
465 ExtClass::Csv => "csv",
466 ExtClass::Pdf => "pdf",
467 ExtClass::Office => "office",
468 ExtClass::Image => "image",
469 ExtClass::Media => "media",
470 ExtClass::Archive => "archive",
471 ExtClass::Asset => "asset",
472 _ => "other",
473 }
474}
475
476pub(crate) const fn source_label(source: RouteSource) -> &'static str {
479 match source {
480 RouteSource::Heuristic => "heuristic",
481 RouteSource::Model => "model",
482 RouteSource::Caller => "caller",
483 RouteSource::Memory => "memory",
484 RouteSource::Explore => "explore",
485 _ => "other",
486 }
487}
488
489#[cfg(test)]
490mod tests {
491 #![allow(
492 clippy::unwrap_used,
493 clippy::expect_used,
494 clippy::panic,
495 clippy::unreachable,
496 clippy::indexing_slicing,
497 clippy::string_slice
498 )]
499 use super::*;
500 use crate::candidates::{generate, Candidate};
501 use crate::features::featurize_edit;
502 use crate::schema::Schema;
503 use crate::testing::Fixture;
504 use serde::de::{Deserializer, IgnoredAny, MapAccess, Visitor};
505 use spider_route::{featurize, RouteInput};
506
507 struct KeyOrder(Vec<String>);
509
510 impl<'de> serde::Deserialize<'de> for KeyOrder {
511 fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<KeyOrder, D::Error> {
512 struct Keys;
513 impl<'de> Visitor<'de> for Keys {
514 type Value = KeyOrder;
515 fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
516 f.write_str("an object")
517 }
518 fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<KeyOrder, A::Error> {
519 let mut keys = Vec::new();
520 while let Some(key) = map.next_key::<String>()? {
521 let _: IgnoredAny = map.next_value()?;
522 keys.push(key);
523 }
524 Ok(KeyOrder(keys))
525 }
526 }
527 deserializer.deserialize_map(Keys)
528 }
529 }
530
531 fn sample(fixture: &Fixture) -> String {
533 let ctx = fixture.ctx();
534 let cand = generate(&Schema::v1(), &ctx)
535 .into_iter()
536 .find(|c| c.edits.get(Key::NetworkBlacklist).is_some())
537 .unwrap_or_else(|| Candidate::keep(&ctx));
538 let feats = featurize_edit(&ctx, &cand);
539 let base = featurize(&RouteInput::new(ctx.url, ctx.need));
540 let (pinned, pinned_hi) = pinned_mask(ctx.caller);
541 comparison_row(&ComparisonRow {
542 pair: u64::MAX,
543 arm: Arm::Candidate,
544 fallback: false,
545 day: 20_000,
546 domain_key: 42,
547 need: ctx.need,
548 ext: ExtClass::Markup,
549 tld: 0,
550 memory: MemoryState::of(ctx.observation.memory.as_ref()),
551 routed: ctx.routed,
552 edit: EditDescriptor::describe(&cand.edits, ctx.observation),
553 pinned,
554 pinned_hi,
555 success: true,
556 status: StatusClass::Ok,
557 millis: 1_234,
558 bytes: 56_789,
559 credits: f64::NAN,
560 attempts: 1,
561 multiplier: cand.multiplier,
562 fields_requested: 2,
563 fields_present: 2,
564 content_ok: Some(true),
565 fields_ok: None,
566 shingle_jaccard: Some(0.97),
567 byte_ratio: Some(f32::INFINITY),
568 base: &base,
569 edit_feats: &feats,
570 })
571 }
572
573 #[test]
574 fn rows_hold_no_url_host_or_body() {
575 let row = sample(&Fixture::observed());
576
577 let edit = format!(
578 "\"edit\":{{\"key\":{},\"op\":1",
579 Key::NetworkBlacklist.index()
580 );
581 assert!(row.contains(&edit), "{row}");
582 for needle in [
583 "http",
584 "example",
585 "articles",
586 "tracker-alpha",
587 "cdn-beta",
588 ".com",
589 "\n",
590 ] {
591 assert!(!row.contains(needle), "{needle:?} reached the row: {row}");
592 }
593 }
594
595 #[test]
596 fn a_row_parses_as_json_with_the_documented_keys() {
597 let mut fixture = Fixture::observed();
598 fixture.caller.stealth = Some(true);
599 fixture.caller.disable_hints = Some(true);
600 let row = sample(&fixture);
601
602 let order: KeyOrder = serde_json::from_str(&row).unwrap();
603 assert_eq!(
604 order.0,
605 [
606 "v",
607 "schema_v",
608 "feat_v",
609 "edit_feat_v",
610 "pair",
611 "arm",
612 "fallback",
613 "day",
614 "dk",
615 "need",
616 "ext",
617 "tld",
618 "mem",
619 "routed",
620 "edit",
621 "pinned",
622 "pinned_hi",
623 "success",
624 "status",
625 "millis",
626 "bytes",
627 "credits",
628 "attempts",
629 "multiplier",
630 "fields_requested",
631 "fields_present",
632 "content_ok",
633 "fields_ok",
634 "shingle_jaccard",
635 "byte_ratio",
636 "base",
637 "edit_feats",
638 ]
639 );
640
641 let value: serde_json::Value = serde_json::from_str(&row).unwrap();
642 let from = row.find("\"routed\":").unwrap() + "\"routed\":".len();
645 let to = from + row[from..].find('}').unwrap() + 1;
646 let routed: KeyOrder = serde_json::from_str(&row[from..to]).unwrap();
647 assert_eq!(
648 routed.0,
649 [
650 "mode",
651 "proxy",
652 "wait_ms",
653 "start_rung",
654 "source",
655 "confidence"
656 ]
657 );
658 assert_eq!(value["pair"].as_u64(), Some(u64::MAX));
659 assert_eq!(value["base"].as_array().unwrap().len(), FEATURES_USED);
660 assert_eq!(value["edit_feats"].as_array().unwrap().len(), 96);
661 assert_eq!(value["credits"], 0, "a NaN is written as zero");
662 assert_eq!(value["byte_ratio"], 0, "an infinity is written as zero");
663 assert!(value["fields_ok"].is_null());
664 assert_eq!(value["pinned"], 1 << Key::Stealth.index());
665 assert_eq!(
666 value["pinned_hi"].as_u64(),
667 Some(1 << (Key::DisableHints.index() - 32))
668 );
669 assert_eq!(value["edit"]["ident"]["third"], true);
670 assert_eq!(value["status"], "ok");
671 }
672}