Skip to main content

spider_optimize/
row.rs

1//! The comparison row: one arm of one paired trial, as a line of JSON.
2//!
3//! A pair is the same request sent twice, once as the baseline and once with
4//! a candidate edit, or once more in shadow. Each arm writes one row. The rows
5//! are what a scorer is trained on, so what a row may hold is the same as what
6//! a feature may hold: numbers, buckets and labels from a fixed vocabulary.
7//! There is no url, host, pattern or body in [`ComparisonRow`], and no field
8//! that could carry one.
9//!
10//! Written by hand rather than with a serializer, so a new column is a
11//! deliberate edit to [`comparison_row`] and shows up in review as one. A
12//! float that JSON cannot hold is written as `0`, and an absent value as
13//! `null`.
14
15use crate::edit::EditSet;
16use crate::features::{value_bucket, EditFeatures, EDIT_FEATURE_VERSION};
17use crate::observe::Observation;
18use crate::params::Params;
19use crate::schema::{Key, SCHEMA_VERSION};
20use spider_route::{
21    DeclaredNeed, ExtClass, FeatureVector, RouteDecision, RouteSource, SiteMemory, StatusClass,
22    FEATURES_USED,
23};
24use std::fmt;
25
26/// The version of the row's columns.
27pub const CMP_ROW_VERSION: u32 = 1;
28
29/// The version of the router feature layout a row's `base` columns follow.
30///
31/// The router crate does not version its layout, so this crate names the one
32/// it was written against: 152 used slots. The assertion below fails the build
33/// when that count moves, which is the moment this number has to move too.
34pub const BASE_FEATURE_VERSION: u16 = 1;
35
36const _: () = assert!(FEATURES_USED == 152);
37
38/// Which arm of a pair a row describes.
39#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
40pub enum Arm {
41    /// The request as it was.
42    Baseline,
43    /// The request with the candidate edit applied.
44    Candidate,
45    /// A copy scored but not acted on.
46    Shadow,
47}
48
49/// How much the caller remembered about the site.
50#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
51pub enum MemoryState {
52    /// Nothing recorded.
53    Cold,
54    /// One or two attempts.
55    Thin,
56    /// Three to nine attempts.
57    Warm,
58    /// Ten or more.
59    Steady,
60}
61
62impl MemoryState {
63    /// The state a memory is in.
64    pub const fn of(memory: Option<&SiteMemory>) -> MemoryState {
65        match memory {
66            None => MemoryState::Cold,
67            Some(memory) => match memory.observations {
68                0 => MemoryState::Cold,
69                1..=2 => MemoryState::Thin,
70                3..=9 => MemoryState::Warm,
71                _ => MemoryState::Steady,
72            },
73        }
74    }
75}
76
77/// An identifier, as buckets.
78#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
79pub struct IdentDescriptor {
80    /// The extension class slot. See [`crate::Identifier::class`].
81    pub class: u8,
82    /// See [`crate::Identifier::share_bucket`].
83    pub share: u8,
84    /// See [`crate::Identifier::count_bucket`].
85    pub count: u8,
86    /// See [`crate::Identifier::third_party`].
87    pub third: bool,
88}
89
90/// An edit, as numbers.
91#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
92pub struct EditDescriptor {
93    /// [`Key::index`].
94    pub key: u8,
95    /// [`crate::Op::code`].
96    pub op: u8,
97    /// The value's index within its kind, or 255 when the op has none.
98    pub bucket: u8,
99    /// The identifier an append names, when it names an observed one.
100    pub ident: Option<IdentDescriptor>,
101}
102
103impl EditDescriptor {
104    /// Describe an edit set, or `None` for keep.
105    ///
106    /// A row holds one descriptor. For a pair that is the blacklist append
107    /// when there is one, since the identifier is what the rest of the row
108    /// cannot recover, and otherwise the first edit in key order. The whole
109    /// set is in the row's edit features either way.
110    pub fn describe(edits: &EditSet, observation: &Observation) -> Option<EditDescriptor> {
111        let edit = edits
112            .get(Key::NetworkBlacklist)
113            .or_else(|| edits.edits().first())?;
114        let (bucket, ident) = match value_bucket(edit, observation) {
115            Some((bucket, ident)) => (bucket.min(254) as u8, ident),
116            None => (u8::MAX, None),
117        };
118        Some(EditDescriptor {
119            key: edit.key.index() as u8,
120            op: edit.op.code(),
121            bucket,
122            ident: ident.map(|ident| IdentDescriptor {
123                class: crate::features::ext_index(ident.class) as u8,
124                share: ident.share_bucket,
125                count: ident.count_bucket,
126                third: ident.third_party,
127            }),
128        })
129    }
130}
131
132/// Which keys the caller set, as two bitmasks over [`Key::index`]: the first
133/// 32 keys, then the rest.
134///
135/// The second mask is 64 bits because version one has 80 keys.
136pub fn pinned_mask(caller: &dyn Params) -> (u32, u64) {
137    let mut low = 0u32;
138    let mut high = 0u64;
139    for key in Key::ALL {
140        if !caller.is_set(*key) {
141            continue;
142        }
143        let at = key.index();
144        if at < 32 {
145            low |= 1 << at;
146        } else if at < 96 {
147            high |= 1 << (at - 32);
148        }
149    }
150    (low, high)
151}
152
153/// One arm of one paired trial.
154#[derive(Debug, Clone, Copy, PartialEq)]
155pub struct ComparisonRow<'a> {
156    /// Which pair this arm belongs to.
157    pub pair: u64,
158    /// Which arm.
159    pub arm: Arm,
160    /// Whether a tripped monitor shadowed a pick that would otherwise have
161    /// been applied. See [`crate::monitor`].
162    pub fallback: bool,
163    /// The day the trial ran, as a count the caller chooses.
164    pub day: u32,
165    /// The caller's opaque key for the site, for splitting train and test by
166    /// site. Never a hash this crate computes from a name.
167    pub domain_key: u64,
168    /// What the caller wanted back.
169    pub need: DeclaredNeed,
170    /// What kind of file the address names.
171    pub ext: ExtClass,
172    /// The label group slot of the host, as the router reads it.
173    pub tld: u8,
174    /// How much the caller remembered.
175    pub memory: MemoryState,
176    /// What the router decided.
177    pub routed: &'a RouteDecision,
178    /// The edit this arm carried, `None` for the baseline.
179    pub edit: Option<EditDescriptor>,
180    /// See [`pinned_mask`], the first 32 keys.
181    pub pinned: u32,
182    /// See [`pinned_mask`], keys 32 and up.
183    pub pinned_hi: u64,
184    /// Whether the arm got what the caller asked for.
185    pub success: bool,
186    /// How it ended.
187    pub status: StatusClass,
188    /// How long it took.
189    pub millis: u32,
190    /// How many bytes came back.
191    pub bytes: u32,
192    /// What it cost.
193    pub credits: f64,
194    /// How many attempts it took.
195    pub attempts: u8,
196    /// The arm's multiplier.
197    pub multiplier: f32,
198    /// How many fields the caller asked for.
199    pub fields_requested: u8,
200    /// How many came back non-empty.
201    pub fields_present: u8,
202    /// Whether the content matched the baseline closely enough.
203    pub content_ok: Option<bool>,
204    /// See [`crate::labels::fields_ok`].
205    pub fields_ok: Option<bool>,
206    /// See [`crate::labels::shingle_jaccard`].
207    pub shingle_jaccard: Option<f32>,
208    /// See [`crate::labels::byte_ratio`].
209    pub byte_ratio: Option<f32>,
210    /// The router's features for the request.
211    pub base: &'a FeatureVector,
212    /// The edit's features.
213    pub edit_feats: &'a EditFeatures,
214}
215
216/// A string built with `write!`, without a trait import.
217struct Out(String);
218
219impl Out {
220    fn write_fmt(&mut self, args: fmt::Arguments<'_>) {
221        // Writing into a `String` cannot fail.
222        let _ = fmt::Write::write_fmt(&mut self.0, args);
223    }
224
225    fn raw(&mut self, text: &str) {
226        self.0.push_str(text);
227    }
228
229    fn float(&mut self, value: f32) {
230        if value.is_finite() {
231            write!(self, "{value}");
232        } else {
233            self.raw("0");
234        }
235    }
236
237    fn double(&mut self, value: f64) {
238        if value.is_finite() {
239            write!(self, "{value}");
240        } else {
241            self.raw("0");
242        }
243    }
244
245    fn flag(&mut self, value: bool) {
246        self.raw(if value { "true" } else { "false" });
247    }
248
249    fn label(&mut self, text: &str) {
250        self.raw("\"");
251        self.raw(text);
252        self.raw("\"");
253    }
254}
255
256/// One row, as a single line of JSON with no trailing newline.
257pub fn comparison_row(r: &ComparisonRow<'_>) -> String {
258    let base = r.base.as_slice();
259    let base = base.get(..FEATURES_USED).unwrap_or(base);
260    let mut out = Out(String::with_capacity(4_096));
261
262    write!(
263        out,
264        "{{\"v\":{CMP_ROW_VERSION},\"schema_v\":{SCHEMA_VERSION},\"feat_v\":{BASE_FEATURE_VERSION},\"edit_feat_v\":{EDIT_FEATURE_VERSION},\"pair\":{},\"arm\":",
265        r.pair
266    );
267    out.label(arm_label(r.arm));
268    out.raw(",\"fallback\":");
269    out.flag(r.fallback);
270    write!(out, ",\"day\":{},\"dk\":{},\"need\":", r.day, r.domain_key);
271    out.label(need_label(r.need));
272    out.raw(",\"ext\":");
273    out.label(ext_label(r.ext));
274    write!(out, ",\"tld\":{},\"mem\":", r.tld);
275    out.label(memory_label(r.memory));
276
277    out.raw(",\"routed\":{\"mode\":");
278    out.label(r.routed.mode().as_str());
279    out.raw(",\"proxy\":");
280    out.label(r.routed.proxy().as_str());
281    write!(
282        out,
283        ",\"wait_ms\":{},\"start_rung\":{},\"source\":",
284        r.routed.wait().millis(),
285        r.routed.start_rung
286    );
287    out.label(source_label(r.routed.source));
288    out.raw(",\"confidence\":");
289    out.float(r.routed.confidence);
290    out.raw("}");
291
292    out.raw(",\"edit\":");
293    match r.edit {
294        None => out.raw("null"),
295        Some(edit) => {
296            write!(
297                out,
298                "{{\"key\":{},\"op\":{},\"bucket\":{},\"ident\":",
299                edit.key, edit.op, edit.bucket
300            );
301            match edit.ident {
302                None => out.raw("null"),
303                Some(ident) => {
304                    write!(
305                        out,
306                        "{{\"class\":{},\"share\":{},\"count\":{},\"third\":",
307                        ident.class, ident.share, ident.count
308                    );
309                    out.flag(ident.third);
310                    out.raw("}");
311                }
312            }
313            out.raw("}");
314        }
315    }
316
317    write!(
318        out,
319        ",\"pinned\":{},\"pinned_hi\":{},\"success\":",
320        r.pinned, r.pinned_hi
321    );
322    out.flag(r.success);
323    out.raw(",\"status\":");
324    out.label(status_label(r.status));
325    write!(
326        out,
327        ",\"millis\":{},\"bytes\":{},\"credits\":",
328        r.millis, r.bytes
329    );
330    out.double(r.credits);
331    write!(out, ",\"attempts\":{},\"multiplier\":", r.attempts);
332    out.float(r.multiplier);
333    write!(
334        out,
335        ",\"fields_requested\":{},\"fields_present\":{},\"content_ok\":",
336        r.fields_requested, r.fields_present
337    );
338    optional_flag(&mut out, r.content_ok);
339    out.raw(",\"fields_ok\":");
340    optional_flag(&mut out, r.fields_ok);
341    out.raw(",\"shingle_jaccard\":");
342    optional_float(&mut out, r.shingle_jaccard);
343    out.raw(",\"byte_ratio\":");
344    optional_float(&mut out, r.byte_ratio);
345
346    out.raw(",\"base\":[");
347    array(&mut out, base);
348    out.raw("],\"edit_feats\":[");
349    array(&mut out, r.edit_feats.as_slice());
350    out.raw("]}");
351
352    out.0
353}
354
355fn optional_flag(out: &mut Out, value: Option<bool>) {
356    match value {
357        Some(value) => out.flag(value),
358        None => out.raw("null"),
359    }
360}
361
362fn optional_float(out: &mut Out, value: Option<f32>) {
363    match value {
364        Some(value) => out.float(value),
365        None => out.raw("null"),
366    }
367}
368
369fn array(out: &mut Out, values: &[f32]) {
370    for (at, value) in values.iter().enumerate() {
371        if at > 0 {
372            out.raw(",");
373        }
374        out.float(*value);
375    }
376}
377
378/// The name an arm is written under.
379pub(crate) const fn arm_label(arm: Arm) -> &'static str {
380    match arm {
381        Arm::Baseline => "baseline",
382        Arm::Candidate => "candidate",
383        Arm::Shadow => "shadow",
384    }
385}
386
387/// The name a memory state is written under.
388pub(crate) const fn memory_label(memory: MemoryState) -> &'static str {
389    match memory {
390        MemoryState::Cold => "cold",
391        MemoryState::Thin => "thin",
392        MemoryState::Warm => "warm",
393        MemoryState::Steady => "steady",
394    }
395}
396
397/// Every need, in the router's slot order.
398#[cfg(test)]
399pub(crate) const ALL_NEEDS: [DeclaredNeed; 8] = [
400    DeclaredNeed::Text,
401    DeclaredNeed::Markdown,
402    DeclaredNeed::Html,
403    DeclaredNeed::Links,
404    DeclaredNeed::Metadata,
405    DeclaredNeed::Fields,
406    DeclaredNeed::Screenshot,
407    DeclaredNeed::Raw,
408];
409
410/// The name a need is written under.
411pub(crate) const fn need_label(need: DeclaredNeed) -> &'static str {
412    match need {
413        DeclaredNeed::Text => "text",
414        DeclaredNeed::Markdown => "markdown",
415        DeclaredNeed::Html => "html",
416        DeclaredNeed::Links => "links",
417        DeclaredNeed::Metadata => "metadata",
418        DeclaredNeed::Fields => "fields",
419        DeclaredNeed::Screenshot => "screenshot",
420        DeclaredNeed::Raw => "raw",
421        _ => "other",
422    }
423}
424
425/// Every status class, in the router's slot order.
426#[cfg(test)]
427pub(crate) const ALL_STATUSES: [StatusClass; 9] = [
428    StatusClass::Unknown,
429    StatusClass::Ok,
430    StatusClass::Empty,
431    StatusClass::BadRequest,
432    StatusClass::NeedsLogin,
433    StatusClass::Blocked,
434    StatusClass::NotFound,
435    StatusClass::RateLimited,
436    StatusClass::ServerError,
437];
438
439/// The name a status class is written under, as the client's route rows
440/// write it.
441pub(crate) const fn status_label(status: StatusClass) -> &'static str {
442    match status {
443        StatusClass::Unknown => "unknown",
444        StatusClass::Ok => "ok",
445        StatusClass::Empty => "empty",
446        StatusClass::BadRequest => "bad_request",
447        StatusClass::NeedsLogin => "needs_login",
448        StatusClass::Blocked => "blocked",
449        StatusClass::NotFound => "not_found",
450        StatusClass::RateLimited => "rate_limited",
451        StatusClass::ServerError => "server_error",
452        _ => "other",
453    }
454}
455
456/// The name an extension class is written under.
457pub(crate) const fn ext_label(ext: ExtClass) -> &'static str {
458    match ext {
459        ExtClass::None => "none",
460        ExtClass::Markup => "markup",
461        ExtClass::Xml => "xml",
462        ExtClass::Json => "json",
463        ExtClass::Feed => "feed",
464        ExtClass::Text => "text",
465        ExtClass::Csv => "csv",
466        ExtClass::Pdf => "pdf",
467        ExtClass::Office => "office",
468        ExtClass::Image => "image",
469        ExtClass::Media => "media",
470        ExtClass::Archive => "archive",
471        ExtClass::Asset => "asset",
472        _ => "other",
473    }
474}
475
476/// The name a decision's source is written under, as the client's route rows
477/// write it.
478pub(crate) const fn source_label(source: RouteSource) -> &'static str {
479    match source {
480        RouteSource::Heuristic => "heuristic",
481        RouteSource::Model => "model",
482        RouteSource::Caller => "caller",
483        RouteSource::Memory => "memory",
484        RouteSource::Explore => "explore",
485        _ => "other",
486    }
487}
488
489#[cfg(test)]
490mod tests {
491    #![allow(
492        clippy::unwrap_used,
493        clippy::expect_used,
494        clippy::panic,
495        clippy::unreachable,
496        clippy::indexing_slicing,
497        clippy::string_slice
498    )]
499    use super::*;
500    use crate::candidates::{generate, Candidate};
501    use crate::features::featurize_edit;
502    use crate::schema::Schema;
503    use crate::testing::Fixture;
504    use serde::de::{Deserializer, IgnoredAny, MapAccess, Visitor};
505    use spider_route::{featurize, RouteInput};
506
507    /// The top level keys of a JSON object, in the order they were written.
508    struct KeyOrder(Vec<String>);
509
510    impl<'de> serde::Deserialize<'de> for KeyOrder {
511        fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<KeyOrder, D::Error> {
512            struct Keys;
513            impl<'de> Visitor<'de> for Keys {
514                type Value = KeyOrder;
515                fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
516                    f.write_str("an object")
517                }
518                fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<KeyOrder, A::Error> {
519                    let mut keys = Vec::new();
520                    while let Some(key) = map.next_key::<String>()? {
521                        let _: IgnoredAny = map.next_value()?;
522                        keys.push(key);
523                    }
524                    Ok(KeyOrder(keys))
525                }
526            }
527            deserializer.deserialize_map(Keys)
528        }
529    }
530
531    /// A row for the observed fixture's first append, with every option set.
532    fn sample(fixture: &Fixture) -> String {
533        let ctx = fixture.ctx();
534        let cand = generate(&Schema::v1(), &ctx)
535            .into_iter()
536            .find(|c| c.edits.get(Key::NetworkBlacklist).is_some())
537            .unwrap_or_else(|| Candidate::keep(&ctx));
538        let feats = featurize_edit(&ctx, &cand);
539        let base = featurize(&RouteInput::new(ctx.url, ctx.need));
540        let (pinned, pinned_hi) = pinned_mask(ctx.caller);
541        comparison_row(&ComparisonRow {
542            pair: u64::MAX,
543            arm: Arm::Candidate,
544            fallback: false,
545            day: 20_000,
546            domain_key: 42,
547            need: ctx.need,
548            ext: ExtClass::Markup,
549            tld: 0,
550            memory: MemoryState::of(ctx.observation.memory.as_ref()),
551            routed: ctx.routed,
552            edit: EditDescriptor::describe(&cand.edits, ctx.observation),
553            pinned,
554            pinned_hi,
555            success: true,
556            status: StatusClass::Ok,
557            millis: 1_234,
558            bytes: 56_789,
559            credits: f64::NAN,
560            attempts: 1,
561            multiplier: cand.multiplier,
562            fields_requested: 2,
563            fields_present: 2,
564            content_ok: Some(true),
565            fields_ok: None,
566            shingle_jaccard: Some(0.97),
567            byte_ratio: Some(f32::INFINITY),
568            base: &base,
569            edit_feats: &feats,
570        })
571    }
572
573    #[test]
574    fn rows_hold_no_url_host_or_body() {
575        let row = sample(&Fixture::observed());
576
577        let edit = format!(
578            "\"edit\":{{\"key\":{},\"op\":1",
579            Key::NetworkBlacklist.index()
580        );
581        assert!(row.contains(&edit), "{row}");
582        for needle in [
583            "http",
584            "example",
585            "articles",
586            "tracker-alpha",
587            "cdn-beta",
588            ".com",
589            "\n",
590        ] {
591            assert!(!row.contains(needle), "{needle:?} reached the row: {row}");
592        }
593    }
594
595    #[test]
596    fn a_row_parses_as_json_with_the_documented_keys() {
597        let mut fixture = Fixture::observed();
598        fixture.caller.stealth = Some(true);
599        fixture.caller.disable_hints = Some(true);
600        let row = sample(&fixture);
601
602        let order: KeyOrder = serde_json::from_str(&row).unwrap();
603        assert_eq!(
604            order.0,
605            [
606                "v",
607                "schema_v",
608                "feat_v",
609                "edit_feat_v",
610                "pair",
611                "arm",
612                "fallback",
613                "day",
614                "dk",
615                "need",
616                "ext",
617                "tld",
618                "mem",
619                "routed",
620                "edit",
621                "pinned",
622                "pinned_hi",
623                "success",
624                "status",
625                "millis",
626                "bytes",
627                "credits",
628                "attempts",
629                "multiplier",
630                "fields_requested",
631                "fields_present",
632                "content_ok",
633                "fields_ok",
634                "shingle_jaccard",
635                "byte_ratio",
636                "base",
637                "edit_feats",
638            ]
639        );
640
641        let value: serde_json::Value = serde_json::from_str(&row).unwrap();
642        // A parsed value sorts its keys, so the nested order is read off the
643        // text. The routed object holds no nested object.
644        let from = row.find("\"routed\":").unwrap() + "\"routed\":".len();
645        let to = from + row[from..].find('}').unwrap() + 1;
646        let routed: KeyOrder = serde_json::from_str(&row[from..to]).unwrap();
647        assert_eq!(
648            routed.0,
649            [
650                "mode",
651                "proxy",
652                "wait_ms",
653                "start_rung",
654                "source",
655                "confidence"
656            ]
657        );
658        assert_eq!(value["pair"].as_u64(), Some(u64::MAX));
659        assert_eq!(value["base"].as_array().unwrap().len(), FEATURES_USED);
660        assert_eq!(value["edit_feats"].as_array().unwrap().len(), 96);
661        assert_eq!(value["credits"], 0, "a NaN is written as zero");
662        assert_eq!(value["byte_ratio"], 0, "an infinity is written as zero");
663        assert!(value["fields_ok"].is_null());
664        assert_eq!(value["pinned"], 1 << Key::Stealth.index());
665        assert_eq!(
666            value["pinned_hi"].as_u64(),
667            Some(1 << (Key::DisableHints.index() - 32))
668        );
669        assert_eq!(value["edit"]["ident"]["third"], true);
670        assert_eq!(value["status"], "ok");
671    }
672}