Skip to main content

spider_optimize/
features.rs

1//! What a scorer is allowed to look at about an edit.
2//!
3//! [`featurize_edit`] turns a candidate in its context into
4//! [`EditFeatures`], a fixed array of [`EDIT_DIM`] floats that sits beside the
5//! router's own [`FeatureVector`] in an [`Input`]. Every block is named by a
6//! constant here, every value is a one-hot or a bucket, and every slot is
7//! finite and within minus one to one.
8//!
9//! The same rule holds as in the router: no slot is a function of which site
10//! is being fetched or which third party it loads. An identifier contributes
11//! its class, its byte share bucket and its party, and never its pattern.
12//! Missing information is a set slot in the `MISSING` block, never a gap.
13
14use crate::candidates::{
15    current_mode, current_proxy, current_wait, effective_flag, Candidate, Context, PAIRS,
16};
17use crate::edit::{Edit, Op, Value};
18use crate::observe::{Identifier, Observation};
19use crate::schema::{learnable_slot, Key, MODES, PROXIES, WAIT_BUCKETS};
20use spider_route::{DeclaredNeed, ExtClass, FeatureVector, RouteSource};
21
22/// The version of the edit feature layout. Recorded on every row.
23pub const EDIT_FEATURE_VERSION: u16 = 1;
24
25/// How many slots an edit feature vector has.
26pub const EDIT_DIM: usize = 96;
27
28/// The edit feature vector.
29///
30/// Fixed width, `Copy`, no heap, so there is no field a host could be stored
31/// in.
32#[derive(Copy, Clone, Debug, PartialEq)]
33pub struct EditFeatures {
34    bits: [f32; EDIT_DIM],
35}
36
37impl Default for EditFeatures {
38    fn default() -> EditFeatures {
39        EditFeatures::zeroed()
40    }
41}
42
43impl EditFeatures {
44    /// Every slot zero.
45    pub const fn zeroed() -> EditFeatures {
46        EditFeatures {
47            bits: [0.0; EDIT_DIM],
48        }
49    }
50
51    /// The slots, in order.
52    pub fn as_slice(&self) -> &[f32] {
53        &self.bits
54    }
55
56    /// One slot, or `None` past the end.
57    pub fn get(&self, slot: usize) -> Option<f32> {
58        self.bits.get(slot).copied()
59    }
60
61    /// Set one slot. Out of range does nothing, because a layout bug should
62    /// not take down a caller's process.
63    fn set(&mut self, slot: usize, value: f32) {
64        if let Some(cell) = self.bits.get_mut(slot) {
65            *cell = value;
66        }
67    }
68
69    /// Set the slot at `base + offset`.
70    fn one_hot(&mut self, base: usize, offset: usize) {
71        self.set(base + offset, 1.0);
72    }
73}
74
75/// What a scorer reads: the router's features for the request and the edit's
76/// own.
77#[derive(Copy, Clone, Debug, PartialEq, Default)]
78pub struct Input {
79    /// The router's features for the request, with no caller pins.
80    pub base: FeatureVector,
81    /// This candidate's features.
82    pub edit: EditFeatures,
83}
84
85// The layout. Each base is derived from the block before it, so a block
86// cannot silently overlap its neighbour.
87
88/// The current value of each learnable key: request 3, proxy 2, idle wait
89/// bucket 4, then two slots (off, on) for each of disable intercept, full
90/// resources, block ads, block analytics and block stylesheets, then one for a
91/// non-empty blacklist, then 4 spare.
92pub const CONFIG: usize = 0;
93pub(crate) const CONFIG_W: usize = 24;
94
95/// Which learnable keys the edit set touches, in key order. Multi-hot for a
96/// pair. 9 used, 7 spare.
97pub const EDIT_KEY: usize = CONFIG + CONFIG_W;
98pub(crate) const EDIT_KEY_W: usize = 16;
99
100/// Which operations the edit set uses: set, append, remove, replace, clear.
101/// Keep sets none.
102pub const EDIT_OP: usize = EDIT_KEY + EDIT_KEY_W;
103pub(crate) const EDIT_OP_W: usize = 5;
104
105/// The index of each new value within its kind: the mode or pool index, the
106/// wait bucket index, off 0 and on 1, or an appended identifier's rank.
107pub const VALUE_BUCKET: usize = EDIT_OP + EDIT_OP_W;
108pub(crate) const VALUE_BUCKET_W: usize = 8;
109
110/// Which compatibility pair the edit set is, in [`PAIRS`] order, then a slot
111/// for not a pair. 5 used, 3 spare.
112pub const PAIR: usize = VALUE_BUCKET + VALUE_BUCKET_W;
113pub(crate) const PAIR_W: usize = 8;
114
115/// What the caller wants back: a body, fields, links or metadata, a picture.
116pub const NEED_BITS: usize = PAIR + PAIR_W;
117pub(crate) const NEED_BITS_W: usize = 4;
118
119/// Third party hosts seen: none, 1 to 2, 3 to 5, 6 to 10, 11 or more.
120pub const OBS_HOSTS: usize = NEED_BITS + NEED_BITS_W;
121pub(crate) const OBS_HOSTS_W: usize = 5;
122
123/// The appended identifier's byte share bucket. See
124/// [`crate::Identifier::share_bucket`].
125pub const OBS_SHARE: usize = OBS_HOSTS + OBS_HOSTS_W;
126pub(crate) const OBS_SHARE_W: usize = 5;
127
128/// The appended identifier's extension class, one-hot.
129pub const OBS_CLASS: usize = OBS_SHARE + OBS_SHARE_W;
130pub(crate) const OBS_CLASS_W: usize = 14;
131
132/// The appended identifier's party: first, third.
133pub const OBS_PARTY: usize = OBS_CLASS + OBS_CLASS_W;
134pub(crate) const OBS_PARTY_W: usize = 2;
135
136/// What is not known: no resource observation, no identifier on this
137/// candidate, cold memory, no routed decision on the request.
138pub const MISSING: usize = OBS_PARTY + OBS_PARTY_W;
139pub(crate) const MISSING_W: usize = 4;
140
141/// Always one.
142pub const BIAS: usize = MISSING + MISSING_W;
143pub(crate) const BIAS_W: usize = 1;
144
145const _: () = assert!(BIAS + BIAS_W == EDIT_DIM);
146
147/// Every block, as (name, base, width), in layout order.
148pub const BLOCKS: [(&str, usize, usize); 12] = [
149    ("config", CONFIG, CONFIG_W),
150    ("edit_key", EDIT_KEY, EDIT_KEY_W),
151    ("edit_op", EDIT_OP, EDIT_OP_W),
152    ("value_bucket", VALUE_BUCKET, VALUE_BUCKET_W),
153    ("pair", PAIR, PAIR_W),
154    ("need_bits", NEED_BITS, NEED_BITS_W),
155    ("obs_hosts", OBS_HOSTS, OBS_HOSTS_W),
156    ("obs_share", OBS_SHARE, OBS_SHARE_W),
157    ("obs_class", OBS_CLASS, OBS_CLASS_W),
158    ("obs_party", OBS_PARTY, OBS_PARTY_W),
159    ("missing", MISSING, MISSING_W),
160    ("bias", BIAS, BIAS_W),
161];
162
163/// The switches in the config block, in key order.
164const SWITCHES: [Key; 5] = [
165    Key::DisableIntercept,
166    Key::FullResources,
167    Key::BlockAds,
168    Key::BlockAnalytics,
169    Key::BlockStylesheets,
170];
171
172/// Turn a candidate in its context into the edit features.
173///
174/// Allocates nothing, reads no clock, and gives the same answer for the same
175/// input.
176pub fn featurize_edit(ctx: &Context<'_>, cand: &Candidate) -> EditFeatures {
177    let mut out = EditFeatures::zeroed();
178    let current = ctx.current;
179
180    // What the request carries now.
181    out.one_hot(CONFIG, mode_index(current_mode(current)));
182    out.one_hot(CONFIG + 3, proxy_index(current_proxy(current)));
183    out.one_hot(CONFIG + 5, wait_index(current_wait(current)));
184    for (at, key) in SWITCHES.iter().enumerate() {
185        out.one_hot(
186            CONFIG + 9 + at * 2,
187            usize::from(effective_flag(current, *key)),
188        );
189    }
190    if current
191        .list(Key::NetworkBlacklist)
192        .is_some_and(|list| !list.is_empty())
193    {
194        out.set(CONFIG + 19, 1.0);
195    }
196
197    // What the edit does.
198    let resources = ctx.observation.resources.as_ref();
199    let mut identifier = None;
200    for edit in cand.edits.edits() {
201        if let Some(slot) = learnable_slot(edit.key) {
202            out.one_hot(EDIT_KEY, slot);
203        }
204        out.one_hot(EDIT_OP, usize::from(edit.op.code()));
205
206        if let Some((bucket, ident)) = value_bucket(edit, ctx.observation) {
207            out.one_hot(VALUE_BUCKET, bucket.min(VALUE_BUCKET_W - 1));
208            if ident.is_some() {
209                identifier = ident;
210            }
211        }
212    }
213
214    if !cand.edits.is_keep() {
215        out.one_hot(PAIR, pair_index(cand));
216    }
217
218    // What the caller wants.
219    match ctx.need {
220        DeclaredNeed::Fields => out.one_hot(NEED_BITS, 1),
221        DeclaredNeed::Links | DeclaredNeed::Metadata => out.one_hot(NEED_BITS, 2),
222        DeclaredNeed::Screenshot => out.one_hot(NEED_BITS, 3),
223        _ => out.one_hot(NEED_BITS, 0),
224    }
225
226    // What was seen.
227    if let Some(resources) = resources {
228        out.one_hot(OBS_HOSTS, hosts_bucket(resources.third_party_hosts));
229    }
230    if let Some(ident) = identifier {
231        out.one_hot(
232            OBS_SHARE,
233            usize::from(ident.share_bucket).min(OBS_SHARE_W - 1),
234        );
235        out.one_hot(OBS_CLASS, ext_index(ident.class));
236        out.one_hot(OBS_PARTY, usize::from(ident.third_party));
237    }
238
239    // What was not.
240    if resources.is_none() {
241        out.one_hot(MISSING, 0);
242    }
243    if identifier.is_none() {
244        out.one_hot(MISSING, 1);
245    }
246    if ctx
247        .observation
248        .memory
249        .is_none_or(|memory| memory.observations == 0)
250    {
251        out.one_hot(MISSING, 2);
252    }
253    if ctx.routed.source == RouteSource::Caller || ctx.current.request().is_none() {
254        out.one_hot(MISSING, 3);
255    }
256
257    out.set(BIAS, 1.0);
258    out
259}
260
261/// The index of an edit's new value within its kind, and the observed
262/// identifier an append names. `None` for an op with no single value, and for
263/// an append whose pattern was not observed.
264pub(crate) fn value_bucket<'o>(
265    edit: &Edit,
266    observation: &'o Observation,
267) -> Option<(usize, Option<&'o Identifier>)> {
268    match &edit.op {
269        Op::Set(Value::Mode(mode)) => Some((mode_index(*mode), None)),
270        Op::Set(Value::Proxy(pool)) => Some((proxy_index(*pool), None)),
271        Op::Set(Value::Millis(millis)) => Some((wait_index(*millis), None)),
272        Op::Set(Value::Bool(value)) => Some((usize::from(*value), None)),
273        Op::Set(Value::Int(_)) => None,
274        Op::Append(entries) => {
275            let (rank, ident) = observation.identifier(entries.first()?)?;
276            Some((rank, Some(ident)))
277        }
278        Op::Remove(_) | Op::Replace(_) | Op::Clear => None,
279    }
280}
281
282/// A mode's index in the request kind. An unknown mode reads as the last.
283fn mode_index(mode: spider_route::RequestMode) -> usize {
284    MODES
285        .iter()
286        .position(|m| *m == mode)
287        .unwrap_or(MODES.len() - 1)
288}
289
290/// A pool's index in the proxy kind. An unknown pool reads as the last.
291fn proxy_index(pool: spider_route::ProxyPool) -> usize {
292    PROXIES
293        .iter()
294        .position(|p| *p == pool)
295        .unwrap_or(PROXIES.len() - 1)
296}
297
298/// The largest wait bucket at or below this many milliseconds.
299fn wait_index(millis: u32) -> usize {
300    WAIT_BUCKETS
301        .iter()
302        .rposition(|bucket| *bucket <= millis)
303        .unwrap_or(0)
304}
305
306/// Which pair an edit set is, or the slot after the pairs for none.
307fn pair_index(cand: &Candidate) -> usize {
308    let edits = cand.edits.edits();
309    if let [a, b] = edits {
310        for (at, (first, second)) in PAIRS.iter().enumerate() {
311            if (a.key == *first && b.key == *second) || (a.key == *second && b.key == *first) {
312                return at;
313            }
314        }
315    }
316    PAIRS.len()
317}
318
319/// Third party hosts, bucketed.
320const fn hosts_bucket(hosts: u16) -> usize {
321    match hosts {
322        0 => 0,
323        1..=2 => 1,
324        3..=5 => 2,
325        6..=10 => 3,
326        _ => 4,
327    }
328}
329
330/// An extension class's slot, in the router's own order. A class this version
331/// has not heard of reads as other.
332pub(crate) const fn ext_index(class: ExtClass) -> usize {
333    match class {
334        ExtClass::None => 0,
335        ExtClass::Markup => 1,
336        ExtClass::Xml => 2,
337        ExtClass::Json => 3,
338        ExtClass::Feed => 4,
339        ExtClass::Text => 5,
340        ExtClass::Csv => 6,
341        ExtClass::Pdf => 7,
342        ExtClass::Office => 8,
343        ExtClass::Image => 9,
344        ExtClass::Media => 10,
345        ExtClass::Archive => 11,
346        ExtClass::Asset => 12,
347        _ => 13,
348    }
349}
350
351#[cfg(test)]
352mod tests {
353    #![allow(
354        clippy::unwrap_used,
355        clippy::expect_used,
356        clippy::panic,
357        clippy::unreachable,
358        clippy::indexing_slicing,
359        clippy::string_slice
360    )]
361    use super::*;
362    use crate::candidates::{generate, MULTIPLIERS};
363    use crate::observe::{Identifier, ResourceSummary};
364    use crate::row::{need_label, status_label, ALL_NEEDS, ALL_STATUSES};
365    use crate::schema::{Schema, SCHEMA_VERSION};
366    use crate::testing::Fixture;
367    use spider_route::{SiteMemory, StatusClass};
368
369    #[test]
370    fn edit_features_hold_no_text() {
371        assert_eq!(
372            std::mem::size_of::<EditFeatures>(),
373            EDIT_DIM * std::mem::size_of::<f32>()
374        );
375    }
376
377    #[test]
378    fn blocks_are_contiguous_and_count_is_12() {
379        let mut next = 0;
380        for (name, base, width) in BLOCKS {
381            assert_eq!(
382                base, next,
383                "{name} does not start where the last block ended"
384            );
385            next = base + width;
386        }
387        assert_eq!(next, EDIT_DIM);
388        assert_eq!(BLOCKS.len(), 12);
389        let widths: Vec<usize> = BLOCKS.iter().map(|(_, _, w)| *w).collect();
390        assert_eq!(widths, [24, 16, 5, 8, 8, 4, 5, 5, 14, 2, 4, 1]);
391        // The vocabularies fit their blocks.
392        assert_eq!(ext_index(ExtClass::Other) + 1, OBS_CLASS_W);
393        assert_eq!(PAIRS.len() + 1, 5);
394        assert_eq!(9 + 10 + 1 + 4, CONFIG_W);
395    }
396
397    fn assert_unit(feats: &EditFeatures, what: &str) {
398        for (slot, value) in feats.as_slice().iter().enumerate() {
399            assert!(
400                value.is_finite() && (-1.0..=1.0).contains(value),
401                "slot {slot} is {value} for {what}"
402            );
403        }
404    }
405
406    #[test]
407    fn all_slots_finite_within_unit_range() {
408        let schema = Schema::v1();
409        let mut fixtures = vec![Fixture::cold(), Fixture::observed(), Fixture::pinned()];
410
411        let mut hostile = Fixture::observed();
412        hostile.observation.resources = Some(ResourceSummary {
413            requests: u16::MAX,
414            third_party_hosts: u16::MAX,
415            third_party_bytes: u64::MAX,
416            first_party_bytes: u64::MAX,
417            candidates: vec![Identifier {
418                pattern: "tracker-alpha.example".into(),
419                class: ExtClass::Other,
420                share_bucket: u8::MAX,
421                count_bucket: u8::MAX,
422                third_party: true,
423            }],
424        });
425        hostile.observation.memory = Some(SiteMemory {
426            observations: u32::MAX,
427            success_rate: f32::NAN,
428            streak: i16::MIN,
429            last_status: StatusClass::ServerError,
430        });
431        hostile.current.wait_for = Some(spider_cloud_agent::params::WaitFor::idle_network(
432            spider_cloud_agent::params::Timeout::from_secs(u64::MAX / 1_000),
433        ));
434        hostile.current.network_blacklist = Some(Vec::new());
435        hostile.cap = f32::INFINITY;
436        fixtures.push(hostile);
437
438        let mut empty = Fixture::observed();
439        empty.observation.resources = Some(ResourceSummary::default());
440        empty.current = Default::default();
441        fixtures.push(empty);
442
443        let mut checked = 0;
444        for fixture in &fixtures {
445            let ctx = fixture.ctx();
446            for cand in generate(&schema, &ctx) {
447                assert_unit(&featurize_edit(&ctx, &cand), &format!("{:?}", cand.edits));
448                checked += 1;
449            }
450        }
451        assert!(checked > 40, "only {checked} candidates checked");
452    }
453
454    #[test]
455    fn keep_has_no_edit_bits() {
456        let fixture = Fixture::observed();
457        let ctx = fixture.ctx();
458        let keep = featurize_edit(&ctx, &Candidate::keep(&ctx));
459
460        for slot in EDIT_KEY..NEED_BITS {
461            assert_eq!(keep.get(slot), Some(0.0), "slot {slot} is set for keep");
462        }
463        for slot in OBS_SHARE..MISSING {
464            assert_eq!(keep.get(slot), Some(0.0), "slot {slot} is set for keep");
465        }
466
467        // Every other candidate sets at least one key and one op.
468        for cand in generate(&Schema::v1(), &ctx).iter().skip(1) {
469            let feats = featurize_edit(&ctx, cand);
470            let keys: f32 = (EDIT_KEY..EDIT_OP).map(|s| feats.get(s).unwrap()).sum();
471            let ops: f32 = (EDIT_OP..VALUE_BUCKET).map(|s| feats.get(s).unwrap()).sum();
472            assert!(keys >= 1.0 && ops >= 1.0, "{:?}", cand.edits);
473        }
474    }
475
476    #[test]
477    fn missing_observation_is_a_value_not_a_gap() {
478        let cold = Fixture::cold();
479        let ctx = cold.ctx();
480        let feats = featurize_edit(&ctx, &Candidate::keep(&ctx));
481        assert_eq!(feats.get(MISSING), Some(1.0));
482        assert_eq!(feats.get(MISSING + 1), Some(1.0));
483        assert_eq!(feats.get(MISSING + 2), Some(1.0));
484        assert_eq!(feats.get(MISSING + 3), Some(0.0));
485        assert_eq!(feats.get(BIAS), Some(1.0));
486        // Unset switches read as the service reads them.
487        assert_eq!(
488            feats.get(CONFIG + 9 + 2 * 2 + 1),
489            Some(1.0),
490            "block ads is on"
491        );
492        assert_eq!(
493            feats.get(CONFIG + 9),
494            Some(1.0),
495            "interception is not disabled"
496        );
497
498        let mut observed = Fixture::observed();
499        observed.observation.memory = Some(SiteMemory {
500            observations: 4,
501            ..SiteMemory::cold()
502        });
503        let ctx = observed.ctx();
504        let append = generate(&Schema::v1(), &ctx)
505            .into_iter()
506            .find(|c| c.edits.edits().len() == 1 && c.edits.get(Key::NetworkBlacklist).is_some())
507            .unwrap();
508        let feats = featurize_edit(&ctx, &append);
509        for slot in MISSING..BIAS {
510            assert_eq!(feats.get(slot), Some(0.0), "missing slot {slot}");
511        }
512        assert_eq!(feats.get(OBS_PARTY + 1), Some(1.0));
513        assert_eq!(feats.get(OBS_HOSTS + 2), Some(1.0), "five hosts");
514
515        let mut caller_routed = Fixture::cold();
516        caller_routed.current = Default::default();
517        let ctx = caller_routed.ctx();
518        let feats = featurize_edit(&ctx, &Candidate::keep(&ctx));
519        assert_eq!(feats.get(MISSING + 3), Some(1.0));
520    }
521
522    /// The layout and vocabulary as a document, for the trainer.
523    fn mirror() -> String {
524        let schema = Schema::v1();
525        let blocks: Vec<_> = BLOCKS
526            .iter()
527            .map(|(name, base, width)| serde_json::json!({"name": name, "base": base, "width": width}))
528            .collect();
529        let keys: Vec<_> = schema
530            .specs()
531            .iter()
532            .map(|spec| {
533                serde_json::json!({
534                    "name": format!("{:?}", spec.key),
535                    "index": spec.key.index(),
536                    "wire": spec.key.wire(),
537                    "learnable": spec.learnable,
538                    "content_changing": spec.content_changing,
539                })
540            })
541            .collect();
542        let ops: Vec<_> = crate::edit::Op::NAMES
543            .iter()
544            .enumerate()
545            .map(|(code, name)| serde_json::json!({"name": name, "code": code}))
546            .collect();
547        let needs: Vec<_> = ALL_NEEDS.iter().map(|need| need_label(*need)).collect();
548        let status: Vec<_> = ALL_STATUSES.iter().map(|s| status_label(*s)).collect();
549        let multipliers: Vec<_> = MULTIPLIERS
550            .iter()
551            .map(|(mode, proxy, wait, cost)| {
552                serde_json::json!({
553                    "mode": mode.as_str(),
554                    "proxy": proxy.as_str(),
555                    "wait_ms": wait,
556                    "multiplier": cost,
557                })
558            })
559            .collect();
560        let doc = serde_json::json!({
561            "schema_version": SCHEMA_VERSION,
562            "edit_feature_version": EDIT_FEATURE_VERSION,
563            "edit_dim": EDIT_DIM,
564            "blocks": blocks,
565            "keys": keys,
566            "ops": ops,
567            "needs": needs,
568            "status": status,
569            "multipliers": multipliers,
570        });
571        let mut text = serde_json::to_string_pretty(&doc).unwrap();
572        text.push('\n');
573        text
574    }
575
576    fn mirror_path() -> std::path::PathBuf {
577        std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
578            .join("..")
579            .join("training/fixtures/schema-v2.json")
580    }
581
582    #[test]
583    fn the_schema_mirror_is_committed() {
584        let expected = mirror();
585        let committed = std::fs::read_to_string(mirror_path()).unwrap_or_default();
586        assert!(
587            committed == expected,
588            "training/fixtures/schema-v2.json does not match the constants in spider-optimize. \
589             Regenerate it with `cargo test -p spider-optimize write_schema_mirror -- --ignored` \
590             and commit the result.",
591        );
592    }
593
594    #[test]
595    #[ignore = "writes training/fixtures/schema-v2.json"]
596    fn write_schema_mirror() {
597        let path = mirror_path();
598        std::fs::create_dir_all(path.parent().unwrap()).unwrap();
599        std::fs::write(path, mirror()).unwrap();
600    }
601}