1use crate::candidates::{
15 current_mode, current_proxy, current_wait, effective_flag, Candidate, Context, PAIRS,
16};
17use crate::edit::{Edit, Op, Value};
18use crate::observe::{Identifier, Observation};
19use crate::schema::{learnable_slot, Key, MODES, PROXIES, WAIT_BUCKETS};
20use spider_route::{DeclaredNeed, ExtClass, FeatureVector, RouteSource};
21
22pub const EDIT_FEATURE_VERSION: u16 = 1;
24
25pub const EDIT_DIM: usize = 96;
27
28#[derive(Copy, Clone, Debug, PartialEq)]
33pub struct EditFeatures {
34 bits: [f32; EDIT_DIM],
35}
36
37impl Default for EditFeatures {
38 fn default() -> EditFeatures {
39 EditFeatures::zeroed()
40 }
41}
42
43impl EditFeatures {
44 pub const fn zeroed() -> EditFeatures {
46 EditFeatures {
47 bits: [0.0; EDIT_DIM],
48 }
49 }
50
51 pub fn as_slice(&self) -> &[f32] {
53 &self.bits
54 }
55
56 pub fn get(&self, slot: usize) -> Option<f32> {
58 self.bits.get(slot).copied()
59 }
60
61 fn set(&mut self, slot: usize, value: f32) {
64 if let Some(cell) = self.bits.get_mut(slot) {
65 *cell = value;
66 }
67 }
68
69 fn one_hot(&mut self, base: usize, offset: usize) {
71 self.set(base + offset, 1.0);
72 }
73}
74
75#[derive(Copy, Clone, Debug, PartialEq, Default)]
78pub struct Input {
79 pub base: FeatureVector,
81 pub edit: EditFeatures,
83}
84
85pub const CONFIG: usize = 0;
93pub(crate) const CONFIG_W: usize = 24;
94
95pub const EDIT_KEY: usize = CONFIG + CONFIG_W;
98pub(crate) const EDIT_KEY_W: usize = 16;
99
100pub const EDIT_OP: usize = EDIT_KEY + EDIT_KEY_W;
103pub(crate) const EDIT_OP_W: usize = 5;
104
105pub const VALUE_BUCKET: usize = EDIT_OP + EDIT_OP_W;
108pub(crate) const VALUE_BUCKET_W: usize = 8;
109
110pub const PAIR: usize = VALUE_BUCKET + VALUE_BUCKET_W;
113pub(crate) const PAIR_W: usize = 8;
114
115pub const NEED_BITS: usize = PAIR + PAIR_W;
117pub(crate) const NEED_BITS_W: usize = 4;
118
119pub const OBS_HOSTS: usize = NEED_BITS + NEED_BITS_W;
121pub(crate) const OBS_HOSTS_W: usize = 5;
122
123pub const OBS_SHARE: usize = OBS_HOSTS + OBS_HOSTS_W;
126pub(crate) const OBS_SHARE_W: usize = 5;
127
128pub const OBS_CLASS: usize = OBS_SHARE + OBS_SHARE_W;
130pub(crate) const OBS_CLASS_W: usize = 14;
131
132pub const OBS_PARTY: usize = OBS_CLASS + OBS_CLASS_W;
134pub(crate) const OBS_PARTY_W: usize = 2;
135
136pub const MISSING: usize = OBS_PARTY + OBS_PARTY_W;
139pub(crate) const MISSING_W: usize = 4;
140
141pub const BIAS: usize = MISSING + MISSING_W;
143pub(crate) const BIAS_W: usize = 1;
144
145const _: () = assert!(BIAS + BIAS_W == EDIT_DIM);
146
147pub const BLOCKS: [(&str, usize, usize); 12] = [
149 ("config", CONFIG, CONFIG_W),
150 ("edit_key", EDIT_KEY, EDIT_KEY_W),
151 ("edit_op", EDIT_OP, EDIT_OP_W),
152 ("value_bucket", VALUE_BUCKET, VALUE_BUCKET_W),
153 ("pair", PAIR, PAIR_W),
154 ("need_bits", NEED_BITS, NEED_BITS_W),
155 ("obs_hosts", OBS_HOSTS, OBS_HOSTS_W),
156 ("obs_share", OBS_SHARE, OBS_SHARE_W),
157 ("obs_class", OBS_CLASS, OBS_CLASS_W),
158 ("obs_party", OBS_PARTY, OBS_PARTY_W),
159 ("missing", MISSING, MISSING_W),
160 ("bias", BIAS, BIAS_W),
161];
162
163const SWITCHES: [Key; 5] = [
165 Key::DisableIntercept,
166 Key::FullResources,
167 Key::BlockAds,
168 Key::BlockAnalytics,
169 Key::BlockStylesheets,
170];
171
172pub fn featurize_edit(ctx: &Context<'_>, cand: &Candidate) -> EditFeatures {
177 let mut out = EditFeatures::zeroed();
178 let current = ctx.current;
179
180 out.one_hot(CONFIG, mode_index(current_mode(current)));
182 out.one_hot(CONFIG + 3, proxy_index(current_proxy(current)));
183 out.one_hot(CONFIG + 5, wait_index(current_wait(current)));
184 for (at, key) in SWITCHES.iter().enumerate() {
185 out.one_hot(
186 CONFIG + 9 + at * 2,
187 usize::from(effective_flag(current, *key)),
188 );
189 }
190 if current
191 .list(Key::NetworkBlacklist)
192 .is_some_and(|list| !list.is_empty())
193 {
194 out.set(CONFIG + 19, 1.0);
195 }
196
197 let resources = ctx.observation.resources.as_ref();
199 let mut identifier = None;
200 for edit in cand.edits.edits() {
201 if let Some(slot) = learnable_slot(edit.key) {
202 out.one_hot(EDIT_KEY, slot);
203 }
204 out.one_hot(EDIT_OP, usize::from(edit.op.code()));
205
206 if let Some((bucket, ident)) = value_bucket(edit, ctx.observation) {
207 out.one_hot(VALUE_BUCKET, bucket.min(VALUE_BUCKET_W - 1));
208 if ident.is_some() {
209 identifier = ident;
210 }
211 }
212 }
213
214 if !cand.edits.is_keep() {
215 out.one_hot(PAIR, pair_index(cand));
216 }
217
218 match ctx.need {
220 DeclaredNeed::Fields => out.one_hot(NEED_BITS, 1),
221 DeclaredNeed::Links | DeclaredNeed::Metadata => out.one_hot(NEED_BITS, 2),
222 DeclaredNeed::Screenshot => out.one_hot(NEED_BITS, 3),
223 _ => out.one_hot(NEED_BITS, 0),
224 }
225
226 if let Some(resources) = resources {
228 out.one_hot(OBS_HOSTS, hosts_bucket(resources.third_party_hosts));
229 }
230 if let Some(ident) = identifier {
231 out.one_hot(
232 OBS_SHARE,
233 usize::from(ident.share_bucket).min(OBS_SHARE_W - 1),
234 );
235 out.one_hot(OBS_CLASS, ext_index(ident.class));
236 out.one_hot(OBS_PARTY, usize::from(ident.third_party));
237 }
238
239 if resources.is_none() {
241 out.one_hot(MISSING, 0);
242 }
243 if identifier.is_none() {
244 out.one_hot(MISSING, 1);
245 }
246 if ctx
247 .observation
248 .memory
249 .is_none_or(|memory| memory.observations == 0)
250 {
251 out.one_hot(MISSING, 2);
252 }
253 if ctx.routed.source == RouteSource::Caller || ctx.current.request().is_none() {
254 out.one_hot(MISSING, 3);
255 }
256
257 out.set(BIAS, 1.0);
258 out
259}
260
261pub(crate) fn value_bucket<'o>(
265 edit: &Edit,
266 observation: &'o Observation,
267) -> Option<(usize, Option<&'o Identifier>)> {
268 match &edit.op {
269 Op::Set(Value::Mode(mode)) => Some((mode_index(*mode), None)),
270 Op::Set(Value::Proxy(pool)) => Some((proxy_index(*pool), None)),
271 Op::Set(Value::Millis(millis)) => Some((wait_index(*millis), None)),
272 Op::Set(Value::Bool(value)) => Some((usize::from(*value), None)),
273 Op::Set(Value::Int(_)) => None,
274 Op::Append(entries) => {
275 let (rank, ident) = observation.identifier(entries.first()?)?;
276 Some((rank, Some(ident)))
277 }
278 Op::Remove(_) | Op::Replace(_) | Op::Clear => None,
279 }
280}
281
282fn mode_index(mode: spider_route::RequestMode) -> usize {
284 MODES
285 .iter()
286 .position(|m| *m == mode)
287 .unwrap_or(MODES.len() - 1)
288}
289
290fn proxy_index(pool: spider_route::ProxyPool) -> usize {
292 PROXIES
293 .iter()
294 .position(|p| *p == pool)
295 .unwrap_or(PROXIES.len() - 1)
296}
297
298fn wait_index(millis: u32) -> usize {
300 WAIT_BUCKETS
301 .iter()
302 .rposition(|bucket| *bucket <= millis)
303 .unwrap_or(0)
304}
305
306fn pair_index(cand: &Candidate) -> usize {
308 let edits = cand.edits.edits();
309 if let [a, b] = edits {
310 for (at, (first, second)) in PAIRS.iter().enumerate() {
311 if (a.key == *first && b.key == *second) || (a.key == *second && b.key == *first) {
312 return at;
313 }
314 }
315 }
316 PAIRS.len()
317}
318
319const fn hosts_bucket(hosts: u16) -> usize {
321 match hosts {
322 0 => 0,
323 1..=2 => 1,
324 3..=5 => 2,
325 6..=10 => 3,
326 _ => 4,
327 }
328}
329
330pub(crate) const fn ext_index(class: ExtClass) -> usize {
333 match class {
334 ExtClass::None => 0,
335 ExtClass::Markup => 1,
336 ExtClass::Xml => 2,
337 ExtClass::Json => 3,
338 ExtClass::Feed => 4,
339 ExtClass::Text => 5,
340 ExtClass::Csv => 6,
341 ExtClass::Pdf => 7,
342 ExtClass::Office => 8,
343 ExtClass::Image => 9,
344 ExtClass::Media => 10,
345 ExtClass::Archive => 11,
346 ExtClass::Asset => 12,
347 _ => 13,
348 }
349}
350
351#[cfg(test)]
352mod tests {
353 #![allow(
354 clippy::unwrap_used,
355 clippy::expect_used,
356 clippy::panic,
357 clippy::unreachable,
358 clippy::indexing_slicing,
359 clippy::string_slice
360 )]
361 use super::*;
362 use crate::candidates::{generate, MULTIPLIERS};
363 use crate::observe::{Identifier, ResourceSummary};
364 use crate::row::{need_label, status_label, ALL_NEEDS, ALL_STATUSES};
365 use crate::schema::{Schema, SCHEMA_VERSION};
366 use crate::testing::Fixture;
367 use spider_route::{SiteMemory, StatusClass};
368
369 #[test]
370 fn edit_features_hold_no_text() {
371 assert_eq!(
372 std::mem::size_of::<EditFeatures>(),
373 EDIT_DIM * std::mem::size_of::<f32>()
374 );
375 }
376
377 #[test]
378 fn blocks_are_contiguous_and_count_is_12() {
379 let mut next = 0;
380 for (name, base, width) in BLOCKS {
381 assert_eq!(
382 base, next,
383 "{name} does not start where the last block ended"
384 );
385 next = base + width;
386 }
387 assert_eq!(next, EDIT_DIM);
388 assert_eq!(BLOCKS.len(), 12);
389 let widths: Vec<usize> = BLOCKS.iter().map(|(_, _, w)| *w).collect();
390 assert_eq!(widths, [24, 16, 5, 8, 8, 4, 5, 5, 14, 2, 4, 1]);
391 assert_eq!(ext_index(ExtClass::Other) + 1, OBS_CLASS_W);
393 assert_eq!(PAIRS.len() + 1, 5);
394 assert_eq!(9 + 10 + 1 + 4, CONFIG_W);
395 }
396
397 fn assert_unit(feats: &EditFeatures, what: &str) {
398 for (slot, value) in feats.as_slice().iter().enumerate() {
399 assert!(
400 value.is_finite() && (-1.0..=1.0).contains(value),
401 "slot {slot} is {value} for {what}"
402 );
403 }
404 }
405
406 #[test]
407 fn all_slots_finite_within_unit_range() {
408 let schema = Schema::v1();
409 let mut fixtures = vec![Fixture::cold(), Fixture::observed(), Fixture::pinned()];
410
411 let mut hostile = Fixture::observed();
412 hostile.observation.resources = Some(ResourceSummary {
413 requests: u16::MAX,
414 third_party_hosts: u16::MAX,
415 third_party_bytes: u64::MAX,
416 first_party_bytes: u64::MAX,
417 candidates: vec![Identifier {
418 pattern: "tracker-alpha.example".into(),
419 class: ExtClass::Other,
420 share_bucket: u8::MAX,
421 count_bucket: u8::MAX,
422 third_party: true,
423 }],
424 });
425 hostile.observation.memory = Some(SiteMemory {
426 observations: u32::MAX,
427 success_rate: f32::NAN,
428 streak: i16::MIN,
429 last_status: StatusClass::ServerError,
430 });
431 hostile.current.wait_for = Some(spider_cloud_agent::params::WaitFor::idle_network(
432 spider_cloud_agent::params::Timeout::from_secs(u64::MAX / 1_000),
433 ));
434 hostile.current.network_blacklist = Some(Vec::new());
435 hostile.cap = f32::INFINITY;
436 fixtures.push(hostile);
437
438 let mut empty = Fixture::observed();
439 empty.observation.resources = Some(ResourceSummary::default());
440 empty.current = Default::default();
441 fixtures.push(empty);
442
443 let mut checked = 0;
444 for fixture in &fixtures {
445 let ctx = fixture.ctx();
446 for cand in generate(&schema, &ctx) {
447 assert_unit(&featurize_edit(&ctx, &cand), &format!("{:?}", cand.edits));
448 checked += 1;
449 }
450 }
451 assert!(checked > 40, "only {checked} candidates checked");
452 }
453
454 #[test]
455 fn keep_has_no_edit_bits() {
456 let fixture = Fixture::observed();
457 let ctx = fixture.ctx();
458 let keep = featurize_edit(&ctx, &Candidate::keep(&ctx));
459
460 for slot in EDIT_KEY..NEED_BITS {
461 assert_eq!(keep.get(slot), Some(0.0), "slot {slot} is set for keep");
462 }
463 for slot in OBS_SHARE..MISSING {
464 assert_eq!(keep.get(slot), Some(0.0), "slot {slot} is set for keep");
465 }
466
467 for cand in generate(&Schema::v1(), &ctx).iter().skip(1) {
469 let feats = featurize_edit(&ctx, cand);
470 let keys: f32 = (EDIT_KEY..EDIT_OP).map(|s| feats.get(s).unwrap()).sum();
471 let ops: f32 = (EDIT_OP..VALUE_BUCKET).map(|s| feats.get(s).unwrap()).sum();
472 assert!(keys >= 1.0 && ops >= 1.0, "{:?}", cand.edits);
473 }
474 }
475
476 #[test]
477 fn missing_observation_is_a_value_not_a_gap() {
478 let cold = Fixture::cold();
479 let ctx = cold.ctx();
480 let feats = featurize_edit(&ctx, &Candidate::keep(&ctx));
481 assert_eq!(feats.get(MISSING), Some(1.0));
482 assert_eq!(feats.get(MISSING + 1), Some(1.0));
483 assert_eq!(feats.get(MISSING + 2), Some(1.0));
484 assert_eq!(feats.get(MISSING + 3), Some(0.0));
485 assert_eq!(feats.get(BIAS), Some(1.0));
486 assert_eq!(
488 feats.get(CONFIG + 9 + 2 * 2 + 1),
489 Some(1.0),
490 "block ads is on"
491 );
492 assert_eq!(
493 feats.get(CONFIG + 9),
494 Some(1.0),
495 "interception is not disabled"
496 );
497
498 let mut observed = Fixture::observed();
499 observed.observation.memory = Some(SiteMemory {
500 observations: 4,
501 ..SiteMemory::cold()
502 });
503 let ctx = observed.ctx();
504 let append = generate(&Schema::v1(), &ctx)
505 .into_iter()
506 .find(|c| c.edits.edits().len() == 1 && c.edits.get(Key::NetworkBlacklist).is_some())
507 .unwrap();
508 let feats = featurize_edit(&ctx, &append);
509 for slot in MISSING..BIAS {
510 assert_eq!(feats.get(slot), Some(0.0), "missing slot {slot}");
511 }
512 assert_eq!(feats.get(OBS_PARTY + 1), Some(1.0));
513 assert_eq!(feats.get(OBS_HOSTS + 2), Some(1.0), "five hosts");
514
515 let mut caller_routed = Fixture::cold();
516 caller_routed.current = Default::default();
517 let ctx = caller_routed.ctx();
518 let feats = featurize_edit(&ctx, &Candidate::keep(&ctx));
519 assert_eq!(feats.get(MISSING + 3), Some(1.0));
520 }
521
522 fn mirror() -> String {
524 let schema = Schema::v1();
525 let blocks: Vec<_> = BLOCKS
526 .iter()
527 .map(|(name, base, width)| serde_json::json!({"name": name, "base": base, "width": width}))
528 .collect();
529 let keys: Vec<_> = schema
530 .specs()
531 .iter()
532 .map(|spec| {
533 serde_json::json!({
534 "name": format!("{:?}", spec.key),
535 "index": spec.key.index(),
536 "wire": spec.key.wire(),
537 "learnable": spec.learnable,
538 "content_changing": spec.content_changing,
539 })
540 })
541 .collect();
542 let ops: Vec<_> = crate::edit::Op::NAMES
543 .iter()
544 .enumerate()
545 .map(|(code, name)| serde_json::json!({"name": name, "code": code}))
546 .collect();
547 let needs: Vec<_> = ALL_NEEDS.iter().map(|need| need_label(*need)).collect();
548 let status: Vec<_> = ALL_STATUSES.iter().map(|s| status_label(*s)).collect();
549 let multipliers: Vec<_> = MULTIPLIERS
550 .iter()
551 .map(|(mode, proxy, wait, cost)| {
552 serde_json::json!({
553 "mode": mode.as_str(),
554 "proxy": proxy.as_str(),
555 "wait_ms": wait,
556 "multiplier": cost,
557 })
558 })
559 .collect();
560 let doc = serde_json::json!({
561 "schema_version": SCHEMA_VERSION,
562 "edit_feature_version": EDIT_FEATURE_VERSION,
563 "edit_dim": EDIT_DIM,
564 "blocks": blocks,
565 "keys": keys,
566 "ops": ops,
567 "needs": needs,
568 "status": status,
569 "multipliers": multipliers,
570 });
571 let mut text = serde_json::to_string_pretty(&doc).unwrap();
572 text.push('\n');
573 text
574 }
575
576 fn mirror_path() -> std::path::PathBuf {
577 std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
578 .join("..")
579 .join("training/fixtures/schema-v2.json")
580 }
581
582 #[test]
583 fn the_schema_mirror_is_committed() {
584 let expected = mirror();
585 let committed = std::fs::read_to_string(mirror_path()).unwrap_or_default();
586 assert!(
587 committed == expected,
588 "training/fixtures/schema-v2.json does not match the constants in spider-optimize. \
589 Regenerate it with `cargo test -p spider-optimize write_schema_mirror -- --ignored` \
590 and commit the result.",
591 );
592 }
593
594 #[test]
595 #[ignore = "writes training/fixtures/schema-v2.json"]
596 fn write_schema_mirror() {
597 let path = mirror_path();
598 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
599 std::fs::write(path, mirror()).unwrap();
600 }
601}