use crate::candidates::{
current_mode, current_proxy, current_wait, effective_flag, Candidate, Context, PAIRS,
};
use crate::edit::{Edit, Op, Value};
use crate::observe::{Identifier, Observation};
use crate::schema::{learnable_slot, Key, MODES, PROXIES, WAIT_BUCKETS};
use spider_route::{DeclaredNeed, ExtClass, FeatureVector, RouteSource};
pub const EDIT_FEATURE_VERSION: u16 = 1;
pub const EDIT_DIM: usize = 96;
#[derive(Copy, Clone, Debug, PartialEq)]
pub struct EditFeatures {
bits: [f32; EDIT_DIM],
}
impl Default for EditFeatures {
fn default() -> EditFeatures {
EditFeatures::zeroed()
}
}
impl EditFeatures {
pub const fn zeroed() -> EditFeatures {
EditFeatures {
bits: [0.0; EDIT_DIM],
}
}
pub fn as_slice(&self) -> &[f32] {
&self.bits
}
pub fn get(&self, slot: usize) -> Option<f32> {
self.bits.get(slot).copied()
}
fn set(&mut self, slot: usize, value: f32) {
if let Some(cell) = self.bits.get_mut(slot) {
*cell = value;
}
}
fn one_hot(&mut self, base: usize, offset: usize) {
self.set(base + offset, 1.0);
}
}
#[derive(Copy, Clone, Debug, PartialEq, Default)]
pub struct Input {
pub base: FeatureVector,
pub edit: EditFeatures,
}
pub const CONFIG: usize = 0;
pub(crate) const CONFIG_W: usize = 24;
pub const EDIT_KEY: usize = CONFIG + CONFIG_W;
pub(crate) const EDIT_KEY_W: usize = 16;
pub const EDIT_OP: usize = EDIT_KEY + EDIT_KEY_W;
pub(crate) const EDIT_OP_W: usize = 5;
pub const VALUE_BUCKET: usize = EDIT_OP + EDIT_OP_W;
pub(crate) const VALUE_BUCKET_W: usize = 8;
pub const PAIR: usize = VALUE_BUCKET + VALUE_BUCKET_W;
pub(crate) const PAIR_W: usize = 8;
pub const NEED_BITS: usize = PAIR + PAIR_W;
pub(crate) const NEED_BITS_W: usize = 4;
pub const OBS_HOSTS: usize = NEED_BITS + NEED_BITS_W;
pub(crate) const OBS_HOSTS_W: usize = 5;
pub const OBS_SHARE: usize = OBS_HOSTS + OBS_HOSTS_W;
pub(crate) const OBS_SHARE_W: usize = 5;
pub const OBS_CLASS: usize = OBS_SHARE + OBS_SHARE_W;
pub(crate) const OBS_CLASS_W: usize = 14;
pub const OBS_PARTY: usize = OBS_CLASS + OBS_CLASS_W;
pub(crate) const OBS_PARTY_W: usize = 2;
pub const MISSING: usize = OBS_PARTY + OBS_PARTY_W;
pub(crate) const MISSING_W: usize = 4;
pub const BIAS: usize = MISSING + MISSING_W;
pub(crate) const BIAS_W: usize = 1;
const _: () = assert!(BIAS + BIAS_W == EDIT_DIM);
pub const BLOCKS: [(&str, usize, usize); 12] = [
("config", CONFIG, CONFIG_W),
("edit_key", EDIT_KEY, EDIT_KEY_W),
("edit_op", EDIT_OP, EDIT_OP_W),
("value_bucket", VALUE_BUCKET, VALUE_BUCKET_W),
("pair", PAIR, PAIR_W),
("need_bits", NEED_BITS, NEED_BITS_W),
("obs_hosts", OBS_HOSTS, OBS_HOSTS_W),
("obs_share", OBS_SHARE, OBS_SHARE_W),
("obs_class", OBS_CLASS, OBS_CLASS_W),
("obs_party", OBS_PARTY, OBS_PARTY_W),
("missing", MISSING, MISSING_W),
("bias", BIAS, BIAS_W),
];
const SWITCHES: [Key; 5] = [
Key::DisableIntercept,
Key::FullResources,
Key::BlockAds,
Key::BlockAnalytics,
Key::BlockStylesheets,
];
pub fn featurize_edit(ctx: &Context<'_>, cand: &Candidate) -> EditFeatures {
let mut out = EditFeatures::zeroed();
let current = ctx.current;
out.one_hot(CONFIG, mode_index(current_mode(current)));
out.one_hot(CONFIG + 3, proxy_index(current_proxy(current)));
out.one_hot(CONFIG + 5, wait_index(current_wait(current)));
for (at, key) in SWITCHES.iter().enumerate() {
out.one_hot(
CONFIG + 9 + at * 2,
usize::from(effective_flag(current, *key)),
);
}
if current
.list(Key::NetworkBlacklist)
.is_some_and(|list| !list.is_empty())
{
out.set(CONFIG + 19, 1.0);
}
let resources = ctx.observation.resources.as_ref();
let mut identifier = None;
for edit in cand.edits.edits() {
if let Some(slot) = learnable_slot(edit.key) {
out.one_hot(EDIT_KEY, slot);
}
out.one_hot(EDIT_OP, usize::from(edit.op.code()));
if let Some((bucket, ident)) = value_bucket(edit, ctx.observation) {
out.one_hot(VALUE_BUCKET, bucket.min(VALUE_BUCKET_W - 1));
if ident.is_some() {
identifier = ident;
}
}
}
if !cand.edits.is_keep() {
out.one_hot(PAIR, pair_index(cand));
}
match ctx.need {
DeclaredNeed::Fields => out.one_hot(NEED_BITS, 1),
DeclaredNeed::Links | DeclaredNeed::Metadata => out.one_hot(NEED_BITS, 2),
DeclaredNeed::Screenshot => out.one_hot(NEED_BITS, 3),
_ => out.one_hot(NEED_BITS, 0),
}
if let Some(resources) = resources {
out.one_hot(OBS_HOSTS, hosts_bucket(resources.third_party_hosts));
}
if let Some(ident) = identifier {
out.one_hot(
OBS_SHARE,
usize::from(ident.share_bucket).min(OBS_SHARE_W - 1),
);
out.one_hot(OBS_CLASS, ext_index(ident.class));
out.one_hot(OBS_PARTY, usize::from(ident.third_party));
}
if resources.is_none() {
out.one_hot(MISSING, 0);
}
if identifier.is_none() {
out.one_hot(MISSING, 1);
}
if ctx
.observation
.memory
.is_none_or(|memory| memory.observations == 0)
{
out.one_hot(MISSING, 2);
}
if ctx.routed.source == RouteSource::Caller || ctx.current.request().is_none() {
out.one_hot(MISSING, 3);
}
out.set(BIAS, 1.0);
out
}
pub(crate) fn value_bucket<'o>(
edit: &Edit,
observation: &'o Observation,
) -> Option<(usize, Option<&'o Identifier>)> {
match &edit.op {
Op::Set(Value::Mode(mode)) => Some((mode_index(*mode), None)),
Op::Set(Value::Proxy(pool)) => Some((proxy_index(*pool), None)),
Op::Set(Value::Millis(millis)) => Some((wait_index(*millis), None)),
Op::Set(Value::Bool(value)) => Some((usize::from(*value), None)),
Op::Set(Value::Int(_)) => None,
Op::Append(entries) => {
let (rank, ident) = observation.identifier(entries.first()?)?;
Some((rank, Some(ident)))
}
Op::Remove(_) | Op::Replace(_) | Op::Clear => None,
}
}
fn mode_index(mode: spider_route::RequestMode) -> usize {
MODES
.iter()
.position(|m| *m == mode)
.unwrap_or(MODES.len() - 1)
}
fn proxy_index(pool: spider_route::ProxyPool) -> usize {
PROXIES
.iter()
.position(|p| *p == pool)
.unwrap_or(PROXIES.len() - 1)
}
fn wait_index(millis: u32) -> usize {
WAIT_BUCKETS
.iter()
.rposition(|bucket| *bucket <= millis)
.unwrap_or(0)
}
fn pair_index(cand: &Candidate) -> usize {
let edits = cand.edits.edits();
if let [a, b] = edits {
for (at, (first, second)) in PAIRS.iter().enumerate() {
if (a.key == *first && b.key == *second) || (a.key == *second && b.key == *first) {
return at;
}
}
}
PAIRS.len()
}
const fn hosts_bucket(hosts: u16) -> usize {
match hosts {
0 => 0,
1..=2 => 1,
3..=5 => 2,
6..=10 => 3,
_ => 4,
}
}
pub(crate) const fn ext_index(class: ExtClass) -> usize {
match class {
ExtClass::None => 0,
ExtClass::Markup => 1,
ExtClass::Xml => 2,
ExtClass::Json => 3,
ExtClass::Feed => 4,
ExtClass::Text => 5,
ExtClass::Csv => 6,
ExtClass::Pdf => 7,
ExtClass::Office => 8,
ExtClass::Image => 9,
ExtClass::Media => 10,
ExtClass::Archive => 11,
ExtClass::Asset => 12,
_ => 13,
}
}
#[cfg(test)]
mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::indexing_slicing,
clippy::string_slice
)]
use super::*;
use crate::candidates::{generate, MULTIPLIERS};
use crate::observe::{Identifier, ResourceSummary};
use crate::row::{need_label, status_label, ALL_NEEDS, ALL_STATUSES};
use crate::schema::{Schema, SCHEMA_VERSION};
use crate::testing::Fixture;
use spider_route::{SiteMemory, StatusClass};
#[test]
fn edit_features_hold_no_text() {
assert_eq!(
std::mem::size_of::<EditFeatures>(),
EDIT_DIM * std::mem::size_of::<f32>()
);
}
#[test]
fn blocks_are_contiguous_and_count_is_12() {
let mut next = 0;
for (name, base, width) in BLOCKS {
assert_eq!(
base, next,
"{name} does not start where the last block ended"
);
next = base + width;
}
assert_eq!(next, EDIT_DIM);
assert_eq!(BLOCKS.len(), 12);
let widths: Vec<usize> = BLOCKS.iter().map(|(_, _, w)| *w).collect();
assert_eq!(widths, [24, 16, 5, 8, 8, 4, 5, 5, 14, 2, 4, 1]);
assert_eq!(ext_index(ExtClass::Other) + 1, OBS_CLASS_W);
assert_eq!(PAIRS.len() + 1, 5);
assert_eq!(9 + 10 + 1 + 4, CONFIG_W);
}
fn assert_unit(feats: &EditFeatures, what: &str) {
for (slot, value) in feats.as_slice().iter().enumerate() {
assert!(
value.is_finite() && (-1.0..=1.0).contains(value),
"slot {slot} is {value} for {what}"
);
}
}
#[test]
fn all_slots_finite_within_unit_range() {
let schema = Schema::v1();
let mut fixtures = vec![Fixture::cold(), Fixture::observed(), Fixture::pinned()];
let mut hostile = Fixture::observed();
hostile.observation.resources = Some(ResourceSummary {
requests: u16::MAX,
third_party_hosts: u16::MAX,
third_party_bytes: u64::MAX,
first_party_bytes: u64::MAX,
candidates: vec![Identifier {
pattern: "tracker-alpha.example".into(),
class: ExtClass::Other,
share_bucket: u8::MAX,
count_bucket: u8::MAX,
third_party: true,
}],
});
hostile.observation.memory = Some(SiteMemory {
observations: u32::MAX,
success_rate: f32::NAN,
streak: i16::MIN,
last_status: StatusClass::ServerError,
});
hostile.current.wait_for = Some(spider_cloud_agent::params::WaitFor::idle_network(
spider_cloud_agent::params::Timeout::from_secs(u64::MAX / 1_000),
));
hostile.current.network_blacklist = Some(Vec::new());
hostile.cap = f32::INFINITY;
fixtures.push(hostile);
let mut empty = Fixture::observed();
empty.observation.resources = Some(ResourceSummary::default());
empty.current = Default::default();
fixtures.push(empty);
let mut checked = 0;
for fixture in &fixtures {
let ctx = fixture.ctx();
for cand in generate(&schema, &ctx) {
assert_unit(&featurize_edit(&ctx, &cand), &format!("{:?}", cand.edits));
checked += 1;
}
}
assert!(checked > 40, "only {checked} candidates checked");
}
#[test]
fn keep_has_no_edit_bits() {
let fixture = Fixture::observed();
let ctx = fixture.ctx();
let keep = featurize_edit(&ctx, &Candidate::keep(&ctx));
for slot in EDIT_KEY..NEED_BITS {
assert_eq!(keep.get(slot), Some(0.0), "slot {slot} is set for keep");
}
for slot in OBS_SHARE..MISSING {
assert_eq!(keep.get(slot), Some(0.0), "slot {slot} is set for keep");
}
for cand in generate(&Schema::v1(), &ctx).iter().skip(1) {
let feats = featurize_edit(&ctx, cand);
let keys: f32 = (EDIT_KEY..EDIT_OP).map(|s| feats.get(s).unwrap()).sum();
let ops: f32 = (EDIT_OP..VALUE_BUCKET).map(|s| feats.get(s).unwrap()).sum();
assert!(keys >= 1.0 && ops >= 1.0, "{:?}", cand.edits);
}
}
#[test]
fn missing_observation_is_a_value_not_a_gap() {
let cold = Fixture::cold();
let ctx = cold.ctx();
let feats = featurize_edit(&ctx, &Candidate::keep(&ctx));
assert_eq!(feats.get(MISSING), Some(1.0));
assert_eq!(feats.get(MISSING + 1), Some(1.0));
assert_eq!(feats.get(MISSING + 2), Some(1.0));
assert_eq!(feats.get(MISSING + 3), Some(0.0));
assert_eq!(feats.get(BIAS), Some(1.0));
assert_eq!(
feats.get(CONFIG + 9 + 2 * 2 + 1),
Some(1.0),
"block ads is on"
);
assert_eq!(
feats.get(CONFIG + 9),
Some(1.0),
"interception is not disabled"
);
let mut observed = Fixture::observed();
observed.observation.memory = Some(SiteMemory {
observations: 4,
..SiteMemory::cold()
});
let ctx = observed.ctx();
let append = generate(&Schema::v1(), &ctx)
.into_iter()
.find(|c| c.edits.edits().len() == 1 && c.edits.get(Key::NetworkBlacklist).is_some())
.unwrap();
let feats = featurize_edit(&ctx, &append);
for slot in MISSING..BIAS {
assert_eq!(feats.get(slot), Some(0.0), "missing slot {slot}");
}
assert_eq!(feats.get(OBS_PARTY + 1), Some(1.0));
assert_eq!(feats.get(OBS_HOSTS + 2), Some(1.0), "five hosts");
let mut caller_routed = Fixture::cold();
caller_routed.current = Default::default();
let ctx = caller_routed.ctx();
let feats = featurize_edit(&ctx, &Candidate::keep(&ctx));
assert_eq!(feats.get(MISSING + 3), Some(1.0));
}
fn mirror() -> String {
let schema = Schema::v1();
let blocks: Vec<_> = BLOCKS
.iter()
.map(|(name, base, width)| serde_json::json!({"name": name, "base": base, "width": width}))
.collect();
let keys: Vec<_> = schema
.specs()
.iter()
.map(|spec| {
serde_json::json!({
"name": format!("{:?}", spec.key),
"index": spec.key.index(),
"wire": spec.key.wire(),
"learnable": spec.learnable,
"content_changing": spec.content_changing,
})
})
.collect();
let ops: Vec<_> = crate::edit::Op::NAMES
.iter()
.enumerate()
.map(|(code, name)| serde_json::json!({"name": name, "code": code}))
.collect();
let needs: Vec<_> = ALL_NEEDS.iter().map(|need| need_label(*need)).collect();
let status: Vec<_> = ALL_STATUSES.iter().map(|s| status_label(*s)).collect();
let multipliers: Vec<_> = MULTIPLIERS
.iter()
.map(|(mode, proxy, wait, cost)| {
serde_json::json!({
"mode": mode.as_str(),
"proxy": proxy.as_str(),
"wait_ms": wait,
"multiplier": cost,
})
})
.collect();
let doc = serde_json::json!({
"schema_version": SCHEMA_VERSION,
"edit_feature_version": EDIT_FEATURE_VERSION,
"edit_dim": EDIT_DIM,
"blocks": blocks,
"keys": keys,
"ops": ops,
"needs": needs,
"status": status,
"multipliers": multipliers,
});
let mut text = serde_json::to_string_pretty(&doc).unwrap();
text.push('\n');
text
}
fn mirror_path() -> std::path::PathBuf {
std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("..")
.join("training/fixtures/schema-v1.json")
}
#[test]
fn the_schema_mirror_is_committed() {
let expected = mirror();
let committed = std::fs::read_to_string(mirror_path()).unwrap_or_default();
assert!(
committed == expected,
"training/fixtures/schema-v1.json does not match the constants in spider-optimize. \
Regenerate it with `cargo test -p spider-optimize write_schema_mirror -- --ignored` \
and commit the result.",
);
}
#[test]
#[ignore = "writes training/fixtures/schema-v1.json"]
fn write_schema_mirror() {
let path = mirror_path();
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(path, mirror()).unwrap();
}
}