Skip to main content

spider_route/
features.rs

1//! What the router is allowed to look at.
2//!
3//! [`featurize`] turns a request into a fixed width array of floats. Every
4//! slot in that array is named by a constant in this file, every value is a
5//! one-hot or a bucket, and the whole table is short enough to print. That is
6//! deliberate: the feature to weight table ships publicly with the weights, so
7//! a slot that cannot be printed and defended does not belong here.
8//!
9//! # The rule that shapes all of it
10//!
11//! No slot may be a function of which site is being fetched. Not the name, not
12//! a hash of the name. A hash is not anonymity when the candidate set is a
13//! published list of a million names: hash the list, look for the cells that
14//! carry weight, and you have read back the sites we crawl. So the host
15//! contributes a label group, a subdomain count and three flags, all decided
16//! in [`crate::domain`], and nothing else.
17//!
18//! [`FeatureVector`] holds `[f32; FEATURE_DIM]` and nothing else. It is
19//! `Copy`, it owns no heap, and a test asserts its size, so adding a field
20//! that could hold a name fails the build rather than a review.
21//!
22//! Per site adaptivity comes back through [`SiteMemory`], which the caller
23//! fills in from its own records. The shipped weights carry "this site has
24//! failed twice already". They never carry which site.
25
26use crate::action::{Country, ProxyPool, RequestMode};
27use crate::domain::{host_shape, HostShape, IpKind, TLD_SLOTS};
28use url::Url;
29
30/// How many slots a feature vector has.
31///
32/// A round number with room left. The slots in use today stop at
33/// [`FEATURES_USED`], and what the rest is held for is written down beside it.
34pub const FEATURE_DIM: usize = 256;
35
36/// What the caller wants back, coarsely.
37///
38/// This mirrors the need the client crate takes, minus the payloads, because
39/// the router only cares which kind of answer is wanted and never what is in
40/// it. A field selector is a strong signal about cost and a weak one about
41/// difficulty, which is why it is one slot rather than a description.
42#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
43#[non_exhaustive]
44pub enum DeclaredNeed {
45    /// The page as plain text.
46    Text,
47    /// The page as markdown.
48    #[default]
49    Markdown,
50    /// The markup.
51    Html,
52    /// The links and none of the content.
53    Links,
54    /// The title, description and the rest of the declared metadata.
55    Metadata,
56    /// Named values pulled out of the page.
57    Fields,
58    /// A picture of the page.
59    Screenshot,
60    /// Whatever the service returns on its own.
61    Raw,
62}
63
64impl DeclaredNeed {
65    /// Which slot in the need block this need sets.
66    const fn index(self) -> usize {
67        match self {
68            DeclaredNeed::Text => 0,
69            DeclaredNeed::Markdown => 1,
70            DeclaredNeed::Html => 2,
71            DeclaredNeed::Links => 3,
72            DeclaredNeed::Metadata => 4,
73            DeclaredNeed::Fields => 5,
74            DeclaredNeed::Screenshot => 6,
75            DeclaredNeed::Raw => 7,
76        }
77    }
78}
79
80/// How the last attempt against a site ended, in the coarsest terms that still
81/// change what to do next.
82#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
83#[non_exhaustive]
84pub enum StatusClass {
85    /// Nothing has been tried yet.
86    #[default]
87    Unknown,
88    /// The page came back and had content.
89    Ok,
90    /// The page came back with nothing in it. The most common quiet failure
91    /// and the one worth its own slot.
92    Empty,
93    /// The request was rejected as malformed.
94    BadRequest,
95    /// The page is behind a sign in.
96    NeedsLogin,
97    /// The site refused the fetch.
98    Blocked,
99    /// There is no such page.
100    NotFound,
101    /// The site asked for a slower pace.
102    RateLimited,
103    /// The site failed on its own side.
104    ServerError,
105}
106
107impl StatusClass {
108    /// Which slot in the memory block this class sets.
109    const fn index(self) -> usize {
110        match self {
111            StatusClass::Unknown => 0,
112            StatusClass::Ok => 1,
113            StatusClass::Empty => 2,
114            StatusClass::BadRequest => 3,
115            StatusClass::NeedsLogin => 4,
116            StatusClass::Blocked => 5,
117            StatusClass::NotFound => 6,
118            StatusClass::RateLimited => 7,
119            StatusClass::ServerError => 8,
120        }
121    }
122
123    /// Whether this class says the site turned the fetch away rather than
124    /// failing to serve it.
125    pub const fn is_refusal(self) -> bool {
126        matches!(self, StatusClass::Blocked | StatusClass::NeedsLogin)
127    }
128}
129
130/// What the caller knows about this site from its own history.
131///
132/// The caller keeps this, keyed however it likes. The router reads it and
133/// forgets it. That split is what lets per site adaptivity exist without a
134/// single site name reaching the weights.
135// Not `non_exhaustive`: the caller fills every field of this from its own
136// records, so it has to be writable as a literal from outside the crate.
137#[derive(Debug, Clone, Copy, PartialEq, Default)]
138pub struct SiteMemory {
139    /// How many attempts the caller has recorded. Zero is the normal case for
140    /// a new site and has its own slot, so cold start is a value rather than a
141    /// gap.
142    pub observations: u32,
143    /// A decaying average of how often attempts succeeded, from zero to one.
144    pub success_rate: f32,
145    /// Consecutive outcomes of the same kind. Positive counts successes,
146    /// negative counts failures.
147    pub streak: i16,
148    /// How the last attempt ended.
149    pub last_status: StatusClass,
150}
151
152impl SiteMemory {
153    /// A memory of nothing, which is what a site that has never been fetched
154    /// gets.
155    pub const fn cold() -> SiteMemory {
156        SiteMemory {
157            observations: 0,
158            success_rate: 0.0,
159            streak: 0,
160            last_status: StatusClass::Unknown,
161        }
162    }
163
164    /// How many failures in a row, zero when the last attempt worked.
165    pub const fn failure_streak(self) -> u16 {
166        if self.streak < 0 {
167            self.streak.unsigned_abs()
168        } else {
169            0
170        }
171    }
172
173    /// Whether there is enough here to act on. One attempt is an anecdote.
174    pub const fn is_informative(self) -> bool {
175        self.observations >= 2
176    }
177}
178
179/// Settings the caller fixed itself.
180///
181/// A pin is not a hint. Whatever is set here is what the first attempt uses,
182/// and the router says so by reporting [`crate::RouteSource::Caller`].
183// Not `non_exhaustive`, for the same reason as `SiteMemory`: this is the
184// caller's own input and a literal is the natural way to write it.
185#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
186pub struct CallerPins<'a> {
187    /// The fetch mode the caller fixed.
188    pub mode: Option<RequestMode>,
189    /// The address pool the caller fixed.
190    pub proxy: Option<ProxyPool>,
191    /// The country the caller fixed.
192    pub country: Option<&'a Country>,
193}
194
195impl<'a> CallerPins<'a> {
196    /// Nothing fixed, which is the usual case.
197    pub const fn none() -> CallerPins<'a> {
198        CallerPins {
199            mode: None,
200            proxy: None,
201            country: None,
202        }
203    }
204
205    /// Whether the caller fixed anything at all.
206    pub const fn any(&self) -> bool {
207        self.mode.is_some() || self.proxy.is_some() || self.country.is_some()
208    }
209}
210
211/// One request, as the router sees it.
212///
213/// The url arrives parsed. Parsing allocates and routing does not, so the
214/// caller pays that cost once where it already has to.
215#[derive(Debug, Clone, Copy, PartialEq)]
216#[non_exhaustive]
217pub struct RouteInput<'a> {
218    /// The address to fetch.
219    pub url: &'a Url,
220    /// What the caller wants back.
221    pub need: DeclaredNeed,
222    /// Settings the caller fixed itself.
223    pub pins: CallerPins<'a>,
224    /// What the caller remembers about this site, when it remembers anything.
225    pub memory: Option<&'a SiteMemory>,
226    /// The hour of the day at the caller, from zero to twenty three.
227    ///
228    /// Refusal rates move with the clock, and two floats is a cheap way to say
229    /// so. This crate reads no clock of its own: a caller that cares sets it,
230    /// and a caller that does not gets the unknown slot.
231    pub hour: Option<u8>,
232}
233
234impl<'a> RouteInput<'a> {
235    /// A request with nothing fixed and nothing remembered.
236    pub const fn new(url: &'a Url, need: DeclaredNeed) -> RouteInput<'a> {
237        RouteInput {
238            url,
239            need,
240            pins: CallerPins::none(),
241            memory: None,
242            hour: None,
243        }
244    }
245
246    /// Fix some settings.
247    pub const fn with_pins(mut self, pins: CallerPins<'a>) -> RouteInput<'a> {
248        self.pins = pins;
249        self
250    }
251
252    /// Supply what the caller remembers about this site.
253    pub const fn with_memory(mut self, memory: &'a SiteMemory) -> RouteInput<'a> {
254        self.memory = Some(memory);
255        self
256    }
257
258    /// Say what time it is where the caller is.
259    pub const fn at_hour(mut self, hour: u8) -> RouteInput<'a> {
260        self.hour = Some(hour);
261        self
262    }
263}
264
265/// What the last path segment looks like.
266///
267/// The classes say how a path was generated, which correlates with what serves
268/// it. A run of hex is a record identifier from a store, a run of words with
269/// hyphens is an article slug from a publishing system, and those two are
270/// served by different software more often than not.
271#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
272#[non_exhaustive]
273pub enum SegmentClass {
274    /// The path ends at a directory, so there is no last segment.
275    #[default]
276    Empty,
277    /// Digits only.
278    Digits,
279    /// Letters only.
280    Alpha,
281    /// Letters and digits, no separators.
282    AlphaNum,
283    /// Words joined with hyphens.
284    Hyphenated,
285    /// Words joined with underscores.
286    Underscored,
287    /// A long run of hex, with or without the dashes an identifier carries.
288    HexIdentifier,
289    /// Carries escaped characters.
290    Escaped,
291    /// Anything else.
292    Other,
293}
294
295impl SegmentClass {
296    /// Which slot in the segment block this class sets.
297    const fn index(self) -> usize {
298        match self {
299            SegmentClass::Empty => 0,
300            SegmentClass::Digits => 1,
301            SegmentClass::Alpha => 2,
302            SegmentClass::AlphaNum => 3,
303            SegmentClass::Hyphenated => 4,
304            SegmentClass::Underscored => 5,
305            SegmentClass::HexIdentifier => 6,
306            SegmentClass::Escaped => 7,
307            SegmentClass::Other => 8,
308        }
309    }
310}
311
312/// What kind of file the path names, if it names one.
313///
314/// This is the single most useful pre-call feature, because most of these
315/// classes settle the question on their own: a rendered fetch cannot change
316/// the bytes of a file that was never markup.
317#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
318#[non_exhaustive]
319pub enum ExtClass {
320    /// No extension. Most pages.
321    #[default]
322    None,
323    /// Markup, including the extensions template engines leave behind.
324    Markup,
325    /// Structured data by extension.
326    Xml,
327    /// A json document.
328    Json,
329    /// A syndication document.
330    Feed,
331    /// Plain text or markdown.
332    Text,
333    /// Separated values.
334    Csv,
335    /// A pdf.
336    Pdf,
337    /// A word processor or spreadsheet document.
338    Office,
339    /// An image.
340    Image,
341    /// Audio or video.
342    Media,
343    /// An archive.
344    Archive,
345    /// Script, style or a compiled asset.
346    Asset,
347    /// An extension we do not know.
348    Other,
349}
350
351impl ExtClass {
352    /// Which slot in the extension block this class sets.
353    const fn index(self) -> usize {
354        match self {
355            ExtClass::None => 0,
356            ExtClass::Markup => 1,
357            ExtClass::Xml => 2,
358            ExtClass::Json => 3,
359            ExtClass::Feed => 4,
360            ExtClass::Text => 5,
361            ExtClass::Csv => 6,
362            ExtClass::Pdf => 7,
363            ExtClass::Office => 8,
364            ExtClass::Image => 9,
365            ExtClass::Media => 10,
366            ExtClass::Archive => 11,
367            ExtClass::Asset => 12,
368            ExtClass::Other => 13,
369        }
370    }
371
372    /// Whether a rendered fetch could possibly change what comes back.
373    ///
374    /// False for every class whose bytes are fixed before any script could
375    /// run. This is what lets the heuristic send a feed or a data file down
376    /// the cheap path with no hedging.
377    pub const fn can_need_rendering(self) -> bool {
378        matches!(self, ExtClass::None | ExtClass::Markup | ExtClass::Other)
379    }
380}
381
382/// What a query key is for, judged from the key alone.
383///
384/// Keys are named by the software that serves the page, so they say something
385/// about it. Values are the user's business and are never read: a search term
386/// or an identifier is exactly the kind of thing that could make a slot
387/// personal.
388#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
389#[non_exhaustive]
390pub enum KeyClass {
391    /// Paging through a list.
392    Pagination,
393    /// Naming one record.
394    Identifier,
395    /// A search.
396    Search,
397    /// Campaign tagging, added by whoever linked here rather than by the site.
398    Tracking,
399    /// A session or a token.
400    Session,
401    /// Language or region.
402    Locale,
403    /// Asking for a particular output format.
404    Format,
405    /// Sorting or filtering a list.
406    Filter,
407    /// A single character key, the mark of a hand rolled query string.
408    SingleChar,
409    /// A key that is only digits.
410    Numeric,
411}
412
413/// How many slots [`KeyClass`] needs.
414const KEY_CLASSES: usize = 10;
415
416impl KeyClass {
417    /// Which slot in the key block this class sets.
418    const fn index(self) -> usize {
419        match self {
420            KeyClass::Pagination => 0,
421            KeyClass::Identifier => 1,
422            KeyClass::Search => 2,
423            KeyClass::Tracking => 3,
424            KeyClass::Session => 4,
425            KeyClass::Locale => 5,
426            KeyClass::Format => 6,
427            KeyClass::Filter => 7,
428            KeyClass::SingleChar => 8,
429            KeyClass::Numeric => 9,
430        }
431    }
432}
433
434/// The feature vector.
435///
436/// Fixed width, `Copy`, no heap. It cannot hold a url. That is the privacy
437/// contract, and `feature_vector_holds_no_text` in this file is what keeps it
438/// true when someone reaches for a field with a name in it.
439#[derive(Copy, Clone, Debug, PartialEq)]
440pub struct FeatureVector {
441    bits: [f32; FEATURE_DIM],
442}
443
444impl Default for FeatureVector {
445    /// Written out rather than derived: the standard library stops deriving
446    /// `Default` for arrays well short of this one.
447    fn default() -> FeatureVector {
448        FeatureVector::zeroed()
449    }
450}
451
452impl FeatureVector {
453    /// Every slot zero.
454    pub const fn zeroed() -> FeatureVector {
455        FeatureVector {
456            bits: [0.0; FEATURE_DIM],
457        }
458    }
459
460    /// The slots, in order.
461    pub const fn as_slice(&self) -> &[f32] {
462        &self.bits
463    }
464
465    /// One slot, or `None` past the end.
466    pub fn get(&self, slot: usize) -> Option<f32> {
467        self.bits.get(slot).copied()
468    }
469
470    /// Set one slot. Out of range does nothing, because a feature layout bug
471    /// should not take down a caller's process.
472    fn set(&mut self, slot: usize, value: f32) {
473        if let Some(cell) = self.bits.get_mut(slot) {
474            *cell = value;
475        }
476    }
477
478    /// Set the slot at `base + offset` of a one-hot block.
479    fn one_hot(&mut self, base: usize, offset: usize) {
480        self.set(base + offset, 1.0);
481    }
482}
483
484// The layout. Each block is a base and a width, and the bases are derived from
485// the widths so a block cannot silently overlap its neighbour. The order is
486// url shape, then host, then what the caller asked for, then what the caller
487// remembers.
488
489/// Path depth, bucketed: 0, 1, 2, 3, 4, 5, 6 or more.
490pub const PATH_DEPTH: usize = 0;
491const PATH_DEPTH_W: usize = 7;
492
493/// Path length in bytes, bucketed: to 16, 32, 64, 128, 256, past 256.
494pub const PATH_LEN: usize = PATH_DEPTH + PATH_DEPTH_W;
495const PATH_LEN_W: usize = 6;
496
497/// What the last path segment looks like. See [`SegmentClass`].
498pub const SEGMENT: usize = PATH_LEN + PATH_LEN_W;
499const SEGMENT_W: usize = 9;
500
501/// The path ends in a slash.
502pub const TRAILING_SLASH: usize = SEGMENT + SEGMENT_W;
503const TRAILING_SLASH_W: usize = 1;
504
505/// What kind of file the path names. See [`ExtClass`].
506pub const EXTENSION: usize = TRAILING_SLASH + TRAILING_SLASH_W;
507const EXTENSION_W: usize = 14;
508
509/// How many query parameters, bucketed: 0, 1, 2, 3, 4 to 6, 7 or more.
510pub const QUERY_ARITY: usize = EXTENSION + EXTENSION_W;
511const QUERY_ARITY_W: usize = 6;
512
513/// Which kinds of query key are present. See [`KeyClass`]. Not one-hot: a url
514/// can carry several kinds at once.
515pub const QUERY_KEYS: usize = QUERY_ARITY + QUERY_ARITY_W;
516const QUERY_KEYS_W: usize = KEY_CLASSES;
517
518/// The longest query key, bucketed: to 2, to 6, to 12, past 12.
519pub const QUERY_KEY_LEN: usize = QUERY_KEYS + QUERY_KEYS_W;
520const QUERY_KEY_LEN_W: usize = 4;
521
522/// Labels in front of the registrable name: 0, 1, 2, 3 or more.
523pub const SUBDOMAINS: usize = QUERY_KEY_LEN + QUERY_KEY_LEN_W;
524const SUBDOMAINS_W: usize = 4;
525
526/// The first label is the conventional web prefix.
527pub const WWW: usize = SUBDOMAINS + SUBDOMAINS_W;
528const WWW_W: usize = 1;
529
530/// The host is a literal address: first slot four numbers, second slot eight
531/// groups.
532pub const IP_LITERAL: usize = WWW + WWW_W;
533const IP_LITERAL_W: usize = 2;
534
535/// A label is in the encoded form that carries non-ascii text.
536pub const PUNYCODE: usize = IP_LITERAL + IP_LITERAL_W;
537const PUNYCODE_W: usize = 1;
538
539/// Four flags about the request itself: secure scheme, a port that is not the
540/// default, credentials in the url, a fragment.
541pub const URL_FLAGS: usize = PUNYCODE + PUNYCODE_W;
542const URL_FLAGS_W: usize = 4;
543
544/// The label group, one-hot. See [`crate::domain::TldSlot`].
545pub const TLD: usize = URL_FLAGS + URL_FLAGS_W;
546const TLD_W: usize = TLD_SLOTS;
547
548/// How well known the site is, in five bands plus unknown.
549///
550/// Always zero today, and deliberately still here. The popularity filters are
551/// about a megabyte and they arrive with the weights, not before them, because
552/// a crate that answers from rules alone should not carry a megabyte it never
553/// reads. The slots exist so the layout does not shift when they land, which
554/// would invalidate every weight trained against it.
555pub const POPULARITY: usize = TLD + TLD_W;
556const POPULARITY_W: usize = 6;
557
558/// What the caller asked for, one-hot. See [`DeclaredNeed`].
559pub const NEED: usize = POPULARITY + POPULARITY_W;
560const NEED_W: usize = 8;
561
562/// What the caller fixed itself: any pin at all, then the fixed mode one-hot,
563/// then the fixed pool one-hot, then a fixed country.
564pub const PINS: usize = NEED + NEED_W;
565const PINS_W: usize = 8;
566
567/// The hour of the day as a pair of waves, then a slot for not knowing.
568///
569/// Two waves rather than twenty four slots because the hours either side of
570/// one another behave alike, and because midnight is next to eleven at night.
571pub const HOUR: usize = PINS + PINS_W;
572const HOUR_W: usize = 3;
573
574/// How many attempts the caller has recorded: none, 1 to 2, 3 to 9, 10 to 49,
575/// 50 or more. The first slot is cold start, which is the normal case.
576pub const MEM_COUNT: usize = HOUR + HOUR_W;
577const MEM_COUNT_W: usize = 5;
578
579/// How often attempts succeeded, in five bands.
580pub const MEM_SUCCESS: usize = MEM_COUNT + MEM_COUNT_W;
581const MEM_SUCCESS_W: usize = 5;
582
583/// Failures in a row: none, 1, 2, 3 or more.
584pub const MEM_FAIL_STREAK: usize = MEM_SUCCESS + MEM_SUCCESS_W;
585const MEM_FAIL_STREAK_W: usize = 4;
586
587/// Successes in a row: none, 1, 2, 3 or more.
588pub const MEM_WIN_STREAK: usize = MEM_FAIL_STREAK + MEM_FAIL_STREAK_W;
589const MEM_WIN_STREAK_W: usize = 4;
590
591/// How the last attempt ended, one-hot. See [`StatusClass`].
592pub const MEM_STATUS: usize = MEM_WIN_STREAK + MEM_WIN_STREAK_W;
593const MEM_STATUS_W: usize = 9;
594
595/// Always one, so a model has an intercept.
596pub const BIAS: usize = MEM_STATUS + MEM_STATUS_W;
597const BIAS_W: usize = 1;
598
599/// The first slot no block claims.
600///
601/// What is left is held for the two blocks that are not written yet: the
602/// signals read off a response, which only the escalation head uses, and the
603/// explicit crosses between blocks that give a linear model its curves. Both
604/// land with the weights. Leaving the room here means adding them does not
605/// move any slot already trained against.
606pub const FEATURES_USED: usize = BIAS + BIAS_W;
607
608const _: () = assert!(FEATURES_USED <= FEATURE_DIM);
609
610/// Turn a request into the vector the router reads.
611///
612/// Allocates nothing, reads no clock and no environment, and gives the same
613/// answer for the same input forever. Two urls of the same shape on two
614/// different sites come out identical, which is the anonymity claim and is
615/// tested as such.
616pub fn featurize(input: &RouteInput<'_>) -> FeatureVector {
617    let mut out = FeatureVector::zeroed();
618    let url = input.url;
619
620    // Path shape.
621    let path = url.path();
622    let mut depth = 0usize;
623    let mut last = "";
624
625    for segment in path.split('/') {
626        if !segment.is_empty() {
627            depth += 1;
628            last = segment;
629        }
630    }
631
632    out.one_hot(PATH_DEPTH, depth.min(6));
633    out.one_hot(PATH_LEN, length_bucket(path.len()));
634
635    let trailing = path.len() > 1 && path.ends_with('/');
636    let segment_class = if trailing || last.is_empty() {
637        SegmentClass::Empty
638    } else {
639        segment_class(last)
640    };
641
642    out.one_hot(SEGMENT, segment_class.index());
643
644    if trailing {
645        out.set(TRAILING_SLASH, 1.0);
646    }
647
648    out.one_hot(EXTENSION, extension_class(last).index());
649
650    // Query shape. Keys are read, values never are.
651    let mut arity = 0usize;
652    let mut longest_key = 0usize;
653
654    for pair in url.query().unwrap_or("").split('&') {
655        if pair.is_empty() {
656            continue;
657        }
658
659        arity += 1;
660
661        let key = match pair.split_once('=') {
662            Some((key, _value)) => key,
663            None => pair,
664        };
665
666        longest_key = longest_key.max(key.chars().count());
667
668        for class in key_classes(key) {
669            out.one_hot(QUERY_KEYS, class.index());
670        }
671    }
672
673    out.one_hot(QUERY_ARITY, arity_bucket(arity));
674
675    if arity > 0 {
676        out.one_hot(QUERY_KEY_LEN, key_length_bucket(longest_key));
677    }
678
679    // Host shape, decided entirely in `domain`.
680    let host: HostShape = host_shape(url);
681
682    out.one_hot(SUBDOMAINS, usize::from(host.subdomains).min(3));
683
684    if host.www {
685        out.set(WWW, 1.0);
686    }
687
688    match host.ip {
689        Some(IpKind::V4) => out.one_hot(IP_LITERAL, 0),
690        Some(IpKind::V6) => out.one_hot(IP_LITERAL, 1),
691        None => {}
692    }
693
694    if host.punycode {
695        out.set(PUNYCODE, 1.0);
696    }
697
698    if let Some(tld) = host.tld {
699        out.one_hot(TLD, tld.index());
700    }
701
702    // The request itself.
703    if url.scheme() == "https" {
704        out.set(URL_FLAGS, 1.0);
705    }
706
707    if url.port().is_some() {
708        out.set(URL_FLAGS + 1, 1.0);
709    }
710
711    if !url.username().is_empty() || url.password().is_some() {
712        out.set(URL_FLAGS + 2, 1.0);
713    }
714
715    if url.fragment().is_some() {
716        out.set(URL_FLAGS + 3, 1.0);
717    }
718
719    // Popularity is left at zero until the filters ship. Named here so the
720    // block is visible in the layout rather than looking like a gap.
721    let _ = POPULARITY;
722
723    // What the caller asked for.
724    out.one_hot(NEED, input.need.index());
725
726    if input.pins.any() {
727        out.set(PINS, 1.0);
728    }
729
730    match input.pins.mode {
731        Some(RequestMode::Http) => out.one_hot(PINS, 1),
732        Some(RequestMode::Smart) => out.one_hot(PINS, 2),
733        Some(RequestMode::Browser) => out.one_hot(PINS, 3),
734        _ => {}
735    }
736
737    match input.pins.proxy {
738        Some(ProxyPool::Isp) => out.one_hot(PINS, 4),
739        Some(ProxyPool::Residential) => out.one_hot(PINS, 5),
740        _ => {}
741    }
742
743    if input.pins.country.is_some() {
744        out.one_hot(PINS, 6);
745    }
746
747    // Slot 7 of the pin block is spare, so the block stays eight wide when a
748    // fourth pin is added.
749
750    match input.hour {
751        Some(hour) => {
752            let turns = f32::from(hour.min(23)) / 24.0 * std::f32::consts::TAU;
753            out.set(HOUR, turns.sin());
754            out.set(HOUR + 1, turns.cos());
755        }
756        None => out.set(HOUR + 2, 1.0),
757    }
758
759    // What the caller remembers.
760    let memory = input.memory.copied().unwrap_or_else(SiteMemory::cold);
761
762    out.one_hot(MEM_COUNT, observation_bucket(memory.observations));
763    out.one_hot(MEM_SUCCESS, success_bucket(memory.success_rate));
764    out.one_hot(MEM_FAIL_STREAK, streak_bucket(memory.failure_streak()));
765    out.one_hot(
766        MEM_WIN_STREAK,
767        streak_bucket(memory.streak.max(0).unsigned_abs()),
768    );
769    out.one_hot(MEM_STATUS, memory.last_status.index());
770
771    out.set(BIAS, 1.0);
772
773    out
774}
775
776/// Path length in bytes, bucketed.
777const fn length_bucket(len: usize) -> usize {
778    match len {
779        0..=16 => 0,
780        17..=32 => 1,
781        33..=64 => 2,
782        65..=128 => 3,
783        129..=256 => 4,
784        _ => 5,
785    }
786}
787
788/// Query parameter count, bucketed.
789const fn arity_bucket(count: usize) -> usize {
790    match count {
791        0 => 0,
792        1 => 1,
793        2 => 2,
794        3 => 3,
795        4..=6 => 4,
796        _ => 5,
797    }
798}
799
800/// Longest query key, bucketed.
801const fn key_length_bucket(len: usize) -> usize {
802    match len {
803        0..=2 => 0,
804        3..=6 => 1,
805        7..=12 => 2,
806        _ => 3,
807    }
808}
809
810/// Recorded attempts, bucketed.
811const fn observation_bucket(count: u32) -> usize {
812    match count {
813        0 => 0,
814        1..=2 => 1,
815        3..=9 => 2,
816        10..=49 => 3,
817        _ => 4,
818    }
819}
820
821/// Success rate, bucketed. Anything outside zero to one is clamped, because a
822/// caller's arithmetic is not this crate's to trust. That includes the NaN a
823/// caller gets from dividing no successes by no attempts: `clamp` passes NaN
824/// through and every comparison below is then false, which would land it in
825/// the top band, so it is read as no history instead.
826fn success_bucket(rate: f32) -> usize {
827    if rate.is_nan() {
828        return 0;
829    }
830
831    let rate = rate.clamp(0.0, 1.0);
832
833    if rate < 0.2 {
834        0
835    } else if rate < 0.4 {
836        1
837    } else if rate < 0.6 {
838        2
839    } else if rate < 0.8 {
840        3
841    } else {
842        4
843    }
844}
845
846/// A run of the same outcome, bucketed.
847const fn streak_bucket(streak: u16) -> usize {
848    match streak {
849        0 => 0,
850        1 => 1,
851        2 => 2,
852        _ => 3,
853    }
854}
855
856/// Classify the last path segment. Reads characters, never bytes, so an
857/// escaped or non-ascii path cannot cut a character in half.
858pub fn segment_class(segment: &str) -> SegmentClass {
859    if segment.contains('%') {
860        return SegmentClass::Escaped;
861    }
862
863    let mut letters = 0usize;
864    let mut digits = 0usize;
865    let mut hyphens = 0usize;
866    let mut underscores = 0usize;
867    let mut hex = 0usize;
868    let mut other = 0usize;
869    let mut total = 0usize;
870
871    for ch in segment.chars() {
872        total += 1;
873
874        if ch.is_ascii_digit() {
875            digits += 1;
876            hex += 1;
877        } else if ch.is_ascii_hexdigit() {
878            letters += 1;
879            hex += 1;
880        } else if ch.is_alphabetic() {
881            letters += 1;
882        } else if ch == '-' {
883            hyphens += 1;
884        } else if ch == '_' {
885            underscores += 1;
886        } else {
887            other += 1;
888        }
889    }
890
891    if total == 0 {
892        return SegmentClass::Empty;
893    }
894
895    // An identifier out of a store: long, hex, dashes at most. Checked before
896    // the word classes so a record id does not read as an article slug.
897    if hex + hyphens == total && hex >= 16 && digits > 0 {
898        return SegmentClass::HexIdentifier;
899    }
900
901    if other > 0 {
902        return SegmentClass::Other;
903    }
904
905    if digits == total {
906        return SegmentClass::Digits;
907    }
908
909    if letters == total {
910        return SegmentClass::Alpha;
911    }
912
913    if hyphens > 0 && underscores == 0 {
914        return SegmentClass::Hyphenated;
915    }
916
917    if underscores > 0 && hyphens == 0 {
918        return SegmentClass::Underscored;
919    }
920
921    if hyphens == 0 && underscores == 0 {
922        return SegmentClass::AlphaNum;
923    }
924
925    SegmentClass::Other
926}
927
928/// Classify the extension of the last path segment.
929pub fn extension_class(segment: &str) -> ExtClass {
930    let ext = match segment.rsplit_once('.') {
931        Some((stem, ext)) if !stem.is_empty() && !ext.is_empty() => ext,
932        _ => return ExtClass::None,
933    };
934
935    // A dotted version or a date in a path is not an extension, and neither is
936    // a run of anything that is not a short alphanumeric word.
937    if ext.chars().count() > 8
938        || ext.chars().any(|ch| !ch.is_ascii_alphanumeric())
939        || ext.chars().all(|ch| ch.is_ascii_digit())
940    {
941        return ExtClass::None;
942    }
943
944    const TABLE: [(&str, ExtClass); 51] = [
945        ("html", ExtClass::Markup),
946        ("htm", ExtClass::Markup),
947        ("xhtml", ExtClass::Markup),
948        ("shtml", ExtClass::Markup),
949        ("php", ExtClass::Markup),
950        ("asp", ExtClass::Markup),
951        ("aspx", ExtClass::Markup),
952        ("jsp", ExtClass::Markup),
953        ("cfm", ExtClass::Markup),
954        ("xml", ExtClass::Xml),
955        ("xsl", ExtClass::Xml),
956        ("json", ExtClass::Json),
957        ("jsonl", ExtClass::Json),
958        ("ndjson", ExtClass::Json),
959        ("rss", ExtClass::Feed),
960        ("atom", ExtClass::Feed),
961        ("txt", ExtClass::Text),
962        ("md", ExtClass::Text),
963        ("rst", ExtClass::Text),
964        ("csv", ExtClass::Csv),
965        ("tsv", ExtClass::Csv),
966        ("pdf", ExtClass::Pdf),
967        ("doc", ExtClass::Office),
968        ("docx", ExtClass::Office),
969        ("xls", ExtClass::Office),
970        ("xlsx", ExtClass::Office),
971        ("ppt", ExtClass::Office),
972        ("pptx", ExtClass::Office),
973        ("odt", ExtClass::Office),
974        ("jpg", ExtClass::Image),
975        ("jpeg", ExtClass::Image),
976        ("png", ExtClass::Image),
977        ("gif", ExtClass::Image),
978        ("webp", ExtClass::Image),
979        ("svg", ExtClass::Image),
980        ("avif", ExtClass::Image),
981        ("ico", ExtClass::Image),
982        ("mp4", ExtClass::Media),
983        ("webm", ExtClass::Media),
984        ("mov", ExtClass::Media),
985        ("mp3", ExtClass::Media),
986        ("wav", ExtClass::Media),
987        ("m3u8", ExtClass::Media),
988        ("zip", ExtClass::Archive),
989        ("gz", ExtClass::Archive),
990        ("tar", ExtClass::Archive),
991        ("rar", ExtClass::Archive),
992        ("7z", ExtClass::Archive),
993        ("js", ExtClass::Asset),
994        ("css", ExtClass::Asset),
995        ("wasm", ExtClass::Asset),
996    ];
997
998    for (name, class) in TABLE {
999        if ext.eq_ignore_ascii_case(name) {
1000            return class;
1001        }
1002    }
1003
1004    ExtClass::Other
1005}
1006
1007/// What kind of file a url names, if it names one.
1008///
1009/// Reads the path and nothing else, so the heuristic can ask this question
1010/// without going anywhere near the host.
1011pub fn extension_of(url: &Url) -> ExtClass {
1012    extension_class(last_segment(url))
1013}
1014
1015/// The last non empty path segment, empty when the path ends at a directory.
1016fn last_segment(url: &Url) -> &str {
1017    let path = url.path();
1018
1019    if path.ends_with('/') {
1020        return "";
1021    }
1022
1023    path.rsplit('/')
1024        .find(|segment| !segment.is_empty())
1025        .unwrap_or_default()
1026}
1027
1028/// Which classes a query key belongs to. A key can fall into more than one,
1029/// and the return is a fixed array so nothing is allocated to say so.
1030fn key_classes(key: &str) -> impl Iterator<Item = KeyClass> {
1031    const PAGINATION: [&str; 8] = [
1032        "page", "p", "pg", "offset", "start", "limit", "per_page", "cursor",
1033    ];
1034    const IDENTIFIER: [&str; 8] = ["id", "pid", "sku", "uid", "item", "product", "ref", "asin"];
1035    const SEARCH: [&str; 6] = ["q", "query", "s", "search", "keyword", "term"];
1036    const SESSION: [&str; 7] = [
1037        "token",
1038        "session",
1039        "sid",
1040        "auth",
1041        "key",
1042        "signature",
1043        "nonce",
1044    ];
1045    const LOCALE: [&str; 7] = [
1046        "lang", "locale", "hl", "gl", "country", "region", "currency",
1047    ];
1048    const FORMAT: [&str; 5] = ["format", "output", "type", "view", "render"];
1049    const FILTER: [&str; 8] = [
1050        "sort", "order", "filter", "category", "tag", "min", "max", "price",
1051    ];
1052
1053    let matches = |table: &[&str]| table.iter().any(|name| key.eq_ignore_ascii_case(name));
1054
1055    let mut found = [None; KEY_CLASSES];
1056    let mut at = 0usize;
1057    let mut push = |class: KeyClass| {
1058        if let Some(cell) = found.get_mut(at) {
1059            *cell = Some(class);
1060            at += 1;
1061        }
1062    };
1063
1064    if matches(&PAGINATION) {
1065        push(KeyClass::Pagination);
1066    }
1067
1068    if matches(&IDENTIFIER) {
1069        push(KeyClass::Identifier);
1070    }
1071
1072    if matches(&SEARCH) {
1073        push(KeyClass::Search);
1074    }
1075
1076    // Campaign tags are added by whoever wrote the link, so they say nothing
1077    // about the site and a lot about the traffic. Worth a slot either way.
1078    if key.len() > 4 && key.get(..4).is_some_and(|p| p.eq_ignore_ascii_case("utm_"))
1079        || key.eq_ignore_ascii_case("gclid")
1080        || key.eq_ignore_ascii_case("fbclid")
1081        || key.eq_ignore_ascii_case("msclkid")
1082    {
1083        push(KeyClass::Tracking);
1084    }
1085
1086    if matches(&SESSION) {
1087        push(KeyClass::Session);
1088    }
1089
1090    if matches(&LOCALE) {
1091        push(KeyClass::Locale);
1092    }
1093
1094    if matches(&FORMAT) {
1095        push(KeyClass::Format);
1096    }
1097
1098    if matches(&FILTER) {
1099        push(KeyClass::Filter);
1100    }
1101
1102    if key.chars().count() == 1 {
1103        push(KeyClass::SingleChar);
1104    }
1105
1106    if !key.is_empty() && key.chars().all(|ch| ch.is_ascii_digit()) {
1107        push(KeyClass::Numeric);
1108    }
1109
1110    found.into_iter().flatten()
1111}
1112
1113#[cfg(test)]
1114mod tests {
1115    // A test may unwrap and may panic: a test that cannot set itself up should
1116    // fail loudly rather than quietly measure nothing.
1117    #![allow(
1118        clippy::unwrap_used,
1119        clippy::expect_used,
1120        clippy::panic,
1121        clippy::unreachable,
1122        clippy::indexing_slicing,
1123        clippy::string_slice
1124    )]
1125    use super::*;
1126
1127    fn vector(url: &str) -> FeatureVector {
1128        let parsed = Url::parse(url).unwrap();
1129        featurize(&RouteInput::new(&parsed, DeclaredNeed::Markdown))
1130    }
1131
1132    #[test]
1133    fn feature_vector_holds_no_text() {
1134        // The whole privacy argument rests on this. A `String` field would add
1135        // a pointer, a length and a capacity, and the assertion would fail
1136        // before anyone had to notice it in review.
1137        assert_eq!(
1138            std::mem::size_of::<FeatureVector>(),
1139            FEATURE_DIM * std::mem::size_of::<f32>()
1140        );
1141    }
1142
1143    #[test]
1144    fn the_layout_fits_and_the_blocks_do_not_overlap() {
1145        let bases = [
1146            ("path depth", PATH_DEPTH, PATH_DEPTH_W),
1147            ("path length", PATH_LEN, PATH_LEN_W),
1148            ("segment", SEGMENT, SEGMENT_W),
1149            ("trailing slash", TRAILING_SLASH, TRAILING_SLASH_W),
1150            ("extension", EXTENSION, EXTENSION_W),
1151            ("query arity", QUERY_ARITY, QUERY_ARITY_W),
1152            ("query keys", QUERY_KEYS, QUERY_KEYS_W),
1153            ("query key length", QUERY_KEY_LEN, QUERY_KEY_LEN_W),
1154            ("subdomains", SUBDOMAINS, SUBDOMAINS_W),
1155            ("www", WWW, WWW_W),
1156            ("ip literal", IP_LITERAL, IP_LITERAL_W),
1157            ("punycode", PUNYCODE, PUNYCODE_W),
1158            ("url flags", URL_FLAGS, URL_FLAGS_W),
1159            ("tld", TLD, TLD_W),
1160            ("popularity", POPULARITY, POPULARITY_W),
1161            ("need", NEED, NEED_W),
1162            ("pins", PINS, PINS_W),
1163            ("hour", HOUR, HOUR_W),
1164            ("memory count", MEM_COUNT, MEM_COUNT_W),
1165            ("memory success", MEM_SUCCESS, MEM_SUCCESS_W),
1166            ("memory failures", MEM_FAIL_STREAK, MEM_FAIL_STREAK_W),
1167            ("memory wins", MEM_WIN_STREAK, MEM_WIN_STREAK_W),
1168            ("memory status", MEM_STATUS, MEM_STATUS_W),
1169            ("bias", BIAS, BIAS_W),
1170        ];
1171
1172        let mut next = 0usize;
1173
1174        for (name, base, width) in bases {
1175            assert_eq!(
1176                base, next,
1177                "{name} does not start where the block before it ends"
1178            );
1179            next = base + width;
1180        }
1181
1182        assert_eq!(next, FEATURES_USED);
1183        assert_eq!(bases.len(), 24);
1184        // Room left over, checked against a vector rather than against the
1185        // constants, which the compiler would fold away.
1186        let room = vector("https://example.com/").as_slice().len() - next;
1187        assert!(room > 0, "the layout has filled the vector");
1188    }
1189
1190    #[test]
1191    fn every_block_is_wide_enough_for_what_it_holds() {
1192        // The bases above are derived from the widths, so a block that is too
1193        // narrow keeps the layout consistent and quietly writes its last value
1194        // into the block next door. The vocabularies are what settle the
1195        // widths, so they are checked against them here.
1196        assert_eq!(SEGMENT_W, 9);
1197        assert_eq!(EXTENSION_W, 14);
1198        assert_eq!(QUERY_KEYS_W, 10);
1199        assert_eq!(NEED_W, 8);
1200        assert_eq!(MEM_STATUS_W, 9);
1201        assert_eq!(TLD_W, 30);
1202
1203        let widest = [
1204            ("segment", SegmentClass::Other.index(), SEGMENT_W),
1205            ("extension", ExtClass::Other.index(), EXTENSION_W),
1206            ("query keys", KeyClass::Numeric.index(), QUERY_KEYS_W),
1207            ("need", DeclaredNeed::Raw.index(), NEED_W),
1208            (
1209                "memory status",
1210                StatusClass::ServerError.index(),
1211                MEM_STATUS_W,
1212            ),
1213            ("tld", crate::domain::TldSlot::NewGtld.index(), TLD_W),
1214        ];
1215
1216        for (name, last, width) in widest {
1217            assert_eq!(
1218                last + 1,
1219                width,
1220                "the {name} block is {width} wide and its last value sits at {last}"
1221            );
1222        }
1223    }
1224
1225    #[test]
1226    fn a_value_at_the_end_of_a_block_stays_inside_it() {
1227        // The other half of the check above, from the outside: the last
1228        // extension class and the first query arity bucket are neighbours, and
1229        // this is what catches one writing over the other.
1230        let last_in_block = vector("https://example.com/thing.wobble");
1231
1232        assert_eq!(
1233            last_in_block.get(EXTENSION + ExtClass::Other.index()),
1234            Some(1.0)
1235        );
1236        assert_eq!(last_in_block.get(QUERY_ARITY), Some(1.0));
1237        assert_eq!(last_in_block.get(QUERY_ARITY + 1), Some(0.0));
1238
1239        let one_query = vector("https://example.com/thing.wobble?a=1");
1240        assert_eq!(one_query.get(QUERY_ARITY), Some(0.0));
1241        assert_eq!(one_query.get(QUERY_ARITY + 1), Some(1.0));
1242    }
1243
1244    #[test]
1245    fn the_popularity_block_is_reserved_and_empty() {
1246        let v = vector("https://www.example.com/a/b/c");
1247
1248        for slot in POPULARITY..POPULARITY + POPULARITY_W {
1249            assert_eq!(
1250                v.get(slot),
1251                Some(0.0),
1252                "popularity slot {slot} carries a value, and the filters have not shipped"
1253            );
1254        }
1255    }
1256
1257    #[test]
1258    fn everything_past_the_last_block_stays_zero() {
1259        let v = vector("https://a.b.example.co.uk/x/y/z/page-12.html?q=hi&utm_source=x#frag");
1260
1261        for slot in FEATURES_USED..FEATURE_DIM {
1262            assert_eq!(
1263                v.get(slot),
1264                Some(0.0),
1265                "slot {slot} is not claimed by a block"
1266            );
1267        }
1268    }
1269
1270    /// Sites with nothing in common but the shape of the address they were
1271    /// asked for. If any pair of these stops colliding, a slot has started to
1272    /// depend on which site it is, which is the one thing this crate promises
1273    /// never to do.
1274    const DISSIMILAR: [&str; 8] = [
1275        "www.nike.com",
1276        "www.pfizer.com",
1277        "www.goldmansachs.com",
1278        "www.weightwatchers.com",
1279        "www.nationalgeographic.com",
1280        "www.bostonglobe.com",
1281        "www.chess.com",
1282        "www.tripadvisor.com",
1283    ];
1284
1285    /// The same, one label group over.
1286    const DISSIMILAR_PAIRED_SUFFIX: [&str; 5] = [
1287        "www.bbc.co.uk",
1288        "www.hsbc.co.uk",
1289        "www.rakuten.co.jp",
1290        "www.woolworths.com.au",
1291        "www.globo.com.br",
1292    ];
1293
1294    #[test]
1295    fn the_same_shape_on_different_sites_is_the_same_vector() {
1296        let paths = [
1297            "/products/air-max-90?color=red&size=10",
1298            "/",
1299            "/news/world/a-long-story-about-things",
1300            "/api/v2/items.json?page=3&limit=50",
1301            "/assets/hero-image.png",
1302            "/search?q=something+private&sort=price",
1303        ];
1304
1305        let mut compared = 0usize;
1306
1307        for group in [DISSIMILAR.as_slice(), DISSIMILAR_PAIRED_SUFFIX.as_slice()] {
1308            for path in paths {
1309                let first_host = group.first().expect("a non empty group");
1310                let reference = vector(&format!("https://{first_host}{path}"));
1311
1312                for host in group.iter().skip(1) {
1313                    let other = vector(&format!("https://{host}{path}"));
1314
1315                    assert_eq!(
1316                        reference, other,
1317                        "{first_host} and {host} produced different vectors for {path}, \
1318                         so some slot is reading the site rather than the shape"
1319                    );
1320                    compared += 1;
1321                }
1322            }
1323        }
1324
1325        // A loop that compared nothing would pass.
1326        assert_eq!(compared, (7 + 4) * 6);
1327    }
1328
1329    #[test]
1330    fn the_shape_itself_still_moves_the_vector() {
1331        // The counterpart to the test above. If everything collided the
1332        // anonymity test would pass while the featurizer did nothing.
1333        let flat = vector("https://www.example.com/");
1334        let deep = vector("https://www.example.com/a/b/c/d/e/f/g");
1335        let data = vector("https://www.example.com/a/b/c/d/e/f/g.json");
1336
1337        assert_ne!(flat, deep);
1338        assert_ne!(deep, data);
1339    }
1340
1341    #[test]
1342    fn path_depth_and_length_are_bucketed() {
1343        let v = vector("https://example.com/a/b/c");
1344        assert_eq!(v.get(PATH_DEPTH + 3), Some(1.0));
1345        assert_eq!(v.get(PATH_DEPTH + 2), Some(0.0));
1346
1347        let deep = vector("https://example.com/a/b/c/d/e/f/g/h/i");
1348        assert_eq!(deep.get(PATH_DEPTH + 6), Some(1.0));
1349    }
1350
1351    #[test]
1352    fn segment_classes_split_the_ways_paths_are_generated() {
1353        assert_eq!(segment_class("12345"), SegmentClass::Digits);
1354        assert_eq!(segment_class("about"), SegmentClass::Alpha);
1355        assert_eq!(segment_class("page2"), SegmentClass::AlphaNum);
1356        assert_eq!(segment_class("a-long-story"), SegmentClass::Hyphenated);
1357        assert_eq!(segment_class("a_long_story"), SegmentClass::Underscored);
1358        assert_eq!(
1359            segment_class("f47ac10b-58cc-4372-a567-0e02b2c3d479"),
1360            SegmentClass::HexIdentifier
1361        );
1362        assert_eq!(segment_class("caf%C3%A9"), SegmentClass::Escaped);
1363        assert_eq!(segment_class("a+b"), SegmentClass::Other);
1364    }
1365
1366    #[test]
1367    fn a_non_ascii_segment_is_classified_without_splitting_a_character() {
1368        // `Url` escapes this on parse, so the feature path sees the escaped
1369        // form. The classifier is written over characters either way.
1370        assert_eq!(segment_class("\u{5546}\u{54c1}"), SegmentClass::Alpha);
1371        let v = vector("https://example.com/\u{5546}\u{54c1}/1");
1372        assert_eq!(v.get(PATH_DEPTH + 2), Some(1.0));
1373    }
1374
1375    #[test]
1376    fn extensions_are_classified_and_versions_are_not_extensions() {
1377        assert_eq!(extension_class("index.html"), ExtClass::Markup);
1378        assert_eq!(extension_class("feed.xml"), ExtClass::Xml);
1379        assert_eq!(extension_class("items.json"), ExtClass::Json);
1380        assert_eq!(extension_class("report.PDF"), ExtClass::Pdf);
1381        assert_eq!(extension_class("hero.png"), ExtClass::Image);
1382        assert_eq!(extension_class("about"), ExtClass::None);
1383        assert_eq!(extension_class("v1.2.3"), ExtClass::None);
1384        assert_eq!(extension_class("archive.tar"), ExtClass::Archive);
1385        assert_eq!(extension_class("thing.wobble"), ExtClass::Other);
1386    }
1387
1388    #[test]
1389    fn query_keys_are_read_and_values_are_not() {
1390        let with_value = vector("https://example.com/s?q=a+very+identifying+search");
1391        let other_value = vector("https://example.com/s?q=b");
1392
1393        assert_eq!(
1394            with_value, other_value,
1395            "two different search terms changed the vector, so a value is being read"
1396        );
1397        assert_eq!(
1398            with_value.get(QUERY_KEYS + KeyClass::Search.index()),
1399            Some(1.0)
1400        );
1401        assert_eq!(
1402            with_value.get(QUERY_KEYS + KeyClass::SingleChar.index()),
1403            Some(1.0)
1404        );
1405    }
1406
1407    #[test]
1408    fn a_key_can_fall_into_several_classes_at_once() {
1409        let v = vector("https://example.com/l?page=2&utm_source=mail&sort=price&id=9");
1410
1411        for class in [
1412            KeyClass::Pagination,
1413            KeyClass::Tracking,
1414            KeyClass::Filter,
1415            KeyClass::Identifier,
1416        ] {
1417            assert_eq!(v.get(QUERY_KEYS + class.index()), Some(1.0), "{class:?}");
1418        }
1419
1420        assert_eq!(v.get(QUERY_ARITY + 4), Some(1.0));
1421    }
1422
1423    #[test]
1424    fn cold_start_is_a_value_rather_than_a_gap() {
1425        let v = vector("https://example.com/");
1426
1427        assert_eq!(v.get(MEM_COUNT), Some(1.0));
1428        assert_eq!(v.get(MEM_STATUS + StatusClass::Unknown.index()), Some(1.0));
1429        assert_eq!(v.get(MEM_FAIL_STREAK), Some(1.0));
1430    }
1431
1432    #[test]
1433    fn memory_lands_in_its_own_block() {
1434        let url = Url::parse("https://example.com/").unwrap();
1435        let memory = SiteMemory {
1436            observations: 12,
1437            success_rate: 0.1,
1438            streak: -3,
1439            last_status: StatusClass::Blocked,
1440        };
1441        let v = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown).with_memory(&memory));
1442
1443        assert_eq!(v.get(MEM_COUNT + 3), Some(1.0));
1444        assert_eq!(v.get(MEM_SUCCESS), Some(1.0));
1445        assert_eq!(v.get(MEM_FAIL_STREAK + 3), Some(1.0));
1446        assert_eq!(v.get(MEM_WIN_STREAK), Some(1.0));
1447        assert_eq!(v.get(MEM_STATUS + StatusClass::Blocked.index()), Some(1.0));
1448    }
1449
1450    #[test]
1451    fn the_hour_is_two_waves_and_an_unknown() {
1452        let url = Url::parse("https://example.com/").unwrap();
1453        let unknown = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown));
1454        assert_eq!(unknown.get(HOUR + 2), Some(1.0));
1455        assert_eq!(unknown.get(HOUR), Some(0.0));
1456
1457        let noon = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown).at_hour(12));
1458        assert_eq!(noon.get(HOUR + 2), Some(0.0));
1459        assert!(noon.get(HOUR + 1).unwrap_or(0.0) < -0.99);
1460
1461        // Both waves have to move, or half the clock is not being recorded.
1462        let dawn = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown).at_hour(6));
1463        assert!(dawn.get(HOUR).unwrap_or(0.0) > 0.99);
1464        assert!(dawn.get(HOUR + 1).unwrap_or(1.0).abs() < 0.01);
1465
1466        let dusk = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown).at_hour(18));
1467        assert!(dusk.get(HOUR).unwrap_or(0.0) < -0.99);
1468
1469        // Eleven at night sits next to midnight, which is the reason for waves
1470        // rather than twenty four slots.
1471        let late = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown).at_hour(23));
1472        let midnight = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown).at_hour(0));
1473        let gap = (late.get(HOUR).unwrap_or(0.0) - midnight.get(HOUR).unwrap_or(0.0)).abs();
1474        assert!(gap < 0.3, "the clock does not wrap, gap was {gap}");
1475    }
1476
1477    #[test]
1478    fn an_out_of_range_hour_is_clamped_rather_than_wrapping_into_a_panic() {
1479        let url = Url::parse("https://example.com/").unwrap();
1480        let v = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown).at_hour(200));
1481        assert_eq!(v.get(HOUR + 2), Some(0.0));
1482    }
1483
1484    #[test]
1485    fn pins_are_recorded_without_being_obeyed_here() {
1486        let url = Url::parse("https://example.com/").unwrap();
1487        let country = Country::new("de").unwrap();
1488        let pins = CallerPins {
1489            mode: Some(RequestMode::Browser),
1490            proxy: Some(ProxyPool::Residential),
1491            country: Some(&country),
1492        };
1493        let v = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown).with_pins(pins));
1494
1495        assert_eq!(v.get(PINS), Some(1.0));
1496        assert_eq!(v.get(PINS + 3), Some(1.0));
1497        assert_eq!(v.get(PINS + 5), Some(1.0));
1498        assert_eq!(v.get(PINS + 6), Some(1.0));
1499    }
1500
1501    #[test]
1502    fn a_site_with_no_path_is_not_a_trailing_slash_site() {
1503        let bare = vector("https://example.com");
1504        let slashed = vector("https://example.com/");
1505
1506        assert_eq!(bare, slashed);
1507        assert_eq!(bare.get(TRAILING_SLASH), Some(0.0));
1508        assert_eq!(bare.get(SEGMENT + SegmentClass::Empty.index()), Some(1.0));
1509    }
1510
1511    #[test]
1512    fn url_flags_read_the_request_and_not_the_site() {
1513        let plain = vector("http://example.com/a");
1514        assert_eq!(plain.get(URL_FLAGS), Some(0.0));
1515
1516        let odd = vector("https://user:pw@example.com:8443/a#top");
1517        assert_eq!(odd.get(URL_FLAGS), Some(1.0));
1518        assert_eq!(odd.get(URL_FLAGS + 1), Some(1.0));
1519        assert_eq!(odd.get(URL_FLAGS + 2), Some(1.0));
1520        assert_eq!(odd.get(URL_FLAGS + 3), Some(1.0));
1521    }
1522
1523    #[test]
1524    fn the_bias_slot_is_always_set() {
1525        assert_eq!(vector("https://example.com/").get(BIAS), Some(1.0));
1526    }
1527}