1use crate::action::{Country, ProxyPool, RequestMode};
27use crate::domain::{host_shape, HostShape, IpKind, TLD_SLOTS};
28use url::Url;
29
30pub const FEATURE_DIM: usize = 256;
35
36#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
43#[non_exhaustive]
44pub enum DeclaredNeed {
45 Text,
47 #[default]
49 Markdown,
50 Html,
52 Links,
54 Metadata,
56 Fields,
58 Screenshot,
60 Raw,
62}
63
64impl DeclaredNeed {
65 const fn index(self) -> usize {
67 match self {
68 DeclaredNeed::Text => 0,
69 DeclaredNeed::Markdown => 1,
70 DeclaredNeed::Html => 2,
71 DeclaredNeed::Links => 3,
72 DeclaredNeed::Metadata => 4,
73 DeclaredNeed::Fields => 5,
74 DeclaredNeed::Screenshot => 6,
75 DeclaredNeed::Raw => 7,
76 }
77 }
78}
79
80#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
83#[non_exhaustive]
84pub enum StatusClass {
85 #[default]
87 Unknown,
88 Ok,
90 Empty,
93 BadRequest,
95 NeedsLogin,
97 Blocked,
99 NotFound,
101 RateLimited,
103 ServerError,
105}
106
107impl StatusClass {
108 const fn index(self) -> usize {
110 match self {
111 StatusClass::Unknown => 0,
112 StatusClass::Ok => 1,
113 StatusClass::Empty => 2,
114 StatusClass::BadRequest => 3,
115 StatusClass::NeedsLogin => 4,
116 StatusClass::Blocked => 5,
117 StatusClass::NotFound => 6,
118 StatusClass::RateLimited => 7,
119 StatusClass::ServerError => 8,
120 }
121 }
122
123 pub const fn is_refusal(self) -> bool {
126 matches!(self, StatusClass::Blocked | StatusClass::NeedsLogin)
127 }
128}
129
130#[derive(Debug, Clone, Copy, PartialEq, Default)]
138pub struct SiteMemory {
139 pub observations: u32,
143 pub success_rate: f32,
145 pub streak: i16,
148 pub last_status: StatusClass,
150}
151
152impl SiteMemory {
153 pub const fn cold() -> SiteMemory {
156 SiteMemory {
157 observations: 0,
158 success_rate: 0.0,
159 streak: 0,
160 last_status: StatusClass::Unknown,
161 }
162 }
163
164 pub const fn failure_streak(self) -> u16 {
166 if self.streak < 0 {
167 self.streak.unsigned_abs()
168 } else {
169 0
170 }
171 }
172
173 pub const fn is_informative(self) -> bool {
175 self.observations >= 2
176 }
177}
178
179#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
186pub struct CallerPins<'a> {
187 pub mode: Option<RequestMode>,
189 pub proxy: Option<ProxyPool>,
191 pub country: Option<&'a Country>,
193}
194
195impl<'a> CallerPins<'a> {
196 pub const fn none() -> CallerPins<'a> {
198 CallerPins {
199 mode: None,
200 proxy: None,
201 country: None,
202 }
203 }
204
205 pub const fn any(&self) -> bool {
207 self.mode.is_some() || self.proxy.is_some() || self.country.is_some()
208 }
209}
210
211#[derive(Debug, Clone, Copy, PartialEq)]
216#[non_exhaustive]
217pub struct RouteInput<'a> {
218 pub url: &'a Url,
220 pub need: DeclaredNeed,
222 pub pins: CallerPins<'a>,
224 pub memory: Option<&'a SiteMemory>,
226 pub hour: Option<u8>,
232}
233
234impl<'a> RouteInput<'a> {
235 pub const fn new(url: &'a Url, need: DeclaredNeed) -> RouteInput<'a> {
237 RouteInput {
238 url,
239 need,
240 pins: CallerPins::none(),
241 memory: None,
242 hour: None,
243 }
244 }
245
246 pub const fn with_pins(mut self, pins: CallerPins<'a>) -> RouteInput<'a> {
248 self.pins = pins;
249 self
250 }
251
252 pub const fn with_memory(mut self, memory: &'a SiteMemory) -> RouteInput<'a> {
254 self.memory = Some(memory);
255 self
256 }
257
258 pub const fn at_hour(mut self, hour: u8) -> RouteInput<'a> {
260 self.hour = Some(hour);
261 self
262 }
263}
264
265#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
272#[non_exhaustive]
273pub enum SegmentClass {
274 #[default]
276 Empty,
277 Digits,
279 Alpha,
281 AlphaNum,
283 Hyphenated,
285 Underscored,
287 HexIdentifier,
289 Escaped,
291 Other,
293}
294
295impl SegmentClass {
296 const fn index(self) -> usize {
298 match self {
299 SegmentClass::Empty => 0,
300 SegmentClass::Digits => 1,
301 SegmentClass::Alpha => 2,
302 SegmentClass::AlphaNum => 3,
303 SegmentClass::Hyphenated => 4,
304 SegmentClass::Underscored => 5,
305 SegmentClass::HexIdentifier => 6,
306 SegmentClass::Escaped => 7,
307 SegmentClass::Other => 8,
308 }
309 }
310}
311
312#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
318#[non_exhaustive]
319pub enum ExtClass {
320 #[default]
322 None,
323 Markup,
325 Xml,
327 Json,
329 Feed,
331 Text,
333 Csv,
335 Pdf,
337 Office,
339 Image,
341 Media,
343 Archive,
345 Asset,
347 Other,
349}
350
351impl ExtClass {
352 const fn index(self) -> usize {
354 match self {
355 ExtClass::None => 0,
356 ExtClass::Markup => 1,
357 ExtClass::Xml => 2,
358 ExtClass::Json => 3,
359 ExtClass::Feed => 4,
360 ExtClass::Text => 5,
361 ExtClass::Csv => 6,
362 ExtClass::Pdf => 7,
363 ExtClass::Office => 8,
364 ExtClass::Image => 9,
365 ExtClass::Media => 10,
366 ExtClass::Archive => 11,
367 ExtClass::Asset => 12,
368 ExtClass::Other => 13,
369 }
370 }
371
372 pub const fn can_need_rendering(self) -> bool {
378 matches!(self, ExtClass::None | ExtClass::Markup | ExtClass::Other)
379 }
380}
381
382#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
389#[non_exhaustive]
390pub enum KeyClass {
391 Pagination,
393 Identifier,
395 Search,
397 Tracking,
399 Session,
401 Locale,
403 Format,
405 Filter,
407 SingleChar,
409 Numeric,
411}
412
413const KEY_CLASSES: usize = 10;
415
416impl KeyClass {
417 const fn index(self) -> usize {
419 match self {
420 KeyClass::Pagination => 0,
421 KeyClass::Identifier => 1,
422 KeyClass::Search => 2,
423 KeyClass::Tracking => 3,
424 KeyClass::Session => 4,
425 KeyClass::Locale => 5,
426 KeyClass::Format => 6,
427 KeyClass::Filter => 7,
428 KeyClass::SingleChar => 8,
429 KeyClass::Numeric => 9,
430 }
431 }
432}
433
434#[derive(Copy, Clone, Debug, PartialEq)]
440pub struct FeatureVector {
441 bits: [f32; FEATURE_DIM],
442}
443
444impl Default for FeatureVector {
445 fn default() -> FeatureVector {
448 FeatureVector::zeroed()
449 }
450}
451
452impl FeatureVector {
453 pub const fn zeroed() -> FeatureVector {
455 FeatureVector {
456 bits: [0.0; FEATURE_DIM],
457 }
458 }
459
460 pub const fn as_slice(&self) -> &[f32] {
462 &self.bits
463 }
464
465 pub fn get(&self, slot: usize) -> Option<f32> {
467 self.bits.get(slot).copied()
468 }
469
470 fn set(&mut self, slot: usize, value: f32) {
473 if let Some(cell) = self.bits.get_mut(slot) {
474 *cell = value;
475 }
476 }
477
478 fn one_hot(&mut self, base: usize, offset: usize) {
480 self.set(base + offset, 1.0);
481 }
482}
483
484pub const PATH_DEPTH: usize = 0;
491const PATH_DEPTH_W: usize = 7;
492
493pub const PATH_LEN: usize = PATH_DEPTH + PATH_DEPTH_W;
495const PATH_LEN_W: usize = 6;
496
497pub const SEGMENT: usize = PATH_LEN + PATH_LEN_W;
499const SEGMENT_W: usize = 9;
500
501pub const TRAILING_SLASH: usize = SEGMENT + SEGMENT_W;
503const TRAILING_SLASH_W: usize = 1;
504
505pub const EXTENSION: usize = TRAILING_SLASH + TRAILING_SLASH_W;
507const EXTENSION_W: usize = 14;
508
509pub const QUERY_ARITY: usize = EXTENSION + EXTENSION_W;
511const QUERY_ARITY_W: usize = 6;
512
513pub const QUERY_KEYS: usize = QUERY_ARITY + QUERY_ARITY_W;
516const QUERY_KEYS_W: usize = KEY_CLASSES;
517
518pub const QUERY_KEY_LEN: usize = QUERY_KEYS + QUERY_KEYS_W;
520const QUERY_KEY_LEN_W: usize = 4;
521
522pub const SUBDOMAINS: usize = QUERY_KEY_LEN + QUERY_KEY_LEN_W;
524const SUBDOMAINS_W: usize = 4;
525
526pub const WWW: usize = SUBDOMAINS + SUBDOMAINS_W;
528const WWW_W: usize = 1;
529
530pub const IP_LITERAL: usize = WWW + WWW_W;
533const IP_LITERAL_W: usize = 2;
534
535pub const PUNYCODE: usize = IP_LITERAL + IP_LITERAL_W;
537const PUNYCODE_W: usize = 1;
538
539pub const URL_FLAGS: usize = PUNYCODE + PUNYCODE_W;
542const URL_FLAGS_W: usize = 4;
543
544pub const TLD: usize = URL_FLAGS + URL_FLAGS_W;
546const TLD_W: usize = TLD_SLOTS;
547
548pub const POPULARITY: usize = TLD + TLD_W;
556const POPULARITY_W: usize = 6;
557
558pub const NEED: usize = POPULARITY + POPULARITY_W;
560const NEED_W: usize = 8;
561
562pub const PINS: usize = NEED + NEED_W;
565const PINS_W: usize = 8;
566
567pub const HOUR: usize = PINS + PINS_W;
572const HOUR_W: usize = 3;
573
574pub const MEM_COUNT: usize = HOUR + HOUR_W;
577const MEM_COUNT_W: usize = 5;
578
579pub const MEM_SUCCESS: usize = MEM_COUNT + MEM_COUNT_W;
581const MEM_SUCCESS_W: usize = 5;
582
583pub const MEM_FAIL_STREAK: usize = MEM_SUCCESS + MEM_SUCCESS_W;
585const MEM_FAIL_STREAK_W: usize = 4;
586
587pub const MEM_WIN_STREAK: usize = MEM_FAIL_STREAK + MEM_FAIL_STREAK_W;
589const MEM_WIN_STREAK_W: usize = 4;
590
591pub const MEM_STATUS: usize = MEM_WIN_STREAK + MEM_WIN_STREAK_W;
593const MEM_STATUS_W: usize = 9;
594
595pub const BIAS: usize = MEM_STATUS + MEM_STATUS_W;
597const BIAS_W: usize = 1;
598
599pub const FEATURES_USED: usize = BIAS + BIAS_W;
607
608const _: () = assert!(FEATURES_USED <= FEATURE_DIM);
609
610pub fn featurize(input: &RouteInput<'_>) -> FeatureVector {
617 let mut out = FeatureVector::zeroed();
618 let url = input.url;
619
620 let path = url.path();
622 let mut depth = 0usize;
623 let mut last = "";
624
625 for segment in path.split('/') {
626 if !segment.is_empty() {
627 depth += 1;
628 last = segment;
629 }
630 }
631
632 out.one_hot(PATH_DEPTH, depth.min(6));
633 out.one_hot(PATH_LEN, length_bucket(path.len()));
634
635 let trailing = path.len() > 1 && path.ends_with('/');
636 let segment_class = if trailing || last.is_empty() {
637 SegmentClass::Empty
638 } else {
639 segment_class(last)
640 };
641
642 out.one_hot(SEGMENT, segment_class.index());
643
644 if trailing {
645 out.set(TRAILING_SLASH, 1.0);
646 }
647
648 out.one_hot(EXTENSION, extension_class(last).index());
649
650 let mut arity = 0usize;
652 let mut longest_key = 0usize;
653
654 for pair in url.query().unwrap_or("").split('&') {
655 if pair.is_empty() {
656 continue;
657 }
658
659 arity += 1;
660
661 let key = match pair.split_once('=') {
662 Some((key, _value)) => key,
663 None => pair,
664 };
665
666 longest_key = longest_key.max(key.chars().count());
667
668 for class in key_classes(key) {
669 out.one_hot(QUERY_KEYS, class.index());
670 }
671 }
672
673 out.one_hot(QUERY_ARITY, arity_bucket(arity));
674
675 if arity > 0 {
676 out.one_hot(QUERY_KEY_LEN, key_length_bucket(longest_key));
677 }
678
679 let host: HostShape = host_shape(url);
681
682 out.one_hot(SUBDOMAINS, usize::from(host.subdomains).min(3));
683
684 if host.www {
685 out.set(WWW, 1.0);
686 }
687
688 match host.ip {
689 Some(IpKind::V4) => out.one_hot(IP_LITERAL, 0),
690 Some(IpKind::V6) => out.one_hot(IP_LITERAL, 1),
691 None => {}
692 }
693
694 if host.punycode {
695 out.set(PUNYCODE, 1.0);
696 }
697
698 if let Some(tld) = host.tld {
699 out.one_hot(TLD, tld.index());
700 }
701
702 if url.scheme() == "https" {
704 out.set(URL_FLAGS, 1.0);
705 }
706
707 if url.port().is_some() {
708 out.set(URL_FLAGS + 1, 1.0);
709 }
710
711 if !url.username().is_empty() || url.password().is_some() {
712 out.set(URL_FLAGS + 2, 1.0);
713 }
714
715 if url.fragment().is_some() {
716 out.set(URL_FLAGS + 3, 1.0);
717 }
718
719 let _ = POPULARITY;
722
723 out.one_hot(NEED, input.need.index());
725
726 if input.pins.any() {
727 out.set(PINS, 1.0);
728 }
729
730 match input.pins.mode {
731 Some(RequestMode::Http) => out.one_hot(PINS, 1),
732 Some(RequestMode::Smart) => out.one_hot(PINS, 2),
733 Some(RequestMode::Browser) => out.one_hot(PINS, 3),
734 _ => {}
735 }
736
737 match input.pins.proxy {
738 Some(ProxyPool::Isp) => out.one_hot(PINS, 4),
739 Some(ProxyPool::Residential) => out.one_hot(PINS, 5),
740 _ => {}
741 }
742
743 if input.pins.country.is_some() {
744 out.one_hot(PINS, 6);
745 }
746
747 match input.hour {
751 Some(hour) => {
752 let turns = f32::from(hour.min(23)) / 24.0 * std::f32::consts::TAU;
753 out.set(HOUR, turns.sin());
754 out.set(HOUR + 1, turns.cos());
755 }
756 None => out.set(HOUR + 2, 1.0),
757 }
758
759 let memory = input.memory.copied().unwrap_or_else(SiteMemory::cold);
761
762 out.one_hot(MEM_COUNT, observation_bucket(memory.observations));
763 out.one_hot(MEM_SUCCESS, success_bucket(memory.success_rate));
764 out.one_hot(MEM_FAIL_STREAK, streak_bucket(memory.failure_streak()));
765 out.one_hot(
766 MEM_WIN_STREAK,
767 streak_bucket(memory.streak.max(0).unsigned_abs()),
768 );
769 out.one_hot(MEM_STATUS, memory.last_status.index());
770
771 out.set(BIAS, 1.0);
772
773 out
774}
775
776const fn length_bucket(len: usize) -> usize {
778 match len {
779 0..=16 => 0,
780 17..=32 => 1,
781 33..=64 => 2,
782 65..=128 => 3,
783 129..=256 => 4,
784 _ => 5,
785 }
786}
787
788const fn arity_bucket(count: usize) -> usize {
790 match count {
791 0 => 0,
792 1 => 1,
793 2 => 2,
794 3 => 3,
795 4..=6 => 4,
796 _ => 5,
797 }
798}
799
800const fn key_length_bucket(len: usize) -> usize {
802 match len {
803 0..=2 => 0,
804 3..=6 => 1,
805 7..=12 => 2,
806 _ => 3,
807 }
808}
809
810const fn observation_bucket(count: u32) -> usize {
812 match count {
813 0 => 0,
814 1..=2 => 1,
815 3..=9 => 2,
816 10..=49 => 3,
817 _ => 4,
818 }
819}
820
821fn success_bucket(rate: f32) -> usize {
827 if rate.is_nan() {
828 return 0;
829 }
830
831 let rate = rate.clamp(0.0, 1.0);
832
833 if rate < 0.2 {
834 0
835 } else if rate < 0.4 {
836 1
837 } else if rate < 0.6 {
838 2
839 } else if rate < 0.8 {
840 3
841 } else {
842 4
843 }
844}
845
846const fn streak_bucket(streak: u16) -> usize {
848 match streak {
849 0 => 0,
850 1 => 1,
851 2 => 2,
852 _ => 3,
853 }
854}
855
856pub fn segment_class(segment: &str) -> SegmentClass {
859 if segment.contains('%') {
860 return SegmentClass::Escaped;
861 }
862
863 let mut letters = 0usize;
864 let mut digits = 0usize;
865 let mut hyphens = 0usize;
866 let mut underscores = 0usize;
867 let mut hex = 0usize;
868 let mut other = 0usize;
869 let mut total = 0usize;
870
871 for ch in segment.chars() {
872 total += 1;
873
874 if ch.is_ascii_digit() {
875 digits += 1;
876 hex += 1;
877 } else if ch.is_ascii_hexdigit() {
878 letters += 1;
879 hex += 1;
880 } else if ch.is_alphabetic() {
881 letters += 1;
882 } else if ch == '-' {
883 hyphens += 1;
884 } else if ch == '_' {
885 underscores += 1;
886 } else {
887 other += 1;
888 }
889 }
890
891 if total == 0 {
892 return SegmentClass::Empty;
893 }
894
895 if hex + hyphens == total && hex >= 16 && digits > 0 {
898 return SegmentClass::HexIdentifier;
899 }
900
901 if other > 0 {
902 return SegmentClass::Other;
903 }
904
905 if digits == total {
906 return SegmentClass::Digits;
907 }
908
909 if letters == total {
910 return SegmentClass::Alpha;
911 }
912
913 if hyphens > 0 && underscores == 0 {
914 return SegmentClass::Hyphenated;
915 }
916
917 if underscores > 0 && hyphens == 0 {
918 return SegmentClass::Underscored;
919 }
920
921 if hyphens == 0 && underscores == 0 {
922 return SegmentClass::AlphaNum;
923 }
924
925 SegmentClass::Other
926}
927
928pub fn extension_class(segment: &str) -> ExtClass {
930 let ext = match segment.rsplit_once('.') {
931 Some((stem, ext)) if !stem.is_empty() && !ext.is_empty() => ext,
932 _ => return ExtClass::None,
933 };
934
935 if ext.chars().count() > 8
938 || ext.chars().any(|ch| !ch.is_ascii_alphanumeric())
939 || ext.chars().all(|ch| ch.is_ascii_digit())
940 {
941 return ExtClass::None;
942 }
943
944 const TABLE: [(&str, ExtClass); 51] = [
945 ("html", ExtClass::Markup),
946 ("htm", ExtClass::Markup),
947 ("xhtml", ExtClass::Markup),
948 ("shtml", ExtClass::Markup),
949 ("php", ExtClass::Markup),
950 ("asp", ExtClass::Markup),
951 ("aspx", ExtClass::Markup),
952 ("jsp", ExtClass::Markup),
953 ("cfm", ExtClass::Markup),
954 ("xml", ExtClass::Xml),
955 ("xsl", ExtClass::Xml),
956 ("json", ExtClass::Json),
957 ("jsonl", ExtClass::Json),
958 ("ndjson", ExtClass::Json),
959 ("rss", ExtClass::Feed),
960 ("atom", ExtClass::Feed),
961 ("txt", ExtClass::Text),
962 ("md", ExtClass::Text),
963 ("rst", ExtClass::Text),
964 ("csv", ExtClass::Csv),
965 ("tsv", ExtClass::Csv),
966 ("pdf", ExtClass::Pdf),
967 ("doc", ExtClass::Office),
968 ("docx", ExtClass::Office),
969 ("xls", ExtClass::Office),
970 ("xlsx", ExtClass::Office),
971 ("ppt", ExtClass::Office),
972 ("pptx", ExtClass::Office),
973 ("odt", ExtClass::Office),
974 ("jpg", ExtClass::Image),
975 ("jpeg", ExtClass::Image),
976 ("png", ExtClass::Image),
977 ("gif", ExtClass::Image),
978 ("webp", ExtClass::Image),
979 ("svg", ExtClass::Image),
980 ("avif", ExtClass::Image),
981 ("ico", ExtClass::Image),
982 ("mp4", ExtClass::Media),
983 ("webm", ExtClass::Media),
984 ("mov", ExtClass::Media),
985 ("mp3", ExtClass::Media),
986 ("wav", ExtClass::Media),
987 ("m3u8", ExtClass::Media),
988 ("zip", ExtClass::Archive),
989 ("gz", ExtClass::Archive),
990 ("tar", ExtClass::Archive),
991 ("rar", ExtClass::Archive),
992 ("7z", ExtClass::Archive),
993 ("js", ExtClass::Asset),
994 ("css", ExtClass::Asset),
995 ("wasm", ExtClass::Asset),
996 ];
997
998 for (name, class) in TABLE {
999 if ext.eq_ignore_ascii_case(name) {
1000 return class;
1001 }
1002 }
1003
1004 ExtClass::Other
1005}
1006
1007pub fn extension_of(url: &Url) -> ExtClass {
1012 extension_class(last_segment(url))
1013}
1014
1015fn last_segment(url: &Url) -> &str {
1017 let path = url.path();
1018
1019 if path.ends_with('/') {
1020 return "";
1021 }
1022
1023 path.rsplit('/')
1024 .find(|segment| !segment.is_empty())
1025 .unwrap_or_default()
1026}
1027
1028fn key_classes(key: &str) -> impl Iterator<Item = KeyClass> {
1031 const PAGINATION: [&str; 8] = [
1032 "page", "p", "pg", "offset", "start", "limit", "per_page", "cursor",
1033 ];
1034 const IDENTIFIER: [&str; 8] = ["id", "pid", "sku", "uid", "item", "product", "ref", "asin"];
1035 const SEARCH: [&str; 6] = ["q", "query", "s", "search", "keyword", "term"];
1036 const SESSION: [&str; 7] = [
1037 "token",
1038 "session",
1039 "sid",
1040 "auth",
1041 "key",
1042 "signature",
1043 "nonce",
1044 ];
1045 const LOCALE: [&str; 7] = [
1046 "lang", "locale", "hl", "gl", "country", "region", "currency",
1047 ];
1048 const FORMAT: [&str; 5] = ["format", "output", "type", "view", "render"];
1049 const FILTER: [&str; 8] = [
1050 "sort", "order", "filter", "category", "tag", "min", "max", "price",
1051 ];
1052
1053 let matches = |table: &[&str]| table.iter().any(|name| key.eq_ignore_ascii_case(name));
1054
1055 let mut found = [None; KEY_CLASSES];
1056 let mut at = 0usize;
1057 let mut push = |class: KeyClass| {
1058 if let Some(cell) = found.get_mut(at) {
1059 *cell = Some(class);
1060 at += 1;
1061 }
1062 };
1063
1064 if matches(&PAGINATION) {
1065 push(KeyClass::Pagination);
1066 }
1067
1068 if matches(&IDENTIFIER) {
1069 push(KeyClass::Identifier);
1070 }
1071
1072 if matches(&SEARCH) {
1073 push(KeyClass::Search);
1074 }
1075
1076 if key.len() > 4 && key.get(..4).is_some_and(|p| p.eq_ignore_ascii_case("utm_"))
1079 || key.eq_ignore_ascii_case("gclid")
1080 || key.eq_ignore_ascii_case("fbclid")
1081 || key.eq_ignore_ascii_case("msclkid")
1082 {
1083 push(KeyClass::Tracking);
1084 }
1085
1086 if matches(&SESSION) {
1087 push(KeyClass::Session);
1088 }
1089
1090 if matches(&LOCALE) {
1091 push(KeyClass::Locale);
1092 }
1093
1094 if matches(&FORMAT) {
1095 push(KeyClass::Format);
1096 }
1097
1098 if matches(&FILTER) {
1099 push(KeyClass::Filter);
1100 }
1101
1102 if key.chars().count() == 1 {
1103 push(KeyClass::SingleChar);
1104 }
1105
1106 if !key.is_empty() && key.chars().all(|ch| ch.is_ascii_digit()) {
1107 push(KeyClass::Numeric);
1108 }
1109
1110 found.into_iter().flatten()
1111}
1112
1113#[cfg(test)]
1114mod tests {
1115 #![allow(
1118 clippy::unwrap_used,
1119 clippy::expect_used,
1120 clippy::panic,
1121 clippy::unreachable,
1122 clippy::indexing_slicing,
1123 clippy::string_slice
1124 )]
1125 use super::*;
1126
1127 fn vector(url: &str) -> FeatureVector {
1128 let parsed = Url::parse(url).unwrap();
1129 featurize(&RouteInput::new(&parsed, DeclaredNeed::Markdown))
1130 }
1131
1132 #[test]
1133 fn feature_vector_holds_no_text() {
1134 assert_eq!(
1138 std::mem::size_of::<FeatureVector>(),
1139 FEATURE_DIM * std::mem::size_of::<f32>()
1140 );
1141 }
1142
1143 #[test]
1144 fn the_layout_fits_and_the_blocks_do_not_overlap() {
1145 let bases = [
1146 ("path depth", PATH_DEPTH, PATH_DEPTH_W),
1147 ("path length", PATH_LEN, PATH_LEN_W),
1148 ("segment", SEGMENT, SEGMENT_W),
1149 ("trailing slash", TRAILING_SLASH, TRAILING_SLASH_W),
1150 ("extension", EXTENSION, EXTENSION_W),
1151 ("query arity", QUERY_ARITY, QUERY_ARITY_W),
1152 ("query keys", QUERY_KEYS, QUERY_KEYS_W),
1153 ("query key length", QUERY_KEY_LEN, QUERY_KEY_LEN_W),
1154 ("subdomains", SUBDOMAINS, SUBDOMAINS_W),
1155 ("www", WWW, WWW_W),
1156 ("ip literal", IP_LITERAL, IP_LITERAL_W),
1157 ("punycode", PUNYCODE, PUNYCODE_W),
1158 ("url flags", URL_FLAGS, URL_FLAGS_W),
1159 ("tld", TLD, TLD_W),
1160 ("popularity", POPULARITY, POPULARITY_W),
1161 ("need", NEED, NEED_W),
1162 ("pins", PINS, PINS_W),
1163 ("hour", HOUR, HOUR_W),
1164 ("memory count", MEM_COUNT, MEM_COUNT_W),
1165 ("memory success", MEM_SUCCESS, MEM_SUCCESS_W),
1166 ("memory failures", MEM_FAIL_STREAK, MEM_FAIL_STREAK_W),
1167 ("memory wins", MEM_WIN_STREAK, MEM_WIN_STREAK_W),
1168 ("memory status", MEM_STATUS, MEM_STATUS_W),
1169 ("bias", BIAS, BIAS_W),
1170 ];
1171
1172 let mut next = 0usize;
1173
1174 for (name, base, width) in bases {
1175 assert_eq!(
1176 base, next,
1177 "{name} does not start where the block before it ends"
1178 );
1179 next = base + width;
1180 }
1181
1182 assert_eq!(next, FEATURES_USED);
1183 assert_eq!(bases.len(), 24);
1184 let room = vector("https://example.com/").as_slice().len() - next;
1187 assert!(room > 0, "the layout has filled the vector");
1188 }
1189
1190 #[test]
1191 fn every_block_is_wide_enough_for_what_it_holds() {
1192 assert_eq!(SEGMENT_W, 9);
1197 assert_eq!(EXTENSION_W, 14);
1198 assert_eq!(QUERY_KEYS_W, 10);
1199 assert_eq!(NEED_W, 8);
1200 assert_eq!(MEM_STATUS_W, 9);
1201 assert_eq!(TLD_W, 30);
1202
1203 let widest = [
1204 ("segment", SegmentClass::Other.index(), SEGMENT_W),
1205 ("extension", ExtClass::Other.index(), EXTENSION_W),
1206 ("query keys", KeyClass::Numeric.index(), QUERY_KEYS_W),
1207 ("need", DeclaredNeed::Raw.index(), NEED_W),
1208 (
1209 "memory status",
1210 StatusClass::ServerError.index(),
1211 MEM_STATUS_W,
1212 ),
1213 ("tld", crate::domain::TldSlot::NewGtld.index(), TLD_W),
1214 ];
1215
1216 for (name, last, width) in widest {
1217 assert_eq!(
1218 last + 1,
1219 width,
1220 "the {name} block is {width} wide and its last value sits at {last}"
1221 );
1222 }
1223 }
1224
1225 #[test]
1226 fn a_value_at_the_end_of_a_block_stays_inside_it() {
1227 let last_in_block = vector("https://example.com/thing.wobble");
1231
1232 assert_eq!(
1233 last_in_block.get(EXTENSION + ExtClass::Other.index()),
1234 Some(1.0)
1235 );
1236 assert_eq!(last_in_block.get(QUERY_ARITY), Some(1.0));
1237 assert_eq!(last_in_block.get(QUERY_ARITY + 1), Some(0.0));
1238
1239 let one_query = vector("https://example.com/thing.wobble?a=1");
1240 assert_eq!(one_query.get(QUERY_ARITY), Some(0.0));
1241 assert_eq!(one_query.get(QUERY_ARITY + 1), Some(1.0));
1242 }
1243
1244 #[test]
1245 fn the_popularity_block_is_reserved_and_empty() {
1246 let v = vector("https://www.example.com/a/b/c");
1247
1248 for slot in POPULARITY..POPULARITY + POPULARITY_W {
1249 assert_eq!(
1250 v.get(slot),
1251 Some(0.0),
1252 "popularity slot {slot} carries a value, and the filters have not shipped"
1253 );
1254 }
1255 }
1256
1257 #[test]
1258 fn everything_past_the_last_block_stays_zero() {
1259 let v = vector("https://a.b.example.co.uk/x/y/z/page-12.html?q=hi&utm_source=x#frag");
1260
1261 for slot in FEATURES_USED..FEATURE_DIM {
1262 assert_eq!(
1263 v.get(slot),
1264 Some(0.0),
1265 "slot {slot} is not claimed by a block"
1266 );
1267 }
1268 }
1269
1270 const DISSIMILAR: [&str; 8] = [
1275 "www.nike.com",
1276 "www.pfizer.com",
1277 "www.goldmansachs.com",
1278 "www.weightwatchers.com",
1279 "www.nationalgeographic.com",
1280 "www.bostonglobe.com",
1281 "www.chess.com",
1282 "www.tripadvisor.com",
1283 ];
1284
1285 const DISSIMILAR_PAIRED_SUFFIX: [&str; 5] = [
1287 "www.bbc.co.uk",
1288 "www.hsbc.co.uk",
1289 "www.rakuten.co.jp",
1290 "www.woolworths.com.au",
1291 "www.globo.com.br",
1292 ];
1293
1294 #[test]
1295 fn the_same_shape_on_different_sites_is_the_same_vector() {
1296 let paths = [
1297 "/products/air-max-90?color=red&size=10",
1298 "/",
1299 "/news/world/a-long-story-about-things",
1300 "/api/v2/items.json?page=3&limit=50",
1301 "/assets/hero-image.png",
1302 "/search?q=something+private&sort=price",
1303 ];
1304
1305 let mut compared = 0usize;
1306
1307 for group in [DISSIMILAR.as_slice(), DISSIMILAR_PAIRED_SUFFIX.as_slice()] {
1308 for path in paths {
1309 let first_host = group.first().expect("a non empty group");
1310 let reference = vector(&format!("https://{first_host}{path}"));
1311
1312 for host in group.iter().skip(1) {
1313 let other = vector(&format!("https://{host}{path}"));
1314
1315 assert_eq!(
1316 reference, other,
1317 "{first_host} and {host} produced different vectors for {path}, \
1318 so some slot is reading the site rather than the shape"
1319 );
1320 compared += 1;
1321 }
1322 }
1323 }
1324
1325 assert_eq!(compared, (7 + 4) * 6);
1327 }
1328
1329 #[test]
1330 fn the_shape_itself_still_moves_the_vector() {
1331 let flat = vector("https://www.example.com/");
1334 let deep = vector("https://www.example.com/a/b/c/d/e/f/g");
1335 let data = vector("https://www.example.com/a/b/c/d/e/f/g.json");
1336
1337 assert_ne!(flat, deep);
1338 assert_ne!(deep, data);
1339 }
1340
1341 #[test]
1342 fn path_depth_and_length_are_bucketed() {
1343 let v = vector("https://example.com/a/b/c");
1344 assert_eq!(v.get(PATH_DEPTH + 3), Some(1.0));
1345 assert_eq!(v.get(PATH_DEPTH + 2), Some(0.0));
1346
1347 let deep = vector("https://example.com/a/b/c/d/e/f/g/h/i");
1348 assert_eq!(deep.get(PATH_DEPTH + 6), Some(1.0));
1349 }
1350
1351 #[test]
1352 fn segment_classes_split_the_ways_paths_are_generated() {
1353 assert_eq!(segment_class("12345"), SegmentClass::Digits);
1354 assert_eq!(segment_class("about"), SegmentClass::Alpha);
1355 assert_eq!(segment_class("page2"), SegmentClass::AlphaNum);
1356 assert_eq!(segment_class("a-long-story"), SegmentClass::Hyphenated);
1357 assert_eq!(segment_class("a_long_story"), SegmentClass::Underscored);
1358 assert_eq!(
1359 segment_class("f47ac10b-58cc-4372-a567-0e02b2c3d479"),
1360 SegmentClass::HexIdentifier
1361 );
1362 assert_eq!(segment_class("caf%C3%A9"), SegmentClass::Escaped);
1363 assert_eq!(segment_class("a+b"), SegmentClass::Other);
1364 }
1365
1366 #[test]
1367 fn a_non_ascii_segment_is_classified_without_splitting_a_character() {
1368 assert_eq!(segment_class("\u{5546}\u{54c1}"), SegmentClass::Alpha);
1371 let v = vector("https://example.com/\u{5546}\u{54c1}/1");
1372 assert_eq!(v.get(PATH_DEPTH + 2), Some(1.0));
1373 }
1374
1375 #[test]
1376 fn extensions_are_classified_and_versions_are_not_extensions() {
1377 assert_eq!(extension_class("index.html"), ExtClass::Markup);
1378 assert_eq!(extension_class("feed.xml"), ExtClass::Xml);
1379 assert_eq!(extension_class("items.json"), ExtClass::Json);
1380 assert_eq!(extension_class("report.PDF"), ExtClass::Pdf);
1381 assert_eq!(extension_class("hero.png"), ExtClass::Image);
1382 assert_eq!(extension_class("about"), ExtClass::None);
1383 assert_eq!(extension_class("v1.2.3"), ExtClass::None);
1384 assert_eq!(extension_class("archive.tar"), ExtClass::Archive);
1385 assert_eq!(extension_class("thing.wobble"), ExtClass::Other);
1386 }
1387
1388 #[test]
1389 fn query_keys_are_read_and_values_are_not() {
1390 let with_value = vector("https://example.com/s?q=a+very+identifying+search");
1391 let other_value = vector("https://example.com/s?q=b");
1392
1393 assert_eq!(
1394 with_value, other_value,
1395 "two different search terms changed the vector, so a value is being read"
1396 );
1397 assert_eq!(
1398 with_value.get(QUERY_KEYS + KeyClass::Search.index()),
1399 Some(1.0)
1400 );
1401 assert_eq!(
1402 with_value.get(QUERY_KEYS + KeyClass::SingleChar.index()),
1403 Some(1.0)
1404 );
1405 }
1406
1407 #[test]
1408 fn a_key_can_fall_into_several_classes_at_once() {
1409 let v = vector("https://example.com/l?page=2&utm_source=mail&sort=price&id=9");
1410
1411 for class in [
1412 KeyClass::Pagination,
1413 KeyClass::Tracking,
1414 KeyClass::Filter,
1415 KeyClass::Identifier,
1416 ] {
1417 assert_eq!(v.get(QUERY_KEYS + class.index()), Some(1.0), "{class:?}");
1418 }
1419
1420 assert_eq!(v.get(QUERY_ARITY + 4), Some(1.0));
1421 }
1422
1423 #[test]
1424 fn cold_start_is_a_value_rather_than_a_gap() {
1425 let v = vector("https://example.com/");
1426
1427 assert_eq!(v.get(MEM_COUNT), Some(1.0));
1428 assert_eq!(v.get(MEM_STATUS + StatusClass::Unknown.index()), Some(1.0));
1429 assert_eq!(v.get(MEM_FAIL_STREAK), Some(1.0));
1430 }
1431
1432 #[test]
1433 fn memory_lands_in_its_own_block() {
1434 let url = Url::parse("https://example.com/").unwrap();
1435 let memory = SiteMemory {
1436 observations: 12,
1437 success_rate: 0.1,
1438 streak: -3,
1439 last_status: StatusClass::Blocked,
1440 };
1441 let v = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown).with_memory(&memory));
1442
1443 assert_eq!(v.get(MEM_COUNT + 3), Some(1.0));
1444 assert_eq!(v.get(MEM_SUCCESS), Some(1.0));
1445 assert_eq!(v.get(MEM_FAIL_STREAK + 3), Some(1.0));
1446 assert_eq!(v.get(MEM_WIN_STREAK), Some(1.0));
1447 assert_eq!(v.get(MEM_STATUS + StatusClass::Blocked.index()), Some(1.0));
1448 }
1449
1450 #[test]
1451 fn the_hour_is_two_waves_and_an_unknown() {
1452 let url = Url::parse("https://example.com/").unwrap();
1453 let unknown = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown));
1454 assert_eq!(unknown.get(HOUR + 2), Some(1.0));
1455 assert_eq!(unknown.get(HOUR), Some(0.0));
1456
1457 let noon = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown).at_hour(12));
1458 assert_eq!(noon.get(HOUR + 2), Some(0.0));
1459 assert!(noon.get(HOUR + 1).unwrap_or(0.0) < -0.99);
1460
1461 let dawn = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown).at_hour(6));
1463 assert!(dawn.get(HOUR).unwrap_or(0.0) > 0.99);
1464 assert!(dawn.get(HOUR + 1).unwrap_or(1.0).abs() < 0.01);
1465
1466 let dusk = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown).at_hour(18));
1467 assert!(dusk.get(HOUR).unwrap_or(0.0) < -0.99);
1468
1469 let late = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown).at_hour(23));
1472 let midnight = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown).at_hour(0));
1473 let gap = (late.get(HOUR).unwrap_or(0.0) - midnight.get(HOUR).unwrap_or(0.0)).abs();
1474 assert!(gap < 0.3, "the clock does not wrap, gap was {gap}");
1475 }
1476
1477 #[test]
1478 fn an_out_of_range_hour_is_clamped_rather_than_wrapping_into_a_panic() {
1479 let url = Url::parse("https://example.com/").unwrap();
1480 let v = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown).at_hour(200));
1481 assert_eq!(v.get(HOUR + 2), Some(0.0));
1482 }
1483
1484 #[test]
1485 fn pins_are_recorded_without_being_obeyed_here() {
1486 let url = Url::parse("https://example.com/").unwrap();
1487 let country = Country::new("de").unwrap();
1488 let pins = CallerPins {
1489 mode: Some(RequestMode::Browser),
1490 proxy: Some(ProxyPool::Residential),
1491 country: Some(&country),
1492 };
1493 let v = featurize(&RouteInput::new(&url, DeclaredNeed::Markdown).with_pins(pins));
1494
1495 assert_eq!(v.get(PINS), Some(1.0));
1496 assert_eq!(v.get(PINS + 3), Some(1.0));
1497 assert_eq!(v.get(PINS + 5), Some(1.0));
1498 assert_eq!(v.get(PINS + 6), Some(1.0));
1499 }
1500
1501 #[test]
1502 fn a_site_with_no_path_is_not_a_trailing_slash_site() {
1503 let bare = vector("https://example.com");
1504 let slashed = vector("https://example.com/");
1505
1506 assert_eq!(bare, slashed);
1507 assert_eq!(bare.get(TRAILING_SLASH), Some(0.0));
1508 assert_eq!(bare.get(SEGMENT + SegmentClass::Empty.index()), Some(1.0));
1509 }
1510
1511 #[test]
1512 fn url_flags_read_the_request_and_not_the_site() {
1513 let plain = vector("http://example.com/a");
1514 assert_eq!(plain.get(URL_FLAGS), Some(0.0));
1515
1516 let odd = vector("https://user:pw@example.com:8443/a#top");
1517 assert_eq!(odd.get(URL_FLAGS), Some(1.0));
1518 assert_eq!(odd.get(URL_FLAGS + 1), Some(1.0));
1519 assert_eq!(odd.get(URL_FLAGS + 2), Some(1.0));
1520 assert_eq!(odd.get(URL_FLAGS + 3), Some(1.0));
1521 }
1522
1523 #[test]
1524 fn the_bias_slot_is_always_set() {
1525 assert_eq!(vector("https://example.com/").get(BIAS), Some(1.0));
1526 }
1527}