spider 2.51.49

A web crawler and scraper, building blocks for data curation workloads.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

use crate::features::chrome_common::Viewport;
use rand::distr::{weighted::WeightedIndex, Distribution};
use rand::prelude::IndexedRandom;
use rand::rngs::ThreadRng;
use rand::Rng;

/// Represents different types of devices for viewport simulation.
///
/// The `DeviceType` enum categorizes devices into three main types:
/// `Mobile`, `Tablet`, and `Desktop`. These categories are typically used
/// to simulate and test varying screen resolutions and viewports in
/// web development, ensuring that web applications render correctly
/// across different platforms.
#[derive(Default, Clone, Copy, Debug)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub enum DeviceType {
    /// Represents a mobile device.
    ///
    /// Mobile devices generally have smaller screens, and their
    /// viewport sizes range from small to medium dimensions, typically
    /// used for smartphones.
    #[cfg_attr(feature = "serde", serde(rename = "mobile"))]
    Mobile,
    /// Represents a tablet device.
    ///
    /// Tablet devices have medium-sized screens, which are larger than
    /// those of mobile devices but smaller than desktop monitors.
    /// Viewport dimensions for tablets are typically larger than mobiles
    /// and intended for medium-scale interfaces.
    #[cfg_attr(feature = "serde", serde(rename = "tablet"))]
    Tablet,
    #[default]
    /// Represents a desktop device.
    ///
    /// Desktop devices have larger screens compared to mobile and
    /// tablet devices. Viewport dimensions for desktops are typically
    /// the largest among the three, intended for full-scale applications
    /// on monitors.
    #[cfg_attr(feature = "serde", serde(rename = "desktop"))]
    Desktop,
}

/// Randomize viewport dimensions based on device type
pub fn randomize_viewport(device: &DeviceType) -> Viewport {
    let mut rng = rand::rng();

    randomize_viewport_rng(device, &mut rng)
}

/// Randomize viewport dimensions based on device type.
pub fn randomize_viewport_rng(device: &DeviceType, rng: &mut ThreadRng) -> Viewport {
    match device {
        DeviceType::Mobile => {
            let width = rng.random_range(320..=480);
            let height = rng.random_range(480..=800);
            Viewport::new(width, height)
        }
        DeviceType::Tablet => {
            let width = rng.random_range(600..=800);
            let height = rng.random_range(800..=1200);
            Viewport::new(width, height)
        }
        DeviceType::Desktop => {
            let width = rng.random_range(1024..=1920);
            let height = rng.random_range(768..=1080);
            Viewport::new(width, height)
        }
    }
}
/// Get a random viewport by selecting a random device type first. The weights are aligned in favor of desktop.
pub fn get_random_viewport() -> Viewport {
    let mut rng = rand::rng();
    let device_types = [DeviceType::Mobile, DeviceType::Tablet, DeviceType::Desktop];

    randomize_viewport(if let Ok(dist) = WeightedIndex::new([1, 1, 3]) {
        &device_types[dist.sample(&mut rng)]
    } else {
        device_types
            .choose(&mut rng)
            .unwrap_or(&DeviceType::Desktop)
    })
}