crawlex 1.0.3

Stealth crawler with Chrome-perfect TLS/H2 fingerprint, render pool, hooks, persistent queue
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

use std::{fmt, str::FromStr};

use super::metadata::LatestVersionsPerMilestone;
use super::{Result, VersionError};
use crate::render::chrome_fetcher::{BrowserHost, BrowserKind, BuildInfo, Runtime};

/// Represents a milestone of a browser (e.g. 113).
#[derive(Clone, Copy, Debug)]
pub struct Milestone(u32);

impl Milestone {
    pub const fn new(milestone: u32) -> Self {
        Self(milestone)
    }

    pub(crate) async fn resolve(&self, kind: BrowserKind, host: &BrowserHost) -> Result<BuildInfo> {
        match kind {
            BrowserKind::Chromium => Err(VersionError::ResolveFailed(anyhow::anyhow!(
                "Milestone not supported for chromium"
            ))),
            BrowserKind::Chrome | BrowserKind::ChromeHeadlessShell => {
                let url = format!(
                    "{host}/chrome-for-testing/latest-versions-per-milestone.json",
                    host = host.metadata
                );
                let latest_versions_per_milestone =
                    Runtime::download_json::<LatestVersionsPerMilestone>(&url)
                        .await
                        .map_err(VersionError::ResolveFailed)?;
                let Some(version) = latest_versions_per_milestone
                    .milestones
                    .get(&self.to_string())
                else {
                    return Err(VersionError::InvalidMilestone(self.to_string()));
                };
                Ok(BuildInfo::both(
                    version.version.clone(),
                    version.revision.parse()?,
                ))
            }
        }
    }
}

impl fmt::Display for Milestone {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}", self.0)
    }
}

impl FromStr for Milestone {
    type Err = VersionError;

    fn from_str(s: &str) -> Result<Self, Self::Err> {
        let milestone = s
            .parse::<u32>()
            .map_err(|_| VersionError::InvalidMilestone(s.to_string()))?;
        Ok(Milestone(milestone))
    }
}

impl TryFrom<String> for Milestone {
    type Error = VersionError;

    fn try_from(value: String) -> Result<Self, Self::Error> {
        Self::from_str(&value)
    }
}

impl From<u32> for Milestone {
    fn from(value: u32) -> Self {
        Self(value)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_milestone_resolve_chrome() {
        let host = BrowserHost::current(BrowserKind::Chrome);
        let build_info = Milestone::new(113)
            .resolve(BrowserKind::Chrome, &host)
            .await
            .unwrap();
        assert_eq!(build_info.id, "113.0.5672.63");
    }
}