gossan-intel 0.3.3

Global Passive and Active Intel pillar for Gossan, part of the security research ecosystem
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

//! URLScan source: URL and domain scanning intelligence.

use async_trait::async_trait;
use serde::Deserialize;

use crate::enrichment::IntelEnrichment;
use crate::sources::IntelSource;

const BASE_URL: &str = "https://urlscan.io/api/v1";

/// URLScan API client.
pub struct UrlScanSource {
    client: reqwest::Client,
    api_key: Option<String>,
}

impl UrlScanSource {
    /// Create a new URLScan source.
    pub fn new(client: reqwest::Client, api_key: Option<String>) -> Self {
        Self { client, api_key }
    }
}

#[async_trait]
impl IntelSource for UrlScanSource {
    fn name(&self) -> &'static str {
        "urlscan"
    }

    async fn query_ip(&self, ip: &str) -> anyhow::Result<IntelEnrichment> {
        // URLScan search by IP
        let mut url = url::Url::parse(&format!("{BASE_URL}/search/"))?;
        url.query_pairs_mut().append_pair("q", &format!("ip:{ip}"));
        let mut req = self.client.get(url);
        if let Some(ref key) = self.api_key {
            req = req.header("API-Key", key);
        }
        let resp = req.send().await?.error_for_status()?;
        // URLScan search responses can include hundreds of historical
        // crawls per query; cap at 8 MiB.
        let body: UrlScanSearchResp = gossan_core::net::bounded_json(resp, super::MAX_INTEL_JSON_BYTES).await?;

        let mut enrichment = IntelEnrichment::new("urlscan", "ip", ip);
        enrichment.classification = body
            .results
            .first()
            .and_then(|r| r.page.as_ref())
            .and_then(|p| p.country.clone());

        for result in body.results.into_iter().take(10) {
            if let Some(server) = result.page.as_ref().and_then(|p| p.server.clone()) {
                enrichment.technologies.push(server);
            }
        }

        enrichment
            .raw
            .insert("total".to_string(), serde_json::to_value(body.total)?);

        Ok(enrichment)
    }

    async fn query_domain(&self, domain: &str) -> anyhow::Result<IntelEnrichment> {
        let mut url = url::Url::parse(&format!("{BASE_URL}/search/"))?;
        url.query_pairs_mut().append_pair("q", &format!("domain:{domain}"));
        let mut req = self.client.get(url);
        if let Some(ref key) = self.api_key {
            req = req.header("API-Key", key);
        }
        let resp = req.send().await?.error_for_status()?;
        // URLScan search responses can include hundreds of historical
        // crawls per query; cap at 8 MiB.
        let body: UrlScanSearchResp = gossan_core::net::bounded_json(resp, super::MAX_INTEL_JSON_BYTES).await?;

        let mut enrichment = IntelEnrichment::new("urlscan", "domain", domain);
        for result in body.results.into_iter().take(10) {
            if let Some(server) = result.page.as_ref().and_then(|p| p.server.clone()) {
                enrichment.technologies.push(server);
            }
        }

        enrichment
            .raw
            .insert("total".to_string(), serde_json::to_value(body.total)?);

        Ok(enrichment)
    }
}

#[derive(Debug, Deserialize)]
struct UrlScanSearchResp {
    #[serde(default)]
    total: u32,
    #[serde(default)]
    results: Vec<UrlScanResult>,
}

#[derive(Debug, Deserialize)]
struct UrlScanResult {
    #[serde(default)]
    page: Option<UrlScanPage>,
}

#[derive(Debug, Deserialize)]
struct UrlScanPage {
    #[serde(default)]
    country: Option<String>,
    #[serde(default)]
    server: Option<String>,
}