crawlex 1.0.4

Stealth crawler with Chrome-perfect TLS/H2 fingerprint, render pool, hooks, persistent queue
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

//! `IntelStorage` — host-level intelligence outputs.
//!
//! The discovery pipeline produces per-host facts (DNS, TLS cert, RDAP
//! registration), per-page asset references, and per-page tech
//! fingerprints. This trait is the sink for those outputs.
//!
//! Default no-ops let backends that don't want to track intel
//! (memory-only test sinks) compile clean.

use crate::storage::HostFacts;
use crate::Result;

/// Discovery / intel outputs sink.
#[async_trait::async_trait]
pub trait IntelStorage: Send + Sync {
    /// Persist a host-level facts bundle (DNS + cert + RDAP). Default no-op.
    async fn save_host_facts(&self, _host: &str, _facts: &HostFacts) -> Result<()> {
        Ok(())
    }

    /// Persist a batch of classified `AssetRef`s extracted from a page.
    /// Default no-op.
    async fn save_asset_refs(
        &self,
        _refs: &[crate::discovery::asset_refs::AssetRef],
    ) -> Result<()> {
        Ok(())
    }

    /// Persist a per-page technology fingerprint report and update any
    /// backend-specific host/domain rollups. Default no-op.
    async fn save_tech_fingerprint(
        &self,
        _report: &crate::discovery::tech_fingerprint::TechFingerprintReport,
    ) -> Result<()> {
        Ok(())
    }
}