domain-status 0.2.1

Concurrent URL status checker that captures comprehensive metadata in SQLite.
Documentation
//! Response data structures.

use std::collections::{HashMap, HashSet};
use std::sync::Arc;

/// Extracted response data from HTTP response.
///
/// `body` is `Arc<str>` because the body is consumed by both the
/// (CPU-bound, `spawn_blocking`-wrapped) HTML parser and the technology
/// detector. With `String` the runtime had to perform a 2 MB-class clone
/// before the spawn; with `Arc<str>` both consumers cheaply share a
/// reference-counted view (clone is a pointer + atomic refcount bump).
#[derive(Debug)]
pub(crate) struct ResponseData {
    pub(crate) initial_url: String,
    pub(crate) final_url: String,
    pub(crate) initial_domain: String,
    pub(crate) final_domain: String,
    pub(crate) host: String,
    pub(crate) status: u16,
    pub(crate) status_desc: String,
    pub(crate) headers: reqwest::header::HeaderMap,
    pub(crate) security_headers: HashMap<String, String>,
    pub(crate) http_headers: HashMap<String, String>,
    pub(crate) body: Arc<str>,
    pub(crate) body_sha256: Option<String>,
    /// True when the response body hit `MAX_RESPONSE_BODY_SIZE` and only a prefix was kept.
    pub(crate) body_truncated: bool,
    pub(crate) content_length: Option<i64>,
    pub(crate) http_version: Option<String>,
    pub(crate) content_type: Option<String>,
}

/// Extracted HTML data from parsed document.
#[derive(Debug)]
pub(crate) struct HtmlData {
    pub(crate) title: String,
    pub(crate) description: Option<String>,
    pub(crate) structured_data: crate::parse::StructuredData,
    pub(crate) social_media_links: Vec<crate::parse::SocialMediaLink>,
    pub(crate) contact_links: Vec<crate::parse::ContactLink>,
    pub(crate) exposed_secrets: Vec<crate::parse::ExposedSecret>,
    pub(crate) analytics_ids: Vec<crate::parse::AnalyticsId>, // Analytics/tracking IDs (GA, Facebook Pixel, GTM, AdSense)
    pub(crate) meta_tags: HashMap<String, Vec<String>>, // Vec to handle multiple meta tags with same name (e.g., multiple generator tags)
    pub(crate) script_sources: Vec<String>,
    pub(crate) script_tag_ids: HashSet<String>, // Script tag IDs (for __NEXT_DATA__ etc.)
    /// Concatenated inline `<script>` text (no `src`), lowercase, for `scripts` patterns.
    pub(crate) inline_script_text: String,
    /// First-party external script candidates considered when scanning is enabled.
    pub(crate) external_scripts_eligible: u32,
    /// External scripts successfully fetched/scanned (0 when flag off).
    pub(crate) external_scripts_scanned: u32,
    pub(crate) favicon_url: Option<String>, // Favicon URL extracted from <link rel="icon"> tags
    pub(crate) canonical_url: Option<String>, // Canonical URL from <link rel="canonical">
    pub(crate) meta_refresh_url: Option<String>, // Meta refresh redirect URL
    /// Content of `<meta name="robots">` when present
    pub(crate) meta_robots: Option<String>,
    pub(crate) resource_hints: Vec<(String, String)>, // (hint_type, href) for preconnect/dns-prefetch/preload/prefetch/modulepreload
}